marquetapage 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 83186079a129d656cf05238db63b8d3b39fb9ce7
+  data.tar.gz: ecfb6c7775de3408543d1d95074490a8ad2fdb9c
+SHA512:
+  metadata.gz: 5635907059b633c5b0cadb964c1be2d378ffd735f971884882dab4f421dcf06e024657914e75596b46e3605b45b0622240e3c8e12533bf508d68a3a4b0fed8c4
+  data.tar.gz: 23e1b4f7400e5631173c589111ab1ed5d087c1777bebb6704ec84bf1f42a97f3e5016a0b16600e8ca11209ab4dc26fab9473ec290f16464d2fea854cad2022a3

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/README.md

@@ -0,0 +1,33 @@
+# marquetapage
+
+Extract from `places.sqlite` firefox bookmarks with specific tag to JSON format.
+
+## Usage
+
+~~~
+marquetapage TAG
+~~~
+
+## Output format
+
+~~~ json
+[
+  {
+    "url": "",
+    "title": "",
+    "content": ""
+  }
+]
+~~~
+
+## Tests
+
+~~~
+bundle exec rake
+~~~
+
+## Resources
+
+* https://github.com/websafe/ffbx
+* https://wiki.mozilla.org/images/d/d5/Places.sqlite.schema3.pdf
+* https://developer.mozilla.org/en-US/docs/Mozilla/Tech/Places/Database

data/bin/marquetapage

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'marquetapage/runner'
+
+exit_status = Marquetapage::Runner.run(ARGV)
+exit(exit_status)

data/lib/marquetapage.rb

@@ -0,0 +1,36 @@
+require 'sequel'
+
+# Marquetapage module handle env, db, and db connection
+module Marquetapage
+  # Get current environment from MARQUETAPAGE_ENV
+  # Default: development
+  # @return String
+  def self.env
+    ENV['MARQUETAPAGE_ENV'] || 'production'
+  end
+
+  # Get places.sqlite from current env
+  # production will search first `.mozilla/firefox/xxx.default/places.sqlite`
+  # other environment will search current directory for `places.ENV.sqlite`
+  # @return String
+  def self.db_path
+    if env == 'production'
+      mozilla_path = "#{ENV['HOME']}/.mozilla/firefox/*.default/places.sqlite"
+      Dir.glob(mozilla_path).first
+    else
+      File.expand_path("../../places.#{env}.sqlite", __FILE__)
+    end
+  end
+
+  # Use DATABASE_URL env var or return sqlite connection string
+  # @return String
+  def self.db
+    ENV['DATABASE_URL'] || "sqlite://#{db_path}"
+  end
+
+  # Sequel connect
+  # @return Sequel::SQLite::Database
+  def self.sequel_connect
+    @sequel_connect ||= Sequel.connect(db)
+  end
+end

data/lib/marquetapage/models.rb

@@ -0,0 +1,42 @@
+require 'marquetapage'
+
+Marquetapage.sequel_connect
+
+Sequel::Model.plugin :json_serializer
+
+# moz_places Sequel class
+class Place < Sequel::Model(:moz_places)
+  # make aliasing work with json_serializer
+  # @return String, NilClass
+  def title
+    self[:title]
+  end
+
+  # make aliasing work with json_serializer
+  # @return String, NilClass
+  def url
+    self[:url]
+  end
+
+  # make aliasing work with json_serializer
+  # @return String, NilClass
+  def content
+    self[:content]
+  end
+end
+
+# moz_bookmarks_roots Sequel class
+class BookmarkRoot < Sequel::Model(:moz_bookmarks_roots)
+end
+
+# moz_bookmarks Sequel class
+class Bookmark < Sequel::Model(:moz_bookmarks)
+end
+
+# moz_items_annos Sequel class
+class ItemAnnotation < Sequel::Model(:moz_items_annos)
+end
+
+# moz_anno_attributes Sequel class
+class AnnotationAttribute < Sequel::Model(:moz_anno_attributes)
+end

data/lib/marquetapage/runner.rb

@@ -0,0 +1,86 @@
+require 'marquetapage'
+require 'marquetapage/models'
+
+module Marquetapage
+  # Execute sql command to sqlite
+  class Runner
+    # Execute query and output as json the result
+    # @example run(['firefox'])
+    # @param [ARGV] args must include a tag
+    # @return [Integer] Exit status
+    def self.run(args)
+      new.run(args)
+    end
+
+    # Execute query and output as json the result
+    # @example run(['firefox'])
+    # @param [ARGV] args must include a tag
+    # @return [Integer] Exit status
+    def run(args)
+      fail ArgumentError unless args
+      usage if args.empty?
+      trap_interrupt
+      @tag_name = args.first
+      output_bookmarks
+    end
+
+    private
+
+    # Print usage
+    def usage
+      puts 'Usage: firefox-bookmarks-extract TAG'
+      exit!(1)
+    end
+
+    # Trac Ctrl-c and exit cleanly
+    def trap_interrupt
+      trap('INT') do
+        STDERR.puts "\nExiting..."
+        exit!(1)
+      end
+    end
+
+    def output_bookmarks
+      puts bookmarks.all.to_json
+      @exit_status = 0
+    rescue Sequel::Error => e
+      puts e
+      @exit_status = 1
+    end
+
+    def bookmarks
+      Place
+        .select(:moz_places__url,
+                :moz_bookmarks__title,
+                :moz_items_annos__content)
+        .from(:moz_places,
+              :moz_bookmarks,
+              :moz_items_annos,
+              :moz_anno_attributes)
+        .where(where_bookmarks_hash)
+    end
+
+    def where_bookmarks_hash
+      {
+        moz_anno_attributes__name: 'bookmarkProperties/description',
+        moz_items_annos__anno_attribute_id: :moz_anno_attributes__id,
+        moz_items_annos__item_id: :moz_bookmarks__id,
+        moz_bookmarks__fk: :moz_places__id,
+        moz_places__id: bookmark_place_ids
+      }
+    end
+
+    def bookmark_place_ids
+      Bookmark.distinct.select(:fk).where(parent: tag)
+    end
+
+    def tag
+      Bookmark.join(
+        :moz_bookmarks_roots,
+        root_name: 'tags',
+        moz_bookmarks__parent: :moz_bookmarks_roots__folder_id,
+        moz_bookmarks__title: @tag_name
+      ).select(:id)
+    end
+  end
+end

data/lib/marquetapage/version.rb

@@ -0,0 +1,5 @@
+# Version file
+module Marquetapage
+  # Version constant
+  VERSION = '0.0.1'
+end

data/test/moz_places.schema

@@ -0,0 +1,47 @@
+
+CREATE TABLE moz_bookmarks (id INTEGER PRIMARY KEY,type INTEGER, fk INTEGER DEFAULT NULL, parent INTEGER, position INTEGER, title LONGVARCHAR, keyword_id INTEGER, folder_type TEXT, dateAdded INTEGER, lastModified INTEGER, guid TEXT);
+CREATE TABLE moz_bookmarks_roots (root_name VARCHAR(16) UNIQUE, folder_id INTEGER);
+CREATE TABLE moz_keywords (id INTEGER PRIMARY KEY AUTOINCREMENT, keyword TEXT UNIQUE, place_id INTEGER, post_data TEXT);
+CREATE TABLE moz_favicons (id INTEGER PRIMARY KEY, url LONGVARCHAR UNIQUE, data BLOB, mime_type VARCHAR(32), expiration LONG, guid TEXT);
+CREATE TABLE moz_annos (id INTEGER PRIMARY KEY,place_id INTEGER NOT NULL,anno_attribute_id INTEGER,mime_type VARCHAR(32) DEFAULT NULL,content LONGVARCHAR, flags INTEGER DEFAULT 0,expiration INTEGER DEFAULT 0,type INTEGER DEFAULT 0,dateAdded INTEGER DEFAULT 0,lastModified INTEGER DEFAULT 0);
+CREATE TABLE moz_anno_attributes (id INTEGER PRIMARY KEY,name VARCHAR(32) UNIQUE NOT NULL);
+CREATE TABLE moz_items_annos (id INTEGER PRIMARY KEY,item_id INTEGER NOT NULL,anno_attribute_id INTEGER,mime_type VARCHAR(32) DEFAULT NULL,content LONGVARCHAR, flags INTEGER DEFAULT 0,expiration INTEGER DEFAULT 0,type INTEGER DEFAULT 0,dateAdded INTEGER DEFAULT 0,lastModified INTEGER DEFAULT 0);
+CREATE TABLE moz_places (id INTEGER PRIMARY KEY, url LONGVARCHAR, title LONGVARCHAR, rev_host LONGVARCHAR, visit_count INTEGER DEFAULT 0, hidden INTEGER DEFAULT 0 NOT NULL, typed INTEGER DEFAULT 0 NOT NULL, favicon_id INTEGER, frecency INTEGER DEFAULT -1 NOT NULL, last_visit_date INTEGER, guid TEXT, foreign_count INTEGER DEFAULT 0 NOT NULL);
+CREATE TABLE moz_historyvisits (id INTEGER PRIMARY KEY, from_visit INTEGER, place_id INTEGER, visit_date INTEGER, visit_type INTEGER, session INTEGER);
+CREATE TABLE moz_inputhistory (place_id INTEGER NOT NULL, input LONGVARCHAR NOT NULL, use_count INTEGER, PRIMARY KEY (place_id, input));
+CREATE TABLE moz_hosts (  id INTEGER PRIMARY KEY, host TEXT NOT NULL UNIQUE, frecency INTEGER, typed INTEGER NOT NULL DEFAULT 0, prefix TEXT);
+CREATE INDEX moz_bookmarks_itemindex ON moz_bookmarks (fk, type);
+CREATE INDEX moz_bookmarks_parentindex ON moz_bookmarks (parent, position);
+CREATE INDEX moz_bookmarks_itemlastmodifiedindex ON moz_bookmarks (fk, lastModified);
+CREATE INDEX moz_places_faviconindex ON moz_places (favicon_id);
+CREATE INDEX moz_places_hostindex ON moz_places (rev_host);
+CREATE INDEX moz_places_visitcount ON moz_places (visit_count);
+CREATE INDEX moz_places_frecencyindex ON moz_places (frecency);
+CREATE INDEX moz_historyvisits_placedateindex ON moz_historyvisits (place_id, visit_date);
+CREATE INDEX moz_historyvisits_fromindex ON moz_historyvisits (from_visit);
+CREATE INDEX moz_historyvisits_dateindex ON moz_historyvisits (visit_date);
+CREATE INDEX moz_places_lastvisitdateindex ON moz_places (last_visit_date);
+CREATE UNIQUE INDEX moz_annos_placeattributeindex ON moz_annos (place_id, anno_attribute_id);
+CREATE UNIQUE INDEX moz_items_annos_itemattributeindex ON moz_items_annos (item_id, anno_attribute_id);
+CREATE UNIQUE INDEX moz_places_url_uniqueindex ON moz_places (url);
+CREATE UNIQUE INDEX moz_bookmarks_guid_uniqueindex ON moz_bookmarks (guid);
+CREATE UNIQUE INDEX moz_places_guid_uniqueindex ON moz_places (guid);
+CREATE UNIQUE INDEX moz_keywords_placepostdata_uniqueindex ON moz_keywords (place_id, post_data);
+ANALYZE sqlite_master;
+ANALYZE sqlite_master;
+INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_guid_uniqueindex','101291 1');
+INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_url_uniqueindex','101291 1');
+INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_lastvisitdateindex','101291 2');
+INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_frecencyindex','101291 94');
+INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_visitcount','101291 719');
+INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_hostindex','101291 13');
+INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_faviconindex','101291 21');
+INSERT INTO sqlite_stat1 VALUES('moz_bookmarks','moz_bookmarks_guid_uniqueindex','470 1');
+INSERT INTO sqlite_stat1 VALUES('moz_bookmarks','moz_bookmarks_itemlastmodifiedindex','470 3 1');
+INSERT INTO sqlite_stat1 VALUES('moz_bookmarks','moz_bookmarks_parentindex','470 11 1');
+INSERT INTO sqlite_stat1 VALUES('moz_bookmarks','moz_bookmarks_itemindex','470 3 3');
+INSERT INTO sqlite_stat1 VALUES('moz_historyvisits','moz_historyvisits_dateindex','177617 2');
+INSERT INTO sqlite_stat1 VALUES('moz_historyvisits','moz_historyvisits_fromindex','177617 2');
+INSERT INTO sqlite_stat1 VALUES('moz_historyvisits','moz_historyvisits_placedateindex','177617 2 2');
+INSERT INTO sqlite_stat1 VALUES('moz_inputhistory','sqlite_autoindex_moz_inputhistory_1','451 2 1');
+ANALYZE sqlite_master;

data/test/runner_test.rb

@@ -0,0 +1,46 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'test_helper'))
+
+describe Marquetapage::Runner do
+  describe '.run' do
+    it 'without arg' do
+      -> { Marquetapage::Runner.run }.must_raise ArgumentError
+    end
+
+    it 'with arg nil' do
+      -> { Marquetapage::Runner.run(nil) }.must_raise ArgumentError
+    end
+
+    describe 'with a present tag' do
+      before do
+        @tag_name = 'firefox'
+        @url = 'https://www.mozilla.org/en-US/firefox/central/'
+        place_id = Place.insert url: @url
+        bookmark_root_folder_id = 4
+        BookmarkRoot.insert root_name: 'tags',
+                            folder_id: bookmark_root_folder_id
+        tag_id = Bookmark.insert type: 2, fk: nil,
+                                 parent: bookmark_root_folder_id,
+                                 title: @tag_name,
+                                 keyword_id: nil, folder_type: nil
+        bookmark_id = Bookmark.insert type: 1, fk: place_id, parent: tag_id,
+                                      position: 8, title: nil, keyword_id: nil,
+                                      folder_type: nil
+        anno_name = 'bookmarkProperties/description'
+        anno_attribute_id = AnnotationAttribute.insert name: anno_name
+        @content = 'description'
+        ItemAnnotation.insert item_id: bookmark_id,
+                              anno_attribute_id: anno_attribute_id,
+                              content: @content
+      end
+
+      it 'with good arg' do
+        out, err = capture_io do
+          Marquetapage::Runner.run([@tag_name])
+        end
+        out.chomp.must_match @url
+        out.chomp.must_match @content
+        err.chomp.must_equal ''
+      end
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,11 @@
+ENV['MARQUETAPAGE_ENV'] = 'test'
+require 'minitest/autorun'
+require 'marquetapage'
+require 'marquetapage/runner'
+
+DB = Marquetapage.sequel_connect
+%i(places anno_attributes items_annos bookmarks bookmarks_roots
+   keywords favicons annos historyvisits inputhistory hosts).each do |t|
+  DB.drop_table(:"moz_#{t}") if DB.table_exists?(:"moz_#{t}")
+end
+DB.run(File.read(File.expand_path('../moz_places.schema', __FILE__)))

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: marquetapage
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Laurent Arnoud
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.27'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.27'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.6'
+- !ruby/object:Gem::Dependency
+  name: minitest-line
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: inch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.34'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.34'
+description: marquetapage
+email: laurent@spkdev.net
+executables:
+- marquetapage
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- Gemfile
+- README.md
+- bin/marquetapage
+- lib/marquetapage.rb
+- lib/marquetapage/models.rb
+- lib/marquetapage/runner.rb
+- lib/marquetapage/version.rb
+- test/moz_places.schema
+- test/runner_test.rb
+- test/test_helper.rb
+homepage: http://github.com/spk/marquetapage
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Extract to JSON bookmarks from `places.sqlite` with specific tag
+test_files:
+- test/runner_test.rb
+has_rdoc: