marquetapage 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 83186079a129d656cf05238db63b8d3b39fb9ce7
4
+ data.tar.gz: ecfb6c7775de3408543d1d95074490a8ad2fdb9c
5
+ SHA512:
6
+ metadata.gz: 5635907059b633c5b0cadb964c1be2d378ffd735f971884882dab4f421dcf06e024657914e75596b46e3605b45b0622240e3c8e12533bf508d68a3a4b0fed8c4
7
+ data.tar.gz: 23e1b4f7400e5631173c589111ab1ed5d087c1777bebb6704ec84bf1f42a97f3e5016a0b16600e8ca11209ab4dc26fab9473ec290f16464d2fea854cad2022a3
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
@@ -0,0 +1,33 @@
1
+ # marquetapage
2
+
3
+ Extract from `places.sqlite` firefox bookmarks with specific tag to JSON format.
4
+
5
+ ## Usage
6
+
7
+ ~~~
8
+ marquetapage TAG
9
+ ~~~
10
+
11
+ ## Output format
12
+
13
+ ~~~ json
14
+ [
15
+ {
16
+ "url": "",
17
+ "title": "",
18
+ "content": ""
19
+ }
20
+ ]
21
+ ~~~
22
+
23
+ ## Tests
24
+
25
+ ~~~
26
+ bundle exec rake
27
+ ~~~
28
+
29
+ ## Resources
30
+
31
+ * https://github.com/websafe/ffbx
32
+ * https://wiki.mozilla.org/images/d/d5/Places.sqlite.schema3.pdf
33
+ * https://developer.mozilla.org/en-US/docs/Mozilla/Tech/Places/Database
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'marquetapage/runner'
4
+
5
+ exit_status = Marquetapage::Runner.run(ARGV)
6
+ exit(exit_status)
@@ -0,0 +1,36 @@
1
+ require 'sequel'
2
+
3
+ # Marquetapage module handle env, db, and db connection
4
+ module Marquetapage
5
+ # Get current environment from MARQUETAPAGE_ENV
6
+ # Default: development
7
+ # @return String
8
+ def self.env
9
+ ENV['MARQUETAPAGE_ENV'] || 'production'
10
+ end
11
+
12
+ # Get places.sqlite from current env
13
+ # production will search first `.mozilla/firefox/xxx.default/places.sqlite`
14
+ # other environment will search current directory for `places.ENV.sqlite`
15
+ # @return String
16
+ def self.db_path
17
+ if env == 'production'
18
+ mozilla_path = "#{ENV['HOME']}/.mozilla/firefox/*.default/places.sqlite"
19
+ Dir.glob(mozilla_path).first
20
+ else
21
+ File.expand_path("../../places.#{env}.sqlite", __FILE__)
22
+ end
23
+ end
24
+
25
+ # Use DATABASE_URL env var or return sqlite connection string
26
+ # @return String
27
+ def self.db
28
+ ENV['DATABASE_URL'] || "sqlite://#{db_path}"
29
+ end
30
+
31
+ # Sequel connect
32
+ # @return Sequel::SQLite::Database
33
+ def self.sequel_connect
34
+ @sequel_connect ||= Sequel.connect(db)
35
+ end
36
+ end
@@ -0,0 +1,42 @@
1
+ require 'marquetapage'
2
+
3
+ Marquetapage.sequel_connect
4
+
5
+ Sequel::Model.plugin :json_serializer
6
+
7
+ # moz_places Sequel class
8
+ class Place < Sequel::Model(:moz_places)
9
+ # make aliasing work with json_serializer
10
+ # @return String, NilClass
11
+ def title
12
+ self[:title]
13
+ end
14
+
15
+ # make aliasing work with json_serializer
16
+ # @return String, NilClass
17
+ def url
18
+ self[:url]
19
+ end
20
+
21
+ # make aliasing work with json_serializer
22
+ # @return String, NilClass
23
+ def content
24
+ self[:content]
25
+ end
26
+ end
27
+
28
+ # moz_bookmarks_roots Sequel class
29
+ class BookmarkRoot < Sequel::Model(:moz_bookmarks_roots)
30
+ end
31
+
32
+ # moz_bookmarks Sequel class
33
+ class Bookmark < Sequel::Model(:moz_bookmarks)
34
+ end
35
+
36
+ # moz_items_annos Sequel class
37
+ class ItemAnnotation < Sequel::Model(:moz_items_annos)
38
+ end
39
+
40
+ # moz_anno_attributes Sequel class
41
+ class AnnotationAttribute < Sequel::Model(:moz_anno_attributes)
42
+ end
@@ -0,0 +1,86 @@
1
+ require 'marquetapage'
2
+ require 'marquetapage/models'
3
+
4
+ module Marquetapage
5
+ # Execute sql command to sqlite
6
+ class Runner
7
+ # Execute query and output as json the result
8
+ # @example run(['firefox'])
9
+ # @param [ARGV] args must include a tag
10
+ # @return [Integer] Exit status
11
+ def self.run(args)
12
+ new.run(args)
13
+ end
14
+
15
+ # Execute query and output as json the result
16
+ # @example run(['firefox'])
17
+ # @param [ARGV] args must include a tag
18
+ # @return [Integer] Exit status
19
+ def run(args)
20
+ fail ArgumentError unless args
21
+ usage if args.empty?
22
+ trap_interrupt
23
+ @tag_name = args.first
24
+ output_bookmarks
25
+ end
26
+
27
+ private
28
+
29
+ # Print usage
30
+ def usage
31
+ puts 'Usage: firefox-bookmarks-extract TAG'
32
+ exit!(1)
33
+ end
34
+
35
+ # Trac Ctrl-c and exit cleanly
36
+ def trap_interrupt
37
+ trap('INT') do
38
+ STDERR.puts "\nExiting..."
39
+ exit!(1)
40
+ end
41
+ end
42
+
43
+ def output_bookmarks
44
+ puts bookmarks.all.to_json
45
+ @exit_status = 0
46
+ rescue Sequel::Error => e
47
+ puts e
48
+ @exit_status = 1
49
+ end
50
+
51
+ def bookmarks
52
+ Place
53
+ .select(:moz_places__url,
54
+ :moz_bookmarks__title,
55
+ :moz_items_annos__content)
56
+ .from(:moz_places,
57
+ :moz_bookmarks,
58
+ :moz_items_annos,
59
+ :moz_anno_attributes)
60
+ .where(where_bookmarks_hash)
61
+ end
62
+
63
+ def where_bookmarks_hash
64
+ {
65
+ moz_anno_attributes__name: 'bookmarkProperties/description',
66
+ moz_items_annos__anno_attribute_id: :moz_anno_attributes__id,
67
+ moz_items_annos__item_id: :moz_bookmarks__id,
68
+ moz_bookmarks__fk: :moz_places__id,
69
+ moz_places__id: bookmark_place_ids
70
+ }
71
+ end
72
+
73
+ def bookmark_place_ids
74
+ Bookmark.distinct.select(:fk).where(parent: tag)
75
+ end
76
+
77
+ def tag
78
+ Bookmark.join(
79
+ :moz_bookmarks_roots,
80
+ root_name: 'tags',
81
+ moz_bookmarks__parent: :moz_bookmarks_roots__folder_id,
82
+ moz_bookmarks__title: @tag_name
83
+ ).select(:id)
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,5 @@
1
+ # Version file
2
+ module Marquetapage
3
+ # Version constant
4
+ VERSION = '0.0.1'
5
+ end
@@ -0,0 +1,47 @@
1
+
2
+ CREATE TABLE moz_bookmarks (id INTEGER PRIMARY KEY,type INTEGER, fk INTEGER DEFAULT NULL, parent INTEGER, position INTEGER, title LONGVARCHAR, keyword_id INTEGER, folder_type TEXT, dateAdded INTEGER, lastModified INTEGER, guid TEXT);
3
+ CREATE TABLE moz_bookmarks_roots (root_name VARCHAR(16) UNIQUE, folder_id INTEGER);
4
+ CREATE TABLE moz_keywords (id INTEGER PRIMARY KEY AUTOINCREMENT, keyword TEXT UNIQUE, place_id INTEGER, post_data TEXT);
5
+ CREATE TABLE moz_favicons (id INTEGER PRIMARY KEY, url LONGVARCHAR UNIQUE, data BLOB, mime_type VARCHAR(32), expiration LONG, guid TEXT);
6
+ CREATE TABLE moz_annos (id INTEGER PRIMARY KEY,place_id INTEGER NOT NULL,anno_attribute_id INTEGER,mime_type VARCHAR(32) DEFAULT NULL,content LONGVARCHAR, flags INTEGER DEFAULT 0,expiration INTEGER DEFAULT 0,type INTEGER DEFAULT 0,dateAdded INTEGER DEFAULT 0,lastModified INTEGER DEFAULT 0);
7
+ CREATE TABLE moz_anno_attributes (id INTEGER PRIMARY KEY,name VARCHAR(32) UNIQUE NOT NULL);
8
+ CREATE TABLE moz_items_annos (id INTEGER PRIMARY KEY,item_id INTEGER NOT NULL,anno_attribute_id INTEGER,mime_type VARCHAR(32) DEFAULT NULL,content LONGVARCHAR, flags INTEGER DEFAULT 0,expiration INTEGER DEFAULT 0,type INTEGER DEFAULT 0,dateAdded INTEGER DEFAULT 0,lastModified INTEGER DEFAULT 0);
9
+ CREATE TABLE moz_places (id INTEGER PRIMARY KEY, url LONGVARCHAR, title LONGVARCHAR, rev_host LONGVARCHAR, visit_count INTEGER DEFAULT 0, hidden INTEGER DEFAULT 0 NOT NULL, typed INTEGER DEFAULT 0 NOT NULL, favicon_id INTEGER, frecency INTEGER DEFAULT -1 NOT NULL, last_visit_date INTEGER, guid TEXT, foreign_count INTEGER DEFAULT 0 NOT NULL);
10
+ CREATE TABLE moz_historyvisits (id INTEGER PRIMARY KEY, from_visit INTEGER, place_id INTEGER, visit_date INTEGER, visit_type INTEGER, session INTEGER);
11
+ CREATE TABLE moz_inputhistory (place_id INTEGER NOT NULL, input LONGVARCHAR NOT NULL, use_count INTEGER, PRIMARY KEY (place_id, input));
12
+ CREATE TABLE moz_hosts ( id INTEGER PRIMARY KEY, host TEXT NOT NULL UNIQUE, frecency INTEGER, typed INTEGER NOT NULL DEFAULT 0, prefix TEXT);
13
+ CREATE INDEX moz_bookmarks_itemindex ON moz_bookmarks (fk, type);
14
+ CREATE INDEX moz_bookmarks_parentindex ON moz_bookmarks (parent, position);
15
+ CREATE INDEX moz_bookmarks_itemlastmodifiedindex ON moz_bookmarks (fk, lastModified);
16
+ CREATE INDEX moz_places_faviconindex ON moz_places (favicon_id);
17
+ CREATE INDEX moz_places_hostindex ON moz_places (rev_host);
18
+ CREATE INDEX moz_places_visitcount ON moz_places (visit_count);
19
+ CREATE INDEX moz_places_frecencyindex ON moz_places (frecency);
20
+ CREATE INDEX moz_historyvisits_placedateindex ON moz_historyvisits (place_id, visit_date);
21
+ CREATE INDEX moz_historyvisits_fromindex ON moz_historyvisits (from_visit);
22
+ CREATE INDEX moz_historyvisits_dateindex ON moz_historyvisits (visit_date);
23
+ CREATE INDEX moz_places_lastvisitdateindex ON moz_places (last_visit_date);
24
+ CREATE UNIQUE INDEX moz_annos_placeattributeindex ON moz_annos (place_id, anno_attribute_id);
25
+ CREATE UNIQUE INDEX moz_items_annos_itemattributeindex ON moz_items_annos (item_id, anno_attribute_id);
26
+ CREATE UNIQUE INDEX moz_places_url_uniqueindex ON moz_places (url);
27
+ CREATE UNIQUE INDEX moz_bookmarks_guid_uniqueindex ON moz_bookmarks (guid);
28
+ CREATE UNIQUE INDEX moz_places_guid_uniqueindex ON moz_places (guid);
29
+ CREATE UNIQUE INDEX moz_keywords_placepostdata_uniqueindex ON moz_keywords (place_id, post_data);
30
+ ANALYZE sqlite_master;
31
+ ANALYZE sqlite_master;
32
+ INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_guid_uniqueindex','101291 1');
33
+ INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_url_uniqueindex','101291 1');
34
+ INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_lastvisitdateindex','101291 2');
35
+ INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_frecencyindex','101291 94');
36
+ INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_visitcount','101291 719');
37
+ INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_hostindex','101291 13');
38
+ INSERT INTO sqlite_stat1 VALUES('moz_places','moz_places_faviconindex','101291 21');
39
+ INSERT INTO sqlite_stat1 VALUES('moz_bookmarks','moz_bookmarks_guid_uniqueindex','470 1');
40
+ INSERT INTO sqlite_stat1 VALUES('moz_bookmarks','moz_bookmarks_itemlastmodifiedindex','470 3 1');
41
+ INSERT INTO sqlite_stat1 VALUES('moz_bookmarks','moz_bookmarks_parentindex','470 11 1');
42
+ INSERT INTO sqlite_stat1 VALUES('moz_bookmarks','moz_bookmarks_itemindex','470 3 3');
43
+ INSERT INTO sqlite_stat1 VALUES('moz_historyvisits','moz_historyvisits_dateindex','177617 2');
44
+ INSERT INTO sqlite_stat1 VALUES('moz_historyvisits','moz_historyvisits_fromindex','177617 2');
45
+ INSERT INTO sqlite_stat1 VALUES('moz_historyvisits','moz_historyvisits_placedateindex','177617 2 2');
46
+ INSERT INTO sqlite_stat1 VALUES('moz_inputhistory','sqlite_autoindex_moz_inputhistory_1','451 2 1');
47
+ ANALYZE sqlite_master;
@@ -0,0 +1,46 @@
1
+ require File.expand_path(File.join(File.dirname(__FILE__), 'test_helper'))
2
+
3
+ describe Marquetapage::Runner do
4
+ describe '.run' do
5
+ it 'without arg' do
6
+ -> { Marquetapage::Runner.run }.must_raise ArgumentError
7
+ end
8
+
9
+ it 'with arg nil' do
10
+ -> { Marquetapage::Runner.run(nil) }.must_raise ArgumentError
11
+ end
12
+
13
+ describe 'with a present tag' do
14
+ before do
15
+ @tag_name = 'firefox'
16
+ @url = 'https://www.mozilla.org/en-US/firefox/central/'
17
+ place_id = Place.insert url: @url
18
+ bookmark_root_folder_id = 4
19
+ BookmarkRoot.insert root_name: 'tags',
20
+ folder_id: bookmark_root_folder_id
21
+ tag_id = Bookmark.insert type: 2, fk: nil,
22
+ parent: bookmark_root_folder_id,
23
+ title: @tag_name,
24
+ keyword_id: nil, folder_type: nil
25
+ bookmark_id = Bookmark.insert type: 1, fk: place_id, parent: tag_id,
26
+ position: 8, title: nil, keyword_id: nil,
27
+ folder_type: nil
28
+ anno_name = 'bookmarkProperties/description'
29
+ anno_attribute_id = AnnotationAttribute.insert name: anno_name
30
+ @content = 'description'
31
+ ItemAnnotation.insert item_id: bookmark_id,
32
+ anno_attribute_id: anno_attribute_id,
33
+ content: @content
34
+ end
35
+
36
+ it 'with good arg' do
37
+ out, err = capture_io do
38
+ Marquetapage::Runner.run([@tag_name])
39
+ end
40
+ out.chomp.must_match @url
41
+ out.chomp.must_match @content
42
+ err.chomp.must_equal ''
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,11 @@
1
+ ENV['MARQUETAPAGE_ENV'] = 'test'
2
+ require 'minitest/autorun'
3
+ require 'marquetapage'
4
+ require 'marquetapage/runner'
5
+
6
+ DB = Marquetapage.sequel_connect
7
+ %i(places anno_attributes items_annos bookmarks bookmarks_roots
8
+ keywords favicons annos historyvisits inputhistory hosts).each do |t|
9
+ DB.drop_table(:"moz_#{t}") if DB.table_exists?(:"moz_#{t}")
10
+ end
11
+ DB.run(File.read(File.expand_path('../moz_places.schema', __FILE__)))
metadata ADDED
@@ -0,0 +1,169 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: marquetapage
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Laurent Arnoud
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-11-01 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: json
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.8'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.8'
27
+ - !ruby/object:Gem::Dependency
28
+ name: sequel
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '4.27'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '4.27'
41
+ - !ruby/object:Gem::Dependency
42
+ name: sqlite3
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.3'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.3'
55
+ - !ruby/object:Gem::Dependency
56
+ name: minitest
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '5.6'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '5.6'
69
+ - !ruby/object:Gem::Dependency
70
+ name: minitest-line
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '0.6'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '0.6'
83
+ - !ruby/object:Gem::Dependency
84
+ name: inch
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '0.7'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '0.7'
97
+ - !ruby/object:Gem::Dependency
98
+ name: rake
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '10'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '10'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rubocop
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.34'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.34'
125
+ description: marquetapage
126
+ email: laurent@spkdev.net
127
+ executables:
128
+ - marquetapage
129
+ extensions: []
130
+ extra_rdoc_files:
131
+ - README.md
132
+ files:
133
+ - Gemfile
134
+ - README.md
135
+ - bin/marquetapage
136
+ - lib/marquetapage.rb
137
+ - lib/marquetapage/models.rb
138
+ - lib/marquetapage/runner.rb
139
+ - lib/marquetapage/version.rb
140
+ - test/moz_places.schema
141
+ - test/runner_test.rb
142
+ - test/test_helper.rb
143
+ homepage: http://github.com/spk/marquetapage
144
+ licenses:
145
+ - MIT
146
+ metadata: {}
147
+ post_install_message:
148
+ rdoc_options: []
149
+ require_paths:
150
+ - lib
151
+ required_ruby_version: !ruby/object:Gem::Requirement
152
+ requirements:
153
+ - - ">="
154
+ - !ruby/object:Gem::Version
155
+ version: 1.9.3
156
+ required_rubygems_version: !ruby/object:Gem::Requirement
157
+ requirements:
158
+ - - ">="
159
+ - !ruby/object:Gem::Version
160
+ version: '0'
161
+ requirements: []
162
+ rubyforge_project:
163
+ rubygems_version: 2.4.5
164
+ signing_key:
165
+ specification_version: 4
166
+ summary: Extract to JSON bookmarks from `places.sqlite` with specific tag
167
+ test_files:
168
+ - test/runner_test.rb
169
+ has_rdoc: