browser.rb 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG +516 -0
- data/Gemfile +3 -0
- data/LICENSE +21 -0
- data/README.md +116 -0
- data/Rakefile +182 -0
- data/browser.rb.gemspec +54 -0
- data/lib/Browser/Base.rb +50 -0
- data/lib/Browser/Bookmark.rb +41 -0
- data/lib/Browser/Bookmarks.rb +112 -0
- data/lib/Browser/Brave.rb +16 -0
- data/lib/Browser/Chrome.rb +16 -0
- data/lib/Browser/Chromium.rb +16 -0
- data/lib/Browser/ChromiumBased.rb +48 -0
- data/lib/Browser/Firefox.rb +16 -0
- data/lib/Browser/FirefoxBased.rb +86 -0
- data/lib/Browser/History.rb +41 -0
- data/lib/Browser/Rendering.rb +108 -0
- data/lib/Browser/Safari.rb +48 -0
- data/lib/Browser/TorBrowser.rb +26 -0
- data/lib/Browser/Unreadable.rb +11 -0
- data/lib/Browser/VERSION.rb +6 -0
- data/lib/browser-classes.rb +12 -0
- data/lib/browser.rb +29 -0
- data/test/Browser/Bookmark_test.rb +72 -0
- data/test/Browser/Brave_test.rb +143 -0
- data/test/Browser/Chrome_test.rb +143 -0
- data/test/Browser/Chromium_test.rb +143 -0
- data/test/Browser/Firefox_test.rb +205 -0
- data/test/Browser/Rendering_test.rb +68 -0
- data/test/Browser/Safari_test.rb +152 -0
- data/test/Browser/TorBrowser_test.rb +147 -0
- data/test/browser_test.rb +33 -0
- data/test/fixtures/Brave_bookmarks.json +47 -0
- data/test/fixtures/ChromiumBased_history.sqlite +0 -0
- data/test/fixtures/Chromium_bookmarks.json +47 -0
- data/test/fixtures/FirefoxBased_places.sqlite +0 -0
- data/test/fixtures/Safari_bookmarks.plist +0 -0
- data/test/fixtures/Safari_history.sqlite +0 -0
- data/test/gemspec_test.rb +42 -0
- data/test/loading_test.rb +35 -0
- metadata +194 -0
data/Rakefile
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
require 'rake/testtask'
|
|
2
|
+
|
|
3
|
+
Rake::TestTask.new do |t|
|
|
4
|
+
t.test_files = FileList['test/**/*_test.rb']
|
|
5
|
+
end
|
|
6
|
+
|
|
7
|
+
task default: :test
|
|
8
|
+
|
|
9
|
+
namespace :fixtures do
|
|
10
|
+
desc "Build test/fixtures/ChromiumBased_history.sqlite"
|
|
11
|
+
task :chromium_based_history do
|
|
12
|
+
require 'fileutils'
|
|
13
|
+
require 'sqlite3'
|
|
14
|
+
path = File.join(__dir__, 'test/fixtures/ChromiumBased_history.sqlite')
|
|
15
|
+
FileUtils.rm_f(path)
|
|
16
|
+
db = SQLite3::Database.new(path)
|
|
17
|
+
# The urls table as Chromium creates it, shared by every Chromium-based browser.
|
|
18
|
+
db.execute_batch(<<~SQL)
|
|
19
|
+
CREATE TABLE urls(
|
|
20
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
21
|
+
url LONGVARCHAR,
|
|
22
|
+
title LONGVARCHAR,
|
|
23
|
+
visit_count INTEGER DEFAULT 0 NOT NULL,
|
|
24
|
+
typed_count INTEGER DEFAULT 0 NOT NULL,
|
|
25
|
+
last_visit_time INTEGER NOT NULL,
|
|
26
|
+
hidden INTEGER DEFAULT 0 NOT NULL
|
|
27
|
+
);
|
|
28
|
+
CREATE INDEX urls_url_index ON urls (url);
|
|
29
|
+
SQL
|
|
30
|
+
# last_visit_time is microseconds since 1601-01-01, which is Chromium's epoch.
|
|
31
|
+
[
|
|
32
|
+
['https://www.chromium.org/chromium-projects/', 'Home', 3, 1, 13403232000000000, 0],
|
|
33
|
+
['chrome://welcome/', 'Welcome', 1, 0, 13403231000000000, 0],
|
|
34
|
+
].each do |row|
|
|
35
|
+
db.execute('INSERT INTO urls (url, title, visit_count, typed_count, last_visit_time, hidden) VALUES (?, ?, ?, ?, ?, ?)', row)
|
|
36
|
+
end
|
|
37
|
+
db.close
|
|
38
|
+
puts "Wrote #{path} (#{File.size(path)} bytes)."
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
desc "Build test/fixtures/FirefoxBased_places.sqlite"
|
|
42
|
+
task :firefox_based_places do
|
|
43
|
+
require 'fileutils'
|
|
44
|
+
require 'sqlite3'
|
|
45
|
+
path = File.join(__dir__, 'test/fixtures/FirefoxBased_places.sqlite')
|
|
46
|
+
FileUtils.rm_f(path)
|
|
47
|
+
db = SQLite3::Database.new(path)
|
|
48
|
+
# moz_places, moz_bookmarks and moz_historyvisits as Firefox creates them. A
|
|
49
|
+
# bookmark's label is moz_bookmarks.title; moz_places.title is the page's own, and
|
|
50
|
+
# is empty until the page has been visited.
|
|
51
|
+
db.execute_batch(<<~SQL)
|
|
52
|
+
CREATE TABLE moz_origins (
|
|
53
|
+
id INTEGER PRIMARY KEY, prefix TEXT NOT NULL, host TEXT NOT NULL,
|
|
54
|
+
frecency INTEGER NOT NULL, recalc_frecency INTEGER NOT NULL DEFAULT 0,
|
|
55
|
+
alt_frecency INTEGER, recalc_alt_frecency INTEGER NOT NULL DEFAULT 0,
|
|
56
|
+
block_until_ms INTEGER, block_pages_until_ms INTEGER, UNIQUE (prefix, host)
|
|
57
|
+
);
|
|
58
|
+
CREATE TABLE moz_places (
|
|
59
|
+
id INTEGER PRIMARY KEY, url LONGVARCHAR, title LONGVARCHAR,
|
|
60
|
+
rev_host LONGVARCHAR, visit_count INTEGER DEFAULT 0,
|
|
61
|
+
hidden INTEGER DEFAULT 0 NOT NULL, typed INTEGER DEFAULT 0 NOT NULL,
|
|
62
|
+
frecency INTEGER DEFAULT -1 NOT NULL, last_visit_date INTEGER, guid TEXT,
|
|
63
|
+
foreign_count INTEGER DEFAULT 0 NOT NULL, url_hash INTEGER DEFAULT 0 NOT NULL,
|
|
64
|
+
description TEXT, preview_image_url TEXT, site_name TEXT,
|
|
65
|
+
origin_id INTEGER REFERENCES moz_origins(id),
|
|
66
|
+
recalc_frecency INTEGER NOT NULL DEFAULT 0, alt_frecency INTEGER,
|
|
67
|
+
recalc_alt_frecency INTEGER NOT NULL DEFAULT 0
|
|
68
|
+
);
|
|
69
|
+
CREATE TABLE moz_bookmarks (
|
|
70
|
+
id INTEGER PRIMARY KEY, type INTEGER, fk INTEGER DEFAULT NULL,
|
|
71
|
+
parent INTEGER, position INTEGER, title LONGVARCHAR, keyword_id INTEGER,
|
|
72
|
+
folder_type TEXT, dateAdded INTEGER, lastModified INTEGER, guid TEXT,
|
|
73
|
+
syncStatus INTEGER NOT NULL DEFAULT 0, syncChangeCounter INTEGER NOT NULL DEFAULT 1
|
|
74
|
+
);
|
|
75
|
+
CREATE TABLE moz_historyvisits (
|
|
76
|
+
id INTEGER PRIMARY KEY, from_visit INTEGER, place_id INTEGER,
|
|
77
|
+
visit_date INTEGER, visit_type INTEGER, session INTEGER,
|
|
78
|
+
source INTEGER DEFAULT 0 NOT NULL, triggeringPlaceId INTEGER
|
|
79
|
+
);
|
|
80
|
+
SQL
|
|
81
|
+
|
|
82
|
+
# dateAdded and visit_date are microseconds since 1970-01-01, which is Firefox's epoch.
|
|
83
|
+
added = 1787449000000000
|
|
84
|
+
|
|
85
|
+
[
|
|
86
|
+
[1, 'https://support.mozilla.org/products/firefox', nil, 'gro.allizom.troppus.'],
|
|
87
|
+
[2, 'https://support.mozilla.org/kb/customize-firefox-controls-buttons-and-toolbars', nil, 'gro.allizom.troppus.'],
|
|
88
|
+
[3, 'https://www.mozilla.org/contribute/', nil, 'gro.allizom.www.'],
|
|
89
|
+
[4, 'https://www.mozilla.org/about/', nil, 'gro.allizom.www.'],
|
|
90
|
+
[5, 'https://addons.mozilla.org/en-US/firefox/', 'Extension Starter Pack', 'gro.allizom.snodda.'],
|
|
91
|
+
].each do |id, url, title, rev_host|
|
|
92
|
+
db.execute('INSERT INTO moz_places (id, url, title, rev_host) VALUES (?, ?, ?, ?)', [id, url, title, rev_host])
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# type 2 is a folder, type 1 a bookmark and type 3 a separator. The roots are
|
|
96
|
+
# fixed ids with padded guids. Under tags, a folder per tag holds a copy of
|
|
97
|
+
# each bookmark so tagged.
|
|
98
|
+
[
|
|
99
|
+
[1, 2, nil, 0, 0, nil, 'root________'],
|
|
100
|
+
[2, 2, nil, 1, 0, 'menu', 'menu________'],
|
|
101
|
+
[3, 2, nil, 1, 1, 'toolbar', 'toolbar_____'],
|
|
102
|
+
[4, 2, nil, 1, 2, 'tags', 'tags________'],
|
|
103
|
+
[5, 2, nil, 1, 3, 'unfiled', 'unfiled_____'],
|
|
104
|
+
[6, 2, nil, 1, 4, 'mobile', 'mobile______'],
|
|
105
|
+
[7, 2, nil, 2, 0, 'Mozilla Firefox', 'j47T12MG8hux'],
|
|
106
|
+
[8, 1, 1, 7, 0, 'Get Help', '50OxxDFUahwc'],
|
|
107
|
+
[9, 1, 2, 7, 1, 'Customize Firefox', 'dTBx_Z-u_6U6'],
|
|
108
|
+
[10, 1, 3, 7, 2, 'Get Involved', 'l00bhg3kKbYX'],
|
|
109
|
+
[11, 1, 4, 7, 3, 'About Us', 'pJ9IZ7ivzMw6'],
|
|
110
|
+
[12, 2, nil, 4, 0, 'firefox', 'Q2r8sT5vWx1y'],
|
|
111
|
+
[13, 1, 1, 12, 0, 'Get Help', 'Zb3cD6eF9gH2'],
|
|
112
|
+
[14, 3, nil, 3, 0, nil, 'Kj4mN7pQ0rS3'],
|
|
113
|
+
[15, 1, 5, 3, 1, 'Extension Starter Pack', 'Tu5vW8xY1zA4'],
|
|
114
|
+
].each do |id, type, fk, parent, position, title, guid|
|
|
115
|
+
db.execute(
|
|
116
|
+
'INSERT INTO moz_bookmarks (id, type, fk, parent, position, title, dateAdded, lastModified, guid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
|
117
|
+
[id, type, fk, parent, position, title, added + id, added + id, guid]
|
|
118
|
+
)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
db.execute('INSERT INTO moz_historyvisits (id, place_id, visit_date, visit_type) VALUES (?, ?, ?, ?)', [1, 5, added + 152741667, 1])
|
|
122
|
+
db.close
|
|
123
|
+
puts "Wrote #{path} (#{File.size(path)} bytes)."
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
desc "Build test/fixtures/Safari_history.sqlite"
|
|
127
|
+
task :safari_history do
|
|
128
|
+
require 'fileutils'
|
|
129
|
+
require 'sqlite3'
|
|
130
|
+
path = File.join(__dir__, 'test/fixtures/Safari_history.sqlite')
|
|
131
|
+
FileUtils.rm_f(path)
|
|
132
|
+
db = SQLite3::Database.new(path)
|
|
133
|
+
# history_items and history_visits as Safari creates them.
|
|
134
|
+
db.execute_batch(<<~SQL)
|
|
135
|
+
CREATE TABLE history_items (
|
|
136
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
137
|
+
url TEXT NOT NULL UNIQUE,
|
|
138
|
+
domain_expansion TEXT NULL,
|
|
139
|
+
visit_count INTEGER NOT NULL,
|
|
140
|
+
daily_visit_counts BLOB NOT NULL,
|
|
141
|
+
weekly_visit_counts BLOB NULL,
|
|
142
|
+
autocomplete_triggers BLOB NULL,
|
|
143
|
+
should_recompute_derived_visit_counts INTEGER NOT NULL,
|
|
144
|
+
visit_count_score INTEGER NOT NULL
|
|
145
|
+
);
|
|
146
|
+
CREATE TABLE history_visits (
|
|
147
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
148
|
+
history_item INTEGER NOT NULL REFERENCES history_items(id) ON DELETE CASCADE,
|
|
149
|
+
visit_time REAL NOT NULL,
|
|
150
|
+
title TEXT NULL,
|
|
151
|
+
load_successful BOOLEAN NOT NULL DEFAULT 1,
|
|
152
|
+
http_non_get BOOLEAN NOT NULL DEFAULT 0,
|
|
153
|
+
synthesized BOOLEAN NOT NULL DEFAULT 0,
|
|
154
|
+
redirect_source INTEGER NULL UNIQUE,
|
|
155
|
+
redirect_destination INTEGER NULL UNIQUE,
|
|
156
|
+
origin INTEGER NOT NULL DEFAULT 0,
|
|
157
|
+
generation INTEGER NOT NULL DEFAULT 0,
|
|
158
|
+
attributes INTEGER NOT NULL DEFAULT 0,
|
|
159
|
+
score INTEGER NOT NULL DEFAULT 0
|
|
160
|
+
);
|
|
161
|
+
CREATE INDEX history_visits_history_item_index ON history_visits (history_item);
|
|
162
|
+
SQL
|
|
163
|
+
[
|
|
164
|
+
[1, 'https://www.apple.com/', 'apple', 3, 1],
|
|
165
|
+
[2, 'https://support.apple.com/', 'support.apple', 1, 0],
|
|
166
|
+
].each do |id, url, domain, visit_count, score|
|
|
167
|
+
db.execute(
|
|
168
|
+
'INSERT INTO history_items (id, url, domain_expansion, visit_count, daily_visit_counts, should_recompute_derived_visit_counts, visit_count_score) VALUES (?, ?, ?, ?, ?, 0, ?)',
|
|
169
|
+
[id, url, domain, visit_count, SQLite3::Blob.new(''), score]
|
|
170
|
+
)
|
|
171
|
+
end
|
|
172
|
+
# visit_time is seconds since 2001-01-01, which is Core Data's epoch.
|
|
173
|
+
[
|
|
174
|
+
[1, 1, 780451200.0, 'Apple'],
|
|
175
|
+
[2, 2, 780450000.0, 'Apple Support'],
|
|
176
|
+
].each do |id, item, time, title|
|
|
177
|
+
db.execute('INSERT INTO history_visits (id, history_item, visit_time, title) VALUES (?, ?, ?, ?)', [id, item, time, title])
|
|
178
|
+
end
|
|
179
|
+
db.close
|
|
180
|
+
puts "Wrote #{path} (#{File.size(path)} bytes)."
|
|
181
|
+
end
|
|
182
|
+
end
|
data/browser.rb.gemspec
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# browser.rb.gemspec
|
|
2
|
+
|
|
3
|
+
require_relative './lib/Browser/VERSION'
|
|
4
|
+
|
|
5
|
+
class Gem::Specification
|
|
6
|
+
def dependencies=(gems)
|
|
7
|
+
gems.each{|gem| add_dependency(*gem)}
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def development_dependencies=(gems)
|
|
11
|
+
gems.each{|gem| add_development_dependency(*gem)}
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
Gem::Specification.new do |spec|
|
|
16
|
+
spec.name = 'browser.rb'
|
|
17
|
+
spec.version = Browser::VERSION
|
|
18
|
+
|
|
19
|
+
spec.summary = "Read web browser bookmarks and history with Ruby."
|
|
20
|
+
spec.description = "Read the bookmarks and history of the web browsers on a machine, Brave, Chrome, Chromium, Firefox, Safari and Tor Browser, and render them to CSV, JSON or a property list. Bookmarks come through as one representation whatever the browser; history as each browser stores it."
|
|
21
|
+
|
|
22
|
+
spec.author = 'thoran'
|
|
23
|
+
spec.email = 'code@thoran.com'
|
|
24
|
+
spec.homepage = 'https://github.com/thoran/browser'
|
|
25
|
+
spec.license = 'MIT'
|
|
26
|
+
|
|
27
|
+
spec.required_ruby_version = '>= 3.2'
|
|
28
|
+
spec.require_paths = ['lib']
|
|
29
|
+
|
|
30
|
+
spec.files = [
|
|
31
|
+
'browser.rb.gemspec',
|
|
32
|
+
Dir['lib/**/*.rb'],
|
|
33
|
+
Dir['test/**/*'],
|
|
34
|
+
'CHANGELOG',
|
|
35
|
+
'Gemfile',
|
|
36
|
+
'LICENSE',
|
|
37
|
+
'Rakefile',
|
|
38
|
+
'README.md',
|
|
39
|
+
].flatten
|
|
40
|
+
|
|
41
|
+
spec.dependencies = %w{
|
|
42
|
+
CFPropertyList
|
|
43
|
+
csv
|
|
44
|
+
json
|
|
45
|
+
sqlite3
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
spec.development_dependencies = %w{
|
|
49
|
+
minitest
|
|
50
|
+
rake
|
|
51
|
+
minitest-spec-context
|
|
52
|
+
rspec-expectations
|
|
53
|
+
}
|
|
54
|
+
end
|
data/lib/Browser/Base.rb
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Browser/Base.rb
|
|
2
|
+
# Browser::Base
|
|
3
|
+
|
|
4
|
+
require_relative './Bookmarks'
|
|
5
|
+
require_relative './History'
|
|
6
|
+
|
|
7
|
+
class Browser
|
|
8
|
+
class Base < Browser
|
|
9
|
+
class << self
|
|
10
|
+
def default_profile_name
|
|
11
|
+
self::DEFAULT_PROFILE_NAME
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def bookmarks
|
|
16
|
+
Browser::Bookmarks.new(self)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def history
|
|
20
|
+
Browser::History.new(self)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Each is what was given, or else derived when first wanted, so that making a
|
|
24
|
+
# browser costs nothing and a default which has to be looked up is looked up
|
|
25
|
+
# only for a read.
|
|
26
|
+
def profile_name
|
|
27
|
+
@profile_name ||= self.class.default_profile_name
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def bookmarks_location
|
|
31
|
+
@bookmarks_location ||= self.class.bookmarks_location(profile_name: profile_name)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def history_location
|
|
35
|
+
@history_location ||= self.class.history_location(profile_name: profile_name)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def name
|
|
39
|
+
self.class.to_s.split('::').last
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def initialize(bookmarks_location: nil, history_location: nil, profile_name: nil)
|
|
45
|
+
@bookmarks_location = bookmarks_location
|
|
46
|
+
@history_location = history_location
|
|
47
|
+
@profile_name = profile_name
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Browser/Bookmark.rb
|
|
2
|
+
# Browser::Bookmark
|
|
3
|
+
|
|
4
|
+
# It knows nothing of how a browser stores bookmarks, only the shape they share
|
|
5
|
+
# once read. The title is what the browser shows; the name is what it stores,
|
|
6
|
+
# which is the title but for Firefox's roots, where menu is shown as Bookmarks
|
|
7
|
+
# Menu, and which writing back into a browser would need.
|
|
8
|
+
|
|
9
|
+
class Browser
|
|
10
|
+
class Bookmark
|
|
11
|
+
attr_accessor :title, :name, :url, :children
|
|
12
|
+
|
|
13
|
+
def leaf?
|
|
14
|
+
!@url.nil?
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def folder?
|
|
18
|
+
!leaf?
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The root's own title is not a folder anyone filed under, so a titleless or
|
|
22
|
+
# rootless node adds nothing to the path.
|
|
23
|
+
def flatten(path = [])
|
|
24
|
+
if leaf?
|
|
25
|
+
[{title: @title, url: @url, folder: path.join('/')}]
|
|
26
|
+
else
|
|
27
|
+
here = @title.to_s.empty? ? path : path + [@title]
|
|
28
|
+
@children.flat_map{|child| child.flatten(here)}
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def initialize(title: nil, name: title, url: nil, children: [])
|
|
35
|
+
@title = title
|
|
36
|
+
@name = name
|
|
37
|
+
@url = url
|
|
38
|
+
@children = children
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# Browser/Bookmarks.rb
|
|
2
|
+
# Browser::Bookmarks
|
|
3
|
+
|
|
4
|
+
require 'json'
|
|
5
|
+
require 'cfpropertylist'
|
|
6
|
+
require 'sqlite3'
|
|
7
|
+
|
|
8
|
+
require_relative './Bookmark'
|
|
9
|
+
require_relative './Rendering'
|
|
10
|
+
require_relative './Unreadable'
|
|
11
|
+
|
|
12
|
+
class Browser
|
|
13
|
+
class Bookmarks
|
|
14
|
+
include Rendering
|
|
15
|
+
|
|
16
|
+
class << self
|
|
17
|
+
|
|
18
|
+
def from_json(browser_instance)
|
|
19
|
+
data = ::JSON.parse(File.read(browser_instance.bookmarks_location))
|
|
20
|
+
roots = data.fetch('roots', {}).values.select{|value| value.is_a?(Hash)}
|
|
21
|
+
Browser::Bookmark.new(children: roots.collect{|root| json_node(root)})
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def from_plist(browser_instance)
|
|
25
|
+
data = CFPropertyList.native_types(CFPropertyList::List.new(file: browser_instance.bookmarks_location).value)
|
|
26
|
+
Browser::Bookmark.new(children: plist_children(data))
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def from_sqlite(browser_instance)
|
|
30
|
+
database = SQLite3::Database.new(browser_instance.bookmarks_location)
|
|
31
|
+
column_names, *rows = database.execute2(browser_instance.class.send(:bookmarks_sql))
|
|
32
|
+
records = rows.collect{|row| column_names.zip(row).to_h}
|
|
33
|
+
children = records.group_by{|record| record['parent']}
|
|
34
|
+
Browser::Bookmark.new(children: Array(children[0]).flat_map{|root| sqlite_children(root, children)})
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def json_node(node)
|
|
40
|
+
if node['url']
|
|
41
|
+
Browser::Bookmark.new(title: node['name'], url: node['url'])
|
|
42
|
+
else
|
|
43
|
+
Browser::Bookmark.new(title: node['name'], children: Array(node['children']).collect{|child| json_node(child)})
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def plist_children(node)
|
|
48
|
+
Array(node['Children']).filter_map{|child| plist_node(child)}
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# moz_bookmarks is one table of folders and bookmarks, each row naming its
|
|
52
|
+
# parent and its position within it, and the root the row with no parent.
|
|
53
|
+
# The tags root is left out: a tag is a folder holding a copy of every
|
|
54
|
+
# bookmark so tagged, and Firefox does not show it as one.
|
|
55
|
+
def sqlite_children(record, children)
|
|
56
|
+
Array(children[record['id']]).filter_map{|child| sqlite_node(child, children)}
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# type 1 is a bookmark, 2 a folder and 3 a separator.
|
|
60
|
+
def sqlite_node(record, children)
|
|
61
|
+
case record['type']
|
|
62
|
+
when 1
|
|
63
|
+
Browser::Bookmark.new(title: record['title'], url: record['url'])
|
|
64
|
+
when 2
|
|
65
|
+
sqlite_folder(record, children) unless record['guid'] == 'tags________'
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# The roots are stored under names Firefox never shows; these are the names it does.
|
|
70
|
+
SQLITE_ROOT_TITLES = {
|
|
71
|
+
'menu________' => 'Bookmarks Menu',
|
|
72
|
+
'toolbar_____' => 'Bookmarks Toolbar',
|
|
73
|
+
'unfiled_____' => 'Other Bookmarks',
|
|
74
|
+
'mobile______' => 'Mobile Bookmarks',
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
def sqlite_folder(record, children)
|
|
78
|
+
title = SQLITE_ROOT_TITLES.fetch(record['guid'], record['title'])
|
|
79
|
+
Browser::Bookmark.new(title: title, name: record['title'], children: sqlite_children(record, children))
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def plist_node(node)
|
|
83
|
+
case node['WebBookmarkType']
|
|
84
|
+
when 'WebBookmarkTypeLeaf'
|
|
85
|
+
Browser::Bookmark.new(title: node.dig('URIDictionary', 'title'), url: node['URLString'])
|
|
86
|
+
when 'WebBookmarkTypeList'
|
|
87
|
+
Browser::Bookmark.new(title: node['Title'], children: plist_children(node))
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def to_objects
|
|
93
|
+
results.flatten
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
private
|
|
97
|
+
|
|
98
|
+
def initialize(browser_instance)
|
|
99
|
+
@browser_instance = browser_instance
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def results
|
|
103
|
+
@results ||= self.class.public_send("from_#{@browser_instance.class.send(:bookmarks_format)}", @browser_instance)
|
|
104
|
+
rescue IOError, SystemCallError, SQLite3::Exception => e
|
|
105
|
+
raise Browser::Unreadable, "#{@browser_instance.bookmarks_location} is not readable: #{e.message}"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def column_names
|
|
109
|
+
%i{title url folder}
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Browser/Brave.rb
|
|
2
|
+
# Browser::Brave
|
|
3
|
+
|
|
4
|
+
require_relative './ChromiumBased'
|
|
5
|
+
|
|
6
|
+
class Browser
|
|
7
|
+
class Brave < ChromiumBased
|
|
8
|
+
class << self
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def root_path
|
|
12
|
+
"~/Library/Application Support/BraveSoftware/Brave-Browser"
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Browser/Chrome.rb
|
|
2
|
+
# Browser::Chrome
|
|
3
|
+
|
|
4
|
+
require_relative './ChromiumBased'
|
|
5
|
+
|
|
6
|
+
class Browser
|
|
7
|
+
class Chrome < ChromiumBased
|
|
8
|
+
class << self
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def root_path
|
|
12
|
+
"~/Library/Application Support/Google/Chrome"
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Browser/Chromium.rb
|
|
2
|
+
# Browser::Chromium
|
|
3
|
+
|
|
4
|
+
require_relative './ChromiumBased'
|
|
5
|
+
|
|
6
|
+
class Browser
|
|
7
|
+
class Chromium < ChromiumBased
|
|
8
|
+
class << self
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def root_path
|
|
12
|
+
"~/Library/Application Support/Chromium"
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Browser/ChromiumBased.rb
|
|
2
|
+
# Browser::ChromiumBased
|
|
3
|
+
|
|
4
|
+
require_relative './Base'
|
|
5
|
+
|
|
6
|
+
class Browser
|
|
7
|
+
class ChromiumBased < Base
|
|
8
|
+
DEFAULT_PROFILE_NAME = "Default"
|
|
9
|
+
|
|
10
|
+
class << self
|
|
11
|
+
def bookmarks_location(profile_name: nil)
|
|
12
|
+
profile_name ||= DEFAULT_PROFILE_NAME
|
|
13
|
+
File.expand_path("#{profiles_path}/#{profile_name}/Bookmarks")
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def history_location(profile_name: nil)
|
|
17
|
+
profile_name ||= DEFAULT_PROFILE_NAME
|
|
18
|
+
File.expand_path("#{profiles_path}/#{profile_name}/History")
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def root_path
|
|
24
|
+
raise NotImplementedError, "#{self} must implement root_path"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def profiles_path
|
|
28
|
+
root_path
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def bookmarks_format
|
|
32
|
+
:json
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def history_format
|
|
36
|
+
:sqlite
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def bookmarks_sql
|
|
40
|
+
nil
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def history_sql
|
|
44
|
+
'SELECT * FROM urls ORDER BY last_visit_time DESC;'
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# Browser/FirefoxBased.rb
|
|
2
|
+
# Browser::FirefoxBased
|
|
3
|
+
|
|
4
|
+
require_relative './Base'
|
|
5
|
+
|
|
6
|
+
class Browser
|
|
7
|
+
class FirefoxBased < Base
|
|
8
|
+
class << self
|
|
9
|
+
def bookmarks_location(profile_name: nil)
|
|
10
|
+
profile_name ||= default_profile_name
|
|
11
|
+
File.expand_path("#{profiles_path}/#{profile_name}/places.sqlite")
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def history_location(profile_name: nil)
|
|
15
|
+
profile_name ||= default_profile_name
|
|
16
|
+
File.expand_path("#{profiles_path}/#{profile_name}/places.sqlite")
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# The profile Firefox itself would open, which profiles.ini records: the one an
|
|
20
|
+
# [Install] section names, as Firefox has written since 67; else the one
|
|
21
|
+
# [Profile] flagged Default=1, as it wrote before and still does; else the only
|
|
22
|
+
# profile there is. A directory is named for a hash, so nothing else could
|
|
23
|
+
# stand in for a name.
|
|
24
|
+
def default_profile_name(profiles_ini = read_profiles_ini)
|
|
25
|
+
File.basename(default_profile_path(profiles_ini))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def profiles_ini_path
|
|
31
|
+
File.expand_path("#{root_path}/profiles.ini")
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def read_profiles_ini
|
|
35
|
+
File.read(profiles_ini_path)
|
|
36
|
+
rescue Errno::ENOENT
|
|
37
|
+
raise Browser::Unreadable, "#{profiles_ini_path} is not there, so no profile is the default: pass profile_name:"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Two installations each naming a default, or two profiles flagged, or three
|
|
41
|
+
# profiles and no flag, are a question only the caller can answer.
|
|
42
|
+
def default_profile_path(profiles_ini)
|
|
43
|
+
candidates = default_profile_candidates(ini_sections(profiles_ini))
|
|
44
|
+
unless candidates.size == 1
|
|
45
|
+
raise Browser::Unreadable, "#{profiles_ini_path} names #{candidates.size} profiles as the default: pass profile_name:"
|
|
46
|
+
end
|
|
47
|
+
candidates.first
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def default_profile_candidates(sections)
|
|
51
|
+
installs = sections.select{|name, _| name.start_with?('Install')}.values.filter_map{|section| section['Default']}
|
|
52
|
+
profiles = sections.select{|name, _| name.start_with?('Profile')}.values
|
|
53
|
+
flagged = profiles.select{|section| section['Default'] == '1'}.filter_map{|section| section['Path']}
|
|
54
|
+
[installs, flagged, profiles.filter_map{|section| section['Path']}].find{|candidates| candidates.any?} || []
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def ini_sections(text)
|
|
58
|
+
text.scan(/^\[([^\]]+)\]\n((?:[^\[\n].*\n?)*)/).to_h{|name, body| [name, body.scan(/^(\w+)=(.*)$/).to_h]}
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def root_path
|
|
62
|
+
raise NotImplementedError, "#{self} must implement root_path"
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def profiles_path
|
|
66
|
+
"#{root_path}/Profiles"
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def bookmarks_format
|
|
70
|
+
:sqlite
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def history_format
|
|
74
|
+
:sqlite
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def bookmarks_sql
|
|
78
|
+
'SELECT moz_bookmarks.id, moz_bookmarks.parent, moz_bookmarks.position, moz_bookmarks.type, moz_bookmarks.title, moz_bookmarks.guid, moz_places.url FROM moz_bookmarks LEFT JOIN moz_places ON moz_bookmarks.fk = moz_places.id ORDER BY moz_bookmarks.parent, moz_bookmarks.position;'
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def history_sql
|
|
82
|
+
'SELECT moz_historyvisits.id, moz_places.url, moz_places.title, moz_historyvisits.visit_date FROM moz_historyvisits LEFT JOIN moz_places ON moz_historyvisits.place_id = moz_places.id ORDER BY moz_historyvisits.visit_date DESC;'
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Browser/History.rb
|
|
2
|
+
# Browser::History
|
|
3
|
+
|
|
4
|
+
require 'sqlite3'
|
|
5
|
+
|
|
6
|
+
require_relative './Rendering'
|
|
7
|
+
require_relative './Unreadable'
|
|
8
|
+
|
|
9
|
+
class Browser
|
|
10
|
+
class History
|
|
11
|
+
include Rendering
|
|
12
|
+
|
|
13
|
+
def to_objects
|
|
14
|
+
rows.collect{|row| column_names.zip(row).to_h}
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def initialize(browser_instance)
|
|
20
|
+
@browser_instance = browser_instance
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def database
|
|
24
|
+
@database ||= SQLite3::Database.new(@browser_instance.history_location)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def results
|
|
28
|
+
@results ||= database.execute2(@browser_instance.class.send(:history_sql))
|
|
29
|
+
rescue IOError, SystemCallError, SQLite3::Exception => e
|
|
30
|
+
raise Browser::Unreadable, "#{@browser_instance.history_location} is not readable: #{e.message}"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def column_names
|
|
34
|
+
results.first
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def rows
|
|
38
|
+
results.drop(1)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|