portage-cli 0.7.5 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +616 -0
  3. data/README.md +302 -5
  4. data/known-stores/categories.yml +1263 -0
  5. data/lib/portage/cli/agent_profile_url.rb +30 -0
  6. data/lib/portage/cli/approval_policy.rb +56 -0
  7. data/lib/portage/cli/approve.rb +135 -0
  8. data/lib/portage/cli/browser_import/categorize.rb +59 -0
  9. data/lib/portage/cli/browser_import/confirm.rb +35 -0
  10. data/lib/portage/cli/browser_import/domains.rb +47 -0
  11. data/lib/portage/cli/browser_import/filter.rb +91 -0
  12. data/lib/portage/cli/browser_import/importer.rb +248 -0
  13. data/lib/portage/cli/browser_import/plist_xml.rb +72 -0
  14. data/lib/portage/cli/browser_import/prober.rb +60 -0
  15. data/lib/portage/cli/browser_import/profiles.rb +114 -0
  16. data/lib/portage/cli/browser_import/readers.rb +179 -0
  17. data/lib/portage/cli/browser_import/saver.rb +62 -0
  18. data/lib/portage/cli/browser_import/sqlite.rb +68 -0
  19. data/lib/portage/cli/browser_import.rb +23 -0
  20. data/lib/portage/cli/browser_opener.rb +38 -0
  21. data/lib/portage/cli/browser_profile/allowlist.rb +40 -0
  22. data/lib/portage/cli/browser_profile/bridge.rb +120 -0
  23. data/lib/portage/cli/browser_profile/browsers.rb +69 -0
  24. data/lib/portage/cli/browser_profile/cdp.rb +67 -0
  25. data/lib/portage/cli/browser_profile/cdp_socket.rb +186 -0
  26. data/lib/portage/cli/browser_profile/errors.rb +26 -0
  27. data/lib/portage/cli/browser_profile/launcher.rb +34 -0
  28. data/lib/portage/cli/browser_profile/profile.rb +93 -0
  29. data/lib/portage/cli/browser_profile.rb +25 -0
  30. data/lib/portage/cli/buy.rb +559 -29
  31. data/lib/portage/cli/checkout_handoff.rb +5 -24
  32. data/lib/portage/cli/classifier.rb +158 -0
  33. data/lib/portage/cli/compare.rb +3 -0
  34. data/lib/portage/cli/doctor.rb +155 -1
  35. data/lib/portage/cli/dot_env.rb +55 -0
  36. data/lib/portage/cli/find.rb +103 -12
  37. data/lib/portage/cli/handoff_agents.rb +186 -0
  38. data/lib/portage/cli/handoff_only.rb +94 -0
  39. data/lib/portage/cli/handoff_reconciler.rb +15 -1
  40. data/lib/portage/cli/handoff_target.rb +61 -0
  41. data/lib/portage/cli/history.rb +47 -2
  42. data/lib/portage/cli/human_prompt.rb +118 -0
  43. data/lib/portage/cli/index/builder.rb +335 -0
  44. data/lib/portage/cli/index/exporter.rb +91 -0
  45. data/lib/portage/cli/index/known_cache.rb +155 -0
  46. data/lib/portage/cli/index/product_store.rb +101 -0
  47. data/lib/portage/cli/index/sources/browser.rb +31 -0
  48. data/lib/portage/cli/index/sources/shopify_catalog.rb +82 -0
  49. data/lib/portage/cli/index/sources/stores_file.rb +58 -0
  50. data/lib/portage/cli/index/sources/webmcp_sweep.rb +29 -0
  51. data/lib/portage/cli/index/sources/wikidata.rb +95 -0
  52. data/lib/portage/cli/index/sources.rb +44 -0
  53. data/lib/portage/cli/index/store.rb +109 -0
  54. data/lib/portage/cli/index.rb +20 -0
  55. data/lib/portage/cli/known_stores_url.rb +15 -0
  56. data/lib/portage/cli/money.rb +18 -0
  57. data/lib/portage/cli/offer_choice.rb +38 -0
  58. data/lib/portage/cli/offer_sources.rb +460 -0
  59. data/lib/portage/cli/payment_methods.rb +24 -3
  60. data/lib/portage/cli/pick.rb +167 -0
  61. data/lib/portage/cli/product_page.rb +84 -0
  62. data/lib/portage/cli/quotes.rb +82 -0
  63. data/lib/portage/cli/search_backends.rb +337 -12
  64. data/lib/portage/cli/setup_wizard/prompt.rb +67 -0
  65. data/lib/portage/cli/setup_wizard/steps/agent_profile.rb +60 -0
  66. data/lib/portage/cli/setup_wizard/steps/browser_import.rb +29 -0
  67. data/lib/portage/cli/setup_wizard/steps/handoff.rb +100 -0
  68. data/lib/portage/cli/setup_wizard/steps/index_build.rb +31 -0
  69. data/lib/portage/cli/setup_wizard/steps/policy.rb +60 -0
  70. data/lib/portage/cli/setup_wizard/steps/retailer_keys.rb +55 -0
  71. data/lib/portage/cli/setup_wizard/steps/search_keys.rb +54 -0
  72. data/lib/portage/cli/setup_wizard/steps/shipping.rb +51 -0
  73. data/lib/portage/cli/setup_wizard.rb +74 -0
  74. data/lib/portage/cli/version.rb +1 -1
  75. data/lib/portage/cli/webmcp.rb +10 -3
  76. data/lib/portage/cli/webmcp_autofill_confirm.rb +38 -0
  77. data/lib/portage/cli/webmcp_autofill_fields.rb +60 -0
  78. data/lib/portage/cli/webmcp_autofill_mode.rb +39 -0
  79. data/lib/portage/cli/webmcp_mapping_confirm.rb +68 -0
  80. data/lib/portage/cli/webmcp_mappings.rb +84 -0
  81. data/lib/portage/cli.rb +930 -43
  82. metadata +67 -2
@@ -0,0 +1,248 @@
1
+ require "uri"
2
+
3
+ require_relative "../classifier"
4
+ require_relative "../index"
5
+ require_relative "../index/builder"
6
+ require_relative "profiles"
7
+ require_relative "readers"
8
+ require_relative "filter"
9
+ require_relative "prober"
10
+ require_relative "categorize"
11
+ require_relative "saver"
12
+ require_relative "domains"
13
+
14
+ module Portage
15
+ module Cli
16
+ module BrowserImport
17
+ # `portage browser import` (docs/plans/buy-skill-and-local-browser.md
18
+ # Phase 3, Tier A): bookmarks and history in, a short list of shop
19
+ # domains out, written to the user's own Index::Store only after the
20
+ # user has seen the list and said yes (Cli.run_browser_import owns
21
+ # that gate; this class only ever writes from #save).
22
+ #
23
+ # #plan reads the profile's allowed files (Profiles::ALLOWED_FILES,
24
+ # nothing else), reduces every row to its domain, and decides each
25
+ # domain locally first — skipped as an obvious non-shop (Filter),
26
+ # excluded by the user, on the hand-off-only list, or already in the
27
+ # index — before spending one of at most `max_probes` Prober probes
28
+ # on an unknown one. A domain is kept only when it answers
29
+ # `/.well-known/ucp`, matches a WebMCP preset, or is hand-off only.
30
+ # Each kept domain is classified from what the user actually looked
31
+ # at there (page titles, bookmark folder names, URL slugs), weighted
32
+ # by visit count.
33
+ #
34
+ # Kept domains land in Index::Store with `sources: ["history"]`/
35
+ # `["bookmark"]` — the same untrusted index every other source
36
+ # writes, so an imported store reaches `find` only as a
37
+ # `source: "index"` candidate (never Policy#merchant_allowlist, never
38
+ # past the `--store`/interactive pick a `--yes` buy requires), and
39
+ # Index::Exporter treats both labels as personal: never exported.
40
+ class Importer
41
+ DEFAULT_HISTORY_DAYS = 90
42
+ MAX_PROBES = 200
43
+
44
+ FULL_DISK_ACCESS = "macOS won't let this terminal read Safari's history and bookmarks without Full Disk " \
45
+ "Access. To allow it, open System Settings > Privacy & Security > Full Disk Access, " \
46
+ "turn it on for the terminal app you run portage from, then quit and reopen that app " \
47
+ "and re-run this command. Portage doesn't try any other way in. Nothing was imported, " \
48
+ "probed or saved.".freeze
49
+
50
+ # Any other browser: the same "explain and stop", without claiming
51
+ # to know which protection said no (macOS privacy settings, a
52
+ # sandboxed terminal, file permissions).
53
+ PERMISSION_DENIED = "Permission denied reading %<browser>s's profile folder. If your terminal runs " \
54
+ "sandboxed or macOS is protecting that folder, allow access (System Settings > " \
55
+ "Privacy & Security > Full Disk Access) and re-run. Portage doesn't try any other way " \
56
+ "in. Nothing was imported, probed or saved.".freeze
57
+
58
+ Options = Struct.new(:browser, :root, :history_days, :include_product_pages, :max_probes, :exclude,
59
+ keyword_init: true) do
60
+ def initialize(history_days: DEFAULT_HISTORY_DAYS, include_product_pages: false, max_probes: MAX_PROBES,
61
+ exclude: [], **)
62
+ super
63
+ end
64
+ end
65
+
66
+ # @param handoff_only_hosts [Array<String>] Tier C hosts (Phase 5
67
+ # will supply the user's own `handoff_only_hosts:` config; nothing
68
+ # ships that list yet, so it defaults to empty). A history domain
69
+ # on it is kept — hand-off only, never probed, never automated.
70
+ # @param webmcp_preset_for [#call, nil] `->(origin) { preset_or_nil }`.
71
+ # Matching a WebMCP preset needs a browser bridge an import doesn't
72
+ # have (same posture as Index::Sources::WebmcpSweep), so nil (the
73
+ # default) skips the check entirely.
74
+ # @param readers [#call] `->(family) { reader }` — see Readers.for.
75
+ def initialize(stores: Index::Store.new, products: Index::ProductStore.new,
76
+ known_cache: Index::KnownCache.new, prober: Prober.new, handoff_only_hosts: [],
77
+ webmcp_preset_for: nil, readers: Readers.method(:for), now: Time.now)
78
+ @stores = stores
79
+ @products = products
80
+ @known_cache = known_cache
81
+ @prober = prober
82
+ @handoff_only_hosts = handoff_only_hosts.map { |h| h.to_s.downcase.delete_prefix("www.") }
83
+ @webmcp_preset_for = webmcp_preset_for
84
+ @readers = readers
85
+ @now = now
86
+ end
87
+
88
+ # @return [Hash] the proposal: counts, `kept` (domains to save) and
89
+ # `products` (only with include_product_pages) — or `error:` /
90
+ # `message:` when the browser's files couldn't be read (Safari
91
+ # without Full Disk Access, no sqlite3, no profile found).
92
+ def plan(options)
93
+ profiles = Profiles.locate(options.browser, root: options.root)
94
+ return no_profile(options) if profiles.empty?
95
+
96
+ rows, opened = read_rows(profiles, since: @now.to_i - (options.history_days.to_i * 86_400))
97
+ summarize(options, profiles, opened, rows)
98
+ rescue Sqlite::PermissionDenied
99
+ permission_denied(options)
100
+ rescue Sqlite::Unavailable => e
101
+ { browser: options.browser, error: "reader_unavailable",
102
+ message: "Couldn't read the browser's files: #{e.message}." }
103
+ end
104
+
105
+ # Writes a #plan's kept domains (and products) into the index — see
106
+ # Saver.
107
+ # @return [Hash] stores:, products: — how many entries were written.
108
+ def save(proposal) = Saver.new(stores: @stores, products: @products, now: @now).save(proposal)
109
+
110
+ private
111
+
112
+ # --- Reading ---
113
+
114
+ def read_rows(profiles, since:)
115
+ opened = []
116
+ rows = profiles.flat_map do |profile|
117
+ reader = @readers.call(profile.family)
118
+ opened.concat([profile.history, profile.bookmarks].compact)
119
+ reader.history(profile, since: since) + reader.bookmarks(profile)
120
+ end
121
+ [rows, opened.uniq]
122
+ end
123
+
124
+ # --- Deciding each domain ---
125
+
126
+ def summarize(options, profiles, opened, rows)
127
+ groups = Domains.group(rows)
128
+ state = { skipped: Hash.new(0), kept: [], cached_miss: 0, not_ucp: 0, unprobed: 0 }
129
+ groups.sort_by { |_key, g| -g[:visits] }.each { |key, group| decide(key, group, options, state) }
130
+ report(options, profiles, opened, rows, groups, state)
131
+ end
132
+
133
+ def decide(key, group, options, state)
134
+ reason = local_skip_reason(key, options)
135
+ return state[:skipped][reason] += 1 if reason
136
+
137
+ verdict = local_verdict(key) || probe_verdict(Domains.origin_for(group), options, state)
138
+ state[:kept] << kept_entry(key, group, verdict) if verdict
139
+ end
140
+
141
+ def local_skip_reason(key, options)
142
+ return :excluded if Array(options.exclude).any? { |h| h.to_s.downcase.delete_prefix("www.") == key }
143
+
144
+ Filter.skip_reason(key)
145
+ end
146
+
147
+ # Everything decidable with no network call at all.
148
+ def local_verdict(key)
149
+ return { verdict: "handoff_only", handoff_only: true } if handoff_only?(key)
150
+
151
+ own = own_entry(key)
152
+ return { verdict: "indexed", origin: own["origin"] } if own
153
+
154
+ known = known_entry(key)
155
+ return unless known
156
+
157
+ { verdict: "known", origin: known["origin"], capabilities: known["capabilities"],
158
+ last_verified: known["last_verified"] }
159
+ end
160
+
161
+ # A cached "no UCP" verdict or the probe cap skips the probe, but
162
+ # not the WebMCP check (when a bridge is injected) — a page can
163
+ # expose WebMCP tools without any manifest at all.
164
+ def probe_verdict(origin, options, state)
165
+ session = ucp_session(origin, options, state)
166
+ return nil if session == :capped
167
+ return { verdict: "ucp", origin: origin, capabilities: Index::Builder.capabilities_of(session) } if session
168
+
169
+ preset = @webmcp_preset_for&.call(origin)
170
+ return { verdict: "webmcp", origin: origin, webmcp_preset: preset } if preset
171
+
172
+ state[:not_ucp] += 1 unless session == false
173
+ nil
174
+ end
175
+
176
+ # @return [Session, nil, false, :capped] a session; nil when the
177
+ # probe found no UCP; false for a cached miss (no request made);
178
+ # :capped once max_probes is spent.
179
+ def ucp_session(origin, options, state)
180
+ if @prober.cached_miss?(origin)
181
+ state[:cached_miss] += 1
182
+ false
183
+ elsif @prober.probes >= options.max_probes.to_i
184
+ state[:unprobed] += 1
185
+ :capped
186
+ else
187
+ @prober.probe(origin)
188
+ end
189
+ end
190
+
191
+ def handoff_only?(key)
192
+ @handoff_only_hosts.any? { |h| key == h || key.end_with?(".#{h}") }
193
+ end
194
+
195
+ def own_entry(key) = @stores.all.find { |e| Domains.key_of(e["origin"]) == key }
196
+
197
+ def known_entry(key) = @known_cache.stores.values.find { |e| Domains.key_of(e["origin"]) == key }
198
+
199
+ # --- The proposal ---
200
+
201
+ def kept_entry(key, group, verdict)
202
+ { domain: key, origin: verdict[:origin] || Domains.origin_for(group), verdict: verdict[:verdict],
203
+ sources: group[:rows].map { |r| r[:kind] }.uniq.sort, visits: group[:visits],
204
+ categories: Categorize.domain(group[:rows]), rows: group[:rows] }
205
+ .merge(verdict.slice(:capabilities, :webmcp_preset, :handoff_only, :last_verified))
206
+ end
207
+
208
+ def report(options, profiles, opened, rows, groups, state)
209
+ kept = state[:kept]
210
+ { browser: options.browser, profiles: profiles.length, files_opened: opened, rows: row_counts(rows),
211
+ domains: groups.length, skipped: state[:skipped].transform_keys(&:to_s),
212
+ already_indexed: count(kept, "indexed"), known: count(kept, "known"), probed: @prober.probes,
213
+ cached_miss: state[:cached_miss], not_ucp: state[:not_ucp], unprobed: state[:unprobed],
214
+ capped: state[:unprobed].positive?, kept: kept.map { |entry| public_entry(entry) },
215
+ products: options.include_product_pages ? Categorize.products(kept) : [] }
216
+ end
217
+
218
+ def row_counts(rows)
219
+ kinds = rows.map { |r| r[:kind] }
220
+ { history: kinds.count("history"), bookmark: kinds.count("bookmark") }
221
+ end
222
+
223
+ def public_entry(entry)
224
+ entry.except(:rows).merge(category_names: Classifier.names_for(entry[:categories].keys))
225
+ end
226
+
227
+ def count(kept, verdict) = kept.count { |e| e[:verdict] == verdict }
228
+
229
+ # --- Helpers ---
230
+
231
+ def no_profile(options)
232
+ where = options.root ? " under #{options.root}" : ""
233
+ { browser: options.browser, error: "no_profile",
234
+ message: "No #{options.browser} profile with history or bookmarks found#{where}." }
235
+ end
236
+
237
+ def permission_denied(options)
238
+ if options.browser == "safari"
239
+ { browser: "safari", error: "full_disk_access_required", message: FULL_DISK_ACCESS }
240
+ else
241
+ { browser: options.browser, error: "permission_denied",
242
+ message: format(PERMISSION_DENIED, browser: options.browser) }
243
+ end
244
+ end
245
+ end
246
+ end
247
+ end
248
+ end
@@ -0,0 +1,72 @@
1
+ require "strscan"
2
+
3
+ module Portage
4
+ module Cli
5
+ module BrowserImport
6
+ # Just enough of Apple's XML property-list format to read Safari's
7
+ # Bookmarks.plist once `plutil -convert xml1` has turned its binary
8
+ # form into XML. `plutil -convert json` would be simpler but refuses
9
+ # any plist holding a <date> (Safari's Reading List entries carry
10
+ # them), and a real XML library (rexml) isn't a runtime dependency of
11
+ # this gem — this is ~50 lines instead of a new one.
12
+ #
13
+ # Dates, data and numbers come back as their raw text; bookmark
14
+ # import only ever reads strings out of the tree.
15
+ module PlistXml
16
+ class ParseError < StandardError; end
17
+
18
+ SCALARS = "string|key|data|date|integer|real".freeze
19
+ ENTITIES = { "amp" => "&", "lt" => "<", "gt" => ">", "quot" => '"', "apos" => "'" }.freeze
20
+
21
+ def self.parse(xml)
22
+ scanner = StringScanner.new(xml.to_s)
23
+ raise ParseError, "not an XML plist" unless scanner.skip_until(/<plist[^>]*>/)
24
+
25
+ value(scanner)
26
+ end
27
+
28
+ def self.value(scanner)
29
+ scanner.skip(/\s*/)
30
+ if scanner.scan(%r{<(dict|array|#{SCALARS})\s*/>}) then empty(scanner[1])
31
+ elsif scanner.scan(%r{<(true|false)\s*/>}) then scanner[1] == "true"
32
+ elsif scanner.scan("<dict>") then dict(scanner)
33
+ elsif scanner.scan("<array>") then array(scanner)
34
+ elsif scanner.scan(%r{<(#{SCALARS})>(.*?)</\1>}m) then unescape(scanner[2])
35
+ else raise ParseError, "unexpected plist content at #{scanner.pos}"
36
+ end
37
+ end
38
+
39
+ def self.dict(scanner)
40
+ result = {}
41
+ until scanner.skip(%r{\s*</dict>})
42
+ key = value(scanner)
43
+ raise ParseError, "dict key isn't a string" unless key.is_a?(String)
44
+
45
+ result[key] = value(scanner)
46
+ end
47
+ result
48
+ end
49
+
50
+ def self.array(scanner)
51
+ result = []
52
+ result << value(scanner) until scanner.skip(%r{\s*</array>})
53
+ result
54
+ end
55
+
56
+ def self.empty(tag)
57
+ { "dict" => {}, "array" => [] }.fetch(tag, "")
58
+ end
59
+
60
+ def self.unescape(text)
61
+ text.gsub(/&(#x?)?(\w+);/) do
62
+ if Regexp.last_match(1) == "#x" then [Regexp.last_match(2).to_i(16)].pack("U")
63
+ elsif Regexp.last_match(1) == "#" then [Regexp.last_match(2).to_i].pack("U")
64
+ else ENTITIES.fetch(Regexp.last_match(2), Regexp.last_match(0))
65
+ end
66
+ end
67
+ end
68
+ private_class_method :value, :dict, :array, :empty, :unescape
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,60 @@
1
+ require "timeout"
2
+ require "portage/ucp/client"
3
+
4
+ require_relative "../user_agent"
5
+ require_relative "../probe_cache"
6
+
7
+ module Portage
8
+ module Cli
9
+ module BrowserImport
10
+ # The only way a browser-imported domain ever leaves this machine:
11
+ # one `GET /.well-known/ucp` through Portage::Ucp::Client.discover
12
+ # (the same discovery path `find` and `index build` use), recorded
13
+ # in the shared ProbeCache so a domain that didn't answer last time
14
+ # isn't asked again. No page, title, path or visit count is ever
15
+ # sent — just the origin, as the host of that one request.
16
+ class Prober
17
+ THROTTLE = 0.1
18
+
19
+ # Net::HTTP's own defaults are 60s each way — fine for one store,
20
+ # not for 200 history domains, some of which will be dead hosts.
21
+ TIMEOUT = 5
22
+
23
+ # @param discover [#call, nil] `->(origin) { session_or_nil }` —
24
+ # injectable so specs never make a real request; nil (the
25
+ # default) uses Portage::Ucp::Client.discover.
26
+ def initialize(cache: ProbeCache.new, discover: nil, throttle: THROTTLE, timeout: TIMEOUT)
27
+ @cache = cache
28
+ @discover = discover || method(:ucp_discover)
29
+ @throttle = throttle
30
+ @timeout = timeout
31
+ @probes = 0
32
+ end
33
+
34
+ attr_reader :probes
35
+
36
+ # A cached "no UCP here" verdict — answered with no request at all.
37
+ def cached_miss?(origin) = @cache.fetch(origin) == false
38
+
39
+ # @return [Portage::Ucp::Client::Session, nil]
40
+ def probe(origin)
41
+ sleep(@throttle) if @probes.positive? && @throttle.to_f.positive?
42
+ @probes += 1
43
+ session = safely { @discover.call(origin) }
44
+ @cache.record(origin, !session.nil?)
45
+ session
46
+ end
47
+
48
+ private
49
+
50
+ def safely(&)
51
+ Timeout.timeout(@timeout, &)
52
+ rescue StandardError # Timeout::Error included
53
+ nil
54
+ end
55
+
56
+ def ucp_discover(origin) = Portage::Ucp::Client.discover(origin, headers: UserAgent.headers)
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,114 @@
1
+ require_relative "sqlite"
2
+
3
+ module Portage
4
+ module Cli
5
+ module BrowserImport
6
+ # Where each supported browser keeps its profiles, and the exact
7
+ # files `portage browser import` is allowed to read from one.
8
+ #
9
+ # ALLOWED_FILES is the whole list — History/Bookmarks for the
10
+ # Chromium family, places.sqlite for Firefox, History.db/
11
+ # Bookmarks.plist for Safari. Nothing else in a profile directory is
12
+ # ever opened, copied or listed: not `Login Data`, `Cookies`,
13
+ # `Web Data` (autofill), `logins.json`, `key4.db`, `cookies.sqlite`,
14
+ # `formhistory.sqlite`, nor the keychain (docs/plans/
15
+ # buy-skill-and-local-browser.md, "Never, in any tier"; profiles_spec/
16
+ # importer_spec assert this against a fixture profile full of decoys).
17
+ # (Sqlite also copies an allowed database's own `-wal` file when one
18
+ # exists — it's part of that same database, holding its newest rows.)
19
+ # A profile's own *directory* is only ever stat'ed for these names,
20
+ # never listed; the one directory listing this does is the profile
21
+ # *root* (which profile folders exist — `Default`, `Profile 1`, …),
22
+ # never a profile's own contents.
23
+ module Profiles
24
+ Profile = Struct.new(:browser, :family, :dir, :history, :bookmarks, keyword_init: true)
25
+
26
+ CHROMIUM = %w[chrome edge brave arc].freeze
27
+
28
+ ALLOWED_FILES = {
29
+ "chromium" => { history: "History", bookmarks: "Bookmarks" },
30
+ "firefox" => { history: "places.sqlite", bookmarks: "places.sqlite" },
31
+ "safari" => { history: "History.db", bookmarks: "Bookmarks.plist" }
32
+ }.freeze
33
+
34
+ # Relative to the user's home. The first existing root wins, so a
35
+ # macOS path and a Linux path can share one entry.
36
+ ROOTS = {
37
+ "chrome" => ["Library/Application Support/Google/Chrome", ".config/google-chrome"],
38
+ "edge" => ["Library/Application Support/Microsoft Edge", ".config/microsoft-edge"],
39
+ "brave" => ["Library/Application Support/BraveSoftware/Brave-Browser", ".config/BraveSoftware/Brave-Browser"],
40
+ "arc" => ["Library/Application Support/Arc/User Data"],
41
+ "firefox" => ["Library/Application Support/Firefox", ".mozilla/firefox"],
42
+ "safari" => ["Library/Safari"]
43
+ }.freeze
44
+
45
+ # Default order when `--browser` isn't given. Safari is last: on
46
+ # macOS its directory exists whether or not the user uses it, and
47
+ # it's the only one that needs Full Disk Access.
48
+ BROWSERS = %w[chrome arc brave edge firefox safari].freeze
49
+
50
+ def self.family(browser)
51
+ return "chromium" if CHROMIUM.include?(browser)
52
+
53
+ browser
54
+ end
55
+
56
+ # @return [String, nil] the first root that exists for `browser`.
57
+ def self.default_root(browser, home: Dir.home)
58
+ ROOTS.fetch(browser, []).map { |rel| File.join(home, rel) }.find { |path| File.directory?(path) }
59
+ end
60
+
61
+ # @return [String, nil] the first browser in BROWSERS with a root on
62
+ # this machine.
63
+ def self.detect(home: Dir.home) = BROWSERS.find { |b| default_root(b, home: home) }
64
+
65
+ # @param root [String] the browser's profile root — injectable so
66
+ # specs (and a live check) point at a fixture or a specific copy.
67
+ # @return [Array<Profile>] every profile under `root` holding at
68
+ # least one of the allowed files.
69
+ def self.locate(browser, root:)
70
+ family = family(browser)
71
+ return [] unless ALLOWED_FILES.key?(family) && root && File.directory?(root)
72
+
73
+ profile_dirs(family, root).filter_map { |dir| profile_for(browser, family, dir) }
74
+ rescue Errno::EPERM, Errno::EACCES => e
75
+ # macOS privacy protection (or a sandbox) refusing even the root
76
+ # listing — reported, never worked around (see Importer).
77
+ raise Sqlite::PermissionDenied, e.message
78
+ end
79
+
80
+ # The only directory listing in this module: the profile root's own
81
+ # folder names. Safari has no per-profile folders — its root *is*
82
+ # the profile, so nothing is listed there at all.
83
+ def self.profile_dirs(family, root)
84
+ case family
85
+ when "chromium" then Dir.glob(File.join(root, "{Default,Profile *}"))
86
+ when "firefox"
87
+ profiles = File.directory?(File.join(root, "Profiles")) ? File.join(root, "Profiles") : root
88
+ Dir.glob(File.join(profiles, "*")).select { |d| File.directory?(d) }.sort
89
+ else [root]
90
+ end
91
+ end
92
+ private_class_method :profile_dirs
93
+
94
+ def self.profile_for(browser, family, dir)
95
+ files = ALLOWED_FILES.fetch(family)
96
+ history = existing(dir, files[:history])
97
+ bookmarks = existing(dir, files[:bookmarks])
98
+ return nil unless history || bookmarks
99
+
100
+ Profile.new(browser: browser, family: family, dir: dir, history: history, bookmarks: bookmarks)
101
+ end
102
+ private_class_method :profile_for
103
+
104
+ # A stat, never an open — File.exist? on a name from ALLOWED_FILES
105
+ # only.
106
+ def self.existing(dir, name)
107
+ path = File.join(dir, name)
108
+ File.exist?(path) ? path : nil
109
+ end
110
+ private_class_method :existing
111
+ end
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,179 @@
1
+ require "json"
2
+ require "open3"
3
+ require "tmpdir"
4
+ require "fileutils"
5
+
6
+ require_relative "sqlite"
7
+ require_relative "plist_xml"
8
+
9
+ module Portage
10
+ module Cli
11
+ module BrowserImport
12
+ # One reader per browser family. Each turns a Profiles::Profile into
13
+ # plain rows — `{ kind: "history"|"bookmark", url:, title:, folder:,
14
+ # visits: }` — and reads nothing but that profile's own allowed files
15
+ # (Profiles::ALLOWED_FILES). A row is just what the browser recorded;
16
+ # reducing it to a domain, filtering and probing is Importer's job.
17
+ module Readers
18
+ # Seconds between the Unix epoch and each browser's own clock:
19
+ # Chromium counts microseconds from 1601-01-01, Safari (Core Data)
20
+ # seconds from 2001-01-01, Firefox microseconds from the Unix epoch.
21
+ WEBKIT_EPOCH_OFFSET = 11_644_473_600
22
+ CORE_DATA_EPOCH_OFFSET = 978_307_200
23
+
24
+ def self.for(family, sqlite: Sqlite.new, plist: Plist.new)
25
+ case family
26
+ when "chromium" then Chromium.new(sqlite: sqlite)
27
+ when "firefox" then Firefox.new(sqlite: sqlite)
28
+ when "safari" then Safari.new(sqlite: sqlite, plist: plist)
29
+ end
30
+ end
31
+
32
+ def self.row(kind, url, title: nil, folder: nil, visits: 1)
33
+ { kind: kind, url: url.to_s, title: title.to_s.strip, folder: folder.to_s.strip,
34
+ visits: [visits.to_i, 1].max }
35
+ end
36
+
37
+ # `History` (SQLite, copied first — Chrome keeps it locked) and
38
+ # `Bookmarks` (plain JSON, read in place: it's rewritten whole, never
39
+ # held open).
40
+ class Chromium
41
+ def initialize(sqlite:)
42
+ @sqlite = sqlite
43
+ end
44
+
45
+ def history(profile, since:)
46
+ return [] unless profile.history
47
+
48
+ cutoff = (since.to_i + WEBKIT_EPOCH_OFFSET) * 1_000_000
49
+ sql = "SELECT url, title, visit_count AS visits FROM urls WHERE last_visit_time >= #{cutoff}"
50
+ @sqlite.with_copy(profile.history) { |query| query.call(sql) }
51
+ .map { |r| Readers.row("history", r["url"], title: r["title"], visits: r["visits"]) }
52
+ end
53
+
54
+ def bookmarks(profile)
55
+ return [] unless profile.bookmarks
56
+
57
+ data = JSON.parse(File.read(profile.bookmarks, encoding: "UTF-8"))
58
+ Hash(data["roots"]).values.grep(Hash).flat_map { |node| walk(node, nil) }
59
+ rescue JSON::ParserError
60
+ []
61
+ rescue Errno::EPERM, Errno::EACCES => e
62
+ raise Sqlite::PermissionDenied, e.message
63
+ end
64
+
65
+ private
66
+
67
+ def walk(node, folder)
68
+ if node["type"] == "url"
69
+ [Readers.row("bookmark", node["url"], title: node["name"], folder: folder)]
70
+ else
71
+ Array(node["children"]).flat_map { |child| walk(child, node["name"]) }
72
+ end
73
+ end
74
+ end
75
+
76
+ # `places.sqlite` holds both history and bookmarks — one copy, two
77
+ # queries.
78
+ class Firefox
79
+ def initialize(sqlite:)
80
+ @sqlite = sqlite
81
+ end
82
+
83
+ def history(profile, since:)
84
+ return [] unless profile.history
85
+
86
+ sql = "SELECT url, title, visit_count AS visits FROM moz_places " \
87
+ "WHERE last_visit_date >= #{since.to_i * 1_000_000}"
88
+ @sqlite.with_copy(profile.history) { |query| query.call(sql) }
89
+ .map { |r| Readers.row("history", r["url"], title: r["title"], visits: r["visits"]) }
90
+ end
91
+
92
+ def bookmarks(profile)
93
+ return [] unless profile.bookmarks
94
+
95
+ sql = "SELECT p.url AS url, b.title AS title, f.title AS folder FROM moz_bookmarks b " \
96
+ "JOIN moz_places p ON p.id = b.fk LEFT JOIN moz_bookmarks f ON f.id = b.parent WHERE b.type = 1"
97
+ @sqlite.with_copy(profile.bookmarks) { |query| query.call(sql) }
98
+ .map { |r| Readers.row("bookmark", r["url"], title: r["title"], folder: r["folder"]) }
99
+ end
100
+ end
101
+
102
+ # `History.db` (SQLite) and `Bookmarks.plist` (binary plist). Both
103
+ # sit under ~/Library/Safari, which macOS guards behind Full Disk
104
+ # Access — a copy attempt without it raises Sqlite::PermissionDenied,
105
+ # which Importer turns into an explanation, never a workaround.
106
+ class Safari
107
+ def initialize(sqlite:, plist:)
108
+ @sqlite = sqlite
109
+ @plist = plist
110
+ end
111
+
112
+ def history(profile, since:)
113
+ return [] unless profile.history
114
+
115
+ cutoff = since.to_i - CORE_DATA_EPOCH_OFFSET
116
+ sql = "SELECT i.url AS url, v.title AS title, i.visit_count AS visits, MAX(v.visit_time) AS latest " \
117
+ "FROM history_items i JOIN history_visits v ON v.history_item = i.id " \
118
+ "WHERE v.visit_time >= #{cutoff} GROUP BY i.id"
119
+ @sqlite.with_copy(profile.history) { |query| query.call(sql) }
120
+ .map { |r| Readers.row("history", r["url"], title: r["title"], visits: r["visits"]) }
121
+ end
122
+
123
+ def bookmarks(profile)
124
+ return [] unless profile.bookmarks
125
+
126
+ walk(@plist.read(profile.bookmarks), nil)
127
+ end
128
+
129
+ private
130
+
131
+ def walk(node, folder)
132
+ return [] unless node.is_a?(Hash)
133
+
134
+ if node["URLString"]
135
+ title = Hash(node["URIDictionary"])["title"]
136
+ [Readers.row("bookmark", node["URLString"], title: title, folder: folder)]
137
+ else
138
+ Array(node["Children"]).flat_map { |child| walk(child, node["Title"]) }
139
+ end
140
+ end
141
+ end
142
+
143
+ # Reads a (binary or XML) plist: copies it into a private tmpdir,
144
+ # converts the copy to XML with macOS's own `plutil`, parses it with
145
+ # PlistXml, and deletes the copy.
146
+ class Plist
147
+ def initialize(command: "plutil")
148
+ @command = command
149
+ end
150
+
151
+ def read(path)
152
+ Dir.mktmpdir("portage-browser-import") do |dir|
153
+ copy = File.join(dir, "copy.plist")
154
+ copy_file(path, copy)
155
+ PlistXml.parse(convert(copy))
156
+ end
157
+ end
158
+
159
+ private
160
+
161
+ def copy_file(path, copy)
162
+ FileUtils.cp(path, copy)
163
+ rescue Errno::EPERM, Errno::EACCES => e
164
+ raise Sqlite::PermissionDenied, e.message
165
+ end
166
+
167
+ def convert(copy)
168
+ out, err, status = Open3.capture3(@command, "-convert", "xml1", "-o", "-", copy)
169
+ raise Sqlite::Unavailable, "plutil failed: #{err.strip}" unless status.success?
170
+
171
+ out.dup.force_encoding("UTF-8").scrub
172
+ rescue Errno::ENOENT
173
+ raise Sqlite::Unavailable, "the plutil command isn't available (it ships with macOS)"
174
+ end
175
+ end
176
+ end
177
+ end
178
+ end
179
+ end