portage-cli 0.7.5 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +616 -0
- data/README.md +302 -5
- data/known-stores/categories.yml +1263 -0
- data/lib/portage/cli/agent_profile_url.rb +30 -0
- data/lib/portage/cli/approval_policy.rb +56 -0
- data/lib/portage/cli/approve.rb +135 -0
- data/lib/portage/cli/browser_import/categorize.rb +59 -0
- data/lib/portage/cli/browser_import/confirm.rb +35 -0
- data/lib/portage/cli/browser_import/domains.rb +47 -0
- data/lib/portage/cli/browser_import/filter.rb +91 -0
- data/lib/portage/cli/browser_import/importer.rb +248 -0
- data/lib/portage/cli/browser_import/plist_xml.rb +72 -0
- data/lib/portage/cli/browser_import/prober.rb +60 -0
- data/lib/portage/cli/browser_import/profiles.rb +114 -0
- data/lib/portage/cli/browser_import/readers.rb +179 -0
- data/lib/portage/cli/browser_import/saver.rb +62 -0
- data/lib/portage/cli/browser_import/sqlite.rb +68 -0
- data/lib/portage/cli/browser_import.rb +23 -0
- data/lib/portage/cli/browser_opener.rb +38 -0
- data/lib/portage/cli/browser_profile/allowlist.rb +40 -0
- data/lib/portage/cli/browser_profile/bridge.rb +120 -0
- data/lib/portage/cli/browser_profile/browsers.rb +69 -0
- data/lib/portage/cli/browser_profile/cdp.rb +67 -0
- data/lib/portage/cli/browser_profile/cdp_socket.rb +186 -0
- data/lib/portage/cli/browser_profile/errors.rb +26 -0
- data/lib/portage/cli/browser_profile/launcher.rb +34 -0
- data/lib/portage/cli/browser_profile/profile.rb +93 -0
- data/lib/portage/cli/browser_profile.rb +25 -0
- data/lib/portage/cli/buy.rb +559 -29
- data/lib/portage/cli/checkout_handoff.rb +5 -24
- data/lib/portage/cli/classifier.rb +158 -0
- data/lib/portage/cli/compare.rb +3 -0
- data/lib/portage/cli/doctor.rb +155 -1
- data/lib/portage/cli/dot_env.rb +55 -0
- data/lib/portage/cli/find.rb +103 -12
- data/lib/portage/cli/handoff_agents.rb +186 -0
- data/lib/portage/cli/handoff_only.rb +94 -0
- data/lib/portage/cli/handoff_reconciler.rb +15 -1
- data/lib/portage/cli/handoff_target.rb +61 -0
- data/lib/portage/cli/history.rb +47 -2
- data/lib/portage/cli/human_prompt.rb +118 -0
- data/lib/portage/cli/index/builder.rb +335 -0
- data/lib/portage/cli/index/exporter.rb +91 -0
- data/lib/portage/cli/index/known_cache.rb +155 -0
- data/lib/portage/cli/index/product_store.rb +101 -0
- data/lib/portage/cli/index/sources/browser.rb +31 -0
- data/lib/portage/cli/index/sources/shopify_catalog.rb +82 -0
- data/lib/portage/cli/index/sources/stores_file.rb +58 -0
- data/lib/portage/cli/index/sources/webmcp_sweep.rb +29 -0
- data/lib/portage/cli/index/sources/wikidata.rb +95 -0
- data/lib/portage/cli/index/sources.rb +44 -0
- data/lib/portage/cli/index/store.rb +109 -0
- data/lib/portage/cli/index.rb +20 -0
- data/lib/portage/cli/known_stores_url.rb +15 -0
- data/lib/portage/cli/money.rb +18 -0
- data/lib/portage/cli/offer_choice.rb +38 -0
- data/lib/portage/cli/offer_sources.rb +460 -0
- data/lib/portage/cli/payment_methods.rb +24 -3
- data/lib/portage/cli/pick.rb +167 -0
- data/lib/portage/cli/product_page.rb +84 -0
- data/lib/portage/cli/quotes.rb +82 -0
- data/lib/portage/cli/search_backends.rb +337 -12
- data/lib/portage/cli/setup_wizard/prompt.rb +67 -0
- data/lib/portage/cli/setup_wizard/steps/agent_profile.rb +60 -0
- data/lib/portage/cli/setup_wizard/steps/browser_import.rb +29 -0
- data/lib/portage/cli/setup_wizard/steps/handoff.rb +100 -0
- data/lib/portage/cli/setup_wizard/steps/index_build.rb +31 -0
- data/lib/portage/cli/setup_wizard/steps/policy.rb +60 -0
- data/lib/portage/cli/setup_wizard/steps/retailer_keys.rb +55 -0
- data/lib/portage/cli/setup_wizard/steps/search_keys.rb +54 -0
- data/lib/portage/cli/setup_wizard/steps/shipping.rb +51 -0
- data/lib/portage/cli/setup_wizard.rb +74 -0
- data/lib/portage/cli/version.rb +1 -1
- data/lib/portage/cli/webmcp.rb +10 -3
- data/lib/portage/cli/webmcp_autofill_confirm.rb +38 -0
- data/lib/portage/cli/webmcp_autofill_fields.rb +60 -0
- data/lib/portage/cli/webmcp_autofill_mode.rb +39 -0
- data/lib/portage/cli/webmcp_mapping_confirm.rb +68 -0
- data/lib/portage/cli/webmcp_mappings.rb +84 -0
- data/lib/portage/cli.rb +930 -43
- metadata +67 -2
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
|
|
3
|
+
require_relative "../classifier"
|
|
4
|
+
require_relative "../index"
|
|
5
|
+
require_relative "../index/builder"
|
|
6
|
+
require_relative "profiles"
|
|
7
|
+
require_relative "readers"
|
|
8
|
+
require_relative "filter"
|
|
9
|
+
require_relative "prober"
|
|
10
|
+
require_relative "categorize"
|
|
11
|
+
require_relative "saver"
|
|
12
|
+
require_relative "domains"
|
|
13
|
+
|
|
14
|
+
module Portage
|
|
15
|
+
module Cli
|
|
16
|
+
module BrowserImport
|
|
17
|
+
# `portage browser import` (docs/plans/buy-skill-and-local-browser.md
|
|
18
|
+
# Phase 3, Tier A): bookmarks and history in, a short list of shop
|
|
19
|
+
# domains out, written to the user's own Index::Store only after the
|
|
20
|
+
# user has seen the list and said yes (Cli.run_browser_import owns
|
|
21
|
+
# that gate; this class only ever writes from #save).
|
|
22
|
+
#
|
|
23
|
+
# #plan reads the profile's allowed files (Profiles::ALLOWED_FILES,
|
|
24
|
+
# nothing else), reduces every row to its domain, and decides each
|
|
25
|
+
# domain locally first — skipped as an obvious non-shop (Filter),
|
|
26
|
+
# excluded by the user, on the hand-off-only list, or already in the
|
|
27
|
+
# index — before spending one of at most `max_probes` Prober probes
|
|
28
|
+
# on an unknown one. A domain is kept only when it answers
|
|
29
|
+
# `/.well-known/ucp`, matches a WebMCP preset, or is hand-off only.
|
|
30
|
+
# Each kept domain is classified from what the user actually looked
|
|
31
|
+
# at there (page titles, bookmark folder names, URL slugs), weighted
|
|
32
|
+
# by visit count.
|
|
33
|
+
#
|
|
34
|
+
# Kept domains land in Index::Store with `sources: ["history"]`/
|
|
35
|
+
# `["bookmark"]` — the same untrusted index every other source
|
|
36
|
+
# writes, so an imported store reaches `find` only as a
|
|
37
|
+
# `source: "index"` candidate (never Policy#merchant_allowlist, never
|
|
38
|
+
# past the `--store`/interactive pick a `--yes` buy requires), and
|
|
39
|
+
# Index::Exporter treats both labels as personal: never exported.
|
|
40
|
+
class Importer
|
|
41
|
+
DEFAULT_HISTORY_DAYS = 90
|
|
42
|
+
MAX_PROBES = 200
|
|
43
|
+
|
|
44
|
+
FULL_DISK_ACCESS = "macOS won't let this terminal read Safari's history and bookmarks without Full Disk " \
|
|
45
|
+
"Access. To allow it, open System Settings > Privacy & Security > Full Disk Access, " \
|
|
46
|
+
"turn it on for the terminal app you run portage from, then quit and reopen that app " \
|
|
47
|
+
"and re-run this command. Portage doesn't try any other way in. Nothing was imported, " \
|
|
48
|
+
"probed or saved.".freeze
|
|
49
|
+
|
|
50
|
+
# Any other browser: the same "explain and stop", without claiming
|
|
51
|
+
# to know which protection said no (macOS privacy settings, a
|
|
52
|
+
# sandboxed terminal, file permissions).
|
|
53
|
+
PERMISSION_DENIED = "Permission denied reading %<browser>s's profile folder. If your terminal runs " \
|
|
54
|
+
"sandboxed or macOS is protecting that folder, allow access (System Settings > " \
|
|
55
|
+
"Privacy & Security > Full Disk Access) and re-run. Portage doesn't try any other way " \
|
|
56
|
+
"in. Nothing was imported, probed or saved.".freeze
|
|
57
|
+
|
|
58
|
+
Options = Struct.new(:browser, :root, :history_days, :include_product_pages, :max_probes, :exclude,
|
|
59
|
+
keyword_init: true) do
|
|
60
|
+
def initialize(history_days: DEFAULT_HISTORY_DAYS, include_product_pages: false, max_probes: MAX_PROBES,
|
|
61
|
+
exclude: [], **)
|
|
62
|
+
super
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# @param handoff_only_hosts [Array<String>] Tier C hosts (Phase 5
|
|
67
|
+
# will supply the user's own `handoff_only_hosts:` config; nothing
|
|
68
|
+
# ships that list yet, so it defaults to empty). A history domain
|
|
69
|
+
# on it is kept — hand-off only, never probed, never automated.
|
|
70
|
+
# @param webmcp_preset_for [#call, nil] `->(origin) { preset_or_nil }`.
|
|
71
|
+
# Matching a WebMCP preset needs a browser bridge an import doesn't
|
|
72
|
+
# have (same posture as Index::Sources::WebmcpSweep), so nil (the
|
|
73
|
+
# default) skips the check entirely.
|
|
74
|
+
# @param readers [#call] `->(family) { reader }` — see Readers.for.
|
|
75
|
+
def initialize(stores: Index::Store.new, products: Index::ProductStore.new,
|
|
76
|
+
known_cache: Index::KnownCache.new, prober: Prober.new, handoff_only_hosts: [],
|
|
77
|
+
webmcp_preset_for: nil, readers: Readers.method(:for), now: Time.now)
|
|
78
|
+
@stores = stores
|
|
79
|
+
@products = products
|
|
80
|
+
@known_cache = known_cache
|
|
81
|
+
@prober = prober
|
|
82
|
+
@handoff_only_hosts = handoff_only_hosts.map { |h| h.to_s.downcase.delete_prefix("www.") }
|
|
83
|
+
@webmcp_preset_for = webmcp_preset_for
|
|
84
|
+
@readers = readers
|
|
85
|
+
@now = now
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# @return [Hash] the proposal: counts, `kept` (domains to save) and
|
|
89
|
+
# `products` (only with include_product_pages) — or `error:` /
|
|
90
|
+
# `message:` when the browser's files couldn't be read (Safari
|
|
91
|
+
# without Full Disk Access, no sqlite3, no profile found).
|
|
92
|
+
def plan(options)
|
|
93
|
+
profiles = Profiles.locate(options.browser, root: options.root)
|
|
94
|
+
return no_profile(options) if profiles.empty?
|
|
95
|
+
|
|
96
|
+
rows, opened = read_rows(profiles, since: @now.to_i - (options.history_days.to_i * 86_400))
|
|
97
|
+
summarize(options, profiles, opened, rows)
|
|
98
|
+
rescue Sqlite::PermissionDenied
|
|
99
|
+
permission_denied(options)
|
|
100
|
+
rescue Sqlite::Unavailable => e
|
|
101
|
+
{ browser: options.browser, error: "reader_unavailable",
|
|
102
|
+
message: "Couldn't read the browser's files: #{e.message}." }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Writes a #plan's kept domains (and products) into the index — see
|
|
106
|
+
# Saver.
|
|
107
|
+
# @return [Hash] stores:, products: — how many entries were written.
|
|
108
|
+
def save(proposal) = Saver.new(stores: @stores, products: @products, now: @now).save(proposal)
|
|
109
|
+
|
|
110
|
+
private
|
|
111
|
+
|
|
112
|
+
# --- Reading ---
|
|
113
|
+
|
|
114
|
+
def read_rows(profiles, since:)
|
|
115
|
+
opened = []
|
|
116
|
+
rows = profiles.flat_map do |profile|
|
|
117
|
+
reader = @readers.call(profile.family)
|
|
118
|
+
opened.concat([profile.history, profile.bookmarks].compact)
|
|
119
|
+
reader.history(profile, since: since) + reader.bookmarks(profile)
|
|
120
|
+
end
|
|
121
|
+
[rows, opened.uniq]
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# --- Deciding each domain ---
|
|
125
|
+
|
|
126
|
+
def summarize(options, profiles, opened, rows)
|
|
127
|
+
groups = Domains.group(rows)
|
|
128
|
+
state = { skipped: Hash.new(0), kept: [], cached_miss: 0, not_ucp: 0, unprobed: 0 }
|
|
129
|
+
groups.sort_by { |_key, g| -g[:visits] }.each { |key, group| decide(key, group, options, state) }
|
|
130
|
+
report(options, profiles, opened, rows, groups, state)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def decide(key, group, options, state)
|
|
134
|
+
reason = local_skip_reason(key, options)
|
|
135
|
+
return state[:skipped][reason] += 1 if reason
|
|
136
|
+
|
|
137
|
+
verdict = local_verdict(key) || probe_verdict(Domains.origin_for(group), options, state)
|
|
138
|
+
state[:kept] << kept_entry(key, group, verdict) if verdict
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def local_skip_reason(key, options)
|
|
142
|
+
return :excluded if Array(options.exclude).any? { |h| h.to_s.downcase.delete_prefix("www.") == key }
|
|
143
|
+
|
|
144
|
+
Filter.skip_reason(key)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Everything decidable with no network call at all.
|
|
148
|
+
def local_verdict(key)
|
|
149
|
+
return { verdict: "handoff_only", handoff_only: true } if handoff_only?(key)
|
|
150
|
+
|
|
151
|
+
own = own_entry(key)
|
|
152
|
+
return { verdict: "indexed", origin: own["origin"] } if own
|
|
153
|
+
|
|
154
|
+
known = known_entry(key)
|
|
155
|
+
return unless known
|
|
156
|
+
|
|
157
|
+
{ verdict: "known", origin: known["origin"], capabilities: known["capabilities"],
|
|
158
|
+
last_verified: known["last_verified"] }
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# A cached "no UCP" verdict or the probe cap skips the probe, but
|
|
162
|
+
# not the WebMCP check (when a bridge is injected) — a page can
|
|
163
|
+
# expose WebMCP tools without any manifest at all.
|
|
164
|
+
def probe_verdict(origin, options, state)
|
|
165
|
+
session = ucp_session(origin, options, state)
|
|
166
|
+
return nil if session == :capped
|
|
167
|
+
return { verdict: "ucp", origin: origin, capabilities: Index::Builder.capabilities_of(session) } if session
|
|
168
|
+
|
|
169
|
+
preset = @webmcp_preset_for&.call(origin)
|
|
170
|
+
return { verdict: "webmcp", origin: origin, webmcp_preset: preset } if preset
|
|
171
|
+
|
|
172
|
+
state[:not_ucp] += 1 unless session == false
|
|
173
|
+
nil
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# @return [Session, nil, false, :capped] a session; nil when the
|
|
177
|
+
# probe found no UCP; false for a cached miss (no request made);
|
|
178
|
+
# :capped once max_probes is spent.
|
|
179
|
+
def ucp_session(origin, options, state)
|
|
180
|
+
if @prober.cached_miss?(origin)
|
|
181
|
+
state[:cached_miss] += 1
|
|
182
|
+
false
|
|
183
|
+
elsif @prober.probes >= options.max_probes.to_i
|
|
184
|
+
state[:unprobed] += 1
|
|
185
|
+
:capped
|
|
186
|
+
else
|
|
187
|
+
@prober.probe(origin)
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def handoff_only?(key)
|
|
192
|
+
@handoff_only_hosts.any? { |h| key == h || key.end_with?(".#{h}") }
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def own_entry(key) = @stores.all.find { |e| Domains.key_of(e["origin"]) == key }
|
|
196
|
+
|
|
197
|
+
def known_entry(key) = @known_cache.stores.values.find { |e| Domains.key_of(e["origin"]) == key }
|
|
198
|
+
|
|
199
|
+
# --- The proposal ---
|
|
200
|
+
|
|
201
|
+
def kept_entry(key, group, verdict)
|
|
202
|
+
{ domain: key, origin: verdict[:origin] || Domains.origin_for(group), verdict: verdict[:verdict],
|
|
203
|
+
sources: group[:rows].map { |r| r[:kind] }.uniq.sort, visits: group[:visits],
|
|
204
|
+
categories: Categorize.domain(group[:rows]), rows: group[:rows] }
|
|
205
|
+
.merge(verdict.slice(:capabilities, :webmcp_preset, :handoff_only, :last_verified))
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def report(options, profiles, opened, rows, groups, state)
|
|
209
|
+
kept = state[:kept]
|
|
210
|
+
{ browser: options.browser, profiles: profiles.length, files_opened: opened, rows: row_counts(rows),
|
|
211
|
+
domains: groups.length, skipped: state[:skipped].transform_keys(&:to_s),
|
|
212
|
+
already_indexed: count(kept, "indexed"), known: count(kept, "known"), probed: @prober.probes,
|
|
213
|
+
cached_miss: state[:cached_miss], not_ucp: state[:not_ucp], unprobed: state[:unprobed],
|
|
214
|
+
capped: state[:unprobed].positive?, kept: kept.map { |entry| public_entry(entry) },
|
|
215
|
+
products: options.include_product_pages ? Categorize.products(kept) : [] }
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def row_counts(rows)
|
|
219
|
+
kinds = rows.map { |r| r[:kind] }
|
|
220
|
+
{ history: kinds.count("history"), bookmark: kinds.count("bookmark") }
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def public_entry(entry)
|
|
224
|
+
entry.except(:rows).merge(category_names: Classifier.names_for(entry[:categories].keys))
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def count(kept, verdict) = kept.count { |e| e[:verdict] == verdict }
|
|
228
|
+
|
|
229
|
+
# --- Helpers ---
|
|
230
|
+
|
|
231
|
+
def no_profile(options)
|
|
232
|
+
where = options.root ? " under #{options.root}" : ""
|
|
233
|
+
{ browser: options.browser, error: "no_profile",
|
|
234
|
+
message: "No #{options.browser} profile with history or bookmarks found#{where}." }
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def permission_denied(options)
|
|
238
|
+
if options.browser == "safari"
|
|
239
|
+
{ browser: "safari", error: "full_disk_access_required", message: FULL_DISK_ACCESS }
|
|
240
|
+
else
|
|
241
|
+
{ browser: options.browser, error: "permission_denied",
|
|
242
|
+
message: format(PERMISSION_DENIED, browser: options.browser) }
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
require "strscan"
|
|
2
|
+
|
|
3
|
+
module Portage
|
|
4
|
+
module Cli
|
|
5
|
+
module BrowserImport
|
|
6
|
+
# Just enough of Apple's XML property-list format to read Safari's
|
|
7
|
+
# Bookmarks.plist once `plutil -convert xml1` has turned its binary
|
|
8
|
+
# form into XML. `plutil -convert json` would be simpler but refuses
|
|
9
|
+
# any plist holding a <date> (Safari's Reading List entries carry
|
|
10
|
+
# them), and a real XML library (rexml) isn't a runtime dependency of
|
|
11
|
+
# this gem — this is ~50 lines instead of a new one.
|
|
12
|
+
#
|
|
13
|
+
# Dates, data and numbers come back as their raw text; bookmark
|
|
14
|
+
# import only ever reads strings out of the tree.
|
|
15
|
+
module PlistXml
|
|
16
|
+
class ParseError < StandardError; end
|
|
17
|
+
|
|
18
|
+
SCALARS = "string|key|data|date|integer|real".freeze
|
|
19
|
+
ENTITIES = { "amp" => "&", "lt" => "<", "gt" => ">", "quot" => '"', "apos" => "'" }.freeze
|
|
20
|
+
|
|
21
|
+
def self.parse(xml)
|
|
22
|
+
scanner = StringScanner.new(xml.to_s)
|
|
23
|
+
raise ParseError, "not an XML plist" unless scanner.skip_until(/<plist[^>]*>/)
|
|
24
|
+
|
|
25
|
+
value(scanner)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def self.value(scanner)
|
|
29
|
+
scanner.skip(/\s*/)
|
|
30
|
+
if scanner.scan(%r{<(dict|array|#{SCALARS})\s*/>}) then empty(scanner[1])
|
|
31
|
+
elsif scanner.scan(%r{<(true|false)\s*/>}) then scanner[1] == "true"
|
|
32
|
+
elsif scanner.scan("<dict>") then dict(scanner)
|
|
33
|
+
elsif scanner.scan("<array>") then array(scanner)
|
|
34
|
+
elsif scanner.scan(%r{<(#{SCALARS})>(.*?)</\1>}m) then unescape(scanner[2])
|
|
35
|
+
else raise ParseError, "unexpected plist content at #{scanner.pos}"
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def self.dict(scanner)
|
|
40
|
+
result = {}
|
|
41
|
+
until scanner.skip(%r{\s*</dict>})
|
|
42
|
+
key = value(scanner)
|
|
43
|
+
raise ParseError, "dict key isn't a string" unless key.is_a?(String)
|
|
44
|
+
|
|
45
|
+
result[key] = value(scanner)
|
|
46
|
+
end
|
|
47
|
+
result
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def self.array(scanner)
|
|
51
|
+
result = []
|
|
52
|
+
result << value(scanner) until scanner.skip(%r{\s*</array>})
|
|
53
|
+
result
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def self.empty(tag)
|
|
57
|
+
{ "dict" => {}, "array" => [] }.fetch(tag, "")
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def self.unescape(text)
|
|
61
|
+
text.gsub(/&(#x?)?(\w+);/) do
|
|
62
|
+
if Regexp.last_match(1) == "#x" then [Regexp.last_match(2).to_i(16)].pack("U")
|
|
63
|
+
elsif Regexp.last_match(1) == "#" then [Regexp.last_match(2).to_i].pack("U")
|
|
64
|
+
else ENTITIES.fetch(Regexp.last_match(2), Regexp.last_match(0))
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
private_class_method :value, :dict, :array, :empty, :unescape
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
require "timeout"
|
|
2
|
+
require "portage/ucp/client"
|
|
3
|
+
|
|
4
|
+
require_relative "../user_agent"
|
|
5
|
+
require_relative "../probe_cache"
|
|
6
|
+
|
|
7
|
+
module Portage
|
|
8
|
+
module Cli
|
|
9
|
+
module BrowserImport
|
|
10
|
+
# The only way a browser-imported domain ever leaves this machine:
|
|
11
|
+
# one `GET /.well-known/ucp` through Portage::Ucp::Client.discover
|
|
12
|
+
# (the same discovery path `find` and `index build` use), recorded
|
|
13
|
+
# in the shared ProbeCache so a domain that didn't answer last time
|
|
14
|
+
# isn't asked again. No page, title, path or visit count is ever
|
|
15
|
+
# sent — just the origin, as the host of that one request.
|
|
16
|
+
class Prober
|
|
17
|
+
THROTTLE = 0.1
|
|
18
|
+
|
|
19
|
+
# Net::HTTP's own defaults are 60s each way — fine for one store,
|
|
20
|
+
# not for 200 history domains, some of which will be dead hosts.
|
|
21
|
+
TIMEOUT = 5
|
|
22
|
+
|
|
23
|
+
# @param discover [#call, nil] `->(origin) { session_or_nil }` —
|
|
24
|
+
# injectable so specs never make a real request; nil (the
|
|
25
|
+
# default) uses Portage::Ucp::Client.discover.
|
|
26
|
+
def initialize(cache: ProbeCache.new, discover: nil, throttle: THROTTLE, timeout: TIMEOUT)
|
|
27
|
+
@cache = cache
|
|
28
|
+
@discover = discover || method(:ucp_discover)
|
|
29
|
+
@throttle = throttle
|
|
30
|
+
@timeout = timeout
|
|
31
|
+
@probes = 0
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
attr_reader :probes
|
|
35
|
+
|
|
36
|
+
# A cached "no UCP here" verdict — answered with no request at all.
|
|
37
|
+
def cached_miss?(origin) = @cache.fetch(origin) == false
|
|
38
|
+
|
|
39
|
+
# @return [Portage::Ucp::Client::Session, nil]
|
|
40
|
+
def probe(origin)
|
|
41
|
+
sleep(@throttle) if @probes.positive? && @throttle.to_f.positive?
|
|
42
|
+
@probes += 1
|
|
43
|
+
session = safely { @discover.call(origin) }
|
|
44
|
+
@cache.record(origin, !session.nil?)
|
|
45
|
+
session
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def safely(&)
|
|
51
|
+
Timeout.timeout(@timeout, &)
|
|
52
|
+
rescue StandardError # Timeout::Error included
|
|
53
|
+
nil
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def ucp_discover(origin) = Portage::Ucp::Client.discover(origin, headers: UserAgent.headers)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
require_relative "sqlite"
|
|
2
|
+
|
|
3
|
+
module Portage
|
|
4
|
+
module Cli
|
|
5
|
+
module BrowserImport
|
|
6
|
+
# Where each supported browser keeps its profiles, and the exact
|
|
7
|
+
# files `portage browser import` is allowed to read from one.
|
|
8
|
+
#
|
|
9
|
+
# ALLOWED_FILES is the whole list — History/Bookmarks for the
|
|
10
|
+
# Chromium family, places.sqlite for Firefox, History.db/
|
|
11
|
+
# Bookmarks.plist for Safari. Nothing else in a profile directory is
|
|
12
|
+
# ever opened, copied or listed: not `Login Data`, `Cookies`,
|
|
13
|
+
# `Web Data` (autofill), `logins.json`, `key4.db`, `cookies.sqlite`,
|
|
14
|
+
# `formhistory.sqlite`, nor the keychain (docs/plans/
|
|
15
|
+
# buy-skill-and-local-browser.md, "Never, in any tier"; profiles_spec/
|
|
16
|
+
# importer_spec assert this against a fixture profile full of decoys).
|
|
17
|
+
# (Sqlite also copies an allowed database's own `-wal` file when one
|
|
18
|
+
# exists — it's part of that same database, holding its newest rows.)
|
|
19
|
+
# A profile's own *directory* is only ever stat'ed for these names,
|
|
20
|
+
# never listed; the one directory listing this does is the profile
|
|
21
|
+
# *root* (which profile folders exist — `Default`, `Profile 1`, …),
|
|
22
|
+
# never a profile's own contents.
|
|
23
|
+
module Profiles
|
|
24
|
+
Profile = Struct.new(:browser, :family, :dir, :history, :bookmarks, keyword_init: true)
|
|
25
|
+
|
|
26
|
+
CHROMIUM = %w[chrome edge brave arc].freeze
|
|
27
|
+
|
|
28
|
+
ALLOWED_FILES = {
|
|
29
|
+
"chromium" => { history: "History", bookmarks: "Bookmarks" },
|
|
30
|
+
"firefox" => { history: "places.sqlite", bookmarks: "places.sqlite" },
|
|
31
|
+
"safari" => { history: "History.db", bookmarks: "Bookmarks.plist" }
|
|
32
|
+
}.freeze
|
|
33
|
+
|
|
34
|
+
# Relative to the user's home. The first existing root wins, so a
|
|
35
|
+
# macOS path and a Linux path can share one entry.
|
|
36
|
+
ROOTS = {
|
|
37
|
+
"chrome" => ["Library/Application Support/Google/Chrome", ".config/google-chrome"],
|
|
38
|
+
"edge" => ["Library/Application Support/Microsoft Edge", ".config/microsoft-edge"],
|
|
39
|
+
"brave" => ["Library/Application Support/BraveSoftware/Brave-Browser", ".config/BraveSoftware/Brave-Browser"],
|
|
40
|
+
"arc" => ["Library/Application Support/Arc/User Data"],
|
|
41
|
+
"firefox" => ["Library/Application Support/Firefox", ".mozilla/firefox"],
|
|
42
|
+
"safari" => ["Library/Safari"]
|
|
43
|
+
}.freeze
|
|
44
|
+
|
|
45
|
+
# Default order when `--browser` isn't given. Safari is last: on
|
|
46
|
+
# macOS its directory exists whether or not the user uses it, and
|
|
47
|
+
# it's the only one that needs Full Disk Access.
|
|
48
|
+
BROWSERS = %w[chrome arc brave edge firefox safari].freeze
|
|
49
|
+
|
|
50
|
+
def self.family(browser)
|
|
51
|
+
return "chromium" if CHROMIUM.include?(browser)
|
|
52
|
+
|
|
53
|
+
browser
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# @return [String, nil] the first root that exists for `browser`.
|
|
57
|
+
def self.default_root(browser, home: Dir.home)
|
|
58
|
+
ROOTS.fetch(browser, []).map { |rel| File.join(home, rel) }.find { |path| File.directory?(path) }
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# @return [String, nil] the first browser in BROWSERS with a root on
|
|
62
|
+
# this machine.
|
|
63
|
+
def self.detect(home: Dir.home) = BROWSERS.find { |b| default_root(b, home: home) }
|
|
64
|
+
|
|
65
|
+
# @param root [String] the browser's profile root — injectable so
|
|
66
|
+
# specs (and a live check) point at a fixture or a specific copy.
|
|
67
|
+
# @return [Array<Profile>] every profile under `root` holding at
|
|
68
|
+
# least one of the allowed files.
|
|
69
|
+
def self.locate(browser, root:)
|
|
70
|
+
family = family(browser)
|
|
71
|
+
return [] unless ALLOWED_FILES.key?(family) && root && File.directory?(root)
|
|
72
|
+
|
|
73
|
+
profile_dirs(family, root).filter_map { |dir| profile_for(browser, family, dir) }
|
|
74
|
+
rescue Errno::EPERM, Errno::EACCES => e
|
|
75
|
+
# macOS privacy protection (or a sandbox) refusing even the root
|
|
76
|
+
# listing — reported, never worked around (see Importer).
|
|
77
|
+
raise Sqlite::PermissionDenied, e.message
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# The only directory listing in this module: the profile root's own
|
|
81
|
+
# folder names. Safari has no per-profile folders — its root *is*
|
|
82
|
+
# the profile, so nothing is listed there at all.
|
|
83
|
+
def self.profile_dirs(family, root)
|
|
84
|
+
case family
|
|
85
|
+
when "chromium" then Dir.glob(File.join(root, "{Default,Profile *}"))
|
|
86
|
+
when "firefox"
|
|
87
|
+
profiles = File.directory?(File.join(root, "Profiles")) ? File.join(root, "Profiles") : root
|
|
88
|
+
Dir.glob(File.join(profiles, "*")).select { |d| File.directory?(d) }.sort
|
|
89
|
+
else [root]
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
private_class_method :profile_dirs
|
|
93
|
+
|
|
94
|
+
def self.profile_for(browser, family, dir)
|
|
95
|
+
files = ALLOWED_FILES.fetch(family)
|
|
96
|
+
history = existing(dir, files[:history])
|
|
97
|
+
bookmarks = existing(dir, files[:bookmarks])
|
|
98
|
+
return nil unless history || bookmarks
|
|
99
|
+
|
|
100
|
+
Profile.new(browser: browser, family: family, dir: dir, history: history, bookmarks: bookmarks)
|
|
101
|
+
end
|
|
102
|
+
private_class_method :profile_for
|
|
103
|
+
|
|
104
|
+
# A stat, never an open — File.exist? on a name from ALLOWED_FILES
|
|
105
|
+
# only.
|
|
106
|
+
def self.existing(dir, name)
|
|
107
|
+
path = File.join(dir, name)
|
|
108
|
+
File.exist?(path) ? path : nil
|
|
109
|
+
end
|
|
110
|
+
private_class_method :existing
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "open3"
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
|
|
6
|
+
require_relative "sqlite"
|
|
7
|
+
require_relative "plist_xml"
|
|
8
|
+
|
|
9
|
+
module Portage
|
|
10
|
+
module Cli
|
|
11
|
+
module BrowserImport
|
|
12
|
+
# One reader per browser family. Each turns a Profiles::Profile into
|
|
13
|
+
# plain rows — `{ kind: "history"|"bookmark", url:, title:, folder:,
|
|
14
|
+
# visits: }` — and reads nothing but that profile's own allowed files
|
|
15
|
+
# (Profiles::ALLOWED_FILES). A row is just what the browser recorded;
|
|
16
|
+
# reducing it to a domain, filtering and probing is Importer's job.
|
|
17
|
+
module Readers
|
|
18
|
+
# Seconds between the Unix epoch and each browser's own clock:
|
|
19
|
+
# Chromium counts microseconds from 1601-01-01, Safari (Core Data)
|
|
20
|
+
# seconds from 2001-01-01, Firefox microseconds from the Unix epoch.
|
|
21
|
+
WEBKIT_EPOCH_OFFSET = 11_644_473_600
|
|
22
|
+
CORE_DATA_EPOCH_OFFSET = 978_307_200
|
|
23
|
+
|
|
24
|
+
def self.for(family, sqlite: Sqlite.new, plist: Plist.new)
|
|
25
|
+
case family
|
|
26
|
+
when "chromium" then Chromium.new(sqlite: sqlite)
|
|
27
|
+
when "firefox" then Firefox.new(sqlite: sqlite)
|
|
28
|
+
when "safari" then Safari.new(sqlite: sqlite, plist: plist)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def self.row(kind, url, title: nil, folder: nil, visits: 1)
|
|
33
|
+
{ kind: kind, url: url.to_s, title: title.to_s.strip, folder: folder.to_s.strip,
|
|
34
|
+
visits: [visits.to_i, 1].max }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# `History` (SQLite, copied first — Chrome keeps it locked) and
|
|
38
|
+
# `Bookmarks` (plain JSON, read in place: it's rewritten whole, never
|
|
39
|
+
# held open).
|
|
40
|
+
class Chromium
|
|
41
|
+
def initialize(sqlite:)
|
|
42
|
+
@sqlite = sqlite
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def history(profile, since:)
|
|
46
|
+
return [] unless profile.history
|
|
47
|
+
|
|
48
|
+
cutoff = (since.to_i + WEBKIT_EPOCH_OFFSET) * 1_000_000
|
|
49
|
+
sql = "SELECT url, title, visit_count AS visits FROM urls WHERE last_visit_time >= #{cutoff}"
|
|
50
|
+
@sqlite.with_copy(profile.history) { |query| query.call(sql) }
|
|
51
|
+
.map { |r| Readers.row("history", r["url"], title: r["title"], visits: r["visits"]) }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def bookmarks(profile)
|
|
55
|
+
return [] unless profile.bookmarks
|
|
56
|
+
|
|
57
|
+
data = JSON.parse(File.read(profile.bookmarks, encoding: "UTF-8"))
|
|
58
|
+
Hash(data["roots"]).values.grep(Hash).flat_map { |node| walk(node, nil) }
|
|
59
|
+
rescue JSON::ParserError
|
|
60
|
+
[]
|
|
61
|
+
rescue Errno::EPERM, Errno::EACCES => e
|
|
62
|
+
raise Sqlite::PermissionDenied, e.message
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
def walk(node, folder)
|
|
68
|
+
if node["type"] == "url"
|
|
69
|
+
[Readers.row("bookmark", node["url"], title: node["name"], folder: folder)]
|
|
70
|
+
else
|
|
71
|
+
Array(node["children"]).flat_map { |child| walk(child, node["name"]) }
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# `places.sqlite` holds both history and bookmarks — one copy, two
|
|
77
|
+
# queries.
|
|
78
|
+
class Firefox
|
|
79
|
+
def initialize(sqlite:)
|
|
80
|
+
@sqlite = sqlite
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def history(profile, since:)
|
|
84
|
+
return [] unless profile.history
|
|
85
|
+
|
|
86
|
+
sql = "SELECT url, title, visit_count AS visits FROM moz_places " \
|
|
87
|
+
"WHERE last_visit_date >= #{since.to_i * 1_000_000}"
|
|
88
|
+
@sqlite.with_copy(profile.history) { |query| query.call(sql) }
|
|
89
|
+
.map { |r| Readers.row("history", r["url"], title: r["title"], visits: r["visits"]) }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def bookmarks(profile)
|
|
93
|
+
return [] unless profile.bookmarks
|
|
94
|
+
|
|
95
|
+
sql = "SELECT p.url AS url, b.title AS title, f.title AS folder FROM moz_bookmarks b " \
|
|
96
|
+
"JOIN moz_places p ON p.id = b.fk LEFT JOIN moz_bookmarks f ON f.id = b.parent WHERE b.type = 1"
|
|
97
|
+
@sqlite.with_copy(profile.bookmarks) { |query| query.call(sql) }
|
|
98
|
+
.map { |r| Readers.row("bookmark", r["url"], title: r["title"], folder: r["folder"]) }
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# `History.db` (SQLite) and `Bookmarks.plist` (binary plist). Both
|
|
103
|
+
# sit under ~/Library/Safari, which macOS guards behind Full Disk
|
|
104
|
+
# Access — a copy attempt without it raises Sqlite::PermissionDenied,
|
|
105
|
+
# which Importer turns into an explanation, never a workaround.
|
|
106
|
+
class Safari
|
|
107
|
+
def initialize(sqlite:, plist:)
|
|
108
|
+
@sqlite = sqlite
|
|
109
|
+
@plist = plist
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def history(profile, since:)
|
|
113
|
+
return [] unless profile.history
|
|
114
|
+
|
|
115
|
+
cutoff = since.to_i - CORE_DATA_EPOCH_OFFSET
|
|
116
|
+
sql = "SELECT i.url AS url, v.title AS title, i.visit_count AS visits, MAX(v.visit_time) AS latest " \
|
|
117
|
+
"FROM history_items i JOIN history_visits v ON v.history_item = i.id " \
|
|
118
|
+
"WHERE v.visit_time >= #{cutoff} GROUP BY i.id"
|
|
119
|
+
@sqlite.with_copy(profile.history) { |query| query.call(sql) }
|
|
120
|
+
.map { |r| Readers.row("history", r["url"], title: r["title"], visits: r["visits"]) }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def bookmarks(profile)
|
|
124
|
+
return [] unless profile.bookmarks
|
|
125
|
+
|
|
126
|
+
walk(@plist.read(profile.bookmarks), nil)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
private
|
|
130
|
+
|
|
131
|
+
def walk(node, folder)
|
|
132
|
+
return [] unless node.is_a?(Hash)
|
|
133
|
+
|
|
134
|
+
if node["URLString"]
|
|
135
|
+
title = Hash(node["URIDictionary"])["title"]
|
|
136
|
+
[Readers.row("bookmark", node["URLString"], title: title, folder: folder)]
|
|
137
|
+
else
|
|
138
|
+
Array(node["Children"]).flat_map { |child| walk(child, node["Title"]) }
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Reads a (binary or XML) plist: copies it into a private tmpdir,
|
|
144
|
+
# converts the copy to XML with macOS's own `plutil`, parses it with
|
|
145
|
+
# PlistXml, and deletes the copy.
|
|
146
|
+
class Plist
|
|
147
|
+
def initialize(command: "plutil")
|
|
148
|
+
@command = command
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def read(path)
|
|
152
|
+
Dir.mktmpdir("portage-browser-import") do |dir|
|
|
153
|
+
copy = File.join(dir, "copy.plist")
|
|
154
|
+
copy_file(path, copy)
|
|
155
|
+
PlistXml.parse(convert(copy))
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
private
|
|
160
|
+
|
|
161
|
+
def copy_file(path, copy)
|
|
162
|
+
FileUtils.cp(path, copy)
|
|
163
|
+
rescue Errno::EPERM, Errno::EACCES => e
|
|
164
|
+
raise Sqlite::PermissionDenied, e.message
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def convert(copy)
|
|
168
|
+
out, err, status = Open3.capture3(@command, "-convert", "xml1", "-o", "-", copy)
|
|
169
|
+
raise Sqlite::Unavailable, "plutil failed: #{err.strip}" unless status.success?
|
|
170
|
+
|
|
171
|
+
out.dup.force_encoding("UTF-8").scrub
|
|
172
|
+
rescue Errno::ENOENT
|
|
173
|
+
raise Sqlite::Unavailable, "the plutil command isn't available (it ships with macOS)"
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|