portage-cli 0.7.4 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +580 -0
- data/README.md +322 -19
- data/exe/portage +5 -0
- data/exe/portage-console +3 -0
- data/known-stores/categories.yml +1263 -0
- data/lib/portage/cli/agent_profile_url.rb +30 -0
- data/lib/portage/cli/browser_import/categorize.rb +59 -0
- data/lib/portage/cli/browser_import/confirm.rb +35 -0
- data/lib/portage/cli/browser_import/domains.rb +47 -0
- data/lib/portage/cli/browser_import/filter.rb +91 -0
- data/lib/portage/cli/browser_import/importer.rb +248 -0
- data/lib/portage/cli/browser_import/plist_xml.rb +72 -0
- data/lib/portage/cli/browser_import/prober.rb +60 -0
- data/lib/portage/cli/browser_import/profiles.rb +114 -0
- data/lib/portage/cli/browser_import/readers.rb +179 -0
- data/lib/portage/cli/browser_import/saver.rb +62 -0
- data/lib/portage/cli/browser_import/sqlite.rb +68 -0
- data/lib/portage/cli/browser_import.rb +23 -0
- data/lib/portage/cli/browser_profile/allowlist.rb +40 -0
- data/lib/portage/cli/browser_profile/bridge.rb +120 -0
- data/lib/portage/cli/browser_profile/browsers.rb +69 -0
- data/lib/portage/cli/browser_profile/cdp.rb +67 -0
- data/lib/portage/cli/browser_profile/cdp_socket.rb +186 -0
- data/lib/portage/cli/browser_profile/errors.rb +26 -0
- data/lib/portage/cli/browser_profile/launcher.rb +34 -0
- data/lib/portage/cli/browser_profile/profile.rb +93 -0
- data/lib/portage/cli/browser_profile.rb +25 -0
- data/lib/portage/cli/buy.rb +521 -29
- data/lib/portage/cli/classifier.rb +158 -0
- data/lib/portage/cli/compare.rb +3 -0
- data/lib/portage/cli/doctor.rb +195 -7
- data/lib/portage/cli/dot_env.rb +117 -0
- data/lib/portage/cli/find.rb +96 -12
- data/lib/portage/cli/handoff_agents.rb +186 -0
- data/lib/portage/cli/handoff_only.rb +94 -0
- data/lib/portage/cli/handoff_reconciler.rb +15 -1
- data/lib/portage/cli/handoff_target.rb +61 -0
- data/lib/portage/cli/index/builder.rb +335 -0
- data/lib/portage/cli/index/exporter.rb +91 -0
- data/lib/portage/cli/index/known_cache.rb +155 -0
- data/lib/portage/cli/index/product_store.rb +101 -0
- data/lib/portage/cli/index/sources/browser.rb +31 -0
- data/lib/portage/cli/index/sources/shopify_catalog.rb +82 -0
- data/lib/portage/cli/index/sources/stores_file.rb +58 -0
- data/lib/portage/cli/index/sources/webmcp_sweep.rb +29 -0
- data/lib/portage/cli/index/sources/wikidata.rb +95 -0
- data/lib/portage/cli/index/sources.rb +44 -0
- data/lib/portage/cli/index/store.rb +109 -0
- data/lib/portage/cli/index.rb +20 -0
- data/lib/portage/cli/known_stores_url.rb +15 -0
- data/lib/portage/cli/offer_sources.rb +460 -0
- data/lib/portage/cli/payment_methods.rb +24 -3
- data/lib/portage/cli/search_backends.rb +337 -12
- data/lib/portage/cli/setup_wizard/prompt.rb +67 -0
- data/lib/portage/cli/setup_wizard/steps/agent_profile.rb +60 -0
- data/lib/portage/cli/setup_wizard/steps/browser_import.rb +29 -0
- data/lib/portage/cli/setup_wizard/steps/handoff.rb +100 -0
- data/lib/portage/cli/setup_wizard/steps/index_build.rb +31 -0
- data/lib/portage/cli/setup_wizard/steps/policy.rb +60 -0
- data/lib/portage/cli/setup_wizard/steps/retailer_keys.rb +55 -0
- data/lib/portage/cli/setup_wizard/steps/search_keys.rb +54 -0
- data/lib/portage/cli/setup_wizard/steps/shipping.rb +51 -0
- data/lib/portage/cli/setup_wizard.rb +74 -0
- data/lib/portage/cli/version.rb +1 -1
- data/lib/portage/cli/webmcp.rb +10 -3
- data/lib/portage/cli/webmcp_autofill_confirm.rb +38 -0
- data/lib/portage/cli/webmcp_autofill_fields.rb +60 -0
- data/lib/portage/cli/webmcp_autofill_mode.rb +39 -0
- data/lib/portage/cli/webmcp_mapping_confirm.rb +68 -0
- data/lib/portage/cli/webmcp_mappings.rb +84 -0
- data/lib/portage/cli.rb +528 -6
- metadata +59 -2
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
require_relative "sqlite"
|
|
2
|
+
|
|
3
|
+
module Portage
|
|
4
|
+
module Cli
|
|
5
|
+
module BrowserImport
|
|
6
|
+
# Where each supported browser keeps its profiles, and the exact
|
|
7
|
+
# files `portage browser import` is allowed to read from one.
|
|
8
|
+
#
|
|
9
|
+
# ALLOWED_FILES is the whole list — History/Bookmarks for the
|
|
10
|
+
# Chromium family, places.sqlite for Firefox, History.db/
|
|
11
|
+
# Bookmarks.plist for Safari. Nothing else in a profile directory is
|
|
12
|
+
# ever opened, copied or listed: not `Login Data`, `Cookies`,
|
|
13
|
+
# `Web Data` (autofill), `logins.json`, `key4.db`, `cookies.sqlite`,
|
|
14
|
+
# `formhistory.sqlite`, nor the keychain (docs/plans/
|
|
15
|
+
# buy-skill-and-local-browser.md, "Never, in any tier"; profiles_spec/
|
|
16
|
+
# importer_spec assert this against a fixture profile full of decoys).
|
|
17
|
+
# (Sqlite also copies an allowed database's own `-wal` file when one
|
|
18
|
+
# exists — it's part of that same database, holding its newest rows.)
|
|
19
|
+
# A profile's own *directory* is only ever stat'ed for these names,
|
|
20
|
+
# never listed; the one directory listing this does is the profile
|
|
21
|
+
# *root* (which profile folders exist — `Default`, `Profile 1`, …),
|
|
22
|
+
# never a profile's own contents.
|
|
23
|
+
module Profiles
|
|
24
|
+
Profile = Struct.new(:browser, :family, :dir, :history, :bookmarks, keyword_init: true)
|
|
25
|
+
|
|
26
|
+
CHROMIUM = %w[chrome edge brave arc].freeze
|
|
27
|
+
|
|
28
|
+
ALLOWED_FILES = {
|
|
29
|
+
"chromium" => { history: "History", bookmarks: "Bookmarks" },
|
|
30
|
+
"firefox" => { history: "places.sqlite", bookmarks: "places.sqlite" },
|
|
31
|
+
"safari" => { history: "History.db", bookmarks: "Bookmarks.plist" }
|
|
32
|
+
}.freeze
|
|
33
|
+
|
|
34
|
+
# Relative to the user's home. The first existing root wins, so a
|
|
35
|
+
# macOS path and a Linux path can share one entry.
|
|
36
|
+
ROOTS = {
|
|
37
|
+
"chrome" => ["Library/Application Support/Google/Chrome", ".config/google-chrome"],
|
|
38
|
+
"edge" => ["Library/Application Support/Microsoft Edge", ".config/microsoft-edge"],
|
|
39
|
+
"brave" => ["Library/Application Support/BraveSoftware/Brave-Browser", ".config/BraveSoftware/Brave-Browser"],
|
|
40
|
+
"arc" => ["Library/Application Support/Arc/User Data"],
|
|
41
|
+
"firefox" => ["Library/Application Support/Firefox", ".mozilla/firefox"],
|
|
42
|
+
"safari" => ["Library/Safari"]
|
|
43
|
+
}.freeze
|
|
44
|
+
|
|
45
|
+
# Default order when `--browser` isn't given. Safari is last: on
|
|
46
|
+
# macOS its directory exists whether or not the user uses it, and
|
|
47
|
+
# it's the only one that needs Full Disk Access.
|
|
48
|
+
BROWSERS = %w[chrome arc brave edge firefox safari].freeze
|
|
49
|
+
|
|
50
|
+
def self.family(browser)
|
|
51
|
+
return "chromium" if CHROMIUM.include?(browser)
|
|
52
|
+
|
|
53
|
+
browser
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# @return [String, nil] the first root that exists for `browser`.
|
|
57
|
+
def self.default_root(browser, home: Dir.home)
|
|
58
|
+
ROOTS.fetch(browser, []).map { |rel| File.join(home, rel) }.find { |path| File.directory?(path) }
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# @return [String, nil] the first browser in BROWSERS with a root on
|
|
62
|
+
# this machine.
|
|
63
|
+
def self.detect(home: Dir.home) = BROWSERS.find { |b| default_root(b, home: home) }
|
|
64
|
+
|
|
65
|
+
# @param root [String] the browser's profile root — injectable so
|
|
66
|
+
# specs (and a live check) point at a fixture or a specific copy.
|
|
67
|
+
# @return [Array<Profile>] every profile under `root` holding at
|
|
68
|
+
# least one of the allowed files.
|
|
69
|
+
def self.locate(browser, root:)
|
|
70
|
+
family = family(browser)
|
|
71
|
+
return [] unless ALLOWED_FILES.key?(family) && root && File.directory?(root)
|
|
72
|
+
|
|
73
|
+
profile_dirs(family, root).filter_map { |dir| profile_for(browser, family, dir) }
|
|
74
|
+
rescue Errno::EPERM, Errno::EACCES => e
|
|
75
|
+
# macOS privacy protection (or a sandbox) refusing even the root
|
|
76
|
+
# listing — reported, never worked around (see Importer).
|
|
77
|
+
raise Sqlite::PermissionDenied, e.message
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# The only directory listing in this module: the profile root's own
|
|
81
|
+
# folder names. Safari has no per-profile folders — its root *is*
|
|
82
|
+
# the profile, so nothing is listed there at all.
|
|
83
|
+
def self.profile_dirs(family, root)
|
|
84
|
+
case family
|
|
85
|
+
when "chromium" then Dir.glob(File.join(root, "{Default,Profile *}"))
|
|
86
|
+
when "firefox"
|
|
87
|
+
profiles = File.directory?(File.join(root, "Profiles")) ? File.join(root, "Profiles") : root
|
|
88
|
+
Dir.glob(File.join(profiles, "*")).select { |d| File.directory?(d) }.sort
|
|
89
|
+
else [root]
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
private_class_method :profile_dirs
|
|
93
|
+
|
|
94
|
+
def self.profile_for(browser, family, dir)
|
|
95
|
+
files = ALLOWED_FILES.fetch(family)
|
|
96
|
+
history = existing(dir, files[:history])
|
|
97
|
+
bookmarks = existing(dir, files[:bookmarks])
|
|
98
|
+
return nil unless history || bookmarks
|
|
99
|
+
|
|
100
|
+
Profile.new(browser: browser, family: family, dir: dir, history: history, bookmarks: bookmarks)
|
|
101
|
+
end
|
|
102
|
+
private_class_method :profile_for
|
|
103
|
+
|
|
104
|
+
# A stat, never an open — File.exist? on a name from ALLOWED_FILES
|
|
105
|
+
# only.
|
|
106
|
+
def self.existing(dir, name)
|
|
107
|
+
path = File.join(dir, name)
|
|
108
|
+
File.exist?(path) ? path : nil
|
|
109
|
+
end
|
|
110
|
+
private_class_method :existing
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "open3"
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
|
|
6
|
+
require_relative "sqlite"
|
|
7
|
+
require_relative "plist_xml"
|
|
8
|
+
|
|
9
|
+
module Portage
|
|
10
|
+
module Cli
|
|
11
|
+
module BrowserImport
|
|
12
|
+
# One reader per browser family. Each turns a Profiles::Profile into
|
|
13
|
+
# plain rows — `{ kind: "history"|"bookmark", url:, title:, folder:,
|
|
14
|
+
# visits: }` — and reads nothing but that profile's own allowed files
|
|
15
|
+
# (Profiles::ALLOWED_FILES). A row is just what the browser recorded;
|
|
16
|
+
# reducing it to a domain, filtering and probing is Importer's job.
|
|
17
|
+
module Readers
|
|
18
|
+
# Seconds between the Unix epoch and each browser's own clock:
|
|
19
|
+
# Chromium counts microseconds from 1601-01-01, Safari (Core Data)
|
|
20
|
+
# seconds from 2001-01-01, Firefox microseconds from the Unix epoch.
|
|
21
|
+
WEBKIT_EPOCH_OFFSET = 11_644_473_600
|
|
22
|
+
CORE_DATA_EPOCH_OFFSET = 978_307_200
|
|
23
|
+
|
|
24
|
+
def self.for(family, sqlite: Sqlite.new, plist: Plist.new)
|
|
25
|
+
case family
|
|
26
|
+
when "chromium" then Chromium.new(sqlite: sqlite)
|
|
27
|
+
when "firefox" then Firefox.new(sqlite: sqlite)
|
|
28
|
+
when "safari" then Safari.new(sqlite: sqlite, plist: plist)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def self.row(kind, url, title: nil, folder: nil, visits: 1)
|
|
33
|
+
{ kind: kind, url: url.to_s, title: title.to_s.strip, folder: folder.to_s.strip,
|
|
34
|
+
visits: [visits.to_i, 1].max }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# `History` (SQLite, copied first — Chrome keeps it locked) and
|
|
38
|
+
# `Bookmarks` (plain JSON, read in place: it's rewritten whole, never
|
|
39
|
+
# held open).
|
|
40
|
+
class Chromium
|
|
41
|
+
def initialize(sqlite:)
|
|
42
|
+
@sqlite = sqlite
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def history(profile, since:)
|
|
46
|
+
return [] unless profile.history
|
|
47
|
+
|
|
48
|
+
cutoff = (since.to_i + WEBKIT_EPOCH_OFFSET) * 1_000_000
|
|
49
|
+
sql = "SELECT url, title, visit_count AS visits FROM urls WHERE last_visit_time >= #{cutoff}"
|
|
50
|
+
@sqlite.with_copy(profile.history) { |query| query.call(sql) }
|
|
51
|
+
.map { |r| Readers.row("history", r["url"], title: r["title"], visits: r["visits"]) }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def bookmarks(profile)
|
|
55
|
+
return [] unless profile.bookmarks
|
|
56
|
+
|
|
57
|
+
data = JSON.parse(File.read(profile.bookmarks, encoding: "UTF-8"))
|
|
58
|
+
Hash(data["roots"]).values.grep(Hash).flat_map { |node| walk(node, nil) }
|
|
59
|
+
rescue JSON::ParserError
|
|
60
|
+
[]
|
|
61
|
+
rescue Errno::EPERM, Errno::EACCES => e
|
|
62
|
+
raise Sqlite::PermissionDenied, e.message
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
def walk(node, folder)
|
|
68
|
+
if node["type"] == "url"
|
|
69
|
+
[Readers.row("bookmark", node["url"], title: node["name"], folder: folder)]
|
|
70
|
+
else
|
|
71
|
+
Array(node["children"]).flat_map { |child| walk(child, node["name"]) }
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# `places.sqlite` holds both history and bookmarks — one copy, two
|
|
77
|
+
# queries.
|
|
78
|
+
class Firefox
|
|
79
|
+
def initialize(sqlite:)
|
|
80
|
+
@sqlite = sqlite
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def history(profile, since:)
|
|
84
|
+
return [] unless profile.history
|
|
85
|
+
|
|
86
|
+
sql = "SELECT url, title, visit_count AS visits FROM moz_places " \
|
|
87
|
+
"WHERE last_visit_date >= #{since.to_i * 1_000_000}"
|
|
88
|
+
@sqlite.with_copy(profile.history) { |query| query.call(sql) }
|
|
89
|
+
.map { |r| Readers.row("history", r["url"], title: r["title"], visits: r["visits"]) }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def bookmarks(profile)
|
|
93
|
+
return [] unless profile.bookmarks
|
|
94
|
+
|
|
95
|
+
sql = "SELECT p.url AS url, b.title AS title, f.title AS folder FROM moz_bookmarks b " \
|
|
96
|
+
"JOIN moz_places p ON p.id = b.fk LEFT JOIN moz_bookmarks f ON f.id = b.parent WHERE b.type = 1"
|
|
97
|
+
@sqlite.with_copy(profile.bookmarks) { |query| query.call(sql) }
|
|
98
|
+
.map { |r| Readers.row("bookmark", r["url"], title: r["title"], folder: r["folder"]) }
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# `History.db` (SQLite) and `Bookmarks.plist` (binary plist). Both
|
|
103
|
+
# sit under ~/Library/Safari, which macOS guards behind Full Disk
|
|
104
|
+
# Access — a copy attempt without it raises Sqlite::PermissionDenied,
|
|
105
|
+
# which Importer turns into an explanation, never a workaround.
|
|
106
|
+
class Safari
|
|
107
|
+
def initialize(sqlite:, plist:)
|
|
108
|
+
@sqlite = sqlite
|
|
109
|
+
@plist = plist
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def history(profile, since:)
|
|
113
|
+
return [] unless profile.history
|
|
114
|
+
|
|
115
|
+
cutoff = since.to_i - CORE_DATA_EPOCH_OFFSET
|
|
116
|
+
sql = "SELECT i.url AS url, v.title AS title, i.visit_count AS visits, MAX(v.visit_time) AS latest " \
|
|
117
|
+
"FROM history_items i JOIN history_visits v ON v.history_item = i.id " \
|
|
118
|
+
"WHERE v.visit_time >= #{cutoff} GROUP BY i.id"
|
|
119
|
+
@sqlite.with_copy(profile.history) { |query| query.call(sql) }
|
|
120
|
+
.map { |r| Readers.row("history", r["url"], title: r["title"], visits: r["visits"]) }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def bookmarks(profile)
|
|
124
|
+
return [] unless profile.bookmarks
|
|
125
|
+
|
|
126
|
+
walk(@plist.read(profile.bookmarks), nil)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
private
|
|
130
|
+
|
|
131
|
+
def walk(node, folder)
|
|
132
|
+
return [] unless node.is_a?(Hash)
|
|
133
|
+
|
|
134
|
+
if node["URLString"]
|
|
135
|
+
title = Hash(node["URIDictionary"])["title"]
|
|
136
|
+
[Readers.row("bookmark", node["URLString"], title: title, folder: folder)]
|
|
137
|
+
else
|
|
138
|
+
Array(node["Children"]).flat_map { |child| walk(child, node["Title"]) }
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Reads a (binary or XML) plist: copies it into a private tmpdir,
|
|
144
|
+
# converts the copy to XML with macOS's own `plutil`, parses it with
|
|
145
|
+
# PlistXml, and deletes the copy.
|
|
146
|
+
class Plist
|
|
147
|
+
def initialize(command: "plutil")
|
|
148
|
+
@command = command
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def read(path)
|
|
152
|
+
Dir.mktmpdir("portage-browser-import") do |dir|
|
|
153
|
+
copy = File.join(dir, "copy.plist")
|
|
154
|
+
copy_file(path, copy)
|
|
155
|
+
PlistXml.parse(convert(copy))
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
private
|
|
160
|
+
|
|
161
|
+
def copy_file(path, copy)
|
|
162
|
+
FileUtils.cp(path, copy)
|
|
163
|
+
rescue Errno::EPERM, Errno::EACCES => e
|
|
164
|
+
raise Sqlite::PermissionDenied, e.message
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def convert(copy)
|
|
168
|
+
out, err, status = Open3.capture3(@command, "-convert", "xml1", "-o", "-", copy)
|
|
169
|
+
raise Sqlite::Unavailable, "plutil failed: #{err.strip}" unless status.success?
|
|
170
|
+
|
|
171
|
+
out.dup.force_encoding("UTF-8").scrub
|
|
172
|
+
rescue Errno::ENOENT
|
|
173
|
+
raise Sqlite::Unavailable, "the plutil command isn't available (it ships with macOS)"
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
require_relative "categorize"
|
|
2
|
+
|
|
3
|
+
module Portage
|
|
4
|
+
module Cli
|
|
5
|
+
module BrowserImport
|
|
6
|
+
# Writes an approved Importer#plan into the user's own index — only
|
|
7
|
+
# ever called after BrowserImport::Confirm said :save. A kept domain
|
|
8
|
+
# becomes an Index::Store entry with `sources: ["history"]`/
|
|
9
|
+
# `["bookmark"]`; one already in the index just gains those labels
|
|
10
|
+
# and the new category weights (its capabilities and last_verified
|
|
11
|
+
# are left for `index refresh` to re-check, same as Index::Builder's
|
|
12
|
+
# #update_existing). A domain kept because the published
|
|
13
|
+
# known-stores list already had it keeps that list's own
|
|
14
|
+
# last_verified — this import never probed it. A kept product page becomes an
|
|
15
|
+
# Index::ProductStore entry with the same `sources` labels, so
|
|
16
|
+
# Index::Exporter never publishes it.
|
|
17
|
+
class Saver
|
|
18
|
+
def initialize(stores:, products:, now: Time.now)
|
|
19
|
+
@stores = stores
|
|
20
|
+
@products = products
|
|
21
|
+
@now = now
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# @return [Hash] stores:, products: — how many entries were written.
|
|
25
|
+
def save(plan)
|
|
26
|
+
kept = Array(plan[:kept])
|
|
27
|
+
products = Array(plan[:products])
|
|
28
|
+
kept.each { |entry| save_store(entry) }
|
|
29
|
+
products.each { |product| save_product(product) }
|
|
30
|
+
{ stores: kept.length, products: products.length }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def save_store(entry)
|
|
36
|
+
existing = @stores.find(entry[:origin])
|
|
37
|
+
fields = { sources: (Array(existing && existing["sources"]) + entry[:sources]).uniq,
|
|
38
|
+
categories: merge_categories(existing && existing["categories"], entry[:categories]) }
|
|
39
|
+
fields.merge!(new_store_fields(entry)) unless existing
|
|
40
|
+
@stores.upsert(entry[:origin], **fields)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def new_store_fields(entry)
|
|
44
|
+
{ platform: nil, capabilities: Array(entry[:capabilities]), webmcp_preset: entry[:webmcp_preset],
|
|
45
|
+
last_verified: entry[:last_verified] || @now.to_i, handoff_only: entry[:handoff_only] ? true : false }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def merge_categories(existing, fresh)
|
|
49
|
+
tally = Hash.new(0)
|
|
50
|
+
[existing, fresh].each { |cats| Hash(cats).each { |id, weight| tally[id.to_s] += weight.to_i } }
|
|
51
|
+
tally.sort_by { |_id, weight| -weight }.first(Categorize::TOP_CATEGORIES).to_h
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def save_product(product)
|
|
55
|
+
@products.upsert(product[:key], origin: product[:origin], seen_at: @now.to_i, title: product[:title],
|
|
56
|
+
brand: nil, gtin: nil, category: product[:category],
|
|
57
|
+
sources: [product[:source]])
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "open3"
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
|
|
6
|
+
module Portage
|
|
7
|
+
module Cli
|
|
8
|
+
module BrowserImport
|
|
9
|
+
# Reads a browser's SQLite history database without a native gem:
|
|
10
|
+
# the database (plus its `-wal` file, when the browser left one — it
|
|
11
|
+
# holds the most recent visits until the next checkpoint) is copied
|
|
12
|
+
# into a private tmpdir first, since a running browser holds it
|
|
13
|
+
# locked, and the copy is queried with the system `sqlite3` CLI in
|
|
14
|
+
# `-readonly -json` mode. The copy is deleted before this returns,
|
|
15
|
+
# success or not. The original file is only ever read by the copy.
|
|
16
|
+
#
|
|
17
|
+
# `sqlite3` ships with macOS and every mainstream Linux distro; when
|
|
18
|
+
# it's missing, Unavailable says so rather than falling back to
|
|
19
|
+
# anything else.
|
|
20
|
+
class Sqlite
|
|
21
|
+
class Unavailable < StandardError; end
|
|
22
|
+
|
|
23
|
+
# Raised when macOS refuses to let this process read the file at
|
|
24
|
+
# all (Safari's History.db without Full Disk Access). Never worked
|
|
25
|
+
# around — see Importer's own handling.
|
|
26
|
+
class PermissionDenied < StandardError; end
|
|
27
|
+
|
|
28
|
+
# @param command [String] the sqlite3 executable — injectable so a
|
|
29
|
+
# spec can prove the missing-binary path without uninstalling it.
|
|
30
|
+
def initialize(command: "sqlite3")
|
|
31
|
+
@command = command
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Copies `path` once and yields a query proc, so one copy serves
|
|
35
|
+
# several queries (Firefox's history and bookmarks both live in
|
|
36
|
+
# places.sqlite).
|
|
37
|
+
# @yieldparam query [Proc] `query.call(sql)` => Array<Hash>
|
|
38
|
+
def with_copy(path)
|
|
39
|
+
Dir.mktmpdir("portage-browser-import") do |dir|
|
|
40
|
+
copy = copy_database(path, dir)
|
|
41
|
+
yield ->(sql) { run(copy, sql) }
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def copy_database(path, dir)
|
|
48
|
+
copy = File.join(dir, "copy.sqlite")
|
|
49
|
+
FileUtils.cp(path, copy)
|
|
50
|
+
FileUtils.cp("#{path}-wal", "#{copy}-wal") if File.exist?("#{path}-wal")
|
|
51
|
+
copy
|
|
52
|
+
rescue Errno::EPERM, Errno::EACCES => e
|
|
53
|
+
raise PermissionDenied, e.message
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def run(copy, sql)
|
|
57
|
+
out, err, status = Open3.capture3(@command, "-readonly", "-json", copy, sql)
|
|
58
|
+
raise Unavailable, "sqlite3 failed: #{err.strip}" unless status.success?
|
|
59
|
+
|
|
60
|
+
text = out.dup.force_encoding("UTF-8").scrub
|
|
61
|
+
text.strip.empty? ? [] : JSON.parse(text)
|
|
62
|
+
rescue Errno::ENOENT
|
|
63
|
+
raise Unavailable, "the sqlite3 command isn't installed (it ships with macOS and most Linux distros)"
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
require_relative "browser_import/filter"
|
|
2
|
+
require_relative "browser_import/profiles"
|
|
3
|
+
require_relative "browser_import/sqlite"
|
|
4
|
+
require_relative "browser_import/plist_xml"
|
|
5
|
+
require_relative "browser_import/readers"
|
|
6
|
+
require_relative "browser_import/prober"
|
|
7
|
+
require_relative "browser_import/importer"
|
|
8
|
+
require_relative "browser_import/confirm"
|
|
9
|
+
|
|
10
|
+
module Portage
|
|
11
|
+
module Cli
|
|
12
|
+
# `portage browser import` — bookmarks and history as index seeds
|
|
13
|
+
# (docs/plans/buy-skill-and-local-browser.md Phase 3, Tier A). Opt-in,
|
|
14
|
+
# local-only, reduced to shop domains, and shown to the user before
|
|
15
|
+
# anything is saved. Never reads a browser's password, cookie or
|
|
16
|
+
# autofill store (see Profiles::ALLOWED_FILES), never attaches to or
|
|
17
|
+
# drives the browser, and never sends anything about the user's
|
|
18
|
+
# history anywhere except one `/.well-known/ucp` probe per unknown
|
|
19
|
+
# domain (Prober).
|
|
20
|
+
module BrowserImport
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
require_relative "../handoff_only"
|
|
2
|
+
|
|
3
|
+
module Portage
|
|
4
|
+
module Cli
|
|
5
|
+
module BrowserProfile
|
|
6
|
+
# The domain allowlist Phase 6 requires: driving the profile browser
|
|
7
|
+
# (evaluating WebMCP tool calls, filling checkout fields, navigating)
|
|
8
|
+
# is limited to the store being bought from, plus its checkout host
|
|
9
|
+
# once one is known. Anything else stops the run — see Bridge and
|
|
10
|
+
# DomainNotAllowedError.
|
|
11
|
+
#
|
|
12
|
+
# Reuses HandoffOnly's own host normalization/matching (a base host
|
|
13
|
+
# matches itself or any subdomain, "www." stripped, a full URL or a
|
|
14
|
+
# bare host both accepted) rather than a second copy of that logic.
|
|
15
|
+
class Allowlist
|
|
16
|
+
def initialize(hosts: [])
|
|
17
|
+
@hosts = hosts.filter_map { |h| HandoffOnly.normalize_entry(h) }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def hosts = @hosts.dup
|
|
21
|
+
|
|
22
|
+
def allowed?(host) = HandoffOnly.matches_any?(host, @hosts)
|
|
23
|
+
|
|
24
|
+
# Adds a host once it's known to be exactly what Portage itself is
|
|
25
|
+
# deliberately navigating to (the checkout URL a hand-off is about
|
|
26
|
+
# to open) — never called for a navigation a driven page made on
|
|
27
|
+
# its own; that's what #allowed? guards against.
|
|
28
|
+
# @return [String, nil] the normalized host that was added (or
|
|
29
|
+
# already present), nil when `host_or_url` didn't parse to one.
|
|
30
|
+
def permit!(host_or_url)
|
|
31
|
+
normalized = HandoffOnly.normalize_entry(host_or_url)
|
|
32
|
+
return nil unless normalized
|
|
33
|
+
|
|
34
|
+
@hosts << normalized unless @hosts.include?(normalized)
|
|
35
|
+
normalized
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
require_relative "allowlist"
|
|
3
|
+
require_relative "errors"
|
|
4
|
+
|
|
5
|
+
module Portage
|
|
6
|
+
module Cli
|
|
7
|
+
module BrowserProfile
|
|
8
|
+
# docs/plans/buy-skill-and-local-browser.md Phase 6: the WebMCP
|
|
9
|
+
# bridge `Cli.run_buy` hands to `Buy.new(webmcp_bridge:)` when
|
|
10
|
+
# `--handoff-target profile` — the Portage browser profile's own
|
|
11
|
+
# page target, reached over CdpSocket, wrapped so every call stays
|
|
12
|
+
# inside the domain allowlist. Everything WebMCP-shaped (#list_tools,
|
|
13
|
+
# #execute_tool, #autofill) is delegated to a real
|
|
14
|
+
# Portage::Ucp::WebMcp::Bridges::ScriptEvaluator built from this
|
|
15
|
+
# bridge's own #evaluate — this class exists to add exactly two
|
|
16
|
+
# things ScriptEvaluator doesn't have: the allowlist check on every
|
|
17
|
+
# call, and #navigate (Portage's own deliberate navigation to a
|
|
18
|
+
# checkout URL, as opposed to a page-driven one).
|
|
19
|
+
#
|
|
20
|
+
# Requires `portage-ucp-webmcp` to already be loaded — only ever
|
|
21
|
+
# built by `Cli.run_buy` after `Portage::Cli::Webmcp.available?`,
|
|
22
|
+
# same guard every other WebMCP call site in this gem already uses
|
|
23
|
+
# (see Buy#webmcp_flow) — so nothing here calls `require` itself.
|
|
24
|
+
#
|
|
25
|
+
# #headless? is always false: this is, by construction, a headed
|
|
26
|
+
# browser the shopper can see and pay in (WebMcp::Autofill's own
|
|
27
|
+
# `headless?(bridge)` check — see docs/plans/
|
|
28
|
+
# webmcp-universal-outbound.md Phase 3 — never returns
|
|
29
|
+
# :needs_headed_browser for it).
|
|
30
|
+
class Bridge
|
|
31
|
+
def initialize(socket:, allowlist:)
|
|
32
|
+
@socket = socket
|
|
33
|
+
@allowlist = allowlist
|
|
34
|
+
@page_enabled = false
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def headless? = false
|
|
38
|
+
|
|
39
|
+
def location
|
|
40
|
+
raw_evaluate("Promise.resolve(window.location.href)")
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Portage's own navigation — to the checkout URL a hand-off is
|
|
44
|
+
# about to show the shopper. Permits that URL's host first (this
|
|
45
|
+
# is exactly "the store's checkout host" the allowlist is meant to
|
|
46
|
+
# grow to include, per the plan), then navigates.
|
|
47
|
+
def navigate(url)
|
|
48
|
+
host = URI.parse(url.to_s).host
|
|
49
|
+
@allowlist.permit!(host)
|
|
50
|
+
ensure_page_enabled!
|
|
51
|
+
@socket.call("Page.navigate", "url" => url.to_s)
|
|
52
|
+
nil
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def list_tools = script_evaluator.list_tools
|
|
56
|
+
def execute_tool(name, input) = script_evaluator.execute_tool(name, input)
|
|
57
|
+
def autofill(fields, selectors: {}) = script_evaluator.autofill(fields, selectors: selectors)
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def script_evaluator
|
|
62
|
+
@script_evaluator ||= Portage::Ucp::WebMcp::Bridges::ScriptEvaluator.new(evaluate: method(:evaluate),
|
|
63
|
+
headless: false)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Every driven call (a WebMCP tool call, an autofill attempt)
|
|
67
|
+
# checks the page's *current* location against the allowlist
|
|
68
|
+
# before running — including a navigation a previous call already
|
|
69
|
+
# caused (e.g. a preset's handoff_checkout tool navigating to
|
|
70
|
+
# checkout): that navigation's own evaluate call already ran
|
|
71
|
+
# before the page moved, so this is the first following call that
|
|
72
|
+
# can actually see where it ended up. That's the plan's "stops the
|
|
73
|
+
# run" in practice — a driven page that jumps somewhere unexpected
|
|
74
|
+
# is caught on its very next use, not mid-navigation (there's no
|
|
75
|
+
# navigation-event hook in this minimal a CDP client).
|
|
76
|
+
def evaluate(expression)
|
|
77
|
+
enforce_allowlist!
|
|
78
|
+
raw_evaluate(expression)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def enforce_allowlist!
|
|
82
|
+
host = current_host
|
|
83
|
+
return if @allowlist.allowed?(host)
|
|
84
|
+
|
|
85
|
+
raise DomainNotAllowedError,
|
|
86
|
+
"the profile browser navigated to #{host.inspect}, outside the allowed domains " \
|
|
87
|
+
"(#{@allowlist.hosts.join(', ')}) — stopping"
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def current_host
|
|
91
|
+
URI.parse(raw_evaluate("Promise.resolve(window.location.href)").to_s).host
|
|
92
|
+
rescue URI::InvalidURIError
|
|
93
|
+
nil
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def raw_evaluate(expression)
|
|
97
|
+
response = @socket.call("Runtime.evaluate", "expression" => expression, "awaitPromise" => true,
|
|
98
|
+
"returnByValue" => true, "userGesture" => true)
|
|
99
|
+
raise_on_exception!(response)
|
|
100
|
+
response.dig("result", "value")
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def raise_on_exception!(response)
|
|
104
|
+
details = response["exceptionDetails"]
|
|
105
|
+
return unless details
|
|
106
|
+
|
|
107
|
+
text = details.dig("exception", "description") || details["text"] || "script evaluation failed"
|
|
108
|
+
raise Portage::Ucp::WebMcp::BridgeError, text
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def ensure_page_enabled!
|
|
112
|
+
return if @page_enabled
|
|
113
|
+
|
|
114
|
+
@socket.call("Page.enable", {})
|
|
115
|
+
@page_enabled = true
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
require_relative "../browser_import/profiles"
|
|
2
|
+
|
|
3
|
+
module Portage
|
|
4
|
+
module Cli
|
|
5
|
+
module BrowserProfile
|
|
6
|
+
# Where to find a Chromium-family browser's own executable, and
|
|
7
|
+
# which family is supported at all (docs/plans/
|
|
8
|
+
# buy-skill-and-local-browser.md Phase 6). Firefox/Safari are out of
|
|
9
|
+
# scope for driving (see the plan's Phase 6 section) even though
|
|
10
|
+
# BrowserImport::Profiles already knows their profile roots — this
|
|
11
|
+
# module only ever launches one of CHROMIUM.
|
|
12
|
+
module Browsers
|
|
13
|
+
CHROMIUM = BrowserImport::Profiles::CHROMIUM
|
|
14
|
+
|
|
15
|
+
# macOS app bundle executables. The first existing path wins.
|
|
16
|
+
MAC_APPS = {
|
|
17
|
+
"chrome" => "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
|
18
|
+
"edge" => "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
|
19
|
+
"brave" => "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
|
|
20
|
+
"arc" => "/Applications/Arc.app/Contents/MacOS/Arc"
|
|
21
|
+
}.freeze
|
|
22
|
+
|
|
23
|
+
# Linux binary names, tried on PATH. Arc has no Linux build.
|
|
24
|
+
LINUX_BINARIES = {
|
|
25
|
+
"chrome" => %w[google-chrome google-chrome-stable chromium chromium-browser],
|
|
26
|
+
"edge" => %w[microsoft-edge microsoft-edge-stable],
|
|
27
|
+
"brave" => %w[brave-browser brave-browser-stable],
|
|
28
|
+
"arc" => []
|
|
29
|
+
}.freeze
|
|
30
|
+
|
|
31
|
+
# @param path [String] PATH-shaped, injectable so a spec can point
|
|
32
|
+
# ".which" at a fixture directory instead of mutating ENV.
|
|
33
|
+
# @return [String, nil] the first launchable binary for `browser`
|
|
34
|
+
# on this machine, or nil when it isn't installed.
|
|
35
|
+
def self.binary_for(browser, darwin: RUBY_PLATFORM.include?("darwin"), path: ENV.fetch("PATH", ""))
|
|
36
|
+
mac_binary(browser, darwin) || linux_binary(browser, path)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def self.mac_binary(browser, darwin)
|
|
40
|
+
return nil unless darwin
|
|
41
|
+
|
|
42
|
+
path = MAC_APPS[browser]
|
|
43
|
+
path if path && File.exist?(path)
|
|
44
|
+
end
|
|
45
|
+
private_class_method :mac_binary
|
|
46
|
+
|
|
47
|
+
def self.linux_binary(browser, path)
|
|
48
|
+
LINUX_BINARIES.fetch(browser, []).filter_map { |name| which(name, path) }.first
|
|
49
|
+
end
|
|
50
|
+
private_class_method :linux_binary
|
|
51
|
+
|
|
52
|
+
def self.which(cmd, path)
|
|
53
|
+
path.split(File::PATH_SEPARATOR).map { |dir| File.join(dir, cmd) }
|
|
54
|
+
.find { |candidate| File.file?(candidate) && File.executable?(candidate) }
|
|
55
|
+
end
|
|
56
|
+
private_class_method :which
|
|
57
|
+
|
|
58
|
+
# @return [String, nil] the first CHROMIUM browser that's actually
|
|
59
|
+
# installed on this machine, reusing BrowserImport::Profiles's
|
|
60
|
+
# own "does this browser's default profile root exist" check
|
|
61
|
+
# purely as an installed-or-not signal — nothing under that root
|
|
62
|
+
# is ever read here.
|
|
63
|
+
def self.detect(home: Dir.home)
|
|
64
|
+
CHROMIUM.find { |b| BrowserImport::Profiles.default_root(b, home: home) }
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|