browserctl 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +66 -0
- data/README.md +2 -1
- data/bin/browserctl +168 -78
- data/bin/browserd +8 -1
- data/lib/browserctl/client.rb +50 -6
- data/lib/browserctl/commands/cli_output.rb +36 -3
- data/lib/browserctl/commands/flow.rb +123 -0
- data/lib/browserctl/commands/migrate.rb +94 -0
- data/lib/browserctl/commands/state.rb +193 -0
- data/lib/browserctl/commands/trace.rb +187 -0
- data/lib/browserctl/commands/workflow.rb +62 -4
- data/lib/browserctl/constants.rb +4 -2
- data/lib/browserctl/crash_report.rb +96 -0
- data/lib/browserctl/detectors/auth_required.rb +128 -0
- data/lib/browserctl/detectors.rb +2 -0
- data/lib/browserctl/error/codes.rb +44 -0
- data/lib/browserctl/error/exit_codes.rb +54 -0
- data/lib/browserctl/error/suggested_actions.rb +41 -0
- data/lib/browserctl/errors.rb +72 -12
- data/lib/browserctl/flow.rb +22 -1
- data/lib/browserctl/flow_registry.rb +66 -0
- data/lib/browserctl/flows/stdlib/basic_auth.rb +30 -0
- data/lib/browserctl/flows/stdlib/cloudflare_solve.rb +59 -0
- data/lib/browserctl/flows/stdlib/magic_link_email.rb +28 -0
- data/lib/browserctl/flows/stdlib/oauth_github.rb +28 -0
- data/lib/browserctl/flows/stdlib/oauth_google.rb +30 -0
- data/lib/browserctl/flows/stdlib/totp_2fa.rb +61 -0
- data/lib/browserctl/format_version.rb +37 -0
- data/lib/browserctl/logger.rb +102 -9
- data/lib/browserctl/migrations.rb +216 -0
- data/lib/browserctl/recording.rb +246 -28
- data/lib/browserctl/redactor.rb +58 -0
- data/lib/browserctl/replay/context.rb +40 -0
- data/lib/browserctl/replay/fingerprint_matcher.rb +86 -0
- data/lib/browserctl/replay/snapshot_diff.rb +51 -0
- data/lib/browserctl/replay/telemetry.rb +60 -0
- data/lib/browserctl/rubocop/cops/typed_error.rb +69 -0
- data/lib/browserctl/runner.rb +50 -10
- data/lib/browserctl/secret_resolver_registry.rb +23 -4
- data/lib/browserctl/server/command_dispatcher.rb +13 -1
- data/lib/browserctl/server/handlers/daemon_control.rb +5 -1
- data/lib/browserctl/server/handlers/error_payload.rb +27 -0
- data/lib/browserctl/server/handlers/interaction.rb +21 -3
- data/lib/browserctl/server/handlers/navigation.rb +50 -5
- data/lib/browserctl/server/handlers/observation.rb +43 -2
- data/lib/browserctl/server/handlers/state.rb +149 -0
- data/lib/browserctl/server/page_session.rb +9 -7
- data/lib/browserctl/server/snapshot_builder.rb +21 -45
- data/lib/browserctl/session.rb +1 -1
- data/lib/browserctl/snapshot/annotator.rb +75 -0
- data/lib/browserctl/snapshot/extractor.rb +21 -0
- data/lib/browserctl/snapshot/fingerprint.rb +88 -0
- data/lib/browserctl/snapshot/ref.rb +70 -0
- data/lib/browserctl/snapshot/serializer.rb +17 -0
- data/lib/browserctl/state/bundle.rb +283 -0
- data/lib/browserctl/state/transport.rb +64 -0
- data/lib/browserctl/state/transports/file.rb +35 -0
- data/lib/browserctl/state/transports/one_password.rb +67 -0
- data/lib/browserctl/state/transports/s3.rb +42 -0
- data/lib/browserctl/state.rb +208 -0
- data/lib/browserctl/version.rb +1 -1
- data/lib/browserctl/workflow/flow_wrapper.rb +81 -0
- data/lib/browserctl/workflow/promoter.rb +96 -0
- data/lib/browserctl/workflow/promotion_ledger.rb +72 -0
- data/lib/browserctl/workflow.rb +235 -16
- metadata +44 -7
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "errors"
|
|
5
|
+
require_relative "error/codes"
|
|
6
|
+
|
|
7
|
+
module Browserctl
|
|
8
|
+
# Migration registry for browserctl persisted formats. Operators run
|
|
9
|
+
# `browserctl migrate <path>` to upgrade an artifact written by an older
|
|
10
|
+
# browserctl to the current build's format version.
|
|
11
|
+
#
|
|
12
|
+
# Distinct from `verify_format_version!` (in bundle/recording/workflow):
|
|
13
|
+
# those gates stay strict — a reader that encounters an unknown version
|
|
14
|
+
# raises {Browserctl::ProtocolMismatch}. This module is the only blessed
|
|
15
|
+
# path to mutate an old artifact in place.
|
|
16
|
+
#
|
|
17
|
+
# The registry ships **empty** in v0.12. The first real migration arrives
|
|
18
|
+
# only when a format actually changes post-1.0. See
|
|
19
|
+
# `docs/reference/format-versions.md` ("Migration registry").
|
|
20
|
+
module Migrations
|
|
21
|
+
# Single registered upgrader. `upgrade` is a Proc invoked with the
|
|
22
|
+
# absolute path of the file being migrated; it is responsible for
|
|
23
|
+
# rewriting that file in place, advancing it from `from_version` to
|
|
24
|
+
# `to_version`.
|
|
25
|
+
Migration = Struct.new(:format, :from_version, :to_version, :upgrade, keyword_init: true)
|
|
26
|
+
|
|
27
|
+
# Result of {.run}. `applied` is the ordered list of {Migration} steps
|
|
28
|
+
# that ran. When the artifact was already at target, `applied` is empty.
|
|
29
|
+
Result = Struct.new(:format, :from, :to, :applied, keyword_init: true)
|
|
30
|
+
|
|
31
|
+
FORMAT_EXTENSIONS = {
|
|
32
|
+
".bctl" => :bundle,
|
|
33
|
+
".jsonl" => :recording,
|
|
34
|
+
".rb" => :workflow
|
|
35
|
+
}.freeze
|
|
36
|
+
|
|
37
|
+
@registry = []
|
|
38
|
+
@mutex = Mutex.new
|
|
39
|
+
|
|
40
|
+
class << self
|
|
41
|
+
# Registers an upgrader for one hop in `format`'s version chain. The
|
|
42
|
+
# block receives the file path as a keyword argument and must rewrite
|
|
43
|
+
# the file in place to the new version.
|
|
44
|
+
def register(format:, from_version:, to_version:, &upgrade)
|
|
45
|
+
raise ArgumentError, "upgrade block required" unless upgrade
|
|
46
|
+
|
|
47
|
+
@mutex.synchronize do
|
|
48
|
+
@registry << Migration.new(format: format, from_version: from_version,
|
|
49
|
+
to_version: to_version, upgrade: upgrade)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# All registered migrations, in registration order.
|
|
54
|
+
def all
|
|
55
|
+
@mutex.synchronize { @registry.dup }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Test-only hook — clears the registry. Not part of the public API.
|
|
59
|
+
def reset!
|
|
60
|
+
@mutex.synchronize { @registry.clear }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Breadth-first search through registered migrations to chain a path
|
|
64
|
+
# for `format` from `from` to `to`. Returns the ordered list of
|
|
65
|
+
# {Migration} hops, or `nil` if no path is reachable. When `from == to`
|
|
66
|
+
# returns an empty array (already at target — no work to do).
|
|
67
|
+
def find_path(format:, from:, to:)
|
|
68
|
+
return [] if from == to
|
|
69
|
+
|
|
70
|
+
all_for_format = all.select { |m| m.format == format }
|
|
71
|
+
queue = [[from, []]]
|
|
72
|
+
seen = { from => true }
|
|
73
|
+
|
|
74
|
+
until queue.empty?
|
|
75
|
+
current, path = queue.shift
|
|
76
|
+
all_for_format.each do |m|
|
|
77
|
+
next unless m.from_version == current
|
|
78
|
+
next if seen[m.to_version]
|
|
79
|
+
|
|
80
|
+
new_path = path + [m]
|
|
81
|
+
return new_path if m.to_version == to
|
|
82
|
+
|
|
83
|
+
seen[m.to_version] = true
|
|
84
|
+
queue << [m.to_version, new_path]
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
nil
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Inspects an artifact path and returns a format symbol — `:bundle`,
|
|
91
|
+
# `:recording`, or `:workflow` — or `nil` when the format cannot be
|
|
92
|
+
# identified. Detection is extension-driven: keep new formats listed
|
|
93
|
+
# in {FORMAT_EXTENSIONS} so this stays a one-line lookup.
|
|
94
|
+
def detect_format(path)
|
|
95
|
+
FORMAT_EXTENSIONS[File.extname(path.to_s).downcase]
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Reads `path` and returns the integer format_version it declares, or
|
|
99
|
+
# `nil` when no version header is present. Format-specific because
|
|
100
|
+
# each format stores the header differently — the bundle in its
|
|
101
|
+
# binary manifest, the recording in a `_meta` JSONL line, the
|
|
102
|
+
# workflow in a Ruby comment.
|
|
103
|
+
def detect_version(path, format)
|
|
104
|
+
case format
|
|
105
|
+
when :bundle then peek_bundle_version(path)
|
|
106
|
+
when :recording then peek_recording_version(path)
|
|
107
|
+
when :workflow then peek_workflow_version(path)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# End-to-end migration. Detects format and current version, finds a
|
|
112
|
+
# chain of registered upgraders to `target_version` (or the latest
|
|
113
|
+
# `to_version` seen for this format if `target_version` is nil), and
|
|
114
|
+
# invokes each in order. Each upgrader rewrites the file in place.
|
|
115
|
+
#
|
|
116
|
+
# Returns a {Result}. When no migrations are needed (or the registry
|
|
117
|
+
# has no entries for this format), `applied` is empty.
|
|
118
|
+
#
|
|
119
|
+
# Raises {Browserctl::ProtocolMismatch} when format detection fails,
|
|
120
|
+
# the version cannot be read, or no chain reaches the target.
|
|
121
|
+
def run(path, target_version: nil)
|
|
122
|
+
format = detect_format(path) or raise_protocol("could not detect format for #{path}")
|
|
123
|
+
current = detect_version(path, format) or raise_protocol("could not read format_version from #{path}")
|
|
124
|
+
target = target_version || latest_known_target(format, current)
|
|
125
|
+
|
|
126
|
+
if current == target
|
|
127
|
+
# If we'd be a no-op but the artifact's declared version is one this
|
|
128
|
+
# build's reader does not support, surface that as PROTOCOL_MISMATCH —
|
|
129
|
+
# there is no migration registered to bring it into range.
|
|
130
|
+
unless format_version_supported?(format, current)
|
|
131
|
+
raise_protocol("#{format} at #{path} declares unsupported format_version=#{current}; " \
|
|
132
|
+
"no migration registered")
|
|
133
|
+
end
|
|
134
|
+
return Result.new(format: format, from: current, to: current, applied: [])
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
chain = find_path(format: format, from: current, to: target)
|
|
138
|
+
raise_protocol("no migration path from #{format} v#{current} to v#{target}") if chain.nil?
|
|
139
|
+
|
|
140
|
+
chain.each { |m| m.upgrade.call(path: path, from_version: m.from_version, to_version: m.to_version) }
|
|
141
|
+
Result.new(format: format, from: current, to: target, applied: chain)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
private
|
|
145
|
+
|
|
146
|
+
# Reflects whether each format's reader currently accepts a given
|
|
147
|
+
# version. Mirrors the SUPPORTED_FORMAT_VERSIONS constant on the
|
|
148
|
+
# corresponding class — kept here so adding a new format is one
|
|
149
|
+
# branch, not a new public API.
|
|
150
|
+
def format_version_supported?(format, version)
|
|
151
|
+
case format
|
|
152
|
+
when :bundle
|
|
153
|
+
require_relative "state/bundle"
|
|
154
|
+
Browserctl::State::Bundle::SUPPORTED_FORMAT_VERSIONS.include?(version)
|
|
155
|
+
when :recording
|
|
156
|
+
require_relative "recording"
|
|
157
|
+
Browserctl::Recording::SUPPORTED_FORMAT_VERSIONS.include?(version)
|
|
158
|
+
when :workflow
|
|
159
|
+
require_relative "workflow"
|
|
160
|
+
Browserctl::SUPPORTED_WORKFLOW_FORMAT_VERSIONS.include?(version)
|
|
161
|
+
else
|
|
162
|
+
false
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def latest_known_target(format, current)
|
|
167
|
+
targets = all.select { |m| m.format == format }.map(&:to_version)
|
|
168
|
+
targets.empty? ? current : targets.max
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def peek_bundle_version(path)
|
|
172
|
+
require_relative "state/bundle"
|
|
173
|
+
manifest = Browserctl::State::Bundle.peek_manifest(File.binread(path))
|
|
174
|
+
manifest[:format_version] || manifest["format_version"]
|
|
175
|
+
rescue Browserctl::ProtocolMismatch
|
|
176
|
+
# `peek_manifest` raises on unknown versions, but we want to *return*
|
|
177
|
+
# the version so a migration can target it. Re-parse the manifest
|
|
178
|
+
# bytes directly when the strict gate refuses.
|
|
179
|
+
peek_bundle_version_loosely(path)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def peek_bundle_version_loosely(path)
|
|
183
|
+
blob = File.binread(path)
|
|
184
|
+
# Manifest length is at byte offset 8 (5 magic + 3 header). Parse
|
|
185
|
+
# the JSON without invoking Bundle's strict version check.
|
|
186
|
+
manifest_len = blob.byteslice(8, 4).unpack1("N")
|
|
187
|
+
manifest_bytes = blob.byteslice(12, manifest_len)
|
|
188
|
+
manifest = JSON.parse(manifest_bytes)
|
|
189
|
+
manifest["format_version"]
|
|
190
|
+
rescue StandardError
|
|
191
|
+
nil
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def peek_recording_version(path)
|
|
195
|
+
first_line = File.foreach(path).first
|
|
196
|
+
return nil unless first_line
|
|
197
|
+
|
|
198
|
+
meta = JSON.parse(first_line, symbolize_names: true)
|
|
199
|
+
return nil unless meta[:cmd] == "_meta"
|
|
200
|
+
|
|
201
|
+
meta[:format_version]
|
|
202
|
+
rescue JSON::ParserError
|
|
203
|
+
nil
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def peek_workflow_version(path)
|
|
207
|
+
require_relative "workflow"
|
|
208
|
+
Browserctl.parse_workflow_format_version(File.read(path))
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def raise_protocol(msg)
|
|
212
|
+
raise Browserctl::ProtocolMismatch.new(msg, code: Browserctl::Error::Codes::PROTOCOL_MISMATCH)
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|
data/lib/browserctl/recording.rb
CHANGED
|
@@ -2,19 +2,46 @@
|
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "date"
|
|
5
|
+
require "time"
|
|
5
6
|
require "fileutils"
|
|
6
7
|
require "tmpdir"
|
|
7
8
|
require "uri"
|
|
9
|
+
require_relative "errors"
|
|
10
|
+
require_relative "error/codes"
|
|
8
11
|
|
|
9
12
|
module Browserctl
|
|
10
|
-
class Recording
|
|
13
|
+
class Recording # rubocop:disable Metrics/ClassLength
|
|
11
14
|
RECORDINGS_DIR = File.join(Dir.tmpdir, "browserctl-recordings")
|
|
12
15
|
STATE_FILE = File.expand_path("~/.browserctl/active_recording")
|
|
13
16
|
|
|
17
|
+
# Recording-log format version, written into the `_meta` header and
|
|
18
|
+
# validated when generate_workflow loads a recording. Distinct from
|
|
19
|
+
# LOG_FORMAT below — that string ("v0.11") tracks the human-readable
|
|
20
|
+
# log shape; this integer is the machine-readable schema gate per the
|
|
21
|
+
# WS-1 format-version convention. See docs/reference/format-versions.md.
|
|
22
|
+
RECORDING_FORMAT_VERSION = 1
|
|
23
|
+
SUPPORTED_FORMAT_VERSIONS = [RECORDING_FORMAT_VERSION].freeze
|
|
24
|
+
|
|
14
25
|
RECORDABLE = %w[page_open navigate fill click screenshot evaluate].freeze
|
|
15
26
|
|
|
16
27
|
SENSITIVE_PARAM_PATTERN = /\A(token|key|secret|auth|code|access_token|api_key|client_secret|state)\z/ix
|
|
17
28
|
|
|
29
|
+
# Selector tokens that signal a fill is targeting a secret-shaped field.
|
|
30
|
+
# The captured group (or matched substring) is used as the inferred field
|
|
31
|
+
# name; that name later drives the generated `secret_ref:` placeholder.
|
|
32
|
+
SECRET_FIELD_PATTERN = /\b(password|passwd|api[_-]?key|token|secret|otp|pin|client[_-]?secret|access[_-]?token)\b/i
|
|
33
|
+
|
|
34
|
+
# Conservative thresholds for inferring an explicit wait between recorded
|
|
35
|
+
# steps. Gaps shorter than the threshold come from natural input cadence;
|
|
36
|
+
# gaps above it usually mean the page actually had work to do.
|
|
37
|
+
WAIT_THRESHOLD_SECONDS = 1.5
|
|
38
|
+
WAIT_PADDING_SECONDS = 5
|
|
39
|
+
WAIT_FLOOR_SECONDS = 5
|
|
40
|
+
|
|
41
|
+
# Bumped when the recording log shape changes in a way that older
|
|
42
|
+
# tooling (workflow generate, replay) cannot read.
|
|
43
|
+
LOG_FORMAT = "v0.11"
|
|
44
|
+
|
|
18
45
|
def self.start(name)
|
|
19
46
|
FileUtils.mkdir_p(RECORDINGS_DIR, mode: 0o700)
|
|
20
47
|
FileUtils.mkdir_p(File.dirname(STATE_FILE))
|
|
@@ -22,12 +49,21 @@ module Browserctl
|
|
|
22
49
|
FileUtils.rm_f(log_path(name))
|
|
23
50
|
FileUtils.touch(log_path(name))
|
|
24
51
|
File.chmod(0o600, log_path(name))
|
|
52
|
+
File.open(log_path(name), "a") do |f|
|
|
53
|
+
f.puts JSON.generate(
|
|
54
|
+
cmd: "_meta",
|
|
55
|
+
format_version: RECORDING_FORMAT_VERSION,
|
|
56
|
+
log_format: LOG_FORMAT,
|
|
57
|
+
recording: name,
|
|
58
|
+
started_at: Time.now.utc.iso8601
|
|
59
|
+
)
|
|
60
|
+
end
|
|
25
61
|
name
|
|
26
62
|
end
|
|
27
63
|
|
|
28
64
|
def self.stop
|
|
29
65
|
name = active
|
|
30
|
-
raise "no active recording — run: browserctl record start <name>" unless name
|
|
66
|
+
raise Browserctl::Error, "no active recording — run: browserctl record start <name>" unless name
|
|
31
67
|
|
|
32
68
|
File.unlink(STATE_FILE)
|
|
33
69
|
name
|
|
@@ -37,69 +73,232 @@ module Browserctl
|
|
|
37
73
|
File.exist?(STATE_FILE) ? File.read(STATE_FILE).strip : nil
|
|
38
74
|
end
|
|
39
75
|
|
|
40
|
-
def self.append(cmd, **attrs)
|
|
76
|
+
def self.append(cmd, response: nil, **attrs)
|
|
41
77
|
name = active
|
|
42
78
|
return unless name
|
|
43
79
|
return unless RECORDABLE.include?(cmd.to_s)
|
|
44
80
|
|
|
45
81
|
if %w[click fill].include?(cmd.to_s) && attrs[:selector].nil?
|
|
46
|
-
record_ref_interaction(name, cmd.to_s, attrs)
|
|
82
|
+
record_ref_interaction(name, cmd.to_s, attrs, response)
|
|
47
83
|
return
|
|
48
84
|
end
|
|
49
85
|
|
|
50
86
|
attrs = prepare_attrs(cmd.to_s, attrs)
|
|
87
|
+
entry = { cmd: cmd.to_s, ts: now }.merge(attrs.transform_keys(&:to_s))
|
|
88
|
+
entry.merge!(replay_metadata(response)) if response
|
|
51
89
|
|
|
52
90
|
File.open(log_path(name), "a") do |f|
|
|
53
|
-
f.puts JSON.generate(
|
|
91
|
+
f.puts JSON.generate(entry)
|
|
54
92
|
end
|
|
55
93
|
end
|
|
56
94
|
|
|
57
|
-
def self.generate_workflow(name, output_path: nil)
|
|
95
|
+
def self.generate_workflow(name, output_path: nil, keep_log: false)
|
|
58
96
|
log = log_path(name)
|
|
59
|
-
raise "no recording found for '#{name}'" unless File.exist?(log)
|
|
97
|
+
raise Browserctl::Error, "no recording found for '#{name}'" unless File.exist?(log)
|
|
60
98
|
|
|
61
|
-
|
|
99
|
+
raw = File.readlines(log).map { |l| JSON.parse(l, symbolize_names: true) }
|
|
100
|
+
verify_format_version!(raw, path: log)
|
|
101
|
+
lines = raw.reject { |l| l[:cmd] == "_meta" }
|
|
62
102
|
ruby = build_workflow_ruby(name, lines)
|
|
63
103
|
File.write(output_path, ruby) if output_path
|
|
104
|
+
warn_about_ref_interactions(lines)
|
|
105
|
+
ruby
|
|
106
|
+
ensure
|
|
107
|
+
FileUtils.rm_f(log) if log && !keep_log
|
|
108
|
+
end
|
|
64
109
|
|
|
110
|
+
def self.warn_about_ref_interactions(lines)
|
|
65
111
|
ref_count = lines.count { |l| l[:cmd] == "_ref_interaction" }
|
|
66
|
-
|
|
67
|
-
warn "Warning: #{ref_count} ref-based interaction(s) were captured but cannot be replayed by ref."
|
|
68
|
-
warn "Search the generated workflow for 'TODO: ref-based' and replace with stable CSS selectors."
|
|
69
|
-
end
|
|
112
|
+
return unless ref_count.positive?
|
|
70
113
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
FileUtils.rm_f(log) if log
|
|
114
|
+
warn "Warning: #{ref_count} ref-based interaction(s) were captured but cannot be replayed by ref."
|
|
115
|
+
warn "Search the generated workflow for 'TODO: ref-based' and replace with stable CSS selectors."
|
|
74
116
|
end
|
|
75
117
|
|
|
76
118
|
class << self
|
|
77
119
|
private
|
|
78
120
|
|
|
121
|
+
# Raises Browserctl::ProtocolMismatch when the recording log's _meta
|
|
122
|
+
# header is missing or declares a format_version this build does not
|
|
123
|
+
# support. Mirrors Browserctl::State::Bundle.verify_format_version!.
|
|
124
|
+
def verify_format_version!(raw_lines, path: nil)
|
|
125
|
+
meta = raw_lines.first
|
|
126
|
+
version = meta && meta[:cmd] == "_meta" ? meta[:format_version] : nil
|
|
127
|
+
return if version && SUPPORTED_FORMAT_VERSIONS.include?(version)
|
|
128
|
+
|
|
129
|
+
where = path ? " at #{path}" : ""
|
|
130
|
+
msg = if version.nil?
|
|
131
|
+
"recording log#{where} is missing format_version " \
|
|
132
|
+
"(supported: #{SUPPORTED_FORMAT_VERSIONS.inspect})"
|
|
133
|
+
else
|
|
134
|
+
"recording log#{where} declares format_version=#{version.inspect}, " \
|
|
135
|
+
"this build supports #{SUPPORTED_FORMAT_VERSIONS.inspect}"
|
|
136
|
+
end
|
|
137
|
+
raise Browserctl::ProtocolMismatch.new(msg, code: Browserctl::Error::Codes::PROTOCOL_MISMATCH)
|
|
138
|
+
end
|
|
139
|
+
|
|
79
140
|
def log_path(name)
|
|
80
141
|
File.join(RECORDINGS_DIR, "#{name}.jsonl")
|
|
81
142
|
end
|
|
82
143
|
|
|
83
|
-
def record_ref_interaction(recording_name, cmd, attrs)
|
|
84
|
-
entry = { cmd: "_ref_interaction", action: cmd, ref: attrs[:ref], name: attrs[:name] }
|
|
144
|
+
def record_ref_interaction(recording_name, cmd, attrs, response)
|
|
145
|
+
entry = { cmd: "_ref_interaction", ts: now, action: cmd, ref: attrs[:ref], name: attrs[:name] }
|
|
146
|
+
entry.merge!(replay_metadata(response)) if response
|
|
85
147
|
File.open(log_path(recording_name), "a") do |f|
|
|
86
148
|
f.puts JSON.generate(entry)
|
|
87
149
|
end
|
|
88
150
|
end
|
|
89
151
|
|
|
152
|
+
# Pulls the replay-relevant fields out of a daemon response. Each
|
|
153
|
+
# is optional — older daemons or non-resolving commands may omit
|
|
154
|
+
# any of them.
|
|
155
|
+
def now
|
|
156
|
+
Time.now.utc.to_f
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def replay_metadata(response)
|
|
160
|
+
meta = {}
|
|
161
|
+
meta[:ref] = response[:ref] if response[:ref]
|
|
162
|
+
meta[:fingerprint] = response[:fingerprint] if response[:fingerprint]
|
|
163
|
+
meta[:snapshot_id] = response[:snapshot_id] if response[:snapshot_id]
|
|
164
|
+
meta[:postcondition_hint] = response[:postcondition_hint] if response[:postcondition_hint]
|
|
165
|
+
meta[:post_snapshot_digest] = response[:post_snapshot_digest] if response[:post_snapshot_digest]
|
|
166
|
+
meta.transform_keys(&:to_s)
|
|
167
|
+
end
|
|
168
|
+
|
|
90
169
|
def build_workflow_ruby(name, commands)
|
|
91
|
-
steps
|
|
170
|
+
steps = annotated_steps(commands).join("\n\n")
|
|
171
|
+
secrets = commands.map { |c| c[:secret_field] }.compact.uniq
|
|
172
|
+
header = secret_header(secrets)
|
|
92
173
|
<<~RUBY
|
|
93
174
|
# frozen_string_literal: true
|
|
94
|
-
|
|
175
|
+
# format_version: #{Browserctl::WORKFLOW_FORMAT_VERSION}
|
|
176
|
+
#{header}
|
|
95
177
|
Browserctl.workflow #{name.inspect} do
|
|
96
178
|
desc "Recorded on #{Date.today}"
|
|
97
|
-
|
|
179
|
+
#{secrets.map { |f| " param :secret_#{f}, secret: true" }.join("\n")}
|
|
98
180
|
#{steps.gsub(/^/, ' ')}
|
|
99
181
|
end
|
|
100
182
|
RUBY
|
|
101
183
|
end
|
|
102
184
|
|
|
185
|
+
# Walks the recorded events and emits the rendered step strings,
|
|
186
|
+
# interleaving inferred waits before selector-driven actions whose
|
|
187
|
+
# preceding gap exceeds WAIT_THRESHOLD_SECONDS, and inferred URL
|
|
188
|
+
# postconditions after click/fill steps that triggered navigation.
|
|
189
|
+
def annotated_steps(commands)
|
|
190
|
+
last_url = {}
|
|
191
|
+
commands.each_with_index.flat_map do |cmd, i|
|
|
192
|
+
rendered = []
|
|
193
|
+
if i.positive? && (wait = inferred_wait_step(commands[i - 1], cmd))
|
|
194
|
+
rendered << wait
|
|
195
|
+
end
|
|
196
|
+
rendered << build_step(cmd)
|
|
197
|
+
if (post = url_postcondition_step(cmd, last_url))
|
|
198
|
+
rendered << post
|
|
199
|
+
end
|
|
200
|
+
if (snap = snapshot_postcondition_step(cmd))
|
|
201
|
+
rendered << snap
|
|
202
|
+
end
|
|
203
|
+
update_last_url!(cmd, last_url)
|
|
204
|
+
rendered
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# Emits a postcondition assertion when a click/fill resulted in a URL
|
|
209
|
+
# change. Compares the canonical (scheme+host+path) form so query
|
|
210
|
+
# strings and fragments don't make every replay flaky.
|
|
211
|
+
def url_postcondition_step(cmd, last_url)
|
|
212
|
+
return nil unless %w[click fill].include?(cmd[:cmd])
|
|
213
|
+
return nil unless cmd[:postcondition_hint] && cmd[:postcondition_hint][:url]
|
|
214
|
+
|
|
215
|
+
page = cmd[:name]
|
|
216
|
+
observed = cmd[:postcondition_hint][:url]
|
|
217
|
+
prior = last_url[page]
|
|
218
|
+
return nil if canonical_url(observed) == canonical_url(prior)
|
|
219
|
+
|
|
220
|
+
prefix = canonical_url(observed)
|
|
221
|
+
return nil unless prefix
|
|
222
|
+
|
|
223
|
+
<<~RUBY.chomp
|
|
224
|
+
step "assert url after #{cmd[:cmd]} on #{page}" do
|
|
225
|
+
current = page(:#{page}).url
|
|
226
|
+
assert current.start_with?(#{prefix.inspect}), "expected URL to start with #{prefix}, got \#{current}"
|
|
227
|
+
end
|
|
228
|
+
RUBY
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# Emits an assert_snapshot_stable step when the recording captured a
|
|
232
|
+
# post-step DOM digest. Under workflow run --check the helper records
|
|
233
|
+
# drift on mismatch instead of raising, so a wiggly page surfaces in
|
|
234
|
+
# the report rather than failing the run outright.
|
|
235
|
+
def snapshot_postcondition_step(cmd)
|
|
236
|
+
return nil unless %w[click fill].include?(cmd[:cmd])
|
|
237
|
+
return nil unless cmd[:post_snapshot_digest]
|
|
238
|
+
|
|
239
|
+
page = cmd[:name]
|
|
240
|
+
digest = cmd[:post_snapshot_digest]
|
|
241
|
+
<<~RUBY.chomp
|
|
242
|
+
step "assert post-snapshot stable on #{page}" do
|
|
243
|
+
assert_snapshot_stable(:#{page}, expected_digest: #{digest.inspect})
|
|
244
|
+
end
|
|
245
|
+
RUBY
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def update_last_url!(cmd, last_url)
|
|
249
|
+
case cmd[:cmd]
|
|
250
|
+
when "navigate", "page_open"
|
|
251
|
+
last_url[cmd[:name]] = cmd[:url] if cmd[:url]
|
|
252
|
+
when "click", "fill"
|
|
253
|
+
observed = cmd[:postcondition_hint] && cmd[:postcondition_hint][:url]
|
|
254
|
+
last_url[cmd[:name]] = observed if observed
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def canonical_url(url)
|
|
259
|
+
return nil if url.nil? || url.empty?
|
|
260
|
+
|
|
261
|
+
uri = URI.parse(url)
|
|
262
|
+
path = uri.path.to_s
|
|
263
|
+
path = "/" if path.empty?
|
|
264
|
+
"#{uri.scheme}://#{uri.host}#{path}"
|
|
265
|
+
rescue URI::InvalidURIError
|
|
266
|
+
nil
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def inferred_wait_step(prev, current)
|
|
270
|
+
return nil unless %w[fill click].include?(current[:cmd])
|
|
271
|
+
return nil unless current[:selector]
|
|
272
|
+
|
|
273
|
+
delta = elapsed(prev, current)
|
|
274
|
+
return nil unless delta && delta >= WAIT_THRESHOLD_SECONDS
|
|
275
|
+
|
|
276
|
+
timeout = [WAIT_FLOOR_SECONDS, delta.ceil + WAIT_PADDING_SECONDS].max
|
|
277
|
+
page = current[:name]
|
|
278
|
+
sel = current[:selector]
|
|
279
|
+
<<~RUBY.chomp
|
|
280
|
+
# inferred wait: prior step took ~#{format('%.1f', delta)}s
|
|
281
|
+
step "wait for #{sel} on #{page}" do
|
|
282
|
+
page(:#{page}).wait(#{sel.inspect}, timeout: #{timeout})
|
|
283
|
+
end
|
|
284
|
+
RUBY
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
def elapsed(prev, current)
|
|
288
|
+
return nil unless prev && current && prev[:ts] && current[:ts]
|
|
289
|
+
|
|
290
|
+
current[:ts] - prev[:ts]
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
def secret_header(secrets)
|
|
294
|
+
return "" if secrets.empty?
|
|
295
|
+
|
|
296
|
+
lines = ["# TODO: review the following secret-shaped fields detected during recording.",
|
|
297
|
+
"# Configure a secret_ref: source for each before running:"]
|
|
298
|
+
secrets.each { |f| lines << "# - secret_#{f}" }
|
|
299
|
+
"\n#{lines.join("\n")}\n"
|
|
300
|
+
end
|
|
301
|
+
|
|
103
302
|
def build_step(cmd)
|
|
104
303
|
label, body = step_parts(cmd)
|
|
105
304
|
|
|
@@ -113,12 +312,13 @@ module Browserctl
|
|
|
113
312
|
"# end"
|
|
114
313
|
end
|
|
115
314
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
315
|
+
prefix = []
|
|
316
|
+
prefix << "# NOTE: sensitive query params were redacted during recording" \
|
|
317
|
+
if cmd[:url].to_s.include?("[REDACTED]")
|
|
318
|
+
prefix << "# fingerprint fallback: #{cmd[:fingerprint].to_json}" if cmd[:fingerprint]
|
|
319
|
+
|
|
320
|
+
head = prefix.empty? ? "" : "#{prefix.join("\n")}\n"
|
|
321
|
+
"#{head}step #{label.inspect} do\n #{body}\nend"
|
|
122
322
|
end
|
|
123
323
|
|
|
124
324
|
def step_parts(cmd)
|
|
@@ -143,8 +343,9 @@ module Browserctl
|
|
|
143
343
|
page = cmd[:name]
|
|
144
344
|
case cmd[:cmd]
|
|
145
345
|
when "fill"
|
|
346
|
+
value_arg = cmd[:secret_field] ? "params[:secret_#{cmd[:secret_field]}]" : "params[:fill_value]"
|
|
146
347
|
["fill #{cmd[:selector]} on #{page}",
|
|
147
|
-
"page(:#{page}).fill(#{cmd[:selector].inspect},
|
|
348
|
+
"page(:#{page}).fill(#{cmd[:selector].inspect}, #{value_arg})"]
|
|
148
349
|
when "click"
|
|
149
350
|
["click #{cmd[:selector]} on #{page}",
|
|
150
351
|
"page(:#{page}).click(#{cmd[:selector].inspect})"]
|
|
@@ -152,11 +353,28 @@ module Browserctl
|
|
|
152
353
|
end
|
|
153
354
|
|
|
154
355
|
def prepare_attrs(cmd, attrs)
|
|
155
|
-
attrs = attrs.except(:
|
|
356
|
+
attrs = attrs.except(:capture_post_snapshot)
|
|
357
|
+
if cmd == "fill"
|
|
358
|
+
attrs = attrs.except(:value)
|
|
359
|
+
field = infer_secret_field(attrs[:selector])
|
|
360
|
+
if field
|
|
361
|
+
attrs[:secret_hint] = true
|
|
362
|
+
attrs[:secret_field] = field
|
|
363
|
+
end
|
|
364
|
+
end
|
|
156
365
|
attrs[:url] = redact_url(attrs[:url]) if %w[navigate page_open].include?(cmd) && attrs[:url]
|
|
157
366
|
attrs
|
|
158
367
|
end
|
|
159
368
|
|
|
369
|
+
def infer_secret_field(selector)
|
|
370
|
+
return nil unless selector
|
|
371
|
+
|
|
372
|
+
match = selector.match(SECRET_FIELD_PATTERN)
|
|
373
|
+
return nil unless match
|
|
374
|
+
|
|
375
|
+
match[1].downcase.gsub(/[^a-z0-9]/, "_")
|
|
376
|
+
end
|
|
377
|
+
|
|
160
378
|
def redact_url(url)
|
|
161
379
|
uri = URI.parse(url)
|
|
162
380
|
return url if uri.query.nil?
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Browserctl
|
|
4
|
+
# Redacts known secret values from arbitrary strings.
|
|
5
|
+
#
|
|
6
|
+
# Used by `browserctl trace` to ensure traces are safe to attach to issues
|
|
7
|
+
# by default. Secret values are sourced from two places:
|
|
8
|
+
# 1. ENV variables whose names match well-known secret patterns
|
|
9
|
+
# (`*_TOKEN`, `*_KEY`, `*_SECRET`, `*_PASSWORD`).
|
|
10
|
+
# 2. Values captured at runtime by `SecretResolverRegistry` (in-memory
|
|
11
|
+
# only — never persisted).
|
|
12
|
+
#
|
|
13
|
+
# Replacement marker is the literal `[REDACTED]`. We considered including
|
|
14
|
+
# a sha256 prefix to distinguish values, but that doubles the surface area
|
|
15
|
+
# for accidental leakage (a determined attacker could brute-force short
|
|
16
|
+
# secrets from the prefix), and the timeline is more scannable with a
|
|
17
|
+
# uniform marker.
|
|
18
|
+
class Redactor
|
|
19
|
+
MARKER = "[REDACTED]"
|
|
20
|
+
MIN_LENGTH = 4
|
|
21
|
+
ENV_PATTERNS = [/_TOKEN\z/, /_KEY\z/, /_SECRET\z/, /_PASSWORD\z/].freeze
|
|
22
|
+
|
|
23
|
+
# @param secrets [Array<String>] secret values to redact.
|
|
24
|
+
def initialize(secrets: [])
|
|
25
|
+
# Filter empty / too-short values; longest-first to avoid partial
|
|
26
|
+
# overlaps (e.g. redacting "abcd" before "abcdef" would leave "ef").
|
|
27
|
+
@secrets = secrets
|
|
28
|
+
.compact
|
|
29
|
+
.map(&:to_s)
|
|
30
|
+
.reject { |s| s.length < MIN_LENGTH }
|
|
31
|
+
.uniq
|
|
32
|
+
.sort_by { |s| -s.length }
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# @param string [String, nil]
|
|
36
|
+
# @return [String, nil]
|
|
37
|
+
def redact(string)
|
|
38
|
+
return string if string.nil?
|
|
39
|
+
|
|
40
|
+
result = string.to_s
|
|
41
|
+
@secrets.each { |secret| result = result.gsub(secret, MARKER) }
|
|
42
|
+
result
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def empty?
|
|
46
|
+
@secrets.empty?
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Build a Redactor from the current ENV using well-known patterns.
|
|
50
|
+
# Optionally merges in additional values (e.g. from runtime instrumentation).
|
|
51
|
+
def self.from_env(env: ENV, extra: [])
|
|
52
|
+
values = env.each_with_object([]) do |(name, value), acc|
|
|
53
|
+
acc << value if ENV_PATTERNS.any? { |re| name =~ re }
|
|
54
|
+
end
|
|
55
|
+
new(secrets: values + Array(extra))
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|