browserctl 0.9.0 → 0.11.0

data/lib/browserctl/replay/telemetry.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require_relative "../constants"
+
+module Browserctl
+  module Replay
+    # Append-only JSONL log of replay drift events for offline analysis.
+    # Local-only; nothing is uploaded. One line per event.
+    module Telemetry
+      LOG_BASENAME = "replay_drift.jsonl"
+
+      module_function
+
+      def log_path
+        File.join(Browserctl::BROWSERCTL_DIR, LOG_BASENAME)
+      end
+
+      # Write each drift event from a Replay::Context as its own JSONL line.
+      # @param ctx [Browserctl::Replay::Context, nil]
+      # @param workflow [String] workflow name for cross-reference
+      # @param path [String] override the destination (testing)
+      # @return [Integer] number of events written
+      def emit(ctx, workflow:, path: log_path)
+        events = ctx&.drift_events
+        return 0 if events.nil? || events.empty?
+
+        ensure_log_file(path)
+        ts = Time.now.utc.iso8601
+        File.open(path, "a") do |f|
+          events.each do |e|
+            f.puts JSON.generate(
+              event: "replay_drift",
+              ts: ts,
+              workflow: workflow,
+              command: e.command.to_s,
+              selector: e.selector,
+              matched_ref: e.matched_ref,
+              score: e.score,
+              reason: e.reason
+            )
+          end
+        end
+        events.size
+      rescue SystemCallError, IOError
+        # Telemetry must never break a run.
+        0
+      end
+
+      def ensure_log_file(path)
+        FileUtils.mkdir_p(File.dirname(path), mode: 0o700)
+        return if File.exist?(path)
+
+        FileUtils.touch(path)
+        File.chmod(0o600, path)
+      end
+    end
+  end
+end

data/lib/browserctl/runner.rb

@@ -2,7 +2,9 @@
 
 require "json"
 require_relative "workflow"
+require_relative "workflow/promotion_ledger"
 require_relative "client"
+require_relative "replay/telemetry"
 
 module Browserctl
   class Runner
@@ -14,13 +16,27 @@ module Browserctl
     # Runs a named workflow with the given parameters.
     # @param name [String] workflow name (must match /\A[a-zA-Z0-9_-]+\z/)
     # @param params [Hash] keyword arguments passed to the workflow
-    # @return [Boolean] true if all steps succeeded
+    # @param check [Boolean] when true, attaches a Replay::Context, renders
+    #   a drift report after the run, and signals drift via exit code 2.
+    # @return [Symbol] :clean (all ok, no drift), :drift (all ok, drift seen), :fail (any step failed)
     # @raise [WorkflowError] if the name is invalid or a step fails
-    def run_workflow(name, **params)
+    def run_workflow(name, check: false, **params)
       defn    = fetch_workflow(name)
-      results = defn.call(params, Client.new)
+      ctx     = check ? Browserctl::Replay::Context.new : nil
+      begin
+        results = defn.call(params, Client.new, replay_context: ctx)
+      rescue StandardError
+        Browserctl::Workflow::PromotionLedger.record(workflow: name.to_s, verdict: :fail) if check
+        raise
+      end
       print_results(results)
-      results.all?(&:ok)
+      v = verdict(results, ctx)
+      if check
+        print_drift_report(ctx)
+        Browserctl::Replay::Telemetry.emit(ctx, workflow: name.to_s)
+        Browserctl::Workflow::PromotionLedger.record(workflow: name.to_s, verdict: v)
+      end
+      v
     end
 
     # Lists all registered workflows from the standard search paths.
@@ -109,6 +125,24 @@ module Browserctl
       $stdout.puts "  #{label} #{msg}"
     end
 
+    def print_drift_report(ctx)
+      events = ctx&.drift_events || []
+      report = {
+        drift: events.any?,
+        rematches: events.count { |e| e.reason == "rematch" },
+        unresolved: events.count { |e| e.reason == "no candidate above threshold" },
+        events: events.map(&:to_h)
+      }
+      $stdout.puts JSON.pretty_generate(report)
+    end
+
+    def verdict(results, ctx)
+      return :fail unless results.all?(&:ok)
+      return :drift if ctx&.drift_events&.any?
+
+      :clean
+    end
+
     def format_params(defn)
       defn.param_defs.transform_values do |p|
         entry = { required: p.required, secret: p.secret, default: p.default }

data/lib/browserctl/server/command_dispatcher.rb

@@ -11,9 +11,11 @@ require_relative "handlers/devtools"
 require_relative "handlers/daemon_control"
 require_relative "handlers/storage"
 require_relative "handlers/session"
+require_relative "handlers/state"
 require_relative "handlers/interaction"
 require_relative "../detectors"
 require_relative "../policy"
+require_relative "../replay/snapshot_diff"
 
 module Browserctl
   class CommandDispatcher
@@ -26,6 +28,7 @@ module Browserctl
     include Handlers::DaemonControl
     include Handlers::Storage
     include Handlers::Session
+    include Handlers::State
     include Handlers::Interaction
 
     COMMAND_MAP = {
@@ -36,6 +39,7 @@ module Browserctl
       "navigate" => :cmd_navigate,
       "wait" => :cmd_wait,
       "snapshot" => :cmd_snapshot,
+      "auth_check" => :cmd_auth_check,
       "evaluate" => :cmd_evaluate,
       "fill" => :cmd_fill,
       "click" => :cmd_click,
@@ -66,7 +70,12 @@ module Browserctl
       "session_save" => :cmd_session_save,
       "session_load" => :cmd_session_load,
       "session_list" => :cmd_session_list,
-      "session_delete" => :cmd_session_delete
+      "session_delete" => :cmd_session_delete,
+      "state_save" => :cmd_state_save,
+      "state_load" => :cmd_state_load,
+      "state_list" => :cmd_state_list,
+      "state_info" => :cmd_state_info,
+      "state_delete" => :cmd_state_delete
     }.freeze
 
     SCREENSHOT_DIR   = File.expand_path("~/.browserctl/screenshots").freeze

data/lib/browserctl/server/handlers/interaction.rb

@@ -27,7 +27,7 @@ module Browserctl
               "return { x: r.left + r.width / 2, y: r.top + r.height / 2 }; " \
               "})(#{sel.to_json})"
             )
-            return { error: "selector not found: #{sel}" } unless coords
+            return { error: "selector not found: #{sel}", code: "selector_not_found" } unless coords
 
             session.page.mouse.move(x: coords["x"], y: coords["y"])
             { ok: true }
@@ -43,7 +43,7 @@ module Browserctl
             return sel if sel.is_a?(Hash)
 
             el = session.page.at_css(sel)
-            return { error: "selector not found: #{sel}" } unless el
+            return { error: "selector not found: #{sel}", code: "selector_not_found" } unless el
 
             el.select_file(path)
             { ok: true }
@@ -56,7 +56,7 @@ module Browserctl
             return sel if sel.is_a?(Hash)
 
             el = session.page.at_css(sel)
-            return { error: "selector not found: #{sel}" } unless el
+            return { error: "selector not found: #{sel}", code: "selector_not_found" } unless el
 
             el.evaluate(
               "this.value = #{req[:value].to_json}; " \

data/lib/browserctl/server/handlers/navigation.rb

@@ -33,7 +33,8 @@ module Browserctl
             sel = resolve_selector_from(session, req)
             return sel if sel.is_a?(Hash)
 
-            type_into(session.page, sel, req[:value])
+            result = type_into(session.page, sel, req[:value])
+            enrich_with_recording_metadata(result, session, sel, req)
           end
         end
 
@@ -42,17 +43,45 @@ module Browserctl
             sel = resolve_selector_from(session, req)
             return sel if sel.is_a?(Hash)
 
-            click_element(session.page, sel)
+            result = click_element(session.page, sel)
+            enrich_with_recording_metadata(result, session, sel, req)
           end
         end
 
+        # Adds ref / fingerprint / snapshot_id / postcondition_hint to a successful
+        # click/fill response. Recording uses these to build a self-healing log.
+        # When req[:capture_post_snapshot] is true, also takes a fresh snapshot
+        # and attaches its digest so workflow run --check can diff DOM state
+        # against the recorded baseline.
+        def enrich_with_recording_metadata(result, session, selector, req)
+          return result unless result[:ok]
+
+          ref = req[:ref] || session.ref_registry.invert[selector]
+          fp  = (ref && session.fingerprint_index[ref]) || session.fingerprint_index[selector]
+          enriched = result.merge(
+            ref: ref,
+            fingerprint: fp,
+            snapshot_id: session.snapshot_id,
+            postcondition_hint: { url: session.page.current_url }
+          )
+          enriched[:post_snapshot_digest] = capture_post_snapshot_digest(session) if req[:capture_post_snapshot]
+          enriched.compact
+        end
+
+        def capture_post_snapshot_digest(session)
+          snapshot = @snapshot_builder.call(session.page)
+          Browserctl::Replay::SnapshotDiff.digest(snapshot)
+        rescue StandardError
+          nil
+        end
+
         def cmd_url(req)
           with_page(req[:name]) { |session| { ok: true, url: session.page.current_url } }
         end
 
         def type_into(page, selector, value)
           el = page.at_css(selector)
-          return { error: "selector not found: #{selector}" } unless el
+          return { error: "selector not found: #{selector}", code: "selector_not_found" } unless el
 
           el.focus
           el.evaluate("this.select()")
@@ -62,7 +91,7 @@ module Browserctl
 
         def click_element(page, selector)
           el = page.at_css(selector)
-          return { error: "selector not found: #{selector}" } unless el
+          return { error: "selector not found: #{selector}", code: "selector_not_found" } unless el
 
           # Use the DOM native click() so JS-only event listeners fire.
           # CDP mouse simulation (el.click) dispatches events at screen coordinates

data/lib/browserctl/server/handlers/observation.rb

@@ -12,6 +12,30 @@ module Browserctl
           with_page(req[:name]) { |session| take_snapshot(session, req[:format], req[:diff]) }
         end
 
+        # Runs the auth_required detector against the page and returns either a
+        # plain `{ ok: true, auth_required: false }` response or a structured
+        # `{ error:, code: "AUTH_REQUIRED", state:, suggested_flow:, reason: }`
+        # error. Callers feed in cookies / suggested_flow when they have a
+        # bundle in hand (see PR 18); without them, only the URL signal fires.
+        def cmd_auth_check(req)
+          with_page(req[:name]) do |session|
+            cookies = session.page.cookies.all.values.map(&:to_h) if req[:include_cookies]
+            result = Browserctl::Detectors.auth_required(
+              session.page,
+              cookies: cookies,
+              suggested_flow: req[:suggested_flow]
+            )
+            next { ok: true, auth_required: false } unless result.triggered
+
+            Browserctl::AuthRequiredError.new(
+              result.reason,
+              state: req[:state],
+              suggested_flow: result.suggested_flow,
+              reason: result.reason
+            ).to_response
+          end
+        end
+
         def take_snapshot(session, format, diff)
           nonce     = SecureRandom.hex(8)
           challenge = Detectors.cloudflare?(session.page)
@@ -20,15 +44,32 @@ module Browserctl
 
           snapshot = @snapshot_builder.call(session.page)
           registry = snapshot.to_h { |el| [el[:ref], el[:selector]] }
+          fp_index = build_fingerprint_index(snapshot)
 
           prev = session.prev_snapshot
-          session.ref_registry  = registry
-          session.prev_snapshot = snapshot
+          session.ref_registry      = registry
+          session.fingerprint_index = fp_index
+          session.snapshot_id       = nonce
+          session.prev_snapshot     = snapshot
           result = diff && prev ? compute_diff(prev, snapshot) : snapshot
 
           { ok: true, snapshot: result, challenge: challenge, nonce: nonce }
         end
 
+        def build_fingerprint_index(snapshot)
+          index = {}
+          snapshot.each do |el|
+            ref = el[:ref]
+            sel = el[:selector]
+            fp  = el[:fingerprint]
+            next unless fp
+
+            index[ref] = fp if ref
+            index[sel] = fp if sel
+          end
+          index
+        end
+
         def compute_diff(prev, current)
           prev_by_sel = prev.to_h { |el| [el[:selector], el] }
           current.reject do |el|

data/lib/browserctl/server/handlers/state.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../../state"
+
+module Browserctl
+  class CommandDispatcher
+    module Handlers
+      # Top-level state management — collapses cookies + localStorage +
+      # sessionStorage into a single `.bctl` bundle. See lib/browserctl/state.rb.
+      module State
+        private
+
+        def cmd_state_save(req)
+          first_session = @global_mutex.synchronize { @pages.values.first }
+          return { error: "no open pages — open a page before saving state" } unless first_session
+
+          payload, captured_origins = capture_state_payload
+          manifest = Browserctl::State.save(
+            req[:name],
+            payload: payload,
+            origins: req[:origins] || captured_origins,
+            flow: req[:flow],
+            flow_version: req[:flow_version],
+            passphrase: req[:passphrase]
+          )
+
+          {
+            ok: true,
+            path: Browserctl::State.path(req[:name]),
+            origins: manifest[:origins],
+            cookies: payload[:cookies].length,
+            encrypted: manifest[:encrypted]
+          }
+        rescue Browserctl::Error, ArgumentError => e
+          { error: e.message }
+        end
+
+        def cmd_state_load(req)
+          data = Browserctl::State.load(req[:name], passphrase: req[:passphrase])
+          target = @global_mutex.synchronize { @pages.values.first }
+          return { error: "no open pages — open a page before loading state" } unless target
+
+          cookies = pluck(data[:payload], :cookies, default: [])
+
+          unless req[:skip_auth_check]
+            auth = Browserctl::Detectors.auth_required(
+              target.page, cookies: cookies, suggested_flow: data[:manifest][:flow]
+            )
+            if auth.triggered
+              return Browserctl::AuthRequiredError.new(
+                auth.reason,
+                state: req[:name],
+                suggested_flow: auth.suggested_flow,
+                reason: auth.reason
+              ).to_response
+            end
+          end
+
+          restore_state_cookies(target, cookies)
+          ls_count = restore_local_storage(pluck(data[:payload], :local_storage, default: {}))
+
+          {
+            ok: true,
+            cookies: cookies.length,
+            local_storage_keys: ls_count,
+            origins: data[:manifest][:origins]
+          }
+        rescue Browserctl::State::Bundle::BundleError, Browserctl::Error, ArgumentError, JSON::ParserError => e
+          { error: e.message }
+        end
+
+        def pluck(hash, sym, default:)
+          hash[sym] || hash[sym.to_s] || default
+        end
+
+        def restore_state_cookies(target, cookies)
+          cookies.each do |raw|
+            c = raw.transform_keys(&:to_sym)
+            target.page.cookies.set(**c.slice(:name, :value, :domain, :path))
+          end
+        end
+
+        def cmd_state_list(_req)
+          { ok: true, state: Browserctl::State.all }
+        end
+
+        def cmd_state_info(req)
+          { ok: true, info: Browserctl::State.info(req[:name]) }
+        rescue Browserctl::State::Bundle::BundleError, Browserctl::Error, ArgumentError => e
+          { error: e.message }
+        end
+
+        def cmd_state_delete(req)
+          Browserctl::State.delete(req[:name])
+          { ok: true }
+        rescue ArgumentError => e
+          { error: e.message }
+        end
+
+        def capture_state_payload
+          first = @global_mutex.synchronize { @pages.values.first }
+          cookies = first.page.cookies.all.values.map(&:to_h)
+
+          local_storage   = {}
+          session_storage = {}
+          captured_origins = []
+
+          @global_mutex.synchronize { @pages.dup }.each_value do |session|
+            session.mutex.synchronize do
+              origin    = session.page.evaluate("location.origin")
+              ls_str    = session.page.evaluate("JSON.stringify({...localStorage})") || "{}"
+              ss_str    = session.page.evaluate("JSON.stringify({...sessionStorage})") || "{}"
+              local_storage[origin]   = JSON.parse(ls_str)
+              session_storage[origin] = JSON.parse(ss_str)
+              captured_origins << origin
+            end
+          end
+
+          payload = {
+            cookies: cookies,
+            local_storage: local_storage,
+            session_storage: session_storage
+          }
+          [payload, captured_origins.uniq]
+        end
+
+        def restore_local_storage(local_storage)
+          count = 0
+          local_storage.each do |origin, keys|
+            next if keys.nil? || keys.empty?
+
+            tmp_page = @driver.create_page
+            begin
+              tmp_page.go_to(origin.to_s)
+              keys.each do |k, v|
+                tmp_page.evaluate("localStorage.setItem(#{k.to_json}, #{v.to_json})")
+                count += 1
+              end
+            ensure
+              tmp_page.close
+            end
+          end
+          count
+        end
+      end
+    end
+  end
+end

data/lib/browserctl/server/page_session.rb

@@ -3,15 +3,17 @@
 module Browserctl
   class PageSession
     attr_reader :page, :mutex, :pause_cv
-    attr_accessor :ref_registry, :prev_snapshot
+    attr_accessor :ref_registry, :prev_snapshot, :fingerprint_index, :snapshot_id
 
     def initialize(page)
-      @page          = page
-      @mutex         = Mutex.new
-      @pause_cv      = ConditionVariable.new
-      @ref_registry  = {}
-      @prev_snapshot = nil
-      @paused        = false
+      @page              = page
+      @mutex             = Mutex.new
+      @pause_cv          = ConditionVariable.new
+      @ref_registry      = {}
+      @fingerprint_index = {}
+      @snapshot_id       = nil
+      @prev_snapshot     = nil
+      @paused            = false
     end
 
     def paused? = @paused

data/lib/browserctl/server/snapshot_builder.rb

@@ -1,55 +1,31 @@
 # frozen_string_literal: true
 
-require "nokogiri"
+require "browserctl/snapshot/extractor"
+require "browserctl/snapshot/annotator"
+require "browserctl/snapshot/serializer"
 
 module Browserctl
+  # Orchestrates the snapshot pipeline:
+  #
+  #   page.body  ──Extractor──▶  [nodes]
+  #              ──Annotator──▶  [entries with ref + fingerprint]
+  #              ──Serializer─▶  wire-shape array
+  #
+  # Each stage is independently testable. Inject alternates via the keyword
+  # args for tests that want to isolate one stage.
   class SnapshotBuilder
-    INTERACTABLE = %w[a button input select textarea
-                      [role=button] [role=link] [role=menuitem]].freeze
-    ATTRS        = %w[type name placeholder href aria-label role].freeze
-
-    def call(page)
-      doc = Nokogiri::HTML(page.body)
-      ref = 0
-      doc.css(INTERACTABLE.join(",")).map { |el| element_entry(el, ref += 1) }
-    end
-
-    private
-
-    def element_entry(elem, ref)
-      { ref: "e#{ref}", tag: elem.name, text: elem.text.strip.slice(0, 80),
-        selector: css_path(elem), attrs: element_attrs(elem) }
-    end
-
-    def element_attrs(elem)
-      elem.attributes.transform_values(&:value).slice(*ATTRS)
-    end
-
-    def css_path(node)
-      ancestors_until_html(node).map { |n| path_segment(n) }.join(" > ")
+    def initialize(extractor: Snapshot::Extractor.new,
+                   annotator: Snapshot::Annotator.new,
+                   serializer: Snapshot::Serializer.new)
+      @extractor = extractor
+      @annotator = annotator
+      @serializer = serializer
     end
 
-    def ancestors_until_html(node)
-      [].tap do |acc|
-        while node && node.name != "html"
-          acc.unshift(node)
-          node = node.parent
-        end
-      end
-    end
-
-    def path_segment(node)
-      node.name + id_fragment(node) + class_fragment(node)
-    end
-
-    def id_fragment(node)
-      (id = node["id"]) && !id.empty? ? "##{id}" : ""
-    end
-
-    def class_fragment(node)
-      return "" if node["id"] && !node["id"].empty?
-
-      (klass = node["class"]&.split&.first) ? ".#{klass}" : ""
+    def call(page)
+      nodes = @extractor.call(page.body)
+      entries = @annotator.call(nodes)
+      @serializer.call(entries)
     end
   end
 end

data/lib/browserctl/snapshot/annotator.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "browserctl/snapshot/ref"
+require "browserctl/snapshot/fingerprint"
+
+module Browserctl
+  module Snapshot
+    # Stage 2 of the snapshot pipeline.
+    #
+    # Takes the list of interactable nodes from Extractor and produces
+    # element entries with stable refs, semantic metadata, a CSS selector
+    # path, and a fingerprint. Each entry is a plain Hash.
+    class Annotator
+      ATTRS = %w[type name placeholder href aria-label role].freeze
+
+      def initialize(ref_deriver: RefDeriver.new, fingerprint: Fingerprint.new)
+        @ref_deriver = ref_deriver
+        @fingerprint = fingerprint
+      end
+
+      def call(nodes)
+        taken = {}
+        nodes.map do |node|
+          ref = @ref_deriver.disambiguate(@ref_deriver.derive(node), taken)
+          taken[ref] = true
+          entry(node, ref)
+        end
+      end
+
+      private
+
+      def entry(node, ref)
+        {
+          ref: ref,
+          tag: node.name,
+          text: node.text.strip.slice(0, 80),
+          selector: css_path(node),
+          attrs: attrs(node),
+          fingerprint: @fingerprint.build(node)
+        }
+      end
+
+      def attrs(node)
+        node.attributes.transform_values(&:value).slice(*ATTRS)
+      end
+
+      def css_path(node)
+        ancestors_until_html(node).map { |n| segment(n) }.join(" > ")
+      end
+
+      def ancestors_until_html(node)
+        [].tap do |acc|
+          while node && node.name != "html"
+            acc.unshift(node)
+            node = node.parent
+          end
+        end
+      end
+
+      def segment(node)
+        node.name + id_fragment(node) + class_fragment(node)
+      end
+
+      def id_fragment(node)
+        (id = node["id"]) && !id.empty? ? "##{id}" : ""
+      end
+
+      def class_fragment(node)
+        return "" if node["id"] && !node["id"].empty?
+
+        (klass = node["class"]&.split&.first) ? ".#{klass}" : ""
+      end
+    end
+  end
+end

data/lib/browserctl/snapshot/extractor.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Browserctl
+  module Snapshot
+    # Stage 1 of the snapshot pipeline.
+    #
+    # Parses raw HTML and returns the set of interactable Nokogiri nodes
+    # that the rest of the pipeline will annotate. This stage knows nothing
+    # about refs, fingerprints, or wire format.
+    class Extractor
+      INTERACTABLE = %w[a button input select textarea
+                        [role=button] [role=link] [role=menuitem]].freeze
+
+      def call(html)
+        Nokogiri::HTML(html).css(INTERACTABLE.join(",")).to_a
+      end
+    end
+  end
+end