browserctl 0.3.1 → 0.5.0

data/lib/browserctl/recording.rb

@@ -4,6 +4,7 @@ require "json"
 require "date"
 require "fileutils"
 require "tmpdir"
+require "uri"
 
 module Browserctl
   class Recording
@@ -12,11 +13,15 @@ module Browserctl
 
     RECORDABLE = %w[open_page goto fill click screenshot evaluate].freeze
 
+    SENSITIVE_PARAM_PATTERN = /\A(token|key|secret|auth|code|access_token|api_key|client_secret|state)\z/ix
+
     def self.start(name)
-      FileUtils.mkdir_p(RECORDINGS_DIR)
+      FileUtils.mkdir_p(RECORDINGS_DIR, mode: 0o700)
       FileUtils.mkdir_p(File.dirname(STATE_FILE))
       File.write(STATE_FILE, name)
       FileUtils.rm_f(log_path(name))
+      FileUtils.touch(log_path(name))
+      File.chmod(0o600, log_path(name))
       name
     end
 
@@ -36,10 +41,13 @@ module Browserctl
       name = active
       return unless name
       return unless RECORDABLE.include?(cmd.to_s)
-      # ref-based interactions have no replayable selector — skip them
-      return if %w[click fill].include?(cmd.to_s) && attrs[:selector].nil?
 
-      attrs = attrs.except(:value) if cmd.to_s == "fill"
+      if %w[click fill].include?(cmd.to_s) && attrs[:selector].nil?
+        record_ref_interaction(name, cmd.to_s, attrs)
+        return
+      end
+
+      attrs = prepare_attrs(cmd.to_s, attrs)
 
       File.open(log_path(name), "a") do |f|
         f.puts JSON.generate({ cmd: cmd.to_s }.merge(attrs.transform_keys(&:to_s)))
@@ -53,6 +61,13 @@ module Browserctl
       lines = File.readlines(log).map { |l| JSON.parse(l, symbolize_names: true) }
       ruby  = build_workflow_ruby(name, lines)
       File.write(output_path, ruby) if output_path
+
+      ref_count = lines.count { |l| l[:cmd] == "_ref_interaction" }
+      if ref_count.positive?
+        warn "Warning: #{ref_count} ref-based interaction(s) were captured but cannot be replayed by ref."
+        warn "Search the generated workflow for 'TODO: ref-based' and replace with stable CSS selectors."
+      end
+
       ruby
     ensure
       FileUtils.rm_f(log) if log
@@ -65,6 +80,13 @@ module Browserctl
         File.join(RECORDINGS_DIR, "#{name}.jsonl")
       end
 
+      def record_ref_interaction(recording_name, cmd, attrs)
+        entry = { cmd: "_ref_interaction", action: cmd, ref: attrs[:ref], name: attrs[:name] }
+        File.open(log_path(recording_name), "a") do |f|
+          f.puts JSON.generate(entry)
+        end
+      end
+
       def build_workflow_ruby(name, commands)
         steps = commands.map { |c| build_step(c) }.join("\n\n")
         <<~RUBY
@@ -80,30 +102,74 @@ module Browserctl
 
       def build_step(cmd)
         label, body = step_parts(cmd)
-        "step #{label.inspect} do\n  #{body}\nend"
+
+        if body.nil?
+          page_sym = cmd[:name].to_s.gsub(/[^a-zA-Z0-9_]/, "_")
+          action   = cmd[:action].to_s.gsub(/[^a-z_]/, "")
+          return "# TODO: ref-based #{action} on #{cmd[:name].inspect} (ref: #{cmd[:ref].inspect}) — " \
+                 "replace with a stable CSS selector\n" \
+                 "# step #{label.inspect} do\n" \
+                 "#   page(:#{page_sym}).#{action}(\"YOUR_SELECTOR_HERE\")\n" \
+                 "# end"
+        end
+
+        url = cmd[:url].to_s
+        if url.include?("[REDACTED]")
+          "# NOTE: sensitive query params were redacted during recording\nstep #{label.inspect} do\n  #{body}\nend"
+        else
+          "step #{label.inspect} do\n  #{body}\nend"
+        end
       end
 
       def step_parts(cmd)
+        return ref_interaction_parts(cmd) if cmd[:cmd] == "_ref_interaction"
+        return selector_parts(cmd) if %w[fill click].include?(cmd[:cmd])
+
+        page = cmd[:name]
+        case cmd[:cmd]
+        when "open_page"  then ["open #{page}", "page(:#{page}).goto(#{cmd[:url].inspect})"]
+        when "goto"       then ["goto #{page}", "page(:#{page}).goto(#{cmd[:url].inspect})"]
+        when "screenshot" then ["screenshot #{page}", "page(:#{page}).screenshot"]
+        when "evaluate"   then ["eval on #{page}", "page(:#{page}).evaluate(#{cmd[:expression].inspect})"]
+        else ["#{cmd[:cmd]} on #{page}", "# unrecognised command: #{cmd.inspect}"]
+        end
+      end
+
+      def ref_interaction_parts(cmd)
+        ["TODO: ref-based #{cmd[:action]} on #{cmd[:name]} (ref: #{cmd[:ref]})", nil]
+      end
+
+      def selector_parts(cmd)
         page = cmd[:name]
         case cmd[:cmd]
-        when "open_page"
-          ["open #{page}", "page(:#{page}).goto(#{cmd[:url].inspect})"]
-        when "goto"
-          ["goto #{page}", "page(:#{page}).goto(#{cmd[:url].inspect})"]
         when "fill"
           ["fill #{cmd[:selector]} on #{page}",
            "page(:#{page}).fill(#{cmd[:selector].inspect}, params[:fill_value])"]
         when "click"
           ["click #{cmd[:selector]} on #{page}",
            "page(:#{page}).click(#{cmd[:selector].inspect})"]
-        when "screenshot"
-          ["screenshot #{page}", "page(:#{page}).screenshot"]
-        when "evaluate"
-          ["eval on #{page}", "page(:#{page}).evaluate(#{cmd[:expression].inspect})"]
-        else
-          ["#{cmd[:cmd]} on #{page}", "# unrecognised command: #{cmd.inspect}"]
         end
       end
+
+      def prepare_attrs(cmd, attrs)
+        attrs = attrs.except(:value) if cmd == "fill"
+        attrs[:url] = redact_url(attrs[:url]) if %w[goto open_page].include?(cmd) && attrs[:url]
+        attrs
+      end
+
+      def redact_url(url)
+        uri = URI.parse(url)
+        return url if uri.query.nil?
+
+        uri.query = uri.query.gsub(/([^&=]+)=([^&]*)/) do |full_match|
+          raw_key = ::Regexp.last_match(1)
+          key = URI.decode_www_form_component(raw_key)
+          key =~ SENSITIVE_PARAM_PATTERN ? "#{raw_key}=[REDACTED]" : full_match
+        end
+        uri.to_s
+      rescue URI::InvalidURIError
+        url
+      end
     end
   end
 end

data/lib/browserctl/runner.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "json"
 require_relative "workflow"
 require_relative "client"
 
@@ -26,7 +27,7 @@ module Browserctl
     # @return [Array<Hash>] array of `{ name:, desc: }` hashes
     def list_workflows
       load_all_workflows
-      REGISTRY.map { |name, defn| { name: name, desc: defn.description } }
+      Browserctl.registry_snapshot.map { |name, defn| { name: name, desc: defn.description } }
     end
 
     # Returns detailed information about a workflow.
@@ -39,6 +40,24 @@ module Browserctl
 
     SAFE_WORKFLOW_NAME = /\A[a-zA-Z0-9_-]+\z/
 
+    def self.load_params_file(path)
+      raise "params file not found: #{path}" unless File.exist?(path)
+
+      case File.extname(path).downcase
+      when ".yml", ".yaml"
+        require "yaml"
+        YAML.safe_load_file(path, symbolize_names: true)
+      when ".json"
+        JSON.parse(File.read(path), symbolize_names: true)
+      else
+        raise "unsupported params file format: #{path} (use .yml, .yaml, or .json)"
+      end
+    rescue Psych::SyntaxError => e
+      raise "invalid YAML in #{path}: #{e.message}"
+    rescue JSON::ParserError => e
+      raise "invalid JSON in #{path}: #{e.message}"
+    end
+
     private
 
     def validate_name!(name)
@@ -48,15 +67,15 @@ module Browserctl
     end
 
     def fetch_workflow(name)
-      return REGISTRY[name.to_s] if REGISTRY.key?(name.to_s)
+      return Browserctl.lookup_workflow(name.to_s) if Browserctl.lookup_workflow(name.to_s)
 
       validate_name!(name)
       load_workflow_file(name)
-      REGISTRY[name.to_s] || raise("workflow '#{name}' not found")
+      Browserctl.lookup_workflow(name.to_s) || raise("workflow '#{name}' not found")
     end
 
     def load_workflow_file(name)
-      return if REGISTRY.key?(name.to_s)
+      return if Browserctl.lookup_workflow(name.to_s)
 
       path = workflow_path(name)
       load path if path

data/lib/browserctl/server/command_dispatcher.rb

@@ -2,9 +2,26 @@
 
 require_relative "snapshot_builder"
 require_relative "page_session"
+require_relative "handlers/page_lifecycle"
+require_relative "handlers/navigation"
+require_relative "handlers/observation"
+require_relative "handlers/cookies"
+require_relative "handlers/hitl"
+require_relative "handlers/devtools"
+require_relative "handlers/daemon_control"
+require_relative "../detectors"
+require_relative "../policy"
 
 module Browserctl
   class CommandDispatcher
+    include Handlers::PageLifecycle
+    include Handlers::Navigation
+    include Handlers::Observation
+    include Handlers::Cookies
+    include Handlers::Hitl
+    include Handlers::DevTools
+    include Handlers::DaemonControl
+
     COMMAND_MAP = {
       "open_page" => :cmd_open_page,
       "close_page" => :cmd_close_page,
@@ -25,25 +42,29 @@ module Browserctl
       "inspect" => :cmd_inspect,
       "cookies" => :cmd_cookies,
       "set_cookie" => :cmd_set_cookie,
-      "clear_cookies" => :cmd_clear_cookies
+      "clear_cookies" => :cmd_clear_cookies,
+      "import_cookies" => :cmd_import_cookies,
+      "store" => :cmd_store,
+      "fetch" => :cmd_fetch
     }.freeze
 
-    SCREENSHOT_DIR  = File.expand_path("~/.browserctl/screenshots").freeze
-    SCREENSHOT_EXTS = %w[.png .jpg .jpeg].freeze
-    CLOUDFLARE_SIGNALS = [
-      "cf-challenge-running",
-      "cf_chl_opt",
-      "__cf_chl_f_tk",
-      "Just a moment..."
-    ].freeze
+    SCREENSHOT_DIR   = File.expand_path("~/.browserctl/screenshots").freeze
+    SCREENSHOT_ROOTS = [SCREENSHOT_DIR, File.expand_path(".")].freeze
+    SCREENSHOT_EXTS  = %w[.png .jpg .jpeg].freeze
 
     def initialize(pages, browser, snapshot_builder = SnapshotBuilder.new, global_mutex: Mutex.new)
       @pages            = pages
       @browser          = browser
       @snapshot_builder = snapshot_builder
       @global_mutex     = global_mutex
+      @kv_store         = {}
+      @kv_mutex         = Mutex.new
     end
 
+    # Dispatches a parsed request to the appropriate handler.
+    # Returns `{ error: String, code: String }` for unknown commands.
+    # @param req [Hash{Symbol => Object}] parsed request; must include `:cmd`
+    # @return [Hash{Symbol => Object}] response; always includes `:ok` or `:error`
     def dispatch(req)
       handler = COMMAND_MAP[req[:cmd]]
       if handler
@@ -51,7 +72,7 @@ module Browserctl
         return send(handler, req)
       end
 
-      if (plugin = Browserctl::PLUGIN_COMMANDS[req[:cmd]])
+      if (plugin = Browserctl.lookup_plugin_command(req[:cmd]))
         Browserctl.logger.debug("plugin:#{req[:cmd]} #{req[:name]}")
         session = req[:name] ? @global_mutex.synchronize { @pages[req[:name]] } : nil
         return plugin.call(session, req)
@@ -62,216 +83,6 @@ module Browserctl
 
     private
 
-    def cmd_open_page(req)
-      session = @global_mutex.synchronize do
-        @pages[req[:name]] ||= PageSession.new(@browser.create_page)
-      end
-      session.page.go_to(req[:url]) if req[:url]
-      { ok: true, name: req[:name] }
-    end
-
-    def cmd_close_page(req)
-      session = @global_mutex.synchronize { @pages.delete(req[:name]) }
-      session&.page&.close
-      { ok: true }
-    end
-
-    def cmd_list_pages(_req)
-      { pages: @global_mutex.synchronize { @pages.keys } }
-    end
-
-    def cmd_goto(req)
-      with_page(req[:name]) do |session|
-        session.page.go_to(req[:url])
-        { ok: true, url: session.page.current_url, challenge: cloudflare_challenge?(session.page) }
-      end
-    end
-
-    def cmd_snapshot(req)
-      with_page(req[:name]) { |session| take_snapshot(session, req[:format], req[:diff]) }
-    end
-
-    def take_snapshot(session, format, diff)
-      challenge = cloudflare_challenge?(session.page)
-
-      return { ok: true, html: session.page.body, challenge: challenge } unless format == "ai"
-
-      snapshot = @snapshot_builder.call(session.page)
-      registry = snapshot.to_h { |el| [el[:ref], el[:selector]] }
-
-      prev = session.prev_snapshot
-      session.ref_registry  = registry
-      session.prev_snapshot = snapshot
-      result = diff && prev ? compute_diff(prev, snapshot) : snapshot
-
-      { ok: true, snapshot: result, challenge: challenge }
-    end
-
-    def compute_diff(prev, current)
-      prev_by_sel = prev.to_h { |el| [el[:selector], el] }
-      current.reject do |el|
-        old = prev_by_sel[el[:selector]]
-        old && old.slice(:text, :attrs) == el.slice(:text, :attrs)
-      end
-    end
-
-    def cmd_evaluate(req)
-      with_page(req[:name]) { |session| { ok: true, result: session.page.evaluate(req[:expression]) } }
-    end
-
-    def cmd_fill(req)
-      with_page(req[:name]) do |session|
-        sel = resolve_selector_from(session, req)
-        return sel if sel.is_a?(Hash)
-
-        type_into(session.page, sel, req[:value])
-      end
-    end
-
-    def type_into(page, selector, value)
-      el = page.at_css(selector)
-      return { error: "selector not found: #{selector}" } unless el
-
-      el.focus
-      el.type(value)
-      { ok: true }
-    end
-
-    def cmd_click(req)
-      with_page(req[:name]) do |session|
-        sel = resolve_selector_from(session, req)
-        return sel if sel.is_a?(Hash)
-
-        click_element(session.page, sel)
-      end
-    end
-
-    def click_element(page, selector)
-      el = page.at_css(selector)
-      return { error: "selector not found: #{selector}" } unless el
-
-      el.click
-      { ok: true }
-    end
-
-    def cmd_screenshot(req)
-      with_page(req[:name]) do |session|
-        path = safe_screenshot_path(req[:path], req[:name])
-        return path if path.is_a?(Hash)
-
-        FileUtils.mkdir_p(File.dirname(path))
-        session.page.screenshot(path: path, full: req.fetch(:full, false))
-        { ok: true, path: path }
-      end
-    end
-
-    def safe_screenshot_path(requested, page_name)
-      if requested
-        expanded = File.expand_path(requested)
-        return { error: "path outside allowed directory (#{SCREENSHOT_DIR})" } \
-          unless expanded.start_with?(SCREENSHOT_DIR)
-        return { error: "invalid extension — use .png, .jpg, or .jpeg" } \
-          unless SCREENSHOT_EXTS.include?(File.extname(expanded).downcase)
-
-        expanded
-      else
-        name_safe = page_name.to_s.gsub(/[^a-zA-Z0-9_-]/, "_")
-        File.join(SCREENSHOT_DIR, "browserctl_shot_#{name_safe}_#{Time.now.to_i}.png")
-      end
-    end
-
-    def cmd_wait_for(req)
-      with_page(req[:name]) { |session| wait_for_selector(session.page, req[:selector], req.fetch(:timeout, 10).to_f) }
-    end
-
-    def cmd_watch(req)
-      with_page(req[:name]) do |session|
-        result = wait_for_selector(session.page, req[:selector], req.fetch(:timeout, 30).to_f)
-        result[:error] ? result : { ok: true, selector: req[:selector] }
-      end
-    end
-
-    def wait_for_selector(page, selector, timeout)
-      deadline = Time.now + timeout
-      loop do
-        found = page.at_css(selector)
-        break { ok: true } if found
-        break { error: "wait_for timeout: selector '#{selector}' not found after #{timeout}s" } if Time.now >= deadline
-
-        sleep 0.2
-      end
-    end
-
-    def cmd_url(req)
-      with_page(req[:name]) { |session| { ok: true, url: session.page.current_url } }
-    end
-
-    def cmd_cookies(req)
-      session = @global_mutex.synchronize { @pages[req[:name]] }
-      return { error: "no page named '#{req[:name]}'" } unless session
-
-      all = session.page.cookies.all
-      { ok: true, cookies: all.values.map(&:to_h) }
-    end
-
-    def cmd_set_cookie(req)
-      session = @global_mutex.synchronize { @pages[req[:name]] }
-      return { error: "no page named '#{req[:name]}'" } unless session
-
-      session.page.cookies.set(
-        name: req[:cookie_name],
-        value: req[:value],
-        domain: req[:domain],
-        path: req.fetch(:path, "/")
-      )
-      { ok: true }
-    end
-
-    def cmd_clear_cookies(req)
-      session = @global_mutex.synchronize { @pages[req[:name]] }
-      return { error: "no page named '#{req[:name]}'" } unless session
-
-      session.page.cookies.clear
-      { ok: true }
-    end
-
-    def cmd_inspect(req)
-      session = @global_mutex.synchronize { @pages[req[:name]] }
-      return { error: "no page named '#{req[:name]}'" } unless session
-
-      port      = @browser.process.port
-      target_id = session.page.target_id
-      devtools_url = "http://127.0.0.1:#{port}/devtools/inspector.html" \
-                     "?ws=127.0.0.1:#{port}/devtools/page/#{target_id}"
-      { ok: true, devtools_url: devtools_url }
-    end
-
-    def cmd_pause(req)
-      session = @global_mutex.synchronize { @pages[req[:name]] }
-      return { error: "no page named '#{req[:name]}'" } unless session
-
-      session.mutex.synchronize { session.pause! }
-      { ok: true, paused: true }
-    end
-
-    def cmd_resume(req)
-      session = @global_mutex.synchronize { @pages[req[:name]] }
-      return { error: "no page named '#{req[:name]}'" } unless session
-
-      session.mutex.synchronize do
-        session.resume!
-        session.pause_cv.signal
-      end
-      { ok: true, paused: false }
-    end
-
-    def cmd_ping(_req) = { ok: true, pid: Process.pid }
-
-    def cmd_shutdown(_req)
-      Process.kill("INT", Process.pid)
-      { ok: true }
-    end
-
     def with_page(name)
       session = @global_mutex.synchronize { @pages[name] }
       return { error: "no page named '#{name}'" } unless session
@@ -281,19 +92,5 @@ module Browserctl
         yield session
       end
     end
-
-    def cloudflare_challenge?(page)
-      url  = page.current_url.to_s
-      body = page.body.to_s
-      url.include?("challenge-platform") ||
-        CLOUDFLARE_SIGNALS.any? { |sig| body.include?(sig) }
-    end
-
-    def resolve_selector_from(session, req)
-      return req[:selector] if req[:selector]
-      return { error: "selector or ref required" } unless req[:ref]
-
-      session.ref_registry[req[:ref]] || { error: "ref '#{req[:ref]}' not found — run snap first" }
-    end
   end
 end

data/lib/browserctl/server/handlers/cookies.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Browserctl
+  class CommandDispatcher
+    module Handlers
+      module Cookies
+        private
+
+        def cmd_cookies(req)
+          session = @global_mutex.synchronize { @pages[req[:name]] }
+          return { error: "no page named '#{req[:name]}'" } unless session
+
+          all = session.page.cookies.all
+          { ok: true, cookies: all.values.map(&:to_h) }
+        end
+
+        def cmd_set_cookie(req)
+          session = @global_mutex.synchronize { @pages[req[:name]] }
+          return { error: "no page named '#{req[:name]}'" } unless session
+
+          session.page.cookies.set(
+            name: req[:cookie_name],
+            value: req[:value],
+            domain: req[:domain],
+            path: req.fetch(:path, "/")
+          )
+          { ok: true }
+        end
+
+        def cmd_clear_cookies(req)
+          session = @global_mutex.synchronize { @pages[req[:name]] }
+          return { error: "no page named '#{req[:name]}'" } unless session
+
+          session.page.cookies.clear
+          { ok: true }
+        end
+
+        def cmd_import_cookies(req)
+          with_page(req[:name]) do |session|
+            req[:cookies].each do |c|
+              session.page.cookies.set(
+                name: c[:name],
+                value: c[:value],
+                domain: c[:domain],
+                path: c.fetch(:path, "/"),
+                httponly: c[:httpOnly] == true,
+                secure: c[:secure] == true,
+                expires: c[:expires] ? Time.at(c[:expires].to_i) : nil
+              )
+            end
+            { ok: true, count: req[:cookies].length }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/browserctl/server/handlers/daemon_control.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Browserctl
+  class CommandDispatcher
+    module Handlers
+      module DaemonControl
+        private
+
+        def cmd_ping(_req) = { ok: true, pid: Process.pid, protocol_version: PROTOCOL_VERSION }
+
+        def cmd_shutdown(_req)
+          Process.kill("INT", Process.pid)
+          { ok: true }
+        end
+
+        def cmd_store(req)
+          @kv_mutex.synchronize { @kv_store[req[:key].to_s] = req[:value] }
+          { ok: true }
+        end
+
+        def cmd_fetch(req)
+          key = req[:key].to_s
+          found = @kv_mutex.synchronize { @kv_store.key?(key) ? { ok: true, value: @kv_store[key] } : nil }
+          found || { error: "key '#{key}' not found", code: "key_not_found" }
+        end
+      end
+    end
+  end
+end

data/lib/browserctl/server/handlers/devtools.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Browserctl
+  class CommandDispatcher
+    module Handlers
+      module DevTools
+        private
+
+        def cmd_inspect(req)
+          session = @global_mutex.synchronize { @pages[req[:name]] }
+          return { error: "no page named '#{req[:name]}'" } unless session
+
+          port      = @browser.process.port
+          target_id = session.page.target_id
+          devtools_url = "http://127.0.0.1:#{port}/devtools/inspector.html" \
+                         "?ws=127.0.0.1:#{port}/devtools/page/#{target_id}"
+          { ok: true, devtools_url: devtools_url }
+        end
+      end
+    end
+  end
+end

data/lib/browserctl/server/handlers/hitl.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Browserctl
+  class CommandDispatcher
+    module Handlers
+      module Hitl
+        private
+
+        def cmd_pause(req)
+          session = @global_mutex.synchronize { @pages[req[:name]] }
+          return { error: "no page named '#{req[:name]}'" } unless session
+
+          session.mutex.synchronize { session.pause! }
+          { ok: true, paused: true }
+        end
+
+        def cmd_resume(req)
+          session = @global_mutex.synchronize { @pages[req[:name]] }
+          return { error: "no page named '#{req[:name]}'" } unless session
+
+          session.mutex.synchronize do
+            session.resume!
+            session.pause_cv.signal
+          end
+          { ok: true, paused: false }
+        end
+      end
+    end
+  end
+end