capybara-lightpanda 0.8.0 → 0.9.0

data/lib/capybara/lightpanda/browser/navigation.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Capybara
+  module Lightpanda
+    class Browser
+      # Navigation lifecycle: go_to / back / forward / refresh and the
+      # Page.loadEventFired + readyState-polling machinery behind them
+      # (the polling fallback is load-bearing — see CLAUDE.md).
+      module Navigation
+        # Navigation with readyState fallback.
+        #
+        # Lightpanda may never fire Page.loadEventFired on complex JS pages
+        # (lightpanda-io/browser#1801, #1832). When the event times out,
+        # we poll document.readyState as a fallback.
+        #
+        # Page.navigate is sent asynchronously because Lightpanda may not
+        # return the command result until the page is fully loaded (unlike
+        # Chrome which returns immediately with frameId/loaderId). If we
+        # waited synchronously, the readyState fallback would never be
+        # reached on pages that fail to fully load.
+        #
+        # Uses a single shared deadline so the worst-case wait is 1x timeout,
+        # not 2x (lightpanda-io/browser#1849).
+        def go_to(url, wait: true)
+          enable_page_events
+
+          if wait
+            wait_for_page_load(url)
+          else
+            page_command("Page.navigate", url: url)
+          end
+        end
+        alias goto go_to
+
+        def back
+          wait_for_navigation { navigate_history(-1) }
+        end
+
+        def forward
+          wait_for_navigation { navigate_history(+1) }
+        end
+
+        def refresh
+          wait_for_navigation { page_command("Page.reload") }
+        end
+        alias reload refresh
+
+        private
+
+        def wait_for_page_load(url, retried: false)
+          deadline = await_navigation do
+            @client.command("Page.navigate", { url: url }, async: true, session_id: @session_id)
+          end
+          handle_navigation_crash(url, deadline, retried: retried)
+        end
+
+        # Lightpanda may kill the WebSocket or crash during complex page
+        # navigation (lightpanda-io/browser#1849, #1854). Reconnect and
+        # retry once. If the retry also crashes, raise a clear error
+        # instead of leaving the client in a dead state.
+        def handle_navigation_crash(url, deadline, retried:)
+          if @client.closed? && !retried
+            begin
+              reconnect
+              remaining = deadline - monotonic_time
+              if remaining.positive?
+                # Equivalent of re-entering go_to without leaking the retry
+                # bookkeeping into its public signature. enable_page_events is
+                # needed again: reconnect's clear_session_state reset the flag.
+                enable_page_events
+                wait_for_page_load(url, retried: true)
+              end
+            rescue DeadBrowserError
+              raise
+            rescue StandardError
+              # reconnect itself failed (process won't restart, port stuck, etc.).
+              # Fall through to the raise below — a second immediate reconnect
+              # attempt would just duplicate the failure we already swallowed.
+            end
+          end
+
+          return unless @client.closed?
+
+          raise DeadBrowserError, "Lightpanda crashed navigating to #{url}"
+        end
+
+        def safe_current_url
+          current_url
+        rescue StandardError
+          nil
+        end
+
+        # Wait for a navigation triggered by the given block.
+        # Uses the same loadEventFired + readyState fallback as go_to.
+        def wait_for_navigation(&)
+          enable_page_events
+          await_navigation(&)
+        end
+
+        # Step the session history by `offset` (-1 = back, +1 = forward) using
+        # native CDP. `Page.getNavigationHistory` returns the entry list and
+        # `currentIndex`; `Page.navigateToHistoryEntry` jumps to the chosen
+        # entry's `id`. No-op when the offset would step past either end so
+        # the behavior matches `history.back()` / `history.forward()` on a
+        # bounded session history.
+        def navigate_history(offset)
+          history = page_command("Page.getNavigationHistory")
+          target_index = history["currentIndex"] + offset
+          entries = history["entries"]
+          return if target_index.negative? || target_index >= entries.length
+
+          page_command("Page.navigateToHistoryEntry", entryId: entries[target_index]["id"])
+        end
+
+        # Common navigation lifecycle shared by `wait_for_page_load` (fresh
+        # `Page.navigate`) and `wait_for_navigation` (back / forward / reload).
+        # Subscribes to Page.loadEventFired, runs the trigger, waits briefly for
+        # the event, falls back to readyState polling for the remaining budget.
+        # The handler is unsubscribed via `ensure` so a raising trigger doesn't
+        # leak a subscription onto the next navigation. Returns the deadline so
+        # the caller can decide whether to attempt crash recovery.
+        def await_navigation
+          starting_url = safe_current_url
+          deadline = monotonic_time + @options.timeout
+          loaded = Utils::Event.new
+          handler = proc { loaded.set }
+          @client.on("Page.loadEventFired", &handler)
+
+          begin
+            yield
+
+            unless loaded.wait([2, @options.timeout].min)
+              remaining = deadline - monotonic_time
+              poll_ready_state(remaining, loaded_event: loaded, starting_url: starting_url) if remaining.positive?
+            end
+          ensure
+            @client.off("Page.loadEventFired", handler)
+          end
+
+          deadline
+        end
+
+        # Poll document.readyState as a fallback when Page.loadEventFired
+        # doesn't fire (CLAUDE.md rules call this out as load-bearing — do
+        # not remove). When starting_url is provided, the poll ignores
+        # readyState values from the old page (e.g. about:blank reports
+        # "complete" while the new page is still loading in the background).
+        def poll_ready_state(timeout, loaded_event: nil, starting_url: nil)
+          # Use a short per-evaluation timeout because Lightpanda may block
+          # all commands while navigating. Without this, a single evaluate()
+          # call would consume the entire @options.timeout, making the poll
+          # loop effectively a single attempt.
+          poll_cmd_timeout = [timeout / 5.0, 2].max
+
+          Utils::Wait.until(timeout: timeout, interval: 0.1) do
+            loaded_event&.set? || @client.closed? || page_ready?(poll_cmd_timeout, starting_url)
+          end
+        rescue TimeoutError
+          # Expected — readyState fallback exhausted its budget. The caller
+          # (await_navigation) keeps going and lets handle_navigation_crash
+          # decide whether the session is recoverable.
+        end
+
+        POLL_STATE_JS = "(function(){return{r:document.readyState,u:location.href}})()"
+        private_constant :POLL_STATE_JS
+
+        def page_ready?(cmd_timeout, starting_url)
+          response = @client.command(
+            "Runtime.evaluate",
+            { expression: POLL_STATE_JS, returnByValue: true, awaitPromise: true },
+            session_id: @session_id,
+            timeout: cmd_timeout
+          )
+          state = response.dig("result", "value")
+          return false unless state
+
+          url_changed = starting_url.nil? || state["u"] != starting_url
+          url_changed && %w[complete interactive].include?(state["r"])
+        rescue Error
+          false
+        end
+      end
+    end
+  end
+end

data/lib/capybara/lightpanda/browser/runtime.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+module Capybara
+  module Lightpanda
+    class Browser
+      # JS evaluation and RemoteObject plumbing: Runtime.evaluate /
+      # callFunctionOn dispatch, result serialization (Ferrum's
+      # Frame::Runtime is the peer-gem equivalent).
+      module Runtime
+        # Evaluate JS and return a serialized value.
+        # No-args fast path uses Runtime.evaluate; with args we wrap as a function
+        # and dispatch via Runtime.callFunctionOn so `arguments[i]` is bound.
+        # Both paths use `returnByValue: false` and unwrap so DOM-node returns
+        # come back as `{ "__lightpanda_node__" => ... }` for the Driver to wrap.
+        #
+        # The no-args path sends the user's text verbatim with `replMode: true`
+        # (V8's DevTools-console REPL mode — Lightpanda forwards Runtime.evaluate
+        # to the V8 inspector, which handles the flag natively). Without it,
+        # top-level `const`/`let` persist in the global lexical environment
+        # across classic scripts — per spec, and Chrome behaves identically —
+        # so a second `const sel = ...` raises `SyntaxError: Identifier 'sel'
+        # has already been declared`. REPL mode keeps the bindings (visible to
+        # later calls, like the DevTools console) but allows redeclaration.
+        # Completion-value semantics cover a bare expression (`'foo'`), a
+        # `throw` statement, and multi-statement scripts alike.
+        def evaluate(expression, *args)
+          if args.empty?
+            response = page_command("Runtime.evaluate", expression: expression, returnByValue: false,
+                                                        awaitPromise: true, replMode: true)
+            raise_on_js_error!("evaluate", expression, response)
+
+            return unwrap_call_result(response["result"])
+          end
+
+          wrapped = "function() { return #{expression} }"
+          call_with_args(wrapped, args)
+        end
+
+        # Execute JS without returning a value.
+        #
+        # Like `evaluate`, the no-args path uses `replMode: true` so top-level
+        # `const`/`let` redeclarations across calls don't raise. Also raises
+        # on JS exceptions so silent failures don't mask test bugs (the
+        # previous fast path swallowed them because `awaitPromise: false` was
+        # checked but `exceptionDetails` was not).
+        def execute(expression, *args)
+          if args.empty?
+            response = page_command("Runtime.evaluate", expression: expression, returnByValue: false,
+                                                        awaitPromise: false, replMode: true)
+            raise_on_js_error!("execute", expression, response)
+            return nil
+          end
+
+          wrapped = "function() { #{expression} }"
+          call_with_args(wrapped, args, return_by_value: false)
+          nil
+        end
+
+        # Single home for the exceptionDetails check on Runtime responses:
+        # optional LIGHTPANDA_DEBUG dump, then JavaScriptError.
+        def raise_on_js_error!(site, expression, response)
+          return unless response["exceptionDetails"]
+
+          debug_js_failure(site, expression, response)
+          raise JavaScriptError, response
+        end
+
+        # When LIGHTPANDA_DEBUG=1 is set, log the JS expression and full CDP
+        # response for every JsException to STDERR. Invaluable for isolating
+        # which exact JS triggers an upstream Lightpanda bug.
+        def debug_js_failure(site, expression, response)
+          return unless ENV["LIGHTPANDA_DEBUG"]
+
+          warn "[lightpanda:#{site}] expression:\n#{expression}\n[lightpanda:#{site}] response:\n#{response.inspect}\n"
+        end
+
+        # Evaluate async JS with a callback. The user's script receives
+        # the callback as its last argument (`arguments[arguments.length - 1]`),
+        # matching Capybara's evaluate_async_script contract.
+        def evaluate_async(expression, *args, wait: @options.timeout)
+          timeout_ms = (wait * 1000).to_i
+          wrapped = <<~JS
+            function() {
+              var __args = Array.prototype.slice.call(arguments);
+              return new Promise(function(__resolve, __reject) {
+                var __timer = setTimeout(function() {
+                  __reject(new Error('Async script timeout after #{timeout_ms}ms'));
+                }, #{timeout_ms});
+                var __done = function(val) { clearTimeout(__timer); __resolve(val); };
+                __args.push(__done);
+                (function() { #{expression} }).apply(null, __args);
+              });
+            }
+          JS
+          call_with_args(wrapped, args)
+        end
+
+        # Evaluate JS and return a RemoteObject reference (for DOM nodes, arrays).
+        def evaluate_with_ref(expression)
+          response = page_command("Runtime.evaluate", expression: expression, returnByValue: false, awaitPromise: true)
+          raise_on_js_error!("evaluate_with_ref", expression, response)
+
+          result = response["result"]
+          return nil if result["type"] == "undefined"
+
+          result
+        end
+
+        # Call a function on a remote object via Runtime.callFunctionOn.
+        # Binds `this` to the DOM element referenced by remote_object_id.
+        def call_function_on(remote_object_id, function_declaration, *args, return_by_value: true)
+          params = {
+            objectId: remote_object_id,
+            functionDeclaration: function_declaration,
+            returnByValue: return_by_value,
+            awaitPromise: true,
+          }
+          params[:arguments] = args.map { |a| serialize_argument(a) } unless args.empty?
+
+          response = page_command("Runtime.callFunctionOn", **params)
+          raise_on_js_error!("call_function_on", function_declaration, response)
+
+          result = response["result"]
+          return nil if result["type"] == "undefined"
+
+          return_by_value ? result["value"] : result
+        end
+
+        # Get properties of a remote object (used to extract array elements).
+        def get_object_properties(remote_object_id)
+          page_command("Runtime.getProperties", objectId: remote_object_id, ownProperties: true)
+        end
+
+        # Release a remote object reference to free V8 memory. Cleanup is
+        # best-effort: callers wrap their work in `ensure release_object(...)`,
+        # so a TimeoutError or transport hiccup here must not propagate out of
+        # the ensure block and bury the original failure.
+        def release_object(remote_object_id)
+          page_command("Runtime.releaseObject", objectId: remote_object_id)
+        rescue Error
+          # Object may already be released, context destroyed, or the CDP call
+          # itself timed out / failed in transport.
+        end
+
+        private
+
+        def serialize_argument(arg)
+          if arg.respond_to?(:remote_object_id)
+            { objectId: arg.remote_object_id }
+          else
+            { value: arg }
+          end
+        end
+
+        # Extract the by-value result of an already-issued Runtime call.
+        def handle_evaluate_response(response, expression)
+          raise_on_js_error!("handle_evaluate_response", expression, response)
+
+          result = response["result"]
+          return nil if result["type"] == "undefined"
+
+          result["value"]
+        end
+
+        # Run a wrapped function via Runtime.callFunctionOn with `arguments` bound.
+        # `args` is converted via `serialize_argument` (Nodes → objectId, scalars → value).
+        # When `return_by_value: false` (the default) the return value is unwrapped via
+        # `unwrap_call_result` so that DOM nodes come back as `{ "__lightpanda_node__" => ... }`
+        # hashes the Driver can wrap as Capybara nodes.
+        def call_with_args(function_declaration, args, return_by_value: false)
+          # document_object_id returns a fresh RemoteObject handle every call.
+          # Release it on the way out so long-running shared-spec sessions don't
+          # accumulate orphaned V8 handles between resets.
+          doc_oid = document_object_id
+          params = {
+            objectId: doc_oid,
+            functionDeclaration: function_declaration,
+            returnByValue: return_by_value,
+            awaitPromise: true,
+            arguments: args.map { |a| serialize_argument(a) },
+          }
+          response = page_command("Runtime.callFunctionOn", **params)
+          raise_on_js_error!("call_with_args", function_declaration, response)
+
+          return unwrap_call_result(response["result"]) unless return_by_value
+
+          handle_evaluate_response(response, function_declaration)
+        ensure
+          release_object(doc_oid) if doc_oid
+        end
+
+        # Translate a non-by-value Runtime result into a plain Ruby value, surfacing
+        # DOM nodes as `{ "__lightpanda_node__" => "..." }` so the Driver can wrap
+        # them. The sentinel key (rather than a plain "objectId") prevents
+        # misclassifying user JS that legitimately returns `{ objectId: "x" }`.
+        #
+        # When the result carries an objectId we can't unwrap (function, regexp,
+        # date, …), release the handle before falling back to `result["value"]`
+        # so V8 doesn't accumulate orphaned references across long sessions.
+        def unwrap_call_result(result)
+          return nil if result["type"] == "undefined"
+          return nil if result["subtype"] == "null"
+
+          object_id = result["objectId"]
+          if object_id
+            return { NODE_MARKER => object_id } if result["subtype"] == "node"
+            return serialize_remote_array(object_id) if result["subtype"] == "array"
+            return serialize_remote_object(object_id) if result["type"] == "object"
+
+            release_object(object_id)
+          end
+
+          result["value"]
+        end
+
+        # Re-fetch a remote object as JSON-serializable value for plain objects/arrays.
+        # Cheaper than walking properties and good enough for shared specs. Releases
+        # the original handle so long-lived sessions don't accumulate leaked objectIds.
+        def serialize_remote_object(object_id)
+          json = page_command(
+            "Runtime.callFunctionOn",
+            objectId: object_id,
+            functionDeclaration: "function() { return this }",
+            returnByValue: true
+          )
+          handle_evaluate_response(json, "function() { return this }")
+        ensure
+          release_object(object_id)
+        end
+
+        # Walk an array's own indexed properties via `Runtime.getProperties`,
+        # unwrapping each element through the regular result pipeline so that
+        # DOM-node entries surface as `{ "__lightpanda_node__" => ... }` instead
+        # of being flattened to `{}` by `returnByValue: true`. Releases the
+        # outer array's objectId once we've harvested its elements.
+        def serialize_remote_array(object_id)
+          properties = get_object_properties(object_id).fetch("result", [])
+          properties
+            .select { |p| p["enumerable"] && p["name"] =~ /\A\d+\z/ }
+            .sort_by { |p| p["name"].to_i }
+            .map { |p| unwrap_call_result(p["value"] || {}) }
+        ensure
+          release_object(object_id)
+        end
+
+        # objectId of `document`, used as the `this` context for callFunctionOn when
+        # we need `arguments` binding but don't care about `this`. Re-resolved per
+        # call because the document objectId is invalidated by navigation.
+        def document_object_id
+          result = page_command("Runtime.evaluate", expression: "document", returnByValue: false)
+          result.dig("result", "objectId")
+        end
+
+        private :raise_on_js_error!, :debug_js_failure, :get_object_properties
+      end
+    end
+  end
+end