@specsage/cli 0.1.8 → 0.1.11

package/lib/cli.rb

@@ -6,6 +6,7 @@
 
 require 'optparse'
 require 'fileutils'
+require 'json'
 
 # Set the package root directory
 SPECSAGE_HOME = File.expand_path('..', __dir__)
@@ -17,7 +18,7 @@ require 'runner'
 require 'results_uploader'
 
 class SpecSageCLI
-  VERSION = '0.1.6'
+  VERSION = JSON.parse(File.read(File.join(SPECSAGE_HOME, 'package.json')))['version']
 
   def initialize(args)
     @args = args
@@ -80,10 +81,6 @@ class SpecSageCLI
     metadata
   end
 
-  def self_test_mode?
-    ENV['SPECSAGE_APP_TEST'] == 'true'
-  end
-
   def run_ci_mode
     website = ENV['TARGET_WEBSITE_SLUG']
     api_key = ENV['SPEC_SAGE_API_KEY']
@@ -104,9 +101,6 @@ class SpecSageCLI
     puts "Version: #{VERSION}"
     puts "Website: #{website}"
     puts "Base URL: #{base_url}"
-    if self_test_mode?
-      puts "Self-test mode: orchestration via #{base_url}"
-    end
     puts ""
 
     publisher = ResultsUploader.new(api_key: api_key)
@@ -123,6 +117,9 @@ class SpecSageCLI
     scenarios.each { |s| puts "  - #{s['name']}" }
     puts ""
 
+    # Build all_scenarios map for pre-scenario lookup
+    all_scenarios = scenarios.each_with_object({}) { |s, map| map[s['id']] = s }
+
     begin
       run_response = publisher.create_ci_run(website, ci_metadata: build_ci_metadata)
       server_run_id = run_response['server_run_id']
@@ -146,7 +143,7 @@ class SpecSageCLI
         record: @options[:record],
         publisher: publisher,
         server_run_id: server_run_id,
-        orchestration_base_url: self_test_mode? ? base_url : nil
+        all_scenarios: all_scenarios
       )
 
       verdict = runner.run

package/lib/results_uploader.rb

@@ -6,7 +6,6 @@ require 'json'
 
 class ResultsUploader
   DEFAULT_BASE_URL = 'https://api.specsage.com'
-  VERSION = '0.1.0'
 
   class UploadError < StandardError; end
 

package/lib/runner.rb

@@ -21,9 +21,8 @@ class Runner
   SAFE_PATH_SEGMENT = /\A[a-zA-Z0-9_-]+\z/
 
   # Initialize runner with scenario data from server
-  # orchestration_base_url: URL for step submissions (LLM orchestration). Defaults to publisher.base_url.
-  #   Set to CI_APP_URL when SPECSAGE_APP_TEST=true to test server code locally.
-  def initialize(scenario_data, visible: false, record: false, publisher: nil, server_run_id: nil, orchestration_base_url: nil)
+  # @param all_scenarios [Hash] optional map of scenario_id => scenario_data for pre-scenario lookup
+  def initialize(scenario_data, visible: false, record: false, publisher: nil, server_run_id: nil, all_scenarios: nil)
     @scenario = normalize_scenario_data(scenario_data)
     @scenario_id = @scenario['id']
     @scenario_name = @scenario['name'] || @scenario['id'] || 'unnamed'
@@ -36,12 +35,12 @@ class Runner
     @next_request_id = 1
     @node_channel_poisoned = false
     @publisher = publisher
-    @orchestration_base_url = orchestration_base_url || publisher&.base_url
     @step_client = nil
     @server_run_id = server_run_id
     @credentials = {} # Credentials received from server { "NAME" => "value" }
     @max_steps = nil # Max browser actions allowed, received from server on first step
     @temp_dir = nil # Unique temp directory for this runner's video recording
+    @all_scenarios = all_scenarios || {} # Map of scenario_id => scenario_data for pre-scenario lookup
   end
 
   def run
@@ -50,37 +49,124 @@ class Runner
     raise ArgumentError, 'server_run_id is required' unless @server_run_id
 
     @step_client = StepClient.new(
-      base_url: @orchestration_base_url,
+      base_url: @publisher.base_url,
       server_run_id: @server_run_id,
       api_key: @publisher.api_key
     )
 
     start_node_process
-    initial_state = navigate_to_base_url
 
-    current_screenshot_base64 = initial_state[:screenshot_base64]
-    interactive_elements = initial_state[:elements]
+    # Run pre-scenarios first (if any)
+    pre_scenario_ids = @scenario['pre_scenario_ids'] || []
+    if pre_scenario_ids.any?
+      log "Running #{pre_scenario_ids.length} pre-scenario(s) before main scenario"
+
+      pre_scenario_ids.each_with_index do |pre_scenario_id, index|
+        pre_scenario_data = @all_scenarios[pre_scenario_id]
+        unless pre_scenario_data
+          log "ERROR: Pre-scenario #{pre_scenario_id} not found in scenarios list"
+          send_main_scenario_verdict('PRE_SCENARIO_ERROR', "Pre-scenario not found: #{pre_scenario_id}")
+          stop_node_process
+          upload_video
+          cleanup_temp_dir
+          return
+        end
+
+        pre_scenario_name = pre_scenario_data['name'] || pre_scenario_id
+        log "Running pre-scenario #{index + 1}/#{pre_scenario_ids.length}: #{pre_scenario_name}"
+
+        # Use a composite scenario_id for pre-scenarios to avoid collision with standalone runs
+        # Format: "pre_{main_scenario_id}_{pre_scenario_id}" ensures uniqueness per run
+        composite_scenario_id = "pre_#{@scenario_id}_#{pre_scenario_id}"
+
+        result = run_single_scenario(
+          scenario_id: composite_scenario_id,
+          scenario_data: pre_scenario_data,
+          pre_scenario_for_id: @scenario_id,
+          execution_order: index + 1
+        )
+
+        if result[:verdict] == 'FAIL'
+          log "Pre-scenario '#{pre_scenario_name}' FAILED - skipping main scenario"
+          send_main_scenario_verdict('PRE_SCENARIO_FAIL', "Pre-scenario '#{pre_scenario_name}' failed")
+          stop_node_process
+          upload_video
+          cleanup_temp_dir
+          return
+        elsif result[:verdict] == 'ERROR'
+          log "Pre-scenario '#{pre_scenario_name}' ERROR - skipping main scenario"
+          send_main_scenario_verdict('PRE_SCENARIO_ERROR', "Pre-scenario '#{pre_scenario_name}' errored: #{result[:reason]}")
+          stop_node_process
+          upload_video
+          cleanup_temp_dir
+          return
+        end
+
+        log "Pre-scenario '#{pre_scenario_name}' PASSED"
+      end
+
+      log "All pre-scenarios passed, running main scenario"
+    end
+
+    # Run the main scenario
+    run_single_scenario(
+      scenario_id: @scenario_id,
+      scenario_data: @scenario,
+      pre_scenario_for_id: nil,
+      execution_order: 0
+    )
+
+    stop_node_process
+    upload_video
+    cleanup_temp_dir
+  rescue StepClient::StepError => e
+    send_client_verdict_if_needed('ERROR', "Server error: #{e.message}")
+    stop_node_process
+    upload_video
+    cleanup_temp_dir
+  rescue StandardError => e
+    send_client_verdict_if_needed('ERROR', e.message)
+    stop_node_process
+    upload_video
+    cleanup_temp_dir
+  end
+
+  # Run a single scenario (either pre-scenario or main scenario)
+  # Returns { verdict: 'PASS'|'FAIL'|'ERROR', reason: String }
+  def run_single_scenario(scenario_id:, scenario_data:, pre_scenario_for_id:, execution_order:)
+    scenario_name = scenario_data['name'] || scenario_id
+    base_url = scenario_data['base_url']
+
+    # Navigate to scenario's base URL
+    initial_state = send_to_node('navigate', { url: base_url })
+    current_screenshot_base64 = initial_state.dig('result', 'screenshot_base64')
+    interactive_elements = initial_state.dig('result', 'elements') || []
     interactive_elements_by_id = build_elements_by_id(interactive_elements)
     previous_action = nil
     action_result = nil
-    final_verdict = 'ERROR' # Default to ERROR if no verdict received
+
+    # Reset credentials and max_steps for each scenario (each has its own)
+    scenario_credentials = {}
+    scenario_max_steps = nil
 
     loop do
       # Get next action from server
       step_result = @step_client.submit_step(
-        scenario_id: @scenario_id,
+        scenario_id: scenario_id,
         screenshot_base64: current_screenshot_base64,
         elements: interactive_elements,
         previous_action: previous_action,
-        action_result: action_result
+        action_result: action_result,
+        pre_scenario_for_id: pre_scenario_for_id,
+        execution_order: execution_order
       )
 
       # Store credentials and max_steps from server on first step only
-      # Server sends these once at scenario start, runner caches them for the duration
       if step_result[:credentials] && !step_result[:credentials].empty?
-        @credentials = step_result[:credentials]
+        scenario_credentials = step_result[:credentials]
+        @credentials = scenario_credentials # Set for credential substitution
       end
-      @max_steps ||= step_result[:max_steps]
+      scenario_max_steps ||= step_result[:max_steps]
 
       action = step_result[:action]
 
@@ -95,12 +181,11 @@ class Runner
         end
       end
 
-      # Log action with element details
-      safe_puts "  [Step #{step_result[:step_number]}] #{action['action']}: #{action.reject { |k, _| k == 'action' }.to_json}"
+      # Log action with element details (using scenario_name for context)
+      puts "[#{scenario_name}]   [Step #{step_result[:step_number]}] #{action['action']}: #{action.reject { |k, _| k == 'action' }.to_json}"
 
       if action['action'] == 'verdict'
-        final_verdict = action['status'] || 'ERROR'
-        break
+        return { verdict: action['status'], reason: action['reason'] }
       end
 
       # Skip browser execution for meta actions
@@ -113,8 +198,6 @@ class Runner
       result = execute_action(action)
 
       # Update state from action result
-      # Note: screenshot may be nil if a dialog is blocking (can't screenshot while dialog is up)
-      # but we still need to update elements (which will contain the DIALOG pseudo-element)
       if result[:screenshot_base64]
         current_screenshot_base64 = result[:screenshot_base64]
       end
@@ -127,35 +210,36 @@ class Runner
       action_result = result[:result]
 
       # Check if step count has reached max_steps limit
-      # This check happens AFTER executing the action, so max_steps=4 means 4 actions execute
       step_number = step_result[:step_number] || 0
-      if @max_steps && step_number >= @max_steps
-        log "Step limit exceeded (#{@max_steps})."
-        send_client_verdict_if_needed('ERROR', "Step limit exceeded (#{@max_steps}).")
-        final_verdict = 'ERROR'
-        break
+      if scenario_max_steps && step_number >= scenario_max_steps
+        puts "[#{scenario_name}] Step limit exceeded (#{scenario_max_steps})."
+        @step_client.set_verdict(
+          scenario_id: scenario_id,
+          status: 'ERROR',
+          reason: "Step limit exceeded (#{scenario_max_steps})."
+        )
+        return { verdict: 'ERROR', reason: "Step limit exceeded (#{scenario_max_steps})" }
       end
 
       # Check if server says we should stop
-      break unless step_result[:continue]
+      unless step_result[:continue]
+        return { verdict: 'ERROR', reason: 'Server stopped scenario unexpectedly' }
+      end
     end
+  end
 
-    stop_node_process
-    upload_video
-    cleanup_temp_dir
-    final_verdict
+  # Send a verdict for the main scenario when pre-scenarios fail
+  def send_main_scenario_verdict(status, reason)
+    return unless @step_client
+
+    log "Setting main scenario verdict to #{status}: #{reason}"
+    @step_client.set_verdict(
+      scenario_id: @scenario_id,
+      status: status,
+      reason: reason
+    )
   rescue StepClient::StepError => e
-    send_client_verdict_if_needed('ERROR', "Server error: #{e.message}")
-    stop_node_process
-    upload_video
-    cleanup_temp_dir
-    'ERROR'
-  rescue StandardError => e
-    send_client_verdict_if_needed('ERROR', e.message)
-    stop_node_process
-    upload_video
-    cleanup_temp_dir
-    'ERROR'
+    log "Warning: Failed to send main scenario verdict: #{e.message}"
   end
 
   private
@@ -166,12 +250,13 @@ class Runner
     {
       'id' => data['id'] || data[:id],
       'name' => data['name'] || data[:name],
-      'base_url' => data['base_url'] || data[:base_url]
+      'base_url' => data['base_url'] || data[:base_url],
+      'pre_scenario_ids' => data['pre_scenario_ids'] || data[:pre_scenario_ids] || []
     }
   end
 
   def start_node_process
-    node_script = File.join(SPECSAGE_HOME, 'lib', 'browser.js')
+    node_script = File.join(__dir__, 'browser.js')
     raise "browser.js not found at #{node_script}" unless File.exist?(node_script)
 
     # Create unique temp directory for this runner's video recording
@@ -242,6 +327,17 @@ class Runner
               video_path = response&.dig('result', 'video_path')
               if video_path
                 if File.exist?(video_path)
+                  # Playwright WebM files lack seek metadata (Cues), so Chrome can't scrub them.
+                  # Remux with ffmpeg to add the seek index without re-encoding.
+                  remuxed_path = video_path.sub(/\.webm$/, '_remuxed.webm')
+                  if system('ffmpeg', '-i', video_path, '-c', 'copy', remuxed_path, '-y', '-loglevel', 'error')
+                    log "Video remuxed for seekability: #{remuxed_path}"
+                    File.delete(video_path) rescue nil
+                    video_path = remuxed_path
+                  else
+                    log "Warning: ffmpeg remux failed, using original video (scrubbing may not work)"
+                  end
+
                   @video_data = File.binread(video_path)
                   log "Video captured: #{video_path} (#{@video_data&.bytesize} bytes)"
                   File.delete(video_path) rescue nil
@@ -364,16 +460,6 @@ class Runner
     end
   end
 
-  def navigate_to_base_url
-    base_url = @scenario['base_url']
-    raise 'base_url not specified in scenario file' unless base_url
-
-    response = send_to_node('navigate', { url: base_url })
-    screenshot_base64 = response.dig('result', 'screenshot_base64')
-    elements = response.dig('result', 'elements') || []
-    { screenshot_base64: screenshot_base64, elements: elements }
-  end
-
   def execute_action(action)
     case action['action']
     when 'navigate'

package/lib/step_client.rb

@@ -41,15 +41,18 @@ class StepClient
 
   # Submit a step to the server
   # Returns: { action: Hash, step_number: Integer, continue: Boolean }
-  def submit_step(scenario_id:, screenshot_base64:, elements:, previous_action: nil, action_result: nil)
+  def submit_step(scenario_id:, screenshot_base64:, elements:, previous_action: nil, action_result: nil,
+                  pre_scenario_for_id: nil, execution_order: 0)
     body = {
       scenario_id: scenario_id,
       screenshot_base64: screenshot_base64,
-      elements: elements
+      elements: elements,
+      execution_order: execution_order
     }
 
     body[:previous_action] = previous_action if previous_action
     body[:action_result] = action_result if action_result
+    body[:pre_scenario_for_id] = pre_scenario_for_id if pre_scenario_for_id
 
     response = post("/api/runs/#{@server_run_id}/step", body)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specsage/cli",
-  "version": "0.1.8",
+  "version": "0.1.11",
   "description": "SpecSage CLI - AI-powered end-to-end testing automation (Node wrapper for Ruby CLI)",
   "type": "module",
   "bin": {