actionagent 1.8.0 → 1.9.0

data/lib/generators/action_agent/templates/migrations/021_create_active_agent_explorations.rb.erb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# Explorations: a walk through a project's running app, by the engine's
+# explorer agent or by an outside agent, and the candidate scenarios it
+# proposed for review. Accepted candidates are merged into the project's
+# evaluation; the rest stay here with their verdicts and provenance.
+#
+# Table names follow ActionAgent.table_name_prefix, and JSON columns are
+# jsonb on PostgreSQL and json elsewhere, the same way as
+# create_active_agent_dashboard_tables.
+class CreateActiveAgentExplorations < ActiveRecord::Migration<%= migration_version %>
+  def change
+    prefix = ActionAgent.table_name_prefix
+
+    create_table "#{prefix}explorations" do |t|
+      # At least one is set: a project's explorations merge into the
+      # project's evaluation, and one without a project into its own.
+      t.bigint :project_id
+      t.bigint :evaluation_id
+      t.bigint :agent_run_id
+      t.bigint :sandbox_session_id
+      t.bigint :session_recording_id
+      # explorer: the engine's explorer agent; external: candidates an
+      # outside agent submitted.
+      t.string :source, null: false, default: "external"
+      t.string :status, null: false, default: "pending"
+      t.string :start_url
+      t.string :stop_reason
+      t.text :error_message
+      t.datetime :started_at
+      t.datetime :finished_at
+      # { minutes, steps, cost }, the limits and what was used of them.
+      t.column :budget, json_type, **json_default({})
+      t.column :usage, json_type, **json_default({})
+      t.column :candidates, json_type, **json_default([])
+      t.bigint :account_id
+      t.bigint :user_id
+      t.timestamps
+      t.index :project_id
+      t.index :evaluation_id
+      t.index :account_id
+      t.index :user_id
+    end
+  end
+
+  private
+
+  def json_type
+    @json_type ||= postgres? ? :jsonb : :json
+  end
+
+  # MySQL rejects a default on a JSON column outright, so the column is
+  # created without one there.
+  def json_default(value, null: nil)
+    return {} unless postgres?
+
+    null.nil? ? { default: value } : { default: value, null: null }
+  end
+
+  def postgres?
+    connection.adapter_name.to_s.downcase.include?("postgres")
+  end
+end

data/lib/generators/action_agent/templates/migrations/023_create_active_agent_draft_pull_requests.rb.erb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# Draft pull requests opened from a checkout sandbox's changes (Settings ->
+# Integrations -> Open draft PR), and the state GitHub last reported for each.
+#
+# Table names follow ActionAgent.table_name_prefix, and JSON columns are
+# jsonb on PostgreSQL and json elsewhere, the same way as
+# create_active_agent_dashboard_tables.
+class CreateActiveAgentDraftPullRequests < ActiveRecord::Migration<%= migration_version %>
+  def change
+    prefix = ActionAgent.table_name_prefix
+
+    create_table "#{prefix}draft_pull_requests" do |t|
+      t.bigint :sandbox_session_id, null: false
+      # "owner/name" on GitHub.
+      t.string :repository, null: false
+      # The branch the pull request merges into, and the branch published.
+      t.string :base_branch
+      t.string :branch, null: false
+      # The commit the sandbox checked out, and the last commit published.
+      t.string :base_commit, null: false
+      t.string :head_commit
+      t.string :title, null: false
+      t.text :body
+      # The message of the last update's commit.
+      t.text :commit_message
+      # The files of the last publish, each { path, status, mode, digest }.
+      t.column :files, json_type, **json_default([])
+      # What the last publish did ("create", "update", "open_draft",
+      # "open_regular") and how it went ("queued", "publishing", "published",
+      # "draft_refused", "failed").
+      t.string :operation, null: false, default: "create"
+      t.string :status, null: false, default: "queued"
+      # Why the last publish failed: a word the dashboard acts on (see
+      # DraftPullRequestPublisher::Refused), and a message for the user.
+      t.string :error_code
+      t.text :error_message
+      # "app" (a GitHub App installation) or "oauth" (the OAuth connection).
+      t.string :credential_kind
+      t.integer :number
+      t.string :url
+      t.string :compare_url
+      # "open", "closed" or "merged", and whether it is a draft, as GitHub
+      # last reported them.
+      t.string :state
+      t.boolean :draft
+      t.datetime :last_checked_at
+      # The user who published, and the owner.
+      t.bigint :user_id
+      t.bigint :account_id
+      t.timestamps
+      t.index :sandbox_session_id, name: "index_#{prefix}draft_pull_requests_on_sandbox_session"
+      t.index :account_id, name: "index_#{prefix}draft_pull_requests_on_account"
+      t.index :user_id, name: "index_#{prefix}draft_pull_requests_on_user"
+    end
+  end
+
+  private
+
+  def json_type
+    @json_type ||= postgres? ? :jsonb : :json
+  end
+
+  # MySQL rejects a default on a JSON column outright, so the column is
+  # created without one there.
+  def json_default(value, null: nil)
+    return {} unless postgres?
+
+    null.nil? ? { default: value } : { default: value, null: null }
+  end
+
+  def postgres?
+    connection.adapter_name.to_s.downcase.include?("postgres")
+  end
+end

data/lib/generators/action_agent/templates/migrations/024_add_kind_to_active_agent_project_secrets.rb.erb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Project secrets of three kinds: an environment variable a boot hands to
+# the app ("env", every secret before this migration), the credentials the
+# explorer signs in with ("sign_in"), and a browser's saved sign-in
+# ("storage_state"). Only env secrets reach the sandbox's environment.
+#
+# Table names follow ActionAgent.table_name_prefix, the same way as
+# create_active_agent_dashboard_tables.
+class AddKindToActiveAgentProjectSecrets < ActiveRecord::Migration<%= migration_version %>
+  def change
+    add_column "#{ActionAgent.table_name_prefix}project_secrets", :kind, :string, null: false, default: "env"
+  end
+end

data/lib/generators/action_agent/templates/migrations/025_create_active_agent_scenario_catalogs.rb.erb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+# Scenario catalogs: the YAML catalogs of evaluation scenarios an owner keeps
+# in the dashboard, each a set of products (an agent or a project under test)
+# with named sets of scenarios. A catalog remembers the document it was
+# imported from and its digest, so a re-import of the same file changes
+# nothing, and a set remembers the evaluation it was last materialized as.
+#
+# Table names follow ActionAgent.table_name_prefix, and JSON columns are
+# jsonb on PostgreSQL and json elsewhere, the same way as
+# create_active_agent_dashboard_tables.
+class CreateActiveAgentScenarioCatalogs < ActiveRecord::Migration<%= migration_version %>
+  def change
+    prefix = ActionAgent.table_name_prefix
+
+    create_table "#{prefix}scenario_catalogs" do |t|
+      t.string :key, null: false
+      t.string :name, null: false
+      t.text :description
+      # upload: pasted or uploaded YAML; repository: read from a connected
+      # repository at a ref; api: built through the API or MCP tools.
+      t.string :source_kind, null: false, default: "upload"
+      # For a repository: "owner/name@ref:path".
+      t.string :source_path
+      # SHA-256 of the canonical document, and the document itself.
+      t.string :digest
+      t.text :document
+      # When the document was last written to Active Storage, and the digest
+      # it carried then.
+      t.datetime :synced_at
+      t.string :synced_digest
+      t.column :metadata, json_type, **json_default({})
+      t.bigint :account_id
+      t.bigint :user_id
+      t.timestamps
+      t.index :key
+      t.index :account_id
+      t.index :user_id
+    end
+
+    create_table "#{prefix}scenario_products" do |t|
+      t.bigint :scenario_catalog_id, null: false
+      t.string :key, null: false
+      t.string :name, null: false
+      t.text :description
+      # What the product's sets run against: a dashboard agent, a project
+      # (whose sandbox the replays reach), or neither yet. agent_name keeps
+      # the document's `agent:` reference for an agent that is not here.
+      t.bigint :agent_id
+      t.bigint :project_id
+      t.string :agent_name
+      t.integer :position
+      t.column :metadata, json_type, **json_default({})
+      t.timestamps
+      t.index [ :scenario_catalog_id, :key ], unique: true
+      t.index :agent_id
+      t.index :project_id
+    end
+
+    create_table "#{prefix}scenario_sets" do |t|
+      t.bigint :scenario_product_id, null: false
+      t.string :key, null: false
+      t.string :name, null: false
+      t.text :description
+      # { kind, model }: the judge of the evaluation the set becomes.
+      t.column :judge, json_type, **json_default({})
+      t.column :criteria, json_type, **json_default([])
+      t.column :metadata, json_type, **json_default({})
+      # The evaluation this set was last materialized as, if any.
+      t.bigint :evaluation_id
+      t.integer :position
+      t.timestamps
+      t.index [ :scenario_product_id, :key ], unique: true
+      t.index :evaluation_id
+    end
+
+    create_table "#{prefix}catalog_scenarios" do |t|
+      t.bigint :scenario_set_id, null: false
+      t.string :key, null: false
+      t.text :prompt, null: false
+      t.text :notes
+      t.column :expectations, json_type, **json_default({})
+      t.column :tags, json_type, **json_default([])
+      t.column :params, json_type, **json_default({})
+      t.boolean :production_only, null: false, default: false
+      t.boolean :enabled, null: false, default: true
+      t.integer :position
+      t.timestamps
+      t.index [ :scenario_set_id, :key ], unique: true
+    end
+  end
+
+  private
+
+  def json_type
+    @json_type ||= postgres? ? :jsonb : :json
+  end
+
+  # MySQL rejects a default on a JSON column outright, so the column is
+  # created without one there.
+  def json_default(value, null: nil)
+    return {} unless postgres?
+
+    null.nil? ? { default: value } : { default: value, null: null }
+  end
+
+  def postgres?
+    connection.adapter_name.to_s.downcase.include?("postgres")
+  end
+end

data/lib/generators/action_agent/templates/migrations/026_add_evaluation_fixes_to_code_sessions.rb.erb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Links a Claude Code session to the evaluation fix it implements (the run,
+# the fix card, the follow-up it retries) and to the run that verified it,
+# and records which user signed in to Claude Code inside a sandbox.
+class AddEvaluationFixesToCodeSessions < ActiveRecord::Migration<%= migration_version %>
+  def change
+    sessions = "#{ActionAgent.table_name_prefix}code_sessions"
+    sandboxes = "#{ActionAgent.table_name_prefix}sandbox_sessions"
+    add_column sessions, :evaluation_run_id, :bigint
+    add_column sessions, :verification_run_id, :bigint
+    add_column sessions, :previous_code_session_id, :bigint
+    # jsonb on PostgreSQL and json elsewhere, like the dashboard tables.
+    add_column sessions, :fix_item, connection.adapter_name.to_s.downcase.include?("postgres") ? :jsonb : :json
+    add_column sessions, :credential_mode, :string
+    add_column sessions, :verification_error, :text
+    add_index sessions, :evaluation_run_id
+    add_index sessions, :verification_run_id
+    add_column sandboxes, :claude_login_user_id, :bigint
+  end
+end

data/lib/tasks/action_agent.rake

@@ -6,6 +6,39 @@ namespace :action_agent do
     ActionAgent::AgentTemplate.seed_defaults!
     puts "Agent template library: #{ActionAgent::AgentTemplate.count} templates"
   end
+
+  # Sample data a workspace can show before it has run anything of its own.
+  # On a multi-tenant install name the workspace: ACCOUNT_ID=<id> (or
+  # USER_ID=<id> where agents belong to users).
+  namespace :sample do
+    desc "Seed the conference-ticket sample: an agent, a week of runs and traces, a replayable handoff, and an evaluation history"
+    task conference_ticket: :environment do
+      ActionAgent::AgentTemplate.seed_defaults!
+      agent = ActionAgent::SampleData::ConferenceTicket.seed!(owner: action_agent_sample_owner)
+      traces = ActionAgent.trace_model.where(agent_id: agent.id).count
+      evaluation = agent.evaluations.find_by(name: ActionAgent::SampleData::ConferenceTicket::EVALUATION_NAME)
+      puts "Conference ticket sample: agent ##{agent.id} (#{agent.slug})"
+      puts "  #{agent.agent_runs.count} runs, #{traces} traces, " \
+           "#{ActionAgent::SessionRecording.where(agent_run_id: agent.agent_runs.select(:id)).count} session recording"
+      puts "  evaluation \"#{evaluation&.name}\": #{evaluation&.evaluation_runs&.count} runs over #{evaluation&.scenarios&.count} scenarios"
+    end
+
+    desc "Remove the conference-ticket sample and everything it created"
+    task clear: :environment do
+      removed = ActionAgent::SampleData::ConferenceTicket.clear!(owner: action_agent_sample_owner)
+      puts removed.zero? ? "No conference ticket sample to remove" : "Conference ticket sample removed"
+    end
+  end
+end
+
+# The workspace the sample belongs to, from ACCOUNT_ID or USER_ID; nil on a
+# single-user install.
+def action_agent_sample_owner
+  if ENV["ACCOUNT_ID"].present?
+    ActionAgent.account_class.to_s.constantize.find(ENV["ACCOUNT_ID"])
+  elsif ENV["USER_ID"].present?
+    ActionAgent.user_class.to_s.constantize.find(ENV["USER_ID"])
+  end
 end
 
 namespace :action_agent do

data/lib/tasks/browser.rake

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+namespace :action_agent do
+  namespace :browser do
+    desc "Install the browser sidecar and its Chromium for the :local sandbox backend, under the local sandbox root"
+    task install: :environment do
+      ActionAgent::BrowserSidecar.install!
+      puts "Installed the browser sidecar under #{ActionAgent::BrowserSidecar.root}"
+    rescue ActionAgent::BrowserSidecar::Error => e
+      abort "action_agent:browser:install: #{e.message}"
+    end
+
+    desc "Check that a sandbox's browser can start on this machine: Node.js, the browser sidecar and Chromium"
+    task doctor: :environment do
+      checks = ActionAgent::BrowserSidecar.checks
+      checks.each { |check| puts "#{check.ok ? '[ok]     ' : '[missing]'} #{check.name}: #{check.message}" }
+      abort "Browser sessions cannot start on this machine yet." unless checks.all?(&:ok)
+
+      puts "Browser sessions can start on this machine."
+    end
+  end
+end

data/lib/tasks/sandbox.rake

@@ -2,7 +2,7 @@
 
 namespace :action_agent do
   namespace :sandbox do
-    desc "Write this app's checkout sandbox runtime manifest ({mcp_path, mcp_token}) to " \
+    desc "Write this app's checkout sandbox runtime manifest ({mcp_path, mcp_token, models}) to " \
          "$ACTION_AGENT_SANDBOX_MANIFEST, or print it"
     task manifest: :environment do
       manifest = JSON.generate(ActionAgent::SandboxManifest.generate)

data/lib/tasks/scenario_catalogs.rake

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# Scenario catalogs from the command line, for a repository that keeps its
+# scenarios in YAML. On a multi-tenant install name the workspace with
+# ACCOUNT_ID=<id> (or USER_ID=<id> where catalogs belong to users).
+#
+#   bin/rails action_agent:catalogs:import FILE=config/evals/support.yml
+#   bin/rails action_agent:catalogs:export KEY=support FILE=tmp/support.yml
+#   bin/rails action_agent:catalogs:run KEY=support PRODUCT=triage SET=smoke [AGENT_ID=1 | PROJECT_ID=2]
+namespace :action_agent do
+  namespace :catalogs do
+    desc "Import a scenario catalog from a YAML file (FILE=path); re-importing the same file changes nothing"
+    task import: :environment do
+      path = ENV.fetch("FILE") { abort "Name the file: FILE=config/evals/support.yml" }
+      abort "No file at #{path}" unless File.file?(path)
+
+      owner = action_agent_catalog_owner
+      catalog = ActionAgent::ScenarioCatalogImport.new(
+        owner: owner, document: File.read(path), name: File.basename(path, File.extname(path)),
+        source_kind: "upload", source_path: path,
+        agents: ActionAgent.agents_for(owner), projects: ActionAgent::Project.for_owner(owner)
+      ).call
+      puts "Catalog #{catalog.key}: #{catalog.products.count} products, #{catalog.scenario_count} scenarios" \
+           "#{catalog.synced? ? ', synced to Active Storage' : ''}"
+    rescue ActionAgent::ScenarioCatalogImport::Invalid => e
+      abort "Import refused: #{e.message}"
+    end
+
+    desc "Write a catalog's canonical YAML (KEY=catalog key, FILE=path; standard output without FILE)"
+    task export: :environment do
+      catalog = action_agent_catalog!
+      yaml = catalog.export_yaml
+      if ENV["FILE"].present?
+        File.write(ENV["FILE"], yaml)
+        puts "Wrote #{catalog.key} (#{catalog.scenario_count} scenarios) to #{ENV['FILE']}"
+      else
+        puts yaml
+      end
+    end
+
+    desc "Run one set of a catalog (KEY, PRODUCT, SET) against AGENT_ID or PROJECT_ID, or the product's own target"
+    task run: :environment do
+      catalog = action_agent_catalog!
+      product = catalog.products.find_by!(key: ENV.fetch("PRODUCT") { abort "Name the product: PRODUCT=<key>" })
+      set = product.sets.find_by!(key: ENV.fetch("SET") { abort "Name the set: SET=<key>" })
+      agent = ENV["AGENT_ID"].present? ? ActionAgent::Agent.find(ENV["AGENT_ID"]) : nil
+      project = ENV["PROJECT_ID"].present? ? ActionAgent::Project.find(ENV["PROJECT_ID"]) : nil
+
+      run = set.run!(agent: agent, project: project)
+      puts "Run ##{run.id} of #{set.evaluation_name} is #{run.status} (evaluation ##{run.evaluation_id})"
+    rescue ActionAgent::ScenarioSet::NoAgent => e
+      abort e.message
+    end
+  end
+end
+
+# The workspace a catalog belongs to, from ACCOUNT_ID or USER_ID; nil on a
+# single-user install.
+def action_agent_catalog_owner
+  if ENV["ACCOUNT_ID"].present?
+    ActionAgent.account_class.to_s.constantize.find(ENV["ACCOUNT_ID"])
+  elsif ENV["USER_ID"].present?
+    ActionAgent.user_class.to_s.constantize.find(ENV["USER_ID"])
+  end
+end
+
+def action_agent_catalog!
+  key = ENV.fetch("KEY") { abort "Name the catalog: KEY=<catalog key>" }
+  ActionAgent::ScenarioCatalog.for_owner(action_agent_catalog_owner).find_by(key: key) ||
+    abort("No catalog #{key.inspect}#{ENV['ACCOUNT_ID'].present? || ENV['USER_ID'].present? ? ' in that workspace' : ''}")
+end