actionagent 0.0.0 → 1.2.0

data/app/models/action_agent/agent_run.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  class AgentRun < ApplicationRecord
+    belongs_to :agent
+
+    # Status enum
+    enum :status, { pending: 0, running: 1, complete: 2, failed: 3, cancelled: 4 }
+
+    # Validations
+    validates :trace_id, presence: true
+
+    # Scopes
+    scope :recent, -> { order(created_at: :desc) }
+    scope :successful, -> { where(status: :complete) }
+    scope :failed_runs, -> { where(status: :failed) }
+    scope :today, -> { where("created_at >= ?", Time.current.beginning_of_day) }
+
+    # Callbacks
+    before_validation :set_trace_id, on: :create
+    after_update_commit :broadcast_update, if: :saved_change_to_status?
+
+    # Add a log entry
+    def add_log(message, level: :info)
+      new_logs = logs || []
+      new_logs << {
+        timestamp: Time.current.iso8601,
+        level: level.to_s,
+        message: message
+      }
+      update!(logs: new_logs)
+    end
+
+    # Appends a progress event to logs mid-run so pollers can stream what the
+    # agent is doing (pending llm/tool/agent calls). Events pair up by eid:
+    # a "started" event is pending until a "done"/"error" with the same eid
+    # lands. update_column: no validations/callbacks, safe from the run's own
+    # execution thread; reads current DB state so add_log interleaves safely.
+    def append_event(eid:, kind:, label:, status: "done", detail: nil, duration_ms: nil)
+      event = {
+        "at" => Time.current.iso8601(3),
+        "eid" => eid,
+        "kind" => kind.to_s,
+        "label" => label.to_s,
+        "status" => status.to_s
+      }
+      event["detail"] = detail.to_s.byteslice(0, 1200).to_s.scrub if detail
+      event["duration_ms"] = duration_ms if duration_ms
+      current = self.class.where(id: id).pick(:logs) || []
+      update_column(:logs, current + [ event ])
+      event
+    end
+
+    # Stable short fingerprint of the instructions this run executed under —
+    # the grouping key (with model) for configuration cohorts when comparing
+    # instruction/model changes.
+    def instructions_digest
+      instructions = output_metadata&.dig("instructions")
+      return nil if instructions.blank?
+
+      Digest::SHA256.hexdigest(instructions).first(8)
+    end
+
+    # Deterministic memorable name for the digest ("calm-heron") — reads far
+    # better than hex when comparing cohorts, and is stable across runs and
+    # deployments because it's derived from the digest alone.
+    CODENAME_ADJECTIVES = %w[
+      calm brisk quiet bold amber coral dusky fresh golden keen
+      lively mellow nimble pale rustic silver tidal vivid wry zesty
+      arid breezy crisp dapper eager foggy hazy icy jolly lunar
+      misty polar
+    ].freeze
+    CODENAME_NOUNS = %w[
+      heron otter falcon cedar willow harbor mesa ridge grove delta
+      prairie summit canyon reef atoll fjord tundra oasis lagoon dune
+      glacier meadow bluff cove marsh basin knoll strait quarry vale
+      hollow crag
+    ].freeze
+
+    def instructions_codename
+      digest = instructions_digest
+      return nil unless digest
+
+      value = digest.to_i(16)
+      "#{CODENAME_ADJECTIVES[value % 32]}-#{CODENAME_NOUNS[(value / 32) % 32]}"
+    end
+
+    # Calculate duration if not set
+    def calculated_duration_ms
+      return duration_ms if duration_ms.present?
+      return nil unless started_at && completed_at
+
+      ((completed_at - started_at) * 1000).to_i
+    end
+
+    # Check if run is still in progress
+    def in_progress?
+      pending? || running?
+    end
+
+    # Check if run is finished
+    def finished?
+      complete? || failed? || cancelled?
+    end
+
+    # Get a summary for display
+    def summary
+      {
+        id: id,
+        status: status,
+        input_preview: input_prompt&.truncate(100),
+        output_preview: output&.truncate(200),
+        duration_ms: calculated_duration_ms,
+        tokens: total_tokens,
+        provider: output_metadata&.dig("provider"),
+        model: output_metadata&.dig("model"),
+        action_name: action_name || output_metadata&.dig("action") || "ask",
+        instructions_digest: instructions_digest,
+        instructions_codename: instructions_codename,
+        instructions_preview: output_metadata&.dig("instructions")&.truncate(120),
+        created_at: created_at,
+        error: error_message
+      }
+    end
+
+    # Stream output updates via ActionCable
+    def broadcast_update
+      payload = { type: "update", run: summary }
+      ActionCable.server.broadcast("agent_run_#{id}", payload)
+      ActionCable.server.broadcast("agent_runs_#{agent_id}", payload)
+    end
+
+    # Cancel a running execution
+    def cancel!
+      return unless in_progress?
+
+      update!(
+        status: :cancelled,
+        completed_at: Time.current,
+        error_message: "Cancelled by user"
+      )
+      broadcast_update
+    end
+
+    private
+
+    def set_trace_id
+      self.trace_id ||= SecureRandom.uuid
+    end
+  end
+end

data/app/models/action_agent/agent_template.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  class AgentTemplate < ApplicationRecord
+    # Validations
+    validates :name, presence: true
+    validates :slug, presence: true, uniqueness: true
+    validates :category, presence: true
+
+    # Scopes
+    scope :featured, -> { where(featured: true) }
+    scope :by_category, ->(cat) { where(category: cat) }
+    scope :popular, -> { order(usage_count: :desc) }
+    scope :public_templates, -> { where(public: true) }
+    scope :free_tier, -> { where(free_tier: true) }
+
+    # Categories
+    CATEGORIES = %w[
+      productivity
+      development
+      research
+      creative
+      data
+      automation
+    ].freeze
+
+    # Create an agent from this template for a user
+    def create_agent_for(user, name: nil)
+      agent = user.agents.build(
+        name: name || self.name,
+        description: description,
+        provider: provider,
+        model: model,
+        instructions: instructions,
+        preset_type: preset_type,
+        appearance: appearance,
+        instruction_sets: instruction_sets,
+        tools: tools,
+        mcp_servers: mcp_servers,
+        model_config: model_config,
+        status: :draft
+      )
+
+      if agent.save
+        increment!(:usage_count)
+      end
+
+      agent
+    end
+
+    # Seed default templates
+    def self.seed_defaults!
+      templates = [
+        {
+          name: "Code Assistant",
+          slug: "code-assistant",
+          description: "A helpful coding assistant that can explain code, suggest improvements, and help debug issues.",
+          category: "development",
+          provider: "openai",
+          model: "gpt-4o",
+          preset_type: "terminal",
+          appearance: { hat: "fedora", heldItem: "terminal" },
+          instruction_sets: %w[github ruby rails typescript],
+          tools: %w[terminal code filesystem],
+          model_config: { temperature: 0.3 },
+          instructions: "You are a senior software engineer with expertise in multiple programming languages. Help users with:\n- Code explanations and reviews\n- Debugging issues\n- Suggesting best practices\n- Writing tests\n\nAlways explain your reasoning and provide examples when helpful.",
+          icon: "💻",
+          featured: true
+        },
+        {
+          name: "Research Assistant",
+          slug: "research-assistant",
+          description: "Helps research topics, summarize information, and organize findings.",
+          category: "research",
+          provider: "anthropic",
+          model: "claude-sonnet-5",
+          preset_type: "research",
+          appearance: { hat: "safari", heldItem: "magnifyingGlass" },
+          instruction_sets: %w[github python],
+          tools: %w[fetch search memory],
+          model_config: { temperature: 0.5 },
+          instructions: "You are a thorough research assistant. Help users by:\n- Searching for relevant information\n- Summarizing complex topics\n- Organizing findings into clear reports\n- Identifying key insights and patterns\n\nAlways cite sources when available and distinguish between facts and opinions.",
+          icon: "🔍",
+          featured: true
+        },
+        {
+          name: "Writing Assistant",
+          slug: "writing-assistant",
+          description: "Helps with writing, editing, and improving text content.",
+          category: "creative",
+          provider: "openai",
+          model: "gpt-4o",
+          preset_type: "writing",
+          appearance: { hat: "fedora", hatAccessory: "feather", heldItem: "scroll" },
+          instruction_sets: [],
+          tools: %w[edit translate],
+          model_config: { temperature: 0.7 },
+          instructions: "You are a skilled writer and editor. Help users with:\n- Writing and editing content\n- Improving clarity and flow\n- Adjusting tone for different audiences\n- Grammar and style corrections\n\nMaintain the author's voice while suggesting improvements.",
+          icon: "✍️",
+          featured: true
+        },
+        {
+          name: "Browser Automation",
+          slug: "browser-automation",
+          description: "Automates web browsing tasks like form filling, data extraction, and testing.",
+          category: "automation",
+          provider: "anthropic",
+          model: "claude-sonnet-5",
+          preset_type: "playwright",
+          appearance: { hat: "fedora", hatAccessory: "theaterMasks", heldItem: "browser" },
+          instruction_sets: %w[typescript],
+          tools: %w[playwright filesystem],
+          model_config: { temperature: 0.2 },
+          instructions: "You are a browser automation specialist. Help users by:\n- Navigating web pages\n- Filling out forms\n- Extracting data from websites\n- Testing web applications\n\nAlways wait for page loads and handle errors gracefully.",
+          icon: "🎭",
+          featured: false
+        },
+        {
+          name: "Data Analyst",
+          slug: "data-analyst",
+          description: "Analyzes data, creates visualizations, and provides insights.",
+          category: "data",
+          provider: "openai",
+          model: "gpt-4o",
+          preset_type: "documentAnalysis",
+          appearance: { hat: "fedora", heldItem: "document" },
+          instruction_sets: %w[python],
+          tools: %w[code database filesystem],
+          model_config: { temperature: 0.3 },
+          instructions: "You are a data analyst. Help users by:\n- Analyzing datasets\n- Creating visualizations\n- Finding patterns and insights\n- Generating reports\n\nExplain your methodology and provide clear interpretations of results.",
+          icon: "📊",
+          featured: true
+        },
+        {
+          name: "DevOps Assistant",
+          slug: "devops-assistant",
+          description: "Helps with infrastructure, deployments, and system administration.",
+          category: "development",
+          provider: "openai",
+          model: "gpt-4o",
+          preset_type: "terminal",
+          appearance: { hat: "fedora", heldItem: "terminal" },
+          instruction_sets: %w[docker kubernetes aws gcp],
+          tools: %w[terminal filesystem code],
+          model_config: { temperature: 0.2 },
+          instructions: "You are a DevOps engineer. Help users with:\n- Infrastructure setup and management\n- CI/CD pipeline configuration\n- Container orchestration\n- Cloud resource management\n\nAlways prioritize security and follow best practices.",
+          icon: "🚀",
+          featured: false
+        },
+        {
+          name: "PlaywrightMCP Demo",
+          slug: "playwright-mcp-demo",
+          description: "Free browser automation demo using Playwright MCP. Navigate sites, take screenshots, and extract content.",
+          category: "automation",
+          provider: "anthropic",
+          model: "claude-sonnet-5",
+          preset_type: "playwright",
+          appearance: { hat: "fedora", hatAccessory: "theaterMasks", heldItem: "browser" },
+          instruction_sets: [],
+          tools: %w[playwright],
+          mcp_servers: {
+            playwright: {
+              command: "npx",
+              args: [ "-y", "@anthropic/mcp-server-playwright" ]
+            }
+          },
+          model_config: { temperature: 0.2, max_tokens: 4096 },
+          instructions: "You are a browser automation assistant using Playwright MCP.\n\nAvailable actions:\n- browser_navigate: Go to a URL\n- browser_snapshot: Get the accessibility tree\n- browser_click: Click on an element\n- browser_type: Type text into an input\n- browser_take_screenshot: Capture the page\n- browser_wait_for: Wait for text or element\n\nGuidelines:\n1. Always take a snapshot first to understand the page\n2. Use element refs from snapshots for interactions\n3. Wait for page loads before taking actions\n4. Handle errors gracefully\n5. Limit yourself to 10 steps maximum\n\nAlways describe what you see and what actions you're taking.",
+          icon: "🎭",
+          featured: true,
+          free_tier: true
+        }
+      ]
+
+      templates.each do |template_attrs|
+        AgentTemplate.find_or_create_by!(slug: template_attrs[:slug]) do |t|
+          t.assign_attributes(template_attrs)
+        end
+      end
+    end
+  end
+end

data/app/models/action_agent/agent_version.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  class AgentVersion < ApplicationRecord
+    belongs_to :agent
+
+    validates :version_number, presence: true, uniqueness: { scope: :agent_id }
+    validates :configuration_snapshot, presence: true
+
+    # Scopes
+    scope :recent, -> { order(version_number: :desc) }
+    scope :by_version, ->(num) { where(version_number: num) }
+
+    # Compare two versions
+    def diff(other_version)
+      return {} unless other_version
+
+      changes = {}
+      configuration_snapshot.each do |key, value|
+        other_value = other_version.configuration_snapshot[key]
+        if value != other_value
+          changes[key] = { from: other_value, to: value }
+        end
+      end
+      changes
+    end
+
+    # Get previous version
+    def previous
+      agent.agent_versions.where("version_number < ?", version_number).order(version_number: :desc).first
+    end
+
+    # Get next version
+    def next_version
+      agent.agent_versions.where("version_number > ?", version_number).order(version_number: :asc).first
+    end
+
+    # Check if this is the latest version
+    def latest?
+      agent.latest_version&.id == id
+    end
+
+    # Check if this is the initial version
+    def initial?
+      version_number == 1
+    end
+  end
+end

data/app/models/action_agent/api_key.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # API key generated from Settings -> API Keys. Authenticates requests to
+  # the telemetry ingest endpoint as a Bearer token.
+  #
+  # The token is generated server-side (never accepted from user input) and
+  # encrypted at rest with Active Record Encryption, so the host app needs
+  # `rails db:encryption:init` before creating keys. Deterministic
+  # encryption keeps find_by(token:) lookups working against the ciphertext.
+  # Set ActionAgent.encrypt_credentials = false to store tokens
+  # in plain text instead; that is a downgrade, not a default.
+  class ApiKey < ApplicationRecord
+    TOKEN_PREFIX = "aa_"
+
+    include Ownable
+    owned_by :account, :user
+
+    encrypts :token, deterministic: true if ActionAgent.encrypt_credentials
+
+    validates :name, presence: true, length: { maximum: 100 }
+    validates :token, presence: true, uniqueness: true
+
+    before_validation :generate_token, on: :create
+
+    # Finds the key for a presented bearer token. Returns nil for blank or
+    # unknown tokens.
+    def self.authenticate(token)
+      return nil if token.blank?
+
+      find_by(token: token)
+    end
+
+    def touch_last_used!
+      # Throttled to avoid a write per ingest request.
+      update_column(:last_used_at, Time.current) if last_used_at.nil? || last_used_at < 1.minute.ago
+    end
+
+    # Safe to display in the dashboard key list: "aa_3xam…k9Q2"
+    def masked_token
+      "#{token_prefix}…#{token.last(4)}"
+    end
+
+    private
+
+    def generate_token
+      return if token.present?
+
+      self.token = "#{TOKEN_PREFIX}#{SecureRandom.base58(32)}"
+      self.token_prefix = token.first(TOKEN_PREFIX.length + 4)
+    end
+  end
+end

data/app/models/action_agent/application_record.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Base class for all Dashboard engine models.
+  #
+  # Table names come from ActionAgent.table_name_prefix through
+  # Rails' standard namespaced-model resolution, so the engine's own
+  # migrations (active_agent_agents, active_agent_agent_runs, ...) and a
+  # host app that already owns the tables unprefixed are both supported
+  # without touching the models.
+  #
+  # Ownership is configurable: multi-tenant installs scope records to an
+  # Account, single-tenant installs to a User, and a single-user install
+  # scopes to nothing at all.
+  class ApplicationRecord < ::ActiveRecord::Base
+    include AdapterAware
+
+    self.abstract_class = true
+
+    # Models that are not themselves owned still answer the ownership
+    # questions, so callers can scope any dashboard relation uniformly.
+    class << self
+      def owner_association = nil
+      def for_owner(_owner) = all
+    end
+  end
+end

data/app/models/action_agent/evaluation.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # An evaluation definition for an agent: a named set of criteria scored
+  # against the agent's recorded behavior — its recent generations
+  # (solid_agent's agent_generations dataset) and its telemetry traces.
+  #
+  # Criteria are stored as an array of { "key", "type", "config" } hashes.
+  # Rule-based criterion types score each sampled generation
+  # deterministically; telemetry criterion types score aggregates over the
+  # agent's traces (error rate, latency); the llm_judge type asks a judge
+  # model to score each sample and requires a configured provider.
+  class Evaluation < ApplicationRecord
+    belongs_to :agent
+    has_many :evaluation_runs, dependent: :destroy
+
+    # judge_defined: the judge model authors the KPI criteria itself from the
+    # agent's instructions + sample interactions on the first run, then scores
+    # against them (criteria stay persisted/editable so scores are comparable
+    # across runs and models).
+    JUDGE_KINDS = %w[rules llm judge_defined].freeze
+
+    RULE_CRITERION_TYPES = %w[
+      response_present min_length max_latency_ms token_budget contains not_contains
+    ].freeze
+    # Scored from the agent's telemetry traces (aggregate, not per-sample).
+    TELEMETRY_CRITERION_TYPES = %w[trace_error_rate trace_latency].freeze
+    CRITERION_TYPES = (RULE_CRITERION_TYPES + TELEMETRY_CRITERION_TYPES + %w[llm_judge]).freeze
+
+    validates :name, presence: true, uniqueness: { scope: :agent_id }
+    validates :judge_kind, inclusion: { in: JUDGE_KINDS }
+    validates :sample_size, numericality: { greater_than: 0, less_than_or_equal_to: 100 }
+    validate :validate_criteria
+
+    scope :recent, -> { order(updated_at: :desc) }
+
+    def latest_run
+      evaluation_runs.order(created_at: :desc).first
+    end
+
+    def judge_defined?
+      judge_kind == "judge_defined"
+    end
+
+    # Candidate models for per-cohort comparison scoring (config, optional).
+    def compare_models
+      Array(config["compare_models"]).map(&:to_s).reject(&:blank?)
+    end
+
+    def run!
+      EvaluationRunnerService.call(self)
+    end
+
+    def llm_criteria
+      criteria.select { |c| c["type"] == "llm_judge" }
+    end
+
+    private
+
+    def validate_criteria
+      if criteria.blank?
+        # judge_defined evaluations start empty — the judge authors the KPIs
+        # on the first run.
+        errors.add(:criteria, "must include at least one criterion") unless judge_defined?
+        return
+      end
+
+      criteria.each do |criterion|
+        unless criterion.is_a?(Hash) && criterion["key"].present?
+          errors.add(:criteria, "entries must have a key")
+          next
+        end
+
+        unless CRITERION_TYPES.include?(criterion["type"])
+          errors.add(:criteria, "unknown criterion type #{criterion['type']}")
+        end
+      end
+    end
+  end
+end

data/app/models/action_agent/evaluation_run.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # One execution of an Evaluation over a sample of the agent's generations.
+  # scores: { criterion_key => { "score", "min", "max", "passed", "total" } }
+  class EvaluationRun < ApplicationRecord
+    belongs_to :evaluation
+
+    enum :status, { pending: 0, running: 1, complete: 2, failed: 3 }
+
+    scope :recent, -> { order(created_at: :desc) }
+
+    def average_score
+      values = scores.values.map { |s| s["score"] }.compact
+      return nil if values.empty?
+
+      (values.sum.to_f / values.size).round(3)
+    end
+  end
+end

data/app/models/action_agent/model_pricing.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Estimates LLM spend from token counts. The activeagent gem's telemetry
+  # records tokens only; the platform layers pricing on top for the cost
+  # figures shown in Traces and Metrics.
+  #
+  # Rates come from RubyLLM's model registry (USD per million tokens,
+  # maintained upstream per model) when the model is known there; the static
+  # pattern table below is the fallback for aliases/self-hosted models, and
+  # a conservative blended rate covers everything else so totals stay
+  # meaningful. Costs are always presented as estimates.
+  class ModelPricing
+    PRICES = [
+      # [pattern, input $/1M, output $/1M]
+      [ /gpt-4o-mini/i, 0.15, 0.60 ],
+      [ /gpt-4o/i, 2.50, 10.00 ],
+      [ /gpt-4\.1-nano/i, 0.10, 0.40 ],
+      [ /gpt-4\.1-mini/i, 0.40, 1.60 ],
+      [ /gpt-4\.1/i, 2.00, 8.00 ],
+      [ /o3-mini|o4-mini/i, 1.10, 4.40 ],
+      [ /claude.*(fable|mythos)/i, 10.00, 50.00 ],
+      [ /claude.*haiku-?4/i, 1.00, 5.00 ],
+      [ /claude.*(haiku)/i, 0.80, 4.00 ],
+      [ /claude.*(sonnet)/i, 3.00, 15.00 ],
+      [ /claude.*opus-(5|4-[5-9])/i, 5.00, 25.00 ],
+      [ /claude.*(opus)/i, 15.00, 75.00 ],
+      [ /gemini.*flash/i, 0.10, 0.40 ],
+      [ /gemini.*pro/i, 1.25, 10.00 ],
+      [ /llama|mistral|mixtral|qwen|deepseek/i, 0.20, 0.60 ],
+      # Zero-prices "mock-*" traces recorded before the mock fallback was
+      # removed, so legacy rows never register as real spend.
+      [ /mock/i, 0.0, 0.0 ]
+    ].freeze
+
+    # Fallback blended rate for unknown models ($/1M input, $/1M output)
+    DEFAULT_RATE = [ 1.00, 4.00 ].freeze
+
+    # @return [Float, nil] estimated USD cost, nil when there is nothing to price
+    def self.estimate(model:, input_tokens:, output_tokens:)
+      input = input_tokens.to_i
+      output = output_tokens.to_i
+      return nil if input.zero? && output.zero?
+
+      input_rate, output_rate = rate_for(model)
+      ((input * input_rate) + (output * output_rate)) / 1_000_000.0
+    end
+
+    def self.rate_for(model)
+      return DEFAULT_RATE if model.blank?
+
+      registry_rate(model) || static_rate(model)
+    end
+
+    # Exact per-model rates from RubyLLM's registry. Lookups are memoized —
+    # the registry scan is not free and trace serialization calls this per
+    # row.
+    def self.registry_rate(model)
+      @registry_rates ||= {}
+      return @registry_rates[model] if @registry_rates.key?(model)
+
+      @registry_rates[model] = begin
+        info = RubyLLM.models.find(model.to_s)
+        tokens = info&.pricing&.text_tokens
+        if tokens&.input && tokens&.output
+          [ tokens.input, tokens.output ]
+        end
+      rescue StandardError
+        nil
+      end
+    end
+
+    def self.static_rate(model)
+      PRICES.each do |pattern, input_rate, output_rate|
+        return [ input_rate, output_rate ] if model.to_s.match?(pattern)
+      end
+      DEFAULT_RATE
+    end
+  end
+end