actionagent 1.8.0 → 1.9.0

data/app/services/action_agent/sandbox_patch.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require "digest"
+require "zlib"
+
+module ActionAgent
+  # Builds diffs and patches from file contents, in Ruby and with no git
+  # process: the dialog's preview of a draft pull request, and the patch a
+  # user downloads when publishing is not available (see
+  # DraftPullRequestPublisher).
+  #
+  # A change is a Hash:
+  #
+  #   path:         relative to the repository root
+  #   status:       "added", "modified" or "deleted"
+  #   base_mode:    the mode before ("100644" or "100755"); nil when added
+  #   mode:         the mode after; nil when deleted
+  #   base_content: the bytes before; nil when added
+  #   content:      the bytes after; nil when deleted
+  #
+  # Text is diffed line by line with three lines of context. A binary file
+  # (one with a NUL byte in its first 8000, as git decides) is written as a
+  # GIT binary patch holding the new content whole, which `git apply` takes.
+  module SandboxPatch
+    CONTEXT_LINES = 3
+    # Past this many differing lines between two versions, a file's diff
+    # replaces every line rather than searching for the shortest edit.
+    MAX_EDIT_DISTANCE = 2_000
+    NULL_ID = "0" * 40
+    BINARY_PROBE_BYTES = 8_000
+    # The characters git's base-85 encoding uses, in order.
+    BASE85 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"
+    # The author `git am` records for a downloaded patch. The domain is
+    # reserved, so nothing is ever sent there.
+    PATCH_AUTHOR = "ActiveAgent sandbox <sandbox@activeagent.invalid>"
+    LINE_PREFIX = { equal: " ".b, delete: "-".b, insert: "+".b }.freeze
+
+    module_function
+
+    # Whether git would treat +content+ as binary.
+    def binary?(content)
+      content.to_s.b.byteslice(0, BINARY_PROBE_BYTES).include?("\0")
+    end
+
+    # The id git gives +content+ as a blob.
+    def blob_id(content)
+      content = content.to_s.b
+      Digest::SHA1.hexdigest("blob #{content.bytesize}\0".b + content)
+    end
+
+    # +change+ as `git diff` prints one file: the header, mode lines, the
+    # index line and the hunks. Binary-encoded.
+    def file_diff(change)
+      path = change.fetch(:path)
+      before = change[:status] == "added" ? nil : change[:base_content].to_s.b
+      after = change[:status] == "deleted" ? nil : change[:content].to_s.b
+      base_mode = change[:base_mode] || (before && change[:mode]) || "100644"
+      mode = change[:mode] || base_mode
+
+      out = String.new(encoding: Encoding::BINARY)
+      out << "diff --git ".b << quote_path("a/#{path}") << " ".b << quote_path("b/#{path}") << "\n".b
+      if before.nil?
+        out << "new file mode #{mode}\n"
+      elsif after.nil?
+        out << "deleted file mode #{base_mode}\n"
+      elsif base_mode != mode
+        out << "old mode #{base_mode}\nnew mode #{mode}\n"
+      end
+
+      old_id = before ? blob_id(before) : NULL_ID
+      new_id = after ? blob_id(after) : NULL_ID
+      return out if old_id == new_id
+
+      out << "index #{old_id}..#{new_id}#{" #{mode}" if before && after && base_mode == mode}\n"
+      if binary?(before) || binary?(after)
+        out << "GIT binary patch\n" << binary_literal(after.to_s) << "\n"
+      elsif !(before.to_s.empty? && after.to_s.empty?)
+        out << "--- ".b << (before ? quote_path("a/#{path}") : "/dev/null".b) << "\n".b
+        out << "+++ ".b << (after ? quote_path("b/#{path}") : "/dev/null".b) << "\n".b
+        out << hunks(lines(before), lines(after))
+      end
+      out
+    end
+
+    # A patch of every change, as `git format-patch` writes one commit:
+    # `git am` applies it as a commit titled +subject+, and `git apply`
+    # applies its changes to a checkout of +base_commit+.
+    def format_patch(changes, subject:, base_commit:, body: nil, date: Time.now.utc)
+      out = String.new(encoding: Encoding::BINARY)
+      out << "From #{base_commit} Mon Sep 17 00:00:00 2001\n".b
+      out << "From: #{PATCH_AUTHOR}\n".b
+      out << "Date: #{date.rfc2822}\n".b
+      out << "Subject: [PATCH] #{subject.to_s.gsub(/\s+/, ' ').strip}\n\n".b
+      out << "#{body.to_s.strip}\n\n".b if body.present?
+      out << "---\n\n".b
+      changes.each { |change| out << file_diff(change) }
+      out << "-- \nActiveAgent\n".b
+    end
+
+    # +path+ as git writes it in a header: quoted, with C escapes, when it
+    # holds a quote, a backslash, a control character or a non-ASCII byte.
+    def quote_path(path)
+      bytes = path.b
+      return bytes unless bytes.match?(/["\\\x00-\x1f\x7f-\xff]/n)
+
+      escaped = bytes.each_byte.map do |byte|
+        case byte
+        when 0x07 then "\\a"
+        when 0x08 then "\\b"
+        when 0x09 then "\\t"
+        when 0x0a then "\\n"
+        when 0x0b then "\\v"
+        when 0x0c then "\\f"
+        when 0x0d then "\\r"
+        when 0x22 then "\\\""
+        when 0x5c then "\\\\"
+        when 0x00..0x1f, 0x7f..0xff then format("\\%03o", byte)
+        else byte.chr
+        end
+      end
+      "\"#{escaped.join}\"".b
+    end
+
+    # +content+ split into lines, each keeping its newline.
+    def lines(content)
+      content.to_s.b.split(/(?<=\n)/n)
+    end
+
+    # The unified hunks that turn +old_lines+ into +new_lines+.
+    def hunks(old_lines, new_lines)
+      edits = line_edits(old_lines, new_lines)
+      changed = edits.each_index.reject { |index| edits[index].first == :equal }
+      return "".b if changed.empty?
+
+      groups = changed.slice_when { |a, b| b - a > CONTEXT_LINES * 2 + 1 }.to_a
+      groups.each_with_object(String.new(encoding: Encoding::BINARY)) do |group, out|
+        first = [ group.first - CONTEXT_LINES, 0 ].max
+        last = [ group.last + CONTEXT_LINES, edits.size - 1 ].min
+        old_before = edits[0...first].count { |op, _| op != :insert }
+        new_before = edits[0...first].count { |op, _| op != :delete }
+        window = edits[first..last]
+        old_count = window.count { |op, _| op != :insert }
+        new_count = window.count { |op, _| op != :delete }
+
+        out << "@@ -#{range(old_before, old_count)} +#{range(new_before, new_count)} @@\n".b
+        window.each do |op, line|
+          out << LINE_PREFIX.fetch(op) << line
+          out << "\n\\n".b unless line.end_with?("\n")
+        end
+      end
+    end
+
+    # "start,count" for a hunk that starts after +before+ lines. An empty
+    # side names the line it would follow.
+    def range(before, count)
+      "#{count.zero? ? before : before + 1},#{count}"
+    end
+
+    # The edit script from +a+ to +b+, as [op, line] pairs with op :equal,
+    # :delete or :insert: the shortest one (Myers' algorithm) between the
+    # lines the two do not share at either end, or a whole replacement of
+    # those lines past MAX_EDIT_DISTANCE.
+    def line_edits(a, b)
+      prefix = 0
+      prefix += 1 while prefix < a.size && prefix < b.size && a[prefix] == b[prefix]
+      suffix = 0
+      suffix += 1 while suffix < a.size - prefix && suffix < b.size - prefix && a[-1 - suffix] == b[-1 - suffix]
+      a_middle = a[prefix...(a.size - suffix)]
+      b_middle = b[prefix...(b.size - suffix)]
+
+      middle = shortest_edits(a_middle, b_middle) ||
+        (a_middle.map { |line| [ :delete, line ] } + b_middle.map { |line| [ :insert, line ] })
+      a.first(prefix).map { |line| [ :equal, line ] } + middle + a.last(suffix).map { |line| [ :equal, line ] }
+    end
+
+    # Myers' O(ND) shortest edit script, or nil when it is longer than
+    # MAX_EDIT_DISTANCE. Each round's furthest-reaching paths are kept for
+    # the walk back, as only the slice of diagonals that round could reach.
+    def shortest_edits(a, b)
+      n = a.size
+      m = b.size
+      return [] if n.zero? && m.zero?
+
+      max = n + m
+      offset = max + 1
+      furthest = Array.new((2 * max) + 3, 0)
+      trace = []
+      found = (0..[ max, MAX_EDIT_DISTANCE ].min).any? do |d|
+        trace << furthest[(offset - d - 1)..(offset + d + 1)]
+        (-d..d).step(2).any? do |k|
+          x = k == -d || (k != d && furthest[offset + k - 1] < furthest[offset + k + 1]) ? furthest[offset + k + 1] : furthest[offset + k - 1] + 1
+          y = x - k
+          while x < n && y < m && a[x] == b[y]
+            x += 1
+            y += 1
+          end
+          furthest[offset + k] = x
+          x >= n && y >= m
+        end
+      end
+      return nil unless found
+
+      walk_back(trace, a, b)
+    end
+
+    def walk_back(trace, a, b)
+      x = a.size
+      y = b.size
+      edits = []
+      trace.each_with_index.reverse_each do |snapshot, d|
+        at = ->(k) { snapshot[k + d + 1] }
+        k = x - y
+        previous_k = k == -d || (k != d && at.call(k - 1) < at.call(k + 1)) ? k + 1 : k - 1
+        previous_x = at.call(previous_k)
+        previous_y = previous_x - previous_k
+        while x > previous_x && y > previous_y
+          x -= 1
+          y -= 1
+          edits << [ :equal, a[x] ]
+        end
+        if d.positive?
+          edits << (x == previous_x ? [ :insert, b[previous_y] ] : [ :delete, a[previous_x] ])
+        end
+        x = previous_x
+        y = previous_y
+      end
+      edits.reverse
+    end
+
+    # +content+ whole, deflated and base-85 encoded the way a GIT binary
+    # patch's "literal" hunk carries it.
+    def binary_literal(content)
+      deflated = Zlib::Deflate.deflate(content.b)
+      out = "literal #{content.bytesize}\n".b
+      deflated.bytes.each_slice(52) do |chunk|
+        length = chunk.size <= 26 ? ("A".ord + chunk.size - 1) : ("a".ord + chunk.size - 27)
+        out << length.chr << encode85(chunk) << "\n"
+      end
+      out
+    end
+
+    def encode85(bytes)
+      bytes.each_slice(4).map do |group|
+        value = group.each_with_index.sum { |byte, index| byte << (24 - (8 * index)) }
+        digits = Array.new(5) do
+          value, digit = value.divmod(85)
+          BASE85[digit]
+        end
+        digits.reverse.join
+      end.join
+    end
+  end
+end

data/app/services/action_agent/scenario_catalog_import.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Imports a scenario catalog document into the owner's records: products,
+  # sets and scenarios are matched by key. In "replace" mode (the default)
+  # the document is the truth, so what it leaves out is removed and
+  # importing it twice changes nothing; in "merge" mode what it leaves out
+  # is kept, so an export of one set or a few scenarios can be imported back
+  # into the whole catalog. The catalog keeps the canonical document and its
+  # digest, and is written to Active Storage when the host has it.
+  #
+  #   ScenarioCatalogImport.new(owner: account, document: yaml, agents: ActionAgent.agents_for(account)).call
+  #
+  # #preview answers what #call would change, by key, without writing.
+  #
+  # A product's `agent:` is resolved among +agents+ by name, and a project
+  # among +projects+ by the product's `repository` or `project` entry, so a
+  # catalog written for a repository finds the project that boots it.
+  class ScenarioCatalogImport
+    # Raised for a document that cannot be imported, with the reason a
+    # person can act on.
+    class Invalid < StandardError; end
+
+    MODES = %w[replace merge].freeze
+    # The changes #preview lists; the counts cover all of them.
+    PREVIEW_LIMIT = 500
+
+    attr_reader :catalog, :source_path
+
+    def initialize(owner:, document:, catalog: nil, name: nil, source_kind: "upload", source_path: nil, agents: nil, projects: nil, mode: "replace")
+      @owner = owner
+      @document = document.to_s
+      @catalog = catalog
+      @name = name
+      @source_kind = source_kind
+      @source_path = source_path
+      @agents = agents
+      @projects = projects
+      @mode = mode.to_s.presence || "replace"
+      raise Invalid, "The import mode is replace or merge" unless MODES.include?(@mode)
+    end
+
+    # @return [ScenarioCatalog] the imported catalog
+    def call
+      parsed = parse
+
+      ScenarioCatalog.transaction do
+        @catalog ||= find_or_initialize(parsed.key)
+        merging = merge? && @catalog.persisted?
+        @catalog.assign_attributes(
+          key: parsed.key, source_kind: @source_kind, source_path: @source_path,
+          name: merging ? (parsed_field(parsed, "name") || @catalog.name) : parsed.name,
+          description: merging ? (parsed.description || @catalog.description) : parsed.description,
+          metadata: merging ? @catalog.metadata.merge(parsed.metadata) : parsed.metadata,
+          document: parsed.to_yaml, digest: parsed.digest
+        )
+        @catalog.save!
+        replace_products(parsed)
+        if merging
+          @catalog.refresh_document!
+          if @catalog.scenario_count > ScenarioCatalog::MAX_SCENARIOS
+            raise Invalid, "The catalog would hold more than #{ScenarioCatalog::MAX_SCENARIOS} scenarios"
+          end
+        end
+      end
+      @catalog.sync_to_storage!
+      @catalog
+    rescue ActiveAgent::Evals::Catalog::InvalidDocument => e
+      raise Invalid, e.message
+    rescue ActiveRecord::RecordInvalid => e
+      raise Invalid, e.record.errors.full_messages.to_sentence
+    end
+
+    # What #call would do, without writing: the catalog it imports into
+    # (created or updated), counts by action, and the changes (up to
+    # PREVIEW_LIMIT), each { level: product|set|scenario, path, action }
+    # where action is create, update (the document changes a record by
+    # that key), unchanged, remove (replace mode drops a record the document
+    # leaves out) or keep (merge mode keeps it).
+    #
+    # @return [Hash]
+    def preview
+      parsed = parse
+      existing = @catalog || ScenarioCatalog.for_owner(@owner).find_by(key: parsed.key)
+      before = existing ? existing.to_catalog.to_h : {}
+      after = parsed.to_h
+      changes = []
+      diff_level(changes, "product", "", list(before["products"]), list(after["products"]), "sets") do |path, old_product, new_product|
+        diff_level(changes, "set", path, list(old_product["sets"]), list(new_product["sets"]), "scenarios") do |set_path, old_set, new_set|
+          diff_level(changes, "scenario", set_path, list(old_set["scenarios"]), list(new_set["scenarios"]), nil)
+        end
+      end
+
+      counts = Hash.new(0)
+      changes.each { |change| counts[change[:action]] += 1 }
+      {
+        mode: @mode,
+        catalog: {
+          id: existing&.id, key: parsed.key, name: parsed.name,
+          action: existing.nil? ? "create" : (catalog_fields(before) == catalog_fields(after) ? "unchanged" : "update")
+        },
+        scenario_count: parsed.scenario_count,
+        counts: counts,
+        changes: changes.reject { |change| change[:action] == "unchanged" }.first(PREVIEW_LIMIT)
+      }
+    rescue ActiveAgent::Evals::Catalog::InvalidDocument => e
+      raise Invalid, e.message
+    end
+
+    private
+
+    def merge?
+      @mode == "merge"
+    end
+
+    def parse
+      if @document.bytesize > ScenarioCatalog::MAX_BYTES
+        raise Invalid, "The catalog is larger than #{ScenarioCatalog::MAX_BYTES / 1.megabyte} MB"
+      end
+
+      parsed = ActiveAgent::Evals::Catalog.parse(@document, name: @name)
+      if parsed.scenario_count > ScenarioCatalog::MAX_SCENARIOS
+        raise Invalid, "The catalog holds #{parsed.scenario_count} scenarios; the limit is #{ScenarioCatalog::MAX_SCENARIOS}"
+      end
+
+      parsed
+    end
+
+    # The document's own name, rather than the key the parser falls back to.
+    def parsed_field(parsed, field)
+      value = parsed.to_h[field]
+      value unless field == "name" && value == parsed.key
+    end
+
+    def list(value)
+      Array(value).select { |entry| entry.is_a?(Hash) }
+    end
+
+    def catalog_fields(document)
+      document.except("products")
+    end
+
+    # Appends the changes between +old_entries+ and +new_entries+ (matched
+    # by key) to +changes+, comparing each entry without its +children+, and
+    # yields the pairs present in the document so their children are
+    # compared too.
+    def diff_level(changes, level, parent_path, old_entries, new_entries, children)
+      old_by_key = old_entries.index_by { |entry| entry["key"].to_s }
+      new_entries.each do |entry|
+        key = entry["key"].to_s
+        path = parent_path.empty? ? key : "#{parent_path}/#{key}"
+        previous = old_by_key.delete(key)
+        action =
+          if previous.nil? then "create"
+          elsif children ? previous.except(children) == entry.except(children) : previous == entry then "unchanged"
+          else "update"
+          end
+        changes << { level: level, path: path, action: action }
+        yield(path, previous || {}, entry) if children && block_given?
+      end
+      old_by_key.each_key do |key|
+        path = parent_path.empty? ? key : "#{parent_path}/#{key}"
+        changes << { level: level, path: path, action: merge? ? "keep" : "remove" }
+      end
+    end
+
+    def find_or_initialize(key)
+      catalog = ScenarioCatalog.for_owner(@owner).find_or_initialize_by(key: key)
+      catalog.owner = @owner if catalog.new_record? && @owner
+      catalog
+    end
+
+    def replace_products(parsed)
+      @catalog.products.where.not(key: parsed.products.map(&:key)).destroy_all unless merge?
+      offset = merge? ? next_position(@catalog.products) : 0
+
+      parsed.products.each do |entry|
+        product = @catalog.products.find_or_initialize_by(key: entry.key)
+        product.assign_attributes(
+          name: entry.name, description: entry.description, agent_name: entry.agent,
+          metadata: entry.metadata, agent: resolve_agent(entry) || (merge? ? product.agent : nil),
+          project: resolve_project(entry) || (merge? ? product.project : nil)
+        )
+        product.position = position_for(product, entry.position, offset)
+        product.save!
+        replace_sets(product, entry)
+      end
+    end
+
+    def replace_sets(product, entry)
+      product.sets.where.not(key: entry.sets.map(&:key)).destroy_all unless merge?
+      offset = merge? ? next_position(product.sets) : 0
+
+      entry.sets.each do |set_entry|
+        set = product.sets.find_or_initialize_by(key: set_entry.key)
+        set.assign_attributes(
+          name: set_entry.name, description: set_entry.description, judge: set_entry.judge,
+          criteria: set_entry.criteria, metadata: set_entry.metadata
+        )
+        set.position = position_for(set, set_entry.position, offset)
+        set.save!
+        replace_scenarios(set, set_entry)
+      end
+    end
+
+    def replace_scenarios(set, set_entry)
+      set.scenarios.where.not(key: set_entry.scenarios.map(&:key)).destroy_all unless merge?
+      offset = merge? ? next_position(set.scenarios) : 0
+
+      set_entry.scenarios.each do |scenario|
+        record = set.scenarios.find_or_initialize_by(key: scenario.key)
+        record.assign_attributes(
+          prompt: scenario.prompt, notes: scenario.notes, expectations: scenario.expectations,
+          tags: scenario.tags, params: scenario.params, production_only: scenario.production_only?
+        )
+        record.position = position_for(record, scenario.position, offset)
+        record.save!
+      end
+    end
+
+    # Replace mode takes the document's order. Merge mode keeps a record's
+    # place and puts a new one after the records already there.
+    def position_for(record, position, offset)
+      return position unless merge?
+
+      record.persisted? ? record.position : offset + position.to_i
+    end
+
+    def next_position(relation)
+      (relation.maximum(:position) || -1) + 1
+    end
+
+    # The owner's agent the product's `agent:` names, kept as the product's
+    # agent; a name that matches nothing stays as agent_name alone.
+    def resolve_agent(entry)
+      return nil if entry.agent.blank? || @agents.nil?
+
+      @agents.find_by(name: entry.agent)
+    end
+
+    # The project the product's `project:` (a project id or name) or
+    # `repository:` (owner/name) names among +projects+.
+    def resolve_project(entry)
+      return nil if @projects.nil?
+
+      reference = entry.metadata["project"]
+      repository = entry.metadata["repository"]
+      if reference.present?
+        reference.to_s.match?(/\A\d+\z/) ? @projects.find_by(id: reference) : @projects.find_by(name: reference.to_s)
+      elsif repository.present?
+        @projects.find_by(repository: repository.to_s)
+      end
+    end
+  end
+end

data/app/services/action_agent/scenario_catalog_repository_import.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Imports the scenario catalogs a connected repository keeps, read through
+  # GitHub's contents API at a ref: every `.yml` under +path+
+  # (`.activeagents/evals` by default, where the install pull request writes
+  # a project's suite), or the one file +path+ names. Each file is a
+  # catalog; its source is recorded as "owner/name@ref:path", so a re-import
+  # of the same ref finds it again, and a pull request's branch can carry
+  # the scenarios that accept it.
+  class ScenarioCatalogRepositoryImport
+    DEFAULT_PATH = ".activeagents/evals"
+    MAX_FILES = 50
+    EXTENSIONS = %w[.yml .yaml].freeze
+
+    def initialize(owner:, connection:, repository:, ref: nil, path: nil, agents: nil, projects: nil, mode: "replace")
+      @owner = owner
+      @connection = connection
+      @repository = repository.to_s
+      @ref = ref.to_s.presence
+      @path = path.to_s.presence || DEFAULT_PATH
+      @agents = agents
+      @projects = projects
+      @mode = mode
+    end
+
+    # @return [Array<ScenarioCatalog>] the imported catalogs, one per file
+    def call
+      imports.map(&:call)
+    end
+
+    # What #call would change, one ScenarioCatalogImport#preview per file,
+    # each with the file's source_path.
+    # @return [Array<Hash>]
+    def preview
+      imports.map { |import| import.preview.merge(source_path: import.source_path) }
+    end
+
+    private
+
+    # One ScenarioCatalogImport per catalog file at the ref.
+    def imports
+      client = @connection.client
+      ref = @ref || default_branch(client)
+      files = catalog_files(client, ref)
+      if files.empty?
+        raise ScenarioCatalogImport::Invalid, "#{@repository} at #{ref} has no catalog at #{@path}: " \
+          "add a .yml file there, or name the file to read"
+      end
+
+      files.first(MAX_FILES).filter_map do |file_path|
+        content = client.file(@repository, file_path, ref: ref)
+        next if content.nil?
+
+        ScenarioCatalogImport.new(
+          owner: @owner, document: content, name: File.basename(file_path, File.extname(file_path)),
+          source_kind: "repository", source_path: "#{@repository}@#{ref}:#{file_path}",
+          agents: @agents, projects: @projects, mode: @mode
+        )
+      end
+    rescue GithubClient::Error => e
+      raise ScenarioCatalogImport::Invalid, "GitHub refused reading #{@repository}: #{e.message}"
+    end
+
+    def default_branch(client)
+      @connection.repository(@repository)&.dig("default_branch").presence || client.repository(@repository)["default_branch"]
+    end
+
+    # The .yml files at @path: the file itself, or the files directly under
+    # the directory, in name order.
+    def catalog_files(client, ref)
+      return [ @path ] if EXTENSIONS.include?(File.extname(@path))
+
+      directory = @path.chomp("/")
+      client.tree(@repository, ref: ref)[:paths]
+        .map { |entry| entry[:path] }
+        .select { |entry| entry.start_with?("#{directory}/") && EXTENSIONS.include?(File.extname(entry)) && !entry.delete_prefix("#{directory}/").include?("/") }
+        .sort
+    end
+  end
+end