agent_eval_planner 0.1.0 → 0.2.0

data/lib/agent_eval_planner/suite_renderer.rb

@@ -19,27 +19,61 @@ module AgentEvalPlanner
     attr_reader :analyzer, :vectors
 
     def row_for(vector)
-      forbidden = forbidden_tools_for(vector)
+      return native_row(vector) if analyzer.harness == "native"
+
+      legacy_row(vector)
+    end
+
+    def legacy_row(vector)
       {
         "id" => stable_id(vector),
         "input" => {
           "target_agent" => analyzer.agent_name,
           "messages" => [{ "role" => "user", "content" => vector.attack_prompt }]
         },
-        "expected" => {
-          "reference_answer" => vector.expected_validation,
-          "expected_tools" => [],
-          "forbidden_tools" => forbidden,
-          "refusal" => vector.refusal
-        },
-        "metadata" => {
-          "intent" => vector.intent,
-          "split" => vector.split,
-          "priority" => vector.severity == "P0" ? "high" : "medium",
-          "tags" => vector.tags,
-          "requires_live_data" => false,
-          "plan_id" => vector.id
-        }
+        "expected" => expectations(vector),
+        "metadata" => metadata(vector, "plan_id")
+      }
+    end
+
+    def native_row(vector)
+      {
+        "id" => stable_id(vector),
+        "target_component" => analyzer.agent_name,
+        "suite" => "regression",
+        "turns" => [
+          {
+            "sequence" => 0,
+            "input" => {
+              "messages" => [{ "role" => "user", "content" => vector.attack_prompt }]
+            },
+            "expectations" => expectations(vector).merge(
+              "source_link" => nil,
+              "ticket_handoff" => false
+            )
+          }
+        ],
+        "metadata" => metadata(vector, "vector_id")
+      }
+    end
+
+    def expectations(vector)
+      {
+        "reference_answer" => vector.expected_validation,
+        "expected_tools" => [],
+        "forbidden_tools" => forbidden_tools_for(vector),
+        "refusal" => vector.refusal
+      }
+    end
+
+    def metadata(vector, link_field)
+      {
+        "intent" => vector.intent,
+        "split" => vector.split,
+        "priority" => vector.severity == "P0" ? "high" : "medium",
+        "tags" => vector.tags,
+        "requires_live_data" => false,
+        link_field => vector.id
       }
     end
 

data/lib/agent_eval_planner/suite_validator.rb

@@ -4,9 +4,10 @@ require "json"
 require "set"
 
 module AgentEvalPlanner
-  # Hard-fail validator for suite JSONL (Ruby port of the skill script).
+  # Hard-fail validator for legacy rows and native rows.
   class SuiteValidator
     PLACEHOLDER_RE = /\{\{[^{}]+\}\}/
+    ID_RE = /\A[A-Za-z0-9][A-Za-z0-9._:-]*\z/
     GUARDRAIL_INTENTS = %w[
       out_of_scope_refusal
       prompt_injection
@@ -15,13 +16,23 @@ module AgentEvalPlanner
       pii_probe
     ].freeze
     GUARDRAIL_TAGS = %w[scope injection role exfil pii guardrail].freeze
+    DEFAULT_SUITES = %w[golden regression routing calibration production-sample].freeze
+    SCHEMAS = %w[auto legacy native].freeze
 
-    def initialize(path:, known_tools: [], agent_has_no_tools: false)
+    def initialize(path:, known_tools: [], bound_tools: nil, agent_has_no_tools: false,
+                   target_components: nil, suites: nil, schema: "auto")
       @path = path
-      @known_tools = Array(known_tools).map(&:to_s).reject(&:empty?)
+      @known_tools = clean_names(known_tools)
+      @bound_tools = clean_names(bound_tools)
       @agent_has_no_tools = agent_has_no_tools
+      @target_components = clean_names(target_components)
+      @suites = clean_names(suites)
+      @schema = schema
+      @warnings = []
     end
 
+    attr_reader :warnings
+
     def validate!
       errors = validate
       raise ValidationError, errors.join("\n") if errors.any?
@@ -33,9 +44,18 @@ module AgentEvalPlanner
       raise InputError, "Arquivo não encontrado: #{path}" unless File.file?(path)
 
       raw = File.read(path, encoding: "UTF-8")
+      @warnings = []
       errors = placeholder_errors(raw) + mode_errors
-      known = known_tools.to_set
-      seen_ids = {}
+      run = {
+        errors: errors,
+        warnings: [],
+        seen_ids: {},
+        seen_schemas: Set.new,
+        inventory: inventory,
+        allowed_components: target_components.to_set,
+        allowed_suites: (suites.any? ? suites : DEFAULT_SUITES).to_set,
+        schema: SCHEMAS.include?(schema) ? schema : "auto"
+      }
 
       raw.each_line.with_index(1) do |line, line_no|
         stripped = line.strip
@@ -43,43 +63,67 @@ module AgentEvalPlanner
 
         row, load_errors = load_row(stripped, line_no)
         if row.nil?
-          errors.concat(load_errors)
+          run[:errors].concat(load_errors)
           next
         end
 
-        row_id, id_errors = row_id_errors(row, line_no, seen_ids)
-        errors.concat(id_errors)
-        next if row_id.nil?
-
-        errors.concat(errors_for_row(row, row_id, known: known))
+        collect_row(run, row, line_no)
       end
 
-      errors
+      run[:errors].concat(mixed_schema_error(run[:seen_schemas]))
+      @warnings = run[:warnings]
+      run[:errors]
     end
 
     private
 
-    attr_reader :path, :known_tools, :agent_has_no_tools
+    attr_reader :path, :known_tools, :bound_tools, :agent_has_no_tools,
+                :target_components, :suites, :schema
+
+    def clean_names(names)
+      Array(names).map(&:to_s).reject(&:empty?)
+    end
+
+    def guardrail_metadata?(metadata)
+      return true if GUARDRAIL_INTENTS.include?(metadata["intent"].to_s)
+
+      tags = Array(metadata["tags"]).map(&:to_s)
+      (tags & GUARDRAIL_TAGS).any?
+    end
 
     def placeholder_errors(raw)
-      matches = raw.scan(PLACEHOLDER_RE).uniq
+      matches = raw.scan(PLACEHOLDER_RE).uniq.sort
       return [] if matches.empty?
 
-      ["placeholders remaining (must be replaced before delivery): #{matches.sort}"]
+      ["placeholders remaining (must be replaced before delivery): #{matches}"]
     end
 
     def mode_errors
-      if agent_has_no_tools && known_tools.any?
-        return ["pass either --agent-has-no-tools OR --known-tools, not both"]
-      end
-      if !agent_has_no_tools && known_tools.empty?
-        return [
-          "agent tool inventory required: pass --known-tools a,b,c " \
-          "or --agent-has-no-tools if the agent truly has no tools"
-        ]
+      findings = []
+      if agent_has_no_tools && (known_tools.any? || bound_tools.any?)
+        findings << "pass either --agent-has-no-tools OR tool flags, not both"
+      elsif !agent_has_no_tools && known_tools.empty? && bound_tools.empty?
+        findings << "agent tool inventory required: pass --bound-tools a,b " \
+                    "(and --known-tools when the product catalog is wider) " \
+                    "or --agent-has-no-tools if the agent truly has no tools"
+      elsif bound_tools.any? && known_tools.any? && !(bound_tools.to_set <= known_tools.to_set)
+        extra = (bound_tools.to_set - known_tools.to_set).to_a.sort
+        findings << "--bound-tools must be a subset of --known-tools; extra: #{extra}"
       end
+      findings << "--schema must be auto, legacy, or native; got #{schema.inspect}" unless SCHEMAS.include?(schema)
+      findings
+    end
 
-      []
+    def inventory
+      known = (known_tools.any? ? known_tools : bound_tools).to_set
+      bound = (bound_tools.any? ? bound_tools : known_tools).to_set
+      separated = bound_tools.any? && known_tools.any? && bound_tools.to_set < known_tools.to_set
+      {
+        known: known,
+        bound: bound,
+        bound_explicit: separated,
+        agent_has_no_tools: agent_has_no_tools
+      }
     end
 
     def load_row(line, line_no)
@@ -91,78 +135,269 @@ module AgentEvalPlanner
       [nil, ["line #{line_no}: invalid JSON (#{e.message})"]]
     end
 
+    def row_schema(row)
+      native = row.key?("turns") || row.key?("target_component") || row.key?("suite")
+      legacy = row.key?("input") || row.key?("expected")
+      return "mixed" if native && legacy
+      return "native" if native
+
+      "legacy"
+    end
+
+    def collect_row(run, row, line_no)
+      kind = row_schema(row)
+      return if reject_schema(run, line_no, kind)
+
+      row_id, id_errors = row_id_errors(row, line_no, run[:seen_ids])
+      run[:errors].concat(id_errors)
+      return if row_id.nil?
+
+      append_shape_errors(run, row, row_id, kind)
+      append_expectation_findings(run, row, row_id, kind)
+    end
+
+    def reject_schema(run, line_no, kind)
+      if kind == "mixed"
+        run[:errors] << "line #{line_no}: row mixes legacy input/expected with native turns"
+        return true
+      end
+      if run[:schema] != "auto" && kind != run[:schema]
+        run[:errors] << "line #{line_no}: expected --schema #{run[:schema]}, found #{kind}"
+        return true
+      end
+
+      run[:seen_schemas] << kind
+      false
+    end
+
     def row_id_errors(row, line_no, seen_ids)
       row_id = row["id"]
       return [nil, ["line #{line_no}: missing id"]] if row_id.nil? || row_id.to_s.empty?
 
       row_id_str = row_id.to_s
       errors = []
+      unless ID_RE.match?(row_id_str)
+        errors << "line #{line_no}: id #{row_id.inspect} must match " \
+                  "[A-Za-z0-9][A-Za-z0-9._:-]* (stable id)"
+      end
       errors << "line #{line_no}: duplicate id #{row_id.inspect}" if seen_ids[row_id_str]
       seen_ids[row_id_str] = true
       [row_id_str, errors]
     end
 
-    def errors_for_row(row, row_id, known:)
-      forbidden, errors = forbidden_tools(row, row_id)
-      return errors if forbidden.nil?
+    def append_shape_errors(run, row, row_id, kind)
+      if kind == "native"
+        run[:errors].concat(native_shape_errors(row, row_id, run))
+      else
+        run[:errors].concat(legacy_component_errors(row, row_id, run[:allowed_components]))
+      end
+    end
+
+    def native_shape_errors(row, row_id, run)
+      errors = []
+      component = row["target_component"]
+      if !component.is_a?(String) || component.empty?
+        errors << "#{row_id}: native row requires target_component"
+      elsif run[:allowed_components].any? && !run[:allowed_components].include?(component)
+        errors << "#{row_id}: target_component #{component.inspect} not in " \
+                  "--target-components #{run[:allowed_components].to_a.sort}"
+      end
+
+      suite_name = row["suite"]
+      if !suite_name.is_a?(String) || suite_name.empty?
+        errors << "#{row_id}: native row requires suite"
+      elsif !run[:allowed_suites].include?(suite_name)
+        errors << "#{row_id}: suite #{suite_name.inspect} not in #{run[:allowed_suites].to_a.sort}"
+      end
+      errors
+    end
+
+    def legacy_component_errors(row, row_id, allowed)
+      return [] if allowed.empty?
+
+      target_input = row["input"]
+      return ["#{row_id}: missing input.target_agent"] unless target_input.is_a?(Hash)
+
+      component = target_input["target_agent"]
+      return [] if allowed.include?(component)
+
+      ["#{row_id}: target_agent #{component.inspect} not in --target-components #{allowed.to_a.sort}"]
+    end
+
+    def append_expectation_findings(run, row, row_id, kind)
+      blocks, block_errors = expectation_blocks(row, row_id, kind)
+      run[:errors].concat(block_errors)
+      blocks.each do |label, expectations, own_guardrail|
+        found, warned = errors_for_expectations("#{row_id} #{label}", expectations, own_guardrail, run[:inventory])
+        run[:errors].concat(found)
+        run[:warnings].concat(warned)
+      end
+      run[:errors].concat(row_guardrail_gap(row, row_id, kind, blocks, run[:inventory]))
+    end
+
+    def expectation_blocks(row, row_id, kind)
+      return legacy_block(row, row_id) if kind == "legacy"
 
-      return errors + no_tools_mode_errors(row_id, forbidden) if agent_has_no_tools
+      turns = row["turns"]
+      return [[], ["#{row_id}: native row requires a non-empty turns list"]] unless turns.is_a?(Array) && turns.any?
 
-      errors +
-        empty_guardrail_forbidden_errors(row, row_id, forbidden) +
-        unknown_tools_errors(row_id, forbidden, known) +
-        placeholder_tools_errors(row_id, forbidden)
+      native_blocks(row_id, turns)
     end
 
-    def forbidden_tools(row, row_id)
-      expected = row["expected"] || {}
-      forbidden = expected["forbidden_tools"]
-      return [nil, ["#{row_id}: missing expected.forbidden_tools"]] if forbidden.nil?
-      return [nil, ["#{row_id}: forbidden_tools must be a list"]] unless forbidden.is_a?(Array)
+    def legacy_block(row, row_id)
+      expected = row["expected"]
+      return [[], ["#{row_id}: missing expected object"]] unless expected.is_a?(Hash)
 
-      [forbidden, []]
+      metadata = row["metadata"].is_a?(Hash) ? row["metadata"] : {}
+      own_guardrail = guardrail_metadata?(metadata) || expected["refusal"] == true
+      [[["expected", expected, own_guardrail]], []]
     end
 
-    def no_tools_mode_errors(row_id, forbidden)
-      return [] if forbidden.empty?
+    def native_blocks(row_id, turns)
+      blocks = []
+      errors = []
+      sequences = []
+      turns.each_with_index do |turn, index|
+        block, sequence, turn_errors = one_turn(row_id, index, turn)
+        errors.concat(turn_errors)
+        sequences << sequence unless sequence.nil?
+        blocks << block unless block.nil?
+      end
+      if sequences.any? && sequences != (0...sequences.length).to_a
+        errors << "#{row_id}: turn sequence must be contiguous from 0; got #{sequences}"
+      end
+      [blocks, errors]
+    end
 
-      ["#{row_id}: agent-has-no-tools but forbidden_tools=#{forbidden.inspect}"]
+    def one_turn(row_id, index, turn)
+      label = "turns[#{index}].expectations"
+      return [nil, nil, ["#{row_id}: #{label} turn must be an object"]] unless turn.is_a?(Hash)
+
+      errors = []
+      sequence = turn["sequence"]
+      recorded = sequence.is_a?(Integer) ? sequence : nil
+      errors << "#{row_id}: turns[#{index}].sequence must be an int" if recorded.nil?
+      expectations = turn["expectations"]
+      unless expectations.is_a?(Hash)
+        errors << "#{row_id}: missing #{label}"
+        return [nil, recorded, errors]
+      end
+
+      turn_meta = turn["metadata"].is_a?(Hash) ? turn["metadata"] : {}
+      own_guardrail = guardrail_metadata?(turn_meta) || expectations["refusal"] == true
+      [[label, expectations, own_guardrail], recorded, errors]
     end
 
-    def unknown_tools_errors(row_id, forbidden, known)
-      unknown = forbidden.reject { |t| known.include?(t.to_s) }
-      return [] if unknown.empty?
+    def errors_for_expectations(where, expectations, guardrail, inventory)
+      forbidden, errors = name_list(expectations, where, "forbidden_tools")
+      expected, expected_errors = name_list(expectations, where, "expected_tools")
+      errors.concat(expected_errors)
+      return [errors, []] if forbidden.nil? || expected.nil?
 
-      ["#{row_id}: forbidden_tools not in --known-tools: #{unknown}"]
+      errors.concat(placeholder_names(where, "forbidden_tools", forbidden))
+      errors.concat(placeholder_names(where, "expected_tools", expected))
+      if inventory[:agent_has_no_tools]
+        return [errors + findings_when_empty(where, forbidden, expected), []]
+      end
+
+      errors.concat(unknown_name_findings(where, forbidden, expected, inventory[:known]))
+      bound_errors, warned = bound_findings(where, expectations, forbidden, expected, inventory, guardrail)
+      [errors + bound_errors, warned]
     end
 
-    def placeholder_tools_errors(row_id, forbidden)
-      forbidden.filter_map do |tool|
-        next unless PLACEHOLDER_RE.match?(tool.to_s)
+    def name_list(expectations, row_id, field)
+      names = expectations[field]
+      return [[], []] if names.nil? && field != "forbidden_tools"
+      return [nil, ["#{row_id}: missing #{field}"]] if names.nil?
+      return [nil, ["#{row_id}: #{field} must be a list"]] unless names.is_a?(Array)
+
+      [names, []]
+    end
 
-        "#{row_id}: placeholder left in forbidden_tools: #{tool.inspect}"
+    def placeholder_names(row_id, field, names)
+      names.filter_map do |name|
+        next unless PLACEHOLDER_RE.match?(name.to_s)
+
+        "#{row_id}: placeholder left in #{field}: #{name.inspect}"
       end
     end
 
-    def empty_guardrail_forbidden_errors(row, row_id, forbidden)
-      return [] unless guardrail_row?(row)
-      return [] unless forbidden.empty?
+    def findings_when_empty(where, forbidden, expected)
+      return [] if forbidden.empty? && expected.empty?
 
       [
-        "#{row_id}: HARD FAIL — guardrail/refusal row has " \
-        "forbidden_tools=[] while agent has tools; suite cannot " \
-        "detect tool abuse (false sense of safety)"
+        "#{where}: agent-has-no-tools but tools were listed " \
+        "(forbidden=#{forbidden.inspect}, expected=#{expected.inspect})"
       ]
     end
 
-    def guardrail_row?(row)
-      expected = row["expected"] || {}
-      metadata = row["metadata"] || {}
-      return true if expected["refusal"] == true
-      return true if GUARDRAIL_INTENTS.include?(metadata["intent"].to_s)
+    def unknown_name_findings(where, forbidden, expected, known)
+      findings = []
+      unknown_forbidden = forbidden.reject { |name| known.include?(name) }
+      unknown_expected = expected.reject { |name| known.include?(name) }
+      if unknown_forbidden.any?
+        findings << "#{where}: forbidden_tools not in --known-tools: #{unknown_forbidden}"
+      end
+      if unknown_expected.any?
+        findings << "#{where}: expected_tools not in --known-tools: #{unknown_expected}"
+      end
+      findings
+    end
 
-      tags = Array(metadata["tags"]).map(&:to_s)
-      (tags & GUARDRAIL_TAGS).any?
+    def bound_findings(where, expectations, forbidden, expected, inventory, guardrail)
+      errors = []
+      outside_bound = expected.reject { |name| inventory[:bound].include?(name) }
+      if outside_bound.any?
+        errors << "#{where}: expected_tools not bound to this target: #{outside_bound}"
+      end
+      if expectations["refusal"] == true && expected.any?
+        errors << "#{where}: refusal true cannot require expected_tools=#{expected.inspect}"
+      end
+      if guardrail && forbidden.none? { |name| inventory[:bound].include?(name) }
+        errors << "#{where}: HARD FAIL - guardrail/refusal forbids no tool this " \
+                  "target can call; suite cannot detect tool abuse"
+      end
+
+      warnings = []
+      if inventory[:bound_explicit]
+        unbound = forbidden.select { |name| inventory[:known].include?(name) && !inventory[:bound].include?(name) }
+        if unbound.any?
+          warnings << "#{where}: WARN - forbidden_tools names tools this target " \
+                      "cannot call (assertion observes nothing): #{unbound}"
+        end
+      end
+      [errors, warnings]
+    end
+
+    def forbids_bound?(expectations, bound)
+      listed = expectations["forbidden_tools"]
+      return false unless listed.is_a?(Array)
+
+      listed.any? { |name| bound.include?(name) }
+    end
+
+    def row_guardrail_gap(row, row_id, kind, blocks, inventory)
+      return [] unless kind == "native"
+      return [] if inventory[:agent_has_no_tools]
+
+      metadata = row["metadata"].is_a?(Hash) ? row["metadata"] : {}
+      return [] unless guardrail_metadata?(metadata)
+      return [] if blocks.any? { |(_, expectations, _)| forbids_bound?(expectations, inventory[:bound]) }
+
+      [
+        "#{row_id}: HARD FAIL - guardrail metadata requires at least one turn " \
+        "to forbid a tool this target can call"
+      ]
+    end
+
+    def mixed_schema_error(seen_schemas)
+      return [] if seen_schemas.size < 2
+
+      names = seen_schemas.to_a.sort.join(", ")
+      [
+        "mixed suite schemas in one file: #{names}. " \
+        "Split legacy rows from native rows."
+      ]
     end
   end
 end

data/lib/agent_eval_planner/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AgentEvalPlanner
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/agent_eval_planner.rb

@@ -18,11 +18,16 @@ module AgentEvalPlanner
       Planner.new(input_path: input_path, raw: raw, **options).call
     end
 
-    def validate_suite(path, known_tools: [], agent_has_no_tools: false)
+    def validate_suite(path, known_tools: [], bound_tools: nil, agent_has_no_tools: false,
+                       target_components: nil, suites: nil, schema: "auto")
       SuiteValidator.new(
         path: path,
         known_tools: known_tools,
-        agent_has_no_tools: agent_has_no_tools
+        bound_tools: bound_tools,
+        agent_has_no_tools: agent_has_no_tools,
+        target_components: target_components,
+        suites: suites,
+        schema: schema
       ).validate
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: agent_eval_planner
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Saulo Filho
@@ -49,9 +49,26 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
 - CHANGELOG.md
+- Gemfile
 - LICENSE.txt
 - README.md
+- Rakefile
+- agent_eval_planner.gemspec
+- docs/.nojekyll
+- docs/examples/sample-contract.md
+- docs/examples/sample-plan.md
+- docs/examples/sample-remediations.md
+- docs/examples/sample-scope-01-native.json
+- docs/examples/sample-scope-01.json
+- docs/examples/sample-suite.jsonl
+- docs/favicon.svg
+- docs/index.html
+- docs/script.js
+- docs/styles.css
 - exe/agent-eval-planner
 - lib/agent_eval_planner.rb
 - lib/agent_eval_planner/cli.rb