activeagent 1.9.6 → 1.9.8

data/lib/active_agent/evals/runner.rb

@@ -30,8 +30,9 @@ module ActiveAgent
       # tool to add, or how to change the instructions.
       DEFAULT_REFINE_FAULTS = %w[missing_capability expected_tool_not_called ungrounded_answer low_quality missing_content].freeze
       DEFAULT_JUDGE_LIMIT = 25
+      DEFAULT_TRIALS = 1
 
-      attr_reader :scenarios, :models, :criteria, :judge, :threshold
+      attr_reader :scenarios, :models, :criteria, :judge, :threshold, :pass_thresholds, :judge_failed_expectations, :trials
 
       # @param scenarios [Array<Scenario>]
       # @param models [Array<ModelSpec>]
@@ -43,6 +44,12 @@ module ActiveAgent
       # @param available_tools [Hash{String=>String}, Array<String>] the agent's tool roster
       # @param instructions [String, nil] the agent's instructions, for the judge
       # @param agent_name [String] how recommendations refer to the agent
+      # @param threshold [Float, nil] the score an answer must reach to pass;
+      #   an explicit value wins over the `pass_thresholds` bar, and without
+      #   either it is PASS_THRESHOLD
+      # @param pass_thresholds [PassThresholds, nil] the bars the suite
+      #   declared (Suite#pass_thresholds): its threshold applies when none
+      #   is given here, and the Report gates each group by it (Report#gates)
       # @param on_result [#call, nil] called with each Result as it lands
       # @param around_evaluation [#call, nil] called with (scenario, spec) and a
       #   block that returns the Result. Establishes context for replay, scoring
@@ -58,20 +65,38 @@ module ActiveAgent
       #   after `judge_limit`; false skips it. A caller that scores each
       #   scenario in its own Runner (a background job per scenario) shares one
       #   run-wide limit this way, since `judge_limit` counts per instance.
+      # @param judge_failed_expectations [Boolean] whether the judge grades a
+      #   replay that missed one of its scenario's expectations (an expected
+      #   tool not called, forbidden or missing content, a failed tool call).
+      #   false skips its task grade, its llm_judge criteria and its judged
+      #   recommendation, since it fails whatever the judge says, and the
+      #   calls saved do not count against `judge_limit`. The default grades
+      #   every answered replay.
+      # @param trials [Integer] how many times each scenario is replayed under
+      #   each model, 1 or more. Every replay is a Result of its own under the
+      #   scenario's key, told apart by Result#trial, so a flaky scenario
+      #   reads as a pass share rather than one pass or fail.
       def initialize(scenarios:, models:, replay:, criteria: [], judge: nil, judge_task: true, available_tools: {},
-                     instructions: nil, agent_name: "The agent", threshold: PASS_THRESHOLD,
+                     instructions: nil, agent_name: "The agent", threshold: nil, pass_thresholds: nil,
                      refine_faults: DEFAULT_REFINE_FAULTS, judge_limit: DEFAULT_JUDGE_LIMIT, on_result: nil,
-                     around_evaluation: nil, require_judge_scores: false, metadata: {}, release: nil, refine_gate: nil)
+                     around_evaluation: nil, require_judge_scores: false, metadata: {}, release: nil, refine_gate: nil,
+                     judge_failed_expectations: true, trials: DEFAULT_TRIALS)
+        unless trials.is_a?(Integer) && trials >= 1
+          raise ArgumentError, "trials must be an Integer of 1 or more, got #{trials.inspect}"
+        end
+
         @scenarios = scenarios
         @models = models
         @replay = replay
+        @trials = trials
         @criteria = criteria
         @judge = judge
         @judge_task = judge_task
         @available_tools = normalize_tools(available_tools)
         @instructions = instructions
         @agent_name = agent_name
-        @threshold = threshold
+        @threshold = threshold || pass_thresholds&.threshold || PASS_THRESHOLD
+        @pass_thresholds = pass_thresholds
         @refine_faults = refine_faults
         @judge_limit = judge_limit
         @on_result = on_result
@@ -80,27 +105,33 @@ module ActiveAgent
         @metadata = metadata
         @release = release
         @refine_gate = refine_gate
+        @judge_failed_expectations = judge_failed_expectations
         @judge_calls = 0
-        @scorer = Scorer.new(criteria: criteria, judge: judge)
+        @scorer = Scorer.new(criteria: criteria, judge: judge, judge_failed_expectations: judge_failed_expectations)
       end
 
+      # Replays every scenario under every model `trials` times: scenario by
+      # scenario, model by model, trial 1 to `trials`.
       def call
         results = @scenarios.flat_map do |scenario|
-          @models.map do |spec|
-            evaluate_with_context(scenario, spec).tap { |result| @on_result&.call(result) }
+          @models.flat_map do |spec|
+            (1..@trials).map do |trial|
+              evaluate_with_context(scenario, spec, trial: trial).tap { |result| @on_result&.call(result) }
+            end
           end
         end
 
         Report.new(results: results, models: @models, judge: @judge, instructions: @instructions,
-                   threshold: @threshold, metadata: @metadata, release: @release)
+                   threshold: @threshold, metadata: @metadata, release: @release, pass_thresholds: @pass_thresholds)
       end
 
       # Scores and diagnoses one replay. Public so a caller that has already run
       # the agent (a background job per scenario, say) can score without the loop.
-      def evaluate(scenario, spec, replay = nil)
+      # `trial` numbers the Result when the caller replays a scenario more than once.
+      def evaluate(scenario, spec, replay = nil, trial: 1)
         replay ||= run_replay(scenario, spec)
         scores = @scorer.score(scenario, replay)
-        if judge_task? && replay.answer.present?
+        if judge_task? && replay.answer.present? && judge_answer?(scores)
           scores["task_completion"] = @judge.score_task(scenario: scenario, answer: replay.answer)
         end
         score = Scorer.mean(scores)
@@ -110,7 +141,7 @@ module ActiveAgent
                                    judge_keys: llm_judge_keys)
         diagnosis ||= unavailable_judge_diagnosis(scores)
         diagnosis_hash = diagnosis&.to_h
-        refine!(diagnosis_hash, scenario, replay, diagnosis) if diagnosis_hash
+        refine!(diagnosis_hash, scenario, replay, diagnosis, scores) if diagnosis_hash
 
         Result.new(
           scenario: scenario,
@@ -119,16 +150,17 @@ module ActiveAgent
           scores: scores,
           score: score,
           status: replay.errored? ? "errored" : (diagnosis ? "failed" : "passed"),
-          diagnosis: diagnosis_hash
+          diagnosis: diagnosis_hash,
+          trial: trial
         )
       end
 
       private
 
-      def evaluate_with_context(scenario, spec)
-        return evaluate(scenario, spec) unless @around_evaluation
+      def evaluate_with_context(scenario, spec, trial:)
+        return evaluate(scenario, spec, trial: trial) unless @around_evaluation
 
-        result = @around_evaluation.call(scenario, spec) { evaluate(scenario, spec) }
+        result = @around_evaluation.call(scenario, spec) { evaluate(scenario, spec, trial: trial) }
         # A wrapper written the natural way — do something, yield, do something
         # after — returns that last value rather than the Result. Left alone it
         # reaches on_result and the Report, and fails somewhere far from the
@@ -144,6 +176,16 @@ module ActiveAgent
         @judge && @judge_task && @criteria.none? { |criterion| criterion.to_h.stringify_keys["type"] == "llm_judge" }
       end
 
+      # Whether the judge is asked about a replay with these scores: always by
+      # default, or only when it met every expectation of its scenario. A
+      # replay that did not already carries one of the faults Diagnosis
+      # assigns ahead of low_quality (tool_error, missing_capability,
+      # expected_tool_not_called, ungrounded_answer, forbidden_content or
+      # missing_content), so its status is failed whatever the judge would say.
+      def judge_answer?(scores)
+        @judge_failed_expectations || Scorer.expectations_met?(scores)
+      end
+
       # The keys in `scores` a judge graded, so a low grade is not averaged
       # away against rule checks.
       def llm_judge_keys
@@ -189,8 +231,9 @@ module ActiveAgent
         end
       end
 
-      def refine!(diagnosis, scenario, replay, result)
+      def refine!(diagnosis, scenario, replay, result, scores)
         return unless @judge && @refine_faults.include?(result.fault)
+        return unless judge_answer?(scores)
         return if @judge_calls >= @judge_limit
         return if @refine_gate && !@refine_gate.call
 

data/lib/active_agent/evals/scenario.rb

@@ -24,16 +24,21 @@ module ActiveAgent
     #   @return [Array<String>] labels a catalog files the scenario under ("billing")
     # @!attribute params
     #   @return [Hash] free-form parameters a replay may read; kept as written
+    # @!attribute metadata
+    #   @return [Hash, nil] opaque extension fields from a catalog document
     Scenario = Struct.new(:key, :group, :group_name, :prompt, :expected_tools, :expected_patterns,
-                          :forbidden_patterns, :notes, :production_only, :position, :tags, :params,
+                          :forbidden_patterns, :notes, :production_only, :position, :tags, :params, :metadata,
                           keyword_init: true) do
       # Builds a scenario from a hash — ScenarioParser output, a suite entry, or
       # a persisted record's attributes. Expectations are read from an
       # `expectations`/`expect` sub-hash (`tools`, `contains`, `not_contains`)
       # or from those keys at the top level.
-      def self.from_hash(attributes, group: nil, group_name: nil)
+      # Pass +metadata+ when extension fields have been separated from a
+      # document; its literal `metadata` key may itself be an extension.
+      def self.from_hash(attributes, group: nil, group_name: nil, metadata: nil)
         attrs = attributes.to_h.deep_stringify_keys
         expect = (attrs["expectations"] || attrs["expect"] || {}).to_h.stringify_keys
+        metadata = attrs["metadata"] if metadata.nil? && attrs["metadata"].is_a?(Hash)
 
         new(
           key: attrs["key"].to_s,
@@ -47,7 +52,8 @@ module ActiveAgent
           production_only: attrs["production_only"] == true,
           position: attrs["position"],
           tags: list(attrs["tags"]),
-          params: (attrs["params"] || {}).to_h.deep_stringify_keys
+          params: (attrs["params"] || {}).to_h.deep_stringify_keys,
+          metadata: (metadata || {}).to_h.deep_stringify_keys
         )
       end
 
@@ -68,8 +74,10 @@ module ActiveAgent
       end
 
       def to_h
-        super.compact.reject { |field, value| %i[tags params].include?(field) && value.blank? }
+        super.compact.reject { |field, value| %i[tags params metadata].include?(field) && value.blank? }
       end
     end
+
+    Scenario::DOCUMENT_KEYS = %w[key prompt expect expectations tools contains not_contains notes tags params production_only position].freeze
   end
 end

data/lib/active_agent/evals/scenario_parser.rb

@@ -138,12 +138,20 @@ module ActiveAgent
           end
         end
 
-        Suite.new([ document ]).all_scenarios.map do |item|
+        suite_from(document).all_scenarios.map do |item|
           scenario(prompt: item.prompt, group: item.group, group_name: item.group_name, key: item.key,
                    notes: item.notes, expectations: item.expectations, production_only: item.production_only?)
         end
       end
 
+      # The document as a Suite; a `criteria` block the Suite refuses is a
+      # ParseError, as every other fault in a pasted document is.
+      def suite_from(document)
+        Suite.new([ document ])
+      rescue ArgumentError => e
+        raise ParseError, e.message
+      end
+
       def scenario_from_hash(entry)
         entry = entry.stringify_keys
         prompt = entry["prompt"] || entry["message"] || entry["input"] || entry["question"]

data/lib/active_agent/evals/scorer.rb

@@ -4,7 +4,8 @@ module ActiveAgent
   module Evals
     # Scores one Replay against an evaluation's criteria and the scenario's
     # expectations. Returns `criterion key => 0.0..1.0`, with nil for a
-    # criterion that could not be scored (an llm_judge criterion with no judge).
+    # criterion that could not be scored (an llm_judge criterion with no judge,
+    # or one the Scorer did not ask because the expectations failed).
     #
     # Criteria are `{ "key", "type", "config" }` hashes:
     #
@@ -22,41 +23,37 @@ module ActiveAgent
     class Scorer
       RULE_CRITERION_TYPES = %w[response_present min_length max_latency_ms token_budget contains not_contains].freeze
       CRITERION_TYPES = (RULE_CRITERION_TYPES + %w[llm_judge]).freeze
+      # The keys `score` adds from the scenario's own expectations.
+      EXPECTATION_KEYS = %w[expected_tools expected_content forbidden_content tools_succeeded].freeze
 
-      attr_reader :criteria, :judge
+      attr_reader :criteria, :judge, :judge_failed_expectations
 
-      def initialize(criteria: [], judge: nil)
+      # @param criteria [Array<Hash>] the criteria every replay is scored on
+      # @param judge [Judge, nil] scores the llm_judge criteria
+      # @param judge_failed_expectations [Boolean] whether an llm_judge criterion
+      #   is asked of a replay that missed one of its scenario's expectations;
+      #   false leaves it nil, since the replay fails on the expectation anyway
+      def initialize(criteria: [], judge: nil, judge_failed_expectations: true)
         @criteria = Array(criteria).map { |criterion| criterion.to_h.deep_stringify_keys }
         @judge = judge
+        @judge_failed_expectations = judge_failed_expectations
       end
 
       def score(scenario, replay)
         scores = {}
         answer = replay.answer.to_s
+        expectations = expectation_scores(scenario, replay)
+        ask_judge = answer.present? && (@judge_failed_expectations || self.class.expectations_met?(expectations))
 
         @criteria.each do |criterion|
-          scores[criterion["key"]] = answer.present? ? score_criterion(criterion, scenario, replay) : 0.0
+          scores[criterion["key"]] =
+            if answer.blank? then 0.0
+            elsif criterion["type"] == "llm_judge" && !ask_judge then nil
+            else score_criterion(criterion, scenario, replay)
+            end
         end
 
-        if scenario.expected_tools.any?
-          scores["expected_tools"] = (scenario.expected_tools & replay.tool_names).any? ? 1.0 : 0.0
-        end
-        if scenario.expected_patterns.any?
-          hits = scenario.expected_patterns.count { |pattern| self.class.matches_pattern?(answer, pattern) }
-          scores["expected_content"] = (hits.to_f / scenario.expected_patterns.size).round(3)
-        end
-        if scenario.forbidden_patterns.any?
-          hit = scenario.forbidden_patterns.any? { |pattern| self.class.matches_pattern?(answer, pattern) }
-          scores["forbidden_content"] = hit ? 0.0 : 1.0
-        end
-        # A tool that ran without erroring is evidence only when it is one the
-        # scenario expected: a wrong tool that succeeded used to outscore
-        # calling nothing at all.
-        if replay.tool_calls.any? && (scenario.expected_tools.empty? || (scenario.expected_tools & replay.tool_names).any?)
-          scores["tools_succeeded"] = replay.failed_tool_calls.any? ? 0.0 : 1.0
-        end
-
-        scores
+        scores.merge(expectations)
       end
 
       # The mean of the scored criteria, or nil when nothing could be scored.
@@ -67,6 +64,12 @@ module ActiveAgent
         (scored.sum / scored.size).round(3)
       end
 
+      # Whether every expectation score in `scores` is a full 1.0. A scores
+      # hash with no expectation keys is met.
+      def self.expectations_met?(scores)
+        scores.slice(*EXPECTATION_KEYS).values.all? { |value| value.to_f >= 1.0 }
+      end
+
       # How long one pattern may take to match one answer. Patterns are
       # whatever the scenario's author typed, and a pathological one must not
       # stall the evaluation.
@@ -90,6 +93,33 @@ module ActiveAgent
 
       private
 
+      # The scores the scenario's own expectations add, in the order they appear
+      # after the criteria.
+      def expectation_scores(scenario, replay)
+        scores = {}
+        answer = replay.answer.to_s
+
+        if scenario.expected_tools.any?
+          scores["expected_tools"] = (scenario.expected_tools & replay.tool_names).any? ? 1.0 : 0.0
+        end
+        if scenario.expected_patterns.any?
+          hits = scenario.expected_patterns.count { |pattern| self.class.matches_pattern?(answer, pattern) }
+          scores["expected_content"] = (hits.to_f / scenario.expected_patterns.size).round(3)
+        end
+        if scenario.forbidden_patterns.any?
+          hit = scenario.forbidden_patterns.any? { |pattern| self.class.matches_pattern?(answer, pattern) }
+          scores["forbidden_content"] = hit ? 0.0 : 1.0
+        end
+        # A tool that ran without erroring is evidence only when it is one the
+        # scenario expected: a wrong tool that succeeded used to outscore
+        # calling nothing at all.
+        if replay.tool_calls.any? && (scenario.expected_tools.empty? || (scenario.expected_tools & replay.tool_names).any?)
+          scores["tools_succeeded"] = replay.failed_tool_calls.any? ? 0.0 : 1.0
+        end
+
+        scores
+      end
+
       def score_criterion(criterion, scenario, replay)
         config = criterion["config"] || {}
         answer = replay.answer.to_s

data/lib/active_agent/evals/suite.rb

@@ -6,13 +6,30 @@ module ActiveAgent
     # layered over earlier ones: a scenario with a key already present replaces
     # it, a new key is appended to its group, and a new group is appended to the
     # suite. That is how an app keeps a shared suite and lets a deployment add or
-    # reword questions.
+    # reword questions. A `pass` block, on the suite and on a group, declares
+    # the bars a run is held to (PassThresholds); later documents override it
+    # key by key at both levels. Named criteria lists layer the same way by
+    # criterion key, and `criteria("shared", "terse")` hands the chosen lists
+    # to Runner.new(criteria:).
     #
     #   suite: support_desk
     #   description: Questions the support team asks every week
+    #   pass:
+    #     threshold: 0.7
+    #     min_pass_rate: 0.8
+    #   criteria:
+    #     shared:
+    #       - key: cites_order
+    #         type: contains
+    #         config: { pattern: "ABC-" }
+    #     terse:
+    #       - key: short
+    #         type: min_length
+    #         config: { chars: 20 }
     #   groups:
     #     - key: open_tickets
     #       name: Open tickets
+    #       pass: { min_pass_rate: 1.0 }
     #       scenarios:
     #         - key: open_tickets_1
     #           prompt: Which open tickets mention a refund?
@@ -23,7 +40,7 @@ module ActiveAgent
     class Suite
       class NotFound < StandardError; end
 
-      attr_reader :name, :description, :groups
+      attr_reader :name, :description, :groups, :pass_thresholds
 
       # Loads the documents at `paths` (missing files are skipped) and raises
       # NotFound when none exist.
@@ -35,20 +52,75 @@ module ActiveAgent
         new(documents, name: name || File.basename(existing.first, ".yml"))
       end
 
+      # The scenarios among `scenarios` whose key is in `keys`, in the order of
+      # `scenarios`. `keys` is an Array of Strings or Symbols or one
+      # comma-separated String; blanks are dropped, a repeated key selects
+      # once, and blank `keys` leave `scenarios` unchanged. Every key must name
+      # a scenario in `known`, which may be wider than `scenarios` (several
+      # suites' scenarios together), else NotFound lists the missing keys in
+      # the order given, "in <name>" appended when `name` is given. The result
+      # is empty when `known` holds the keys but `scenarios` does not.
+      def self.select(scenarios, keys:, known: scenarios, name: nil)
+        keys = keys.to_s.split(",") unless keys.is_a?(Array)
+        keys = keys.map { |key| key.to_s.strip }.reject(&:empty?).uniq
+        return scenarios if keys.empty?
+
+        missing = keys - known.map(&:key)
+        if missing.any?
+          where = name.present? ? " in #{name}" : ""
+          raise NotFound, "no scenario #{missing.join(', ')}#{where}"
+        end
+
+        scenarios.select { |scenario| keys.include?(scenario.key) }
+      end
+
       # @param documents [Array<Hash>] parsed YAML documents, base first
       def initialize(documents, name: nil)
         documents = Array(documents).map(&:deep_stringify_keys)
         @name = documents.filter_map { |doc| doc["suite"] }.last || name
         @description = documents.filter_map { |doc| doc["description"] }.last
+        @pass_thresholds = PassThresholds.from_documents(documents)
         @groups = merge_groups(documents)
+        @criteria_lists = merge_criteria_lists(documents)
+      end
+
+      # The names of the suite's criteria lists, in document order.
+      def criteria_names
+        @criteria_lists.keys
+      end
+
+      # The criteria of the named lists in the order given, a name given twice
+      # contributing once, or of every list when no names are given. A
+      # criterion whose key an earlier list already holds replaces it in
+      # place, so an audience's list can restate a shared criterion with its
+      # own bar and the answer is scored on it once. A name no document
+      # declared contributes nothing, so a caller can ask for "shared" plus an
+      # audience that has no list of its own, and a nil or blank name is
+      # skipped, so an audience left unset narrows nothing. The result is
+      # ready for Runner.new(criteria:).
+      def criteria(*names)
+        names = names.flatten.filter_map { |list_name| list_name.to_s.presence }.uniq
+        names = criteria_names if names.empty?
+        names.each_with_object([]) do |list_name, chosen|
+          @criteria_lists.fetch(list_name, []).each do |criterion|
+            index = chosen.index { |existing| existing["key"] == criterion["key"] }
+            index ? chosen[index] = criterion : chosen << criterion
+          end
+        end
       end
 
       # Scenarios narrowed by group keys, scenario keys, or both; `production_only`
-      # scenarios are dropped unless `include_production_only` is true.
-      def scenarios(groups: nil, keys: nil, include_production_only: true)
+      # scenarios are dropped unless `include_production_only` is true. With
+      # `strict`, every key in `keys` must name a scenario of the suite (see
+      # Suite.select), and a key outside the chosen groups selects nothing.
+      def scenarios(groups: nil, keys: nil, include_production_only: true, strict: false)
         selected = all_scenarios
         selected = selected.select { |scenario| Array(groups).map(&:to_s).include?(scenario.group) } if groups.present?
-        selected = selected.select { |scenario| Array(keys).map(&:to_s).include?(scenario.key) } if keys.present?
+        if strict
+          selected = Suite.select(selected, keys: keys, known: all_scenarios, name: name)
+        elsif keys.present?
+          selected = selected.select { |scenario| Array(keys).map(&:to_s).include?(scenario.key) }
+        end
         selected = selected.reject(&:production_only?) unless include_production_only
         selected
       end
@@ -56,7 +128,8 @@ module ActiveAgent
       def all_scenarios
         @all_scenarios ||= @groups.flat_map do |group|
           group["scenarios"].each_with_index.map do |entry, index|
-            Scenario.from_hash(entry.merge("position" => index), group: group["key"], group_name: group["name"])
+            Scenario.from_hash(entry.merge("position" => index), group: group["key"], group_name: group["name"],
+              metadata: entry.except(*Scenario::DOCUMENT_KEYS, "group", "group_name"))
           end
         end
       end
@@ -80,6 +153,7 @@ module ActiveAgent
             if existing
               existing["name"] = incoming["name"] if incoming["name"].present?
               existing["description"] = incoming["description"] if incoming["description"].present?
+              existing["pass"] = (existing["pass"] || {}).merge(incoming["pass"]) if incoming["pass"].present?
               merge_scenarios(existing, Array(incoming["scenarios"]))
             else
               groups << incoming.merge("scenarios" => Array(incoming["scenarios"]))
@@ -94,6 +168,43 @@ module ActiveAgent
           index ? group["scenarios"][index] = scenario : group["scenarios"] << scenario
         end
       end
+
+      # Each document's criteria lists layered over the earlier ones by name
+      # and, within a name, by criterion key.
+      def merge_criteria_lists(documents)
+        documents.each_with_object({}) do |document, lists|
+          criteria_lists(document).each do |list_name, incoming|
+            list = (lists[list_name] ||= [])
+            incoming.each do |criterion|
+              index = list.index { |existing| existing["key"] == criterion["key"] }
+              index ? list[index] = criterion : list << criterion
+            end
+          end
+        end
+      end
+
+      # A document's `criteria` as `name => list`; a bare list is the one list
+      # named "default", and an empty one declares no list.
+      def criteria_lists(document)
+        value = document["criteria"]
+        return {} if value.nil? || value == []
+
+        value = { "default" => value } if value.is_a?(Array)
+        raise ArgumentError, "the criteria of suite #{name.inspect} are not a list or a mapping of lists" unless value.is_a?(Hash)
+
+        value.to_h do |list_name, list|
+          raise ArgumentError, "criteria list #{list_name.inspect} of suite #{name.inspect} is not a list" unless list.is_a?(Array)
+
+          [ list_name.to_s, list.map { |criterion| normalize_criterion(criterion, list_name) } ]
+        end
+      end
+
+      def normalize_criterion(criterion, list_name)
+        raise ArgumentError, "a criterion in list #{list_name.inspect} of suite #{name.inspect} is not a mapping" unless criterion.is_a?(Hash)
+        raise ArgumentError, "a criterion in list #{list_name.inspect} of suite #{name.inspect} has no key" if criterion["key"].blank?
+
+        criterion.merge("key" => criterion["key"].to_s)
+      end
     end
   end
 end