@root-signals/scorable-cli 0.8.0 → 0.9.1

package/README.md

@@ -302,6 +302,110 @@ scorable otel-filter list
 scorable otel-filter delete <filter_id>
 ```
 
+### Custom extractor rules (when input/output isn't in `gen_ai.*`)
+
+For the common case the flags above are everything you need: matching traces are evaluated and their `gen_ai.input.messages` / `gen_ai.output.messages` are fed to the evaluator. If your traces don't follow that shape — Claude Code, OpenInference, custom instrumentations — you tell the evaluator where input/output live by attaching **`extractor_rules`** to the filter. Filters without `extractor_rules` keep the default behavior.
+
+Rules are carried in a YAML manifest and applied with `-f`:
+
+```bash
+scorable otel-filter create -f filter.yaml
+scorable otel-filter update <id> -f filter.yaml
+scorable otel-filter validate -f filter.yaml   # dry-run; exit 2 on schema error
+```
+
+CLI flags override values from the file when both are provided.
+
+#### Manifest shape
+
+```yaml
+name: <string>
+evaluator_id: <uuid> # exactly one of evaluator_id / judge_id
+judge_id: <uuid>
+sampling_rate: <0.0-1.0> # optional, default 1.0
+delay_seconds: <int> # optional, default 10
+is_active: <bool> # optional, default true
+filter_criteria: # which traces to evaluate (same shape as --filter-criteria)
+  conditions:
+    - column: <string> # span_name, has_error, kind, status, attribute, resource, …
+      operator: <string> # =, !=, contains, starts with, any of, none of, …
+      value: <string|number|bool>
+      key: <string> # required for "attribute" / "resource" sentinel columns
+extractor_rules: # optional; how to extract input/output from matching spans
+  - emit: <text|request_response|tool_pair|genai_messages>
+    match: # optional; same shape as filter_criteria — empty matches every span
+      conditions: [...]
+    # … emit-specific fields, see below …
+```
+
+#### `extractor_rules` reference
+
+Each rule has an `emit` kind, an optional `match` filter (same shape as `filter_criteria.conditions`), and emit-specific fields. Spans are walked in timestamp order; per span, the **first** rule whose `match` passes wins.
+
+A rule set must be able to produce both user-side and agent-side content (a `request_response` or `genai_messages` rule alone qualifies; a single `text` `role: user` rule does not). Validation rejects sets that can't.
+
+**`text`** — emit one `MessageTurn` per matching span.
+
+```yaml
+- emit: text
+  match: # optional
+    conditions:
+      - { column: span_name, operator: "=", value: claude_code.interaction }
+  role: user # user | assistant
+  locator: # where the value lives
+    kind: span_attr # span_attr | event_attr | resource_attr
+    key: user_prompt # attribute name
+    event_name: <string> # required when kind: event_attr
+    value_path: $.foo # optional JSONPath into the located value
+  tool_name: <string> # only valid when role: assistant
+```
+
+**`request_response`** — emit a user turn + an assistant turn from one span. Common when an agent stores prompt and response as flat attributes (OpenInference's `input.value` / `output.value`).
+
+```yaml
+- emit: request_response
+  match: { ... } # optional
+  input_locator: { kind: span_attr, key: input.value } # → user turn
+  output_locator: { kind: span_attr, key: output.value } # → assistant turn
+```
+
+**`tool_pair`** — emit one assistant turn per matching span carrying a tool call (input + output combined into the turn content). Built for tool-execution spans like Claude Code's `claude_code.tool` event.
+
+```yaml
+- emit: tool_pair
+  match: { ... } # optional
+  input_locator: { kind: event_attr, event_name: tool.output, key: bash_command }
+  output_locator: { kind: event_attr, event_name: tool.output, key: output }
+  tool_name: Bash # static; OR
+  tool_name_locator: { kind: span_attr, key: tool_name } # dynamic
+```
+
+**`genai_messages`** — same parsing as the default extractor, but on attribute keys you choose. Use this if your service emits the standard `gen_ai` JSON-array shape under non-default attribute names.
+
+```yaml
+- emit: genai_messages
+  match: { ... } # optional
+  input_locator: { kind: span_attr, key: my_framework.input.json }
+  output_locator: { kind: span_attr, key: my_framework.output.json }
+```
+
+#### Locator details
+
+| Field        | Meaning                                                                                           |
+| ------------ | ------------------------------------------------------------------------------------------------- |
+| `kind`       | `span_attr` (top-level), `event_attr` (inside a named event), or `resource_attr` (resource-level) |
+| `key`        | Attribute name to read                                                                            |
+| `event_name` | Only for `event_attr` — the OTel event whose attributes to read (e.g. `tool.output`)              |
+| `value_path` | Optional JSONPath into the located value; used when the attribute is itself JSON                  |
+
+#### Reference manifests
+
+`examples/otel-filters/`:
+
+- `openinference-agent.yaml` — single-span agent with `input.value` / `output.value` (`request_response`).
+- `claude-code.yaml` — Claude Code's interaction span + tool spans (`text` + `tool_pair`).
+- `genai-explicit.yaml` — gen_ai messages under custom attribute keys.
+
 ## OTEL Trace Querying
 
 ### List traces

package/dist/commands/otel-filter/create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/otel-filter/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgBpC,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,OAAO,GAAG,IAAI,CAmH/D"}
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/otel-filter/create.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkBpC,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,OAAO,GAAG,IAAI,CA6J/D"}

package/dist/commands/otel-filter/create.js

@@ -1,18 +1,21 @@
+import fs from "node:fs";
 import ora from "ora";
 import { requireApiKey } from "../../auth.js";
 import { apiRequest } from "../../client.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { loadFilterYaml } from "../../lib/filter-yaml.js";
 export function registerCreateCommand(otelFilter) {
     otelFilter
         .command("create")
         .description("Create a new OTEL trace evaluation filter. Wires either an evaluator OR a judge to incoming traces.")
-        .requiredOption("--name <name>", "Filter name")
+        .option("--name <name>", "Filter name")
         .option("--evaluator-id <id>", "Evaluator UUID to run on matching traces (mutually exclusive with --judge-id)")
         .option("--judge-id <id>", "Judge UUID to run on matching traces (mutually exclusive with --evaluator-id)")
         .option("--filter-criteria <json>", 'Filter conditions as JSON, e.g. \'{"conditions":[{"column":"name","type":"string","key":"name","operator":"contains","value":"agent"}]}\'')
-        .option("--sampling-rate <rate>", "Sampling rate between 0.0 and 1.0", "1.0")
-        .option("--delay-seconds <seconds>", "Delay before evaluation (allows late spans)", "10")
-        .option("--inactive", "Create the filter as inactive", false)
+        .option("--sampling-rate <rate>", "Sampling rate between 0.0 and 1.0")
+        .option("--delay-seconds <seconds>", "Delay before evaluation (allows late spans)")
+        .option("--inactive", "Create the filter as inactive")
+        .option("-f, --from-file <path>", "Load filter spec from a YAML/JSON file. Required only when the filter needs extractor_rules (custom input/output extraction).")
         .addHelpText("after", `
 Target — exactly one of --evaluator-id or --judge-id is required.
   --evaluator-id  Runs a single evaluator (one score, one justification).
@@ -34,28 +37,39 @@ Examples:
       --filter-criteria '{"conditions":[{"column":"resource","type":"string","key":"service.name","operator":"=","value":"construction_assistant_agent"}]}' \\
       --delay-seconds 5
 
-  # Match only traces from a specific service (resource attribute), evaluator target
-  $ scorable otel-filter create \\
-      --name "construction-truthfulness" \\
-      --evaluator-id <evaluator-uuid> \\
-      --filter-criteria '{"conditions":[{"column":"resource","type":"string","key":"service.name","operator":"=","value":"construction_assistant_agent"}]}' \\
-      --delay-seconds 5
+YAML manifest (only needed for custom extractor_rules):
+  When your traces don't follow the OTel GenAI shape (Claude Code, OpenInference,
+  custom instrumentations) the eval needs to be told where input/output lives.
+  Carry that mapping as extractor_rules inside a YAML file. The flag-based
+  invocation above stays unchanged for everything else.
 
-  # Match by span attribute (gen_ai semantic conventions)
-  $ scorable otel-filter create \\
-      --name "agent-truthfulness" \\
-      --evaluator-id <evaluator-uuid> \\
-      --filter-criteria '{"conditions":[{"column":"gen_ai.agent.name","type":"string","key":"gen_ai.agent.name","operator":"=","value":"my_agent"}]}'`)
+  $ scorable otel-filter create -f ./filter.yaml
+  $ scorable otel-filter create -f ./filter.yaml --inactive    # override one field
+
+  See examples/otel-filters/ for reference manifests and the README's
+  "OTEL Trace Evaluation Filters" section for the extractor_rules schema.`)
         .action(async (opts) => {
-        if (!opts.evaluatorId && !opts.judgeId) {
-            printError("Either --evaluator-id or --judge-id is required.");
-            process.exit(1);
-        }
-        if (opts.evaluatorId && opts.judgeId) {
-            printError("--evaluator-id and --judge-id are mutually exclusive.");
-            process.exit(1);
+        let filePayload = {};
+        if (opts.fromFile) {
+            let source;
+            try {
+                source = fs.readFileSync(opts.fromFile, "utf8");
+            }
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                printError(`Failed to read --from-file: ${msg}`);
+                process.exit(1);
+            }
+            try {
+                filePayload = loadFilterYaml(source);
+            }
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                printError(msg);
+                process.exit(1);
+            }
         }
-        let filterCriteria = {};
+        let filterCriteria;
         if (opts.filterCriteria) {
             try {
                 filterCriteria = JSON.parse(opts.filterCriteria);
@@ -65,27 +79,62 @@ Examples:
                 process.exit(1);
             }
         }
-        const samplingRate = parseFloat(opts.samplingRate ?? "1.0");
-        if (Number.isNaN(samplingRate) || samplingRate < 0 || samplingRate > 1) {
-            printError("--sampling-rate must be a number between 0.0 and 1.0.");
+        let samplingRate;
+        if (opts.samplingRate !== undefined) {
+            const raw = opts.samplingRate.trim();
+            samplingRate = Number(raw);
+            if (raw === "" || !Number.isFinite(samplingRate) || samplingRate < 0 || samplingRate > 1) {
+                printError("--sampling-rate must be a number between 0.0 and 1.0.");
+                process.exit(1);
+            }
+        }
+        let delaySeconds;
+        if (opts.delaySeconds !== undefined) {
+            const raw = opts.delaySeconds.trim();
+            delaySeconds = Number(raw);
+            if (raw === "" || !Number.isInteger(delaySeconds) || delaySeconds < 0) {
+                printError("--delay-seconds must be a non-negative integer.");
+                process.exit(1);
+            }
+        }
+        const flagPayload = {};
+        if (opts.name !== undefined)
+            flagPayload.name = opts.name;
+        if (opts.evaluatorId !== undefined)
+            flagPayload.evaluator_id = opts.evaluatorId;
+        if (opts.judgeId !== undefined)
+            flagPayload.judge_id = opts.judgeId;
+        if (filterCriteria !== undefined)
+            flagPayload.filter_criteria = filterCriteria;
+        if (samplingRate !== undefined)
+            flagPayload.sampling_rate = samplingRate;
+        if (delaySeconds !== undefined)
+            flagPayload.delay_seconds = delaySeconds;
+        if (opts.inactive !== undefined)
+            flagPayload.is_active = !opts.inactive;
+        const payload = { ...filePayload, ...flagPayload };
+        if (!opts.fromFile) {
+            if (payload.sampling_rate === undefined)
+                payload.sampling_rate = 1.0;
+            if (payload.delay_seconds === undefined)
+                payload.delay_seconds = 10;
+            if (payload.is_active === undefined)
+                payload.is_active = true;
+            if (payload.filter_criteria === undefined)
+                payload.filter_criteria = {};
+        }
+        if (typeof payload.name !== "string" || payload.name.length === 0) {
+            printError("--name is required (provide via --name or --from-file).");
             process.exit(1);
         }
-        const delaySeconds = parseInt(opts.delaySeconds ?? "10", 10);
-        if (Number.isNaN(delaySeconds) || delaySeconds < 0) {
-            printError("--delay-seconds must be a non-negative integer.");
+        if (!payload.evaluator_id && !payload.judge_id) {
+            printError("Either --evaluator-id or --judge-id is required.");
+            process.exit(1);
+        }
+        if (payload.evaluator_id && payload.judge_id) {
+            printError("--evaluator-id and --judge-id are mutually exclusive.");
             process.exit(1);
         }
-        const payload = {
-            name: opts.name,
-            filter_criteria: filterCriteria,
-            sampling_rate: samplingRate,
-            delay_seconds: delaySeconds,
-            is_active: !opts.inactive,
-        };
-        if (opts.evaluatorId)
-            payload.evaluator_id = opts.evaluatorId;
-        if (opts.judgeId)
-            payload.judge_id = opts.judgeId;
         const spinner = ora("Creating filter...").start();
         try {
             const apiKey = await requireApiKey();

package/dist/commands/otel-filter/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/otel-filter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAQjE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/otel-filter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAUjE"}

package/dist/commands/otel-filter/index.js

@@ -1,11 +1,15 @@
 import { registerCreateCommand } from "./create.js";
 import { registerListCommand } from "./list.js";
 import { registerDeleteCommand } from "./delete.js";
+import { registerUpdateCommand } from "./update.js";
+import { registerValidateCommand } from "./validate.js";
 export function registerOtelFilterCommands(program) {
     const otelFilter = program
         .command("otel-filter")
         .description("Manage OTEL trace evaluation filters");
     registerCreateCommand(otelFilter);
     registerListCommand(otelFilter);
+    registerUpdateCommand(otelFilter);
     registerDeleteCommand(otelFilter);
+    registerValidateCommand(otelFilter);
 }

package/dist/commands/otel-filter/update.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerUpdateCommand(otelFilter: Command): void;
+//# sourceMappingURL=update.d.ts.map

package/dist/commands/otel-filter/update.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/otel-filter/update.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,OAAO,GAAG,IAAI,CAoD/D"}

package/dist/commands/otel-filter/update.js

@@ -0,0 +1,53 @@
+import fs from "node:fs";
+import ora from "ora";
+import { requireApiKey } from "../../auth.js";
+import { apiRequest } from "../../client.js";
+import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { loadFilterYaml } from "../../lib/filter-yaml.js";
+export function registerUpdateCommand(otelFilter) {
+    otelFilter
+        .command("update <id>")
+        .description("Update an OTEL trace evaluation filter from a YAML/JSON manifest")
+        .requiredOption("-f, --from-file <path>", "Filter spec path (YAML or JSON)")
+        .addHelpText("after", `
+Pairs with the YAML-manifest workflow used by \`create -f\`. Edit the file,
+re-apply with this command.
+
+Example:
+  $ scorable otel-filter update <filter-id> -f ./filter.yaml`)
+        .action(async (id, opts) => {
+        let source;
+        try {
+            source = fs.readFileSync(opts.fromFile, "utf8");
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            printError(`Failed to read --from-file: ${msg}`);
+            process.exit(1);
+        }
+        let payload;
+        try {
+            payload = loadFilterYaml(source);
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            printError(msg);
+            process.exit(1);
+        }
+        const spinner = ora("Updating filter...").start();
+        try {
+            const apiKey = await requireApiKey();
+            const result = await apiRequest("PATCH", `v1/otel/evaluation-filters/${encodeURIComponent(id)}`, { payload, apiKey });
+            spinner.stop();
+            if (result === null) {
+                process.exit(1);
+            }
+            printSuccess(`Filter ${id} updated.`);
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/otel-filter/validate.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerValidateCommand(otelFilter: Command): void;
+//# sourceMappingURL=validate.d.ts.map

package/dist/commands/otel-filter/validate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../../src/commands/otel-filter/validate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,OAAO,GAAG,IAAI,CAmEjE"}

package/dist/commands/otel-filter/validate.js

@@ -0,0 +1,72 @@
+import fs from "node:fs";
+import ora from "ora";
+import { requireApiKey } from "../../auth.js";
+import { apiRequestStatus } from "../../client.js";
+import { printError, printSuccess, handleSdkError } from "../../output.js";
+import { loadFilterYaml } from "../../lib/filter-yaml.js";
+import { CliError } from "../../types.js";
+export function registerValidateCommand(otelFilter) {
+    otelFilter
+        .command("validate")
+        .description("Validate an OTEL trace evaluation filter spec without saving")
+        .requiredOption("-f, --from-file <path>", "Filter spec path (YAML or JSON)")
+        .addHelpText("after", `
+Useful in CI to fail builds on a bad manifest before applying.
+
+Exit codes:
+  0   schema valid; no warnings
+  0   schema valid; warnings written to stderr (e.g. rules don't match any recent span)
+  2   schema invalid (local Zod failure or server 400)
+
+Example:
+  $ scorable otel-filter validate -f ./filter.yaml`)
+        .action(async (opts) => {
+        let source;
+        try {
+            source = fs.readFileSync(opts.fromFile, "utf8");
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            printError(`Failed to read --from-file: ${msg}`);
+            throw new CliError(1, msg);
+        }
+        let payload;
+        try {
+            payload = loadFilterYaml(source);
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            printError(msg);
+            throw new CliError(2, msg);
+        }
+        const spinner = ora("Validating filter...").start();
+        try {
+            const apiKey = await requireApiKey();
+            const result = await apiRequestStatus("POST", "v1/otel/evaluation-filters/validate", {
+                payload,
+                apiKey,
+            });
+            spinner.stop();
+            if (!result.ok) {
+                if (result.status === 400) {
+                    throw new CliError(2, "Filter schema validation failed.");
+                }
+                throw new CliError(1, `Validation request failed (status ${result.status}).`);
+            }
+            const data = (result.data ?? {});
+            const warnings = Array.isArray(data.warnings) ? data.warnings : [];
+            if (warnings.length > 0) {
+                for (const w of warnings) {
+                    console.error(`warning: ${w}`);
+                }
+            }
+            printSuccess("OK");
+        }
+        catch (e) {
+            spinner.stop();
+            if (e instanceof CliError)
+                throw e;
+            handleSdkError(e);
+        }
+    });
+}

package/dist/lib/filter-yaml.d.ts

@@ -0,0 +1,179 @@
+import { z } from "zod";
+export declare const FilterYamlSchema: z.ZodObject<{
+    name: z.ZodString;
+    evaluator_id: z.ZodOptional<z.ZodString>;
+    judge_id: z.ZodOptional<z.ZodString>;
+    filter_criteria: z.ZodDefault<z.ZodObject<{
+        conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+            column: z.ZodString;
+            operator: z.ZodString;
+            value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+            key: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>>;
+    }, z.core.$strict>>;
+    sampling_rate: z.ZodOptional<z.ZodNumber>;
+    delay_seconds: z.ZodOptional<z.ZodNumber>;
+    is_active: z.ZodOptional<z.ZodBoolean>;
+    extractor_rules: z.ZodDefault<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        emit: z.ZodLiteral<"text">;
+        match: z.ZodOptional<z.ZodObject<{
+            conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                column: z.ZodString;
+                operator: z.ZodString;
+                value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+                key: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>>;
+        role: z.ZodEnum<{
+            user: "user";
+            assistant: "assistant";
+        }>;
+        locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+        tool_name: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>, z.ZodObject<{
+        emit: z.ZodLiteral<"request_response">;
+        match: z.ZodOptional<z.ZodObject<{
+            conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                column: z.ZodString;
+                operator: z.ZodString;
+                value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+                key: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>>;
+        input_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+        output_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+    }, z.core.$strict>, z.ZodObject<{
+        emit: z.ZodLiteral<"tool_pair">;
+        match: z.ZodOptional<z.ZodObject<{
+            conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                column: z.ZodString;
+                operator: z.ZodString;
+                value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+                key: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>>;
+        input_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+        output_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+        tool_name: z.ZodOptional<z.ZodString>;
+        tool_name_locator: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">>;
+    }, z.core.$strict>, z.ZodObject<{
+        emit: z.ZodLiteral<"genai_messages">;
+        match: z.ZodOptional<z.ZodObject<{
+            conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                column: z.ZodString;
+                operator: z.ZodString;
+                value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+                key: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>>;
+        input_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+        output_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+    }, z.core.$strict>], "emit">>>;
+}, z.core.$strict>;
+export type FilterYaml = z.infer<typeof FilterYamlSchema>;
+export declare function loadFilterYaml(source: string): FilterYaml;
+//# sourceMappingURL=filter-yaml.d.ts.map

package/dist/lib/filter-yaml.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter-yaml.d.ts","sourceRoot":"","sources":["../../src/lib/filter-yaml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyFxB,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAczB,CAAC;AAEL,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAczD"}

package/dist/lib/filter-yaml.js

@@ -0,0 +1,111 @@
+import { z } from "zod";
+import yaml from "js-yaml";
+const MatchCondition = z
+    .object({
+    column: z.string(),
+    operator: z.string(),
+    value: z.union([z.string(), z.number(), z.boolean()]).optional(),
+    key: z.string().optional(),
+})
+    .strict();
+const Match = z
+    .object({
+    conditions: z.array(MatchCondition).default([]),
+})
+    .strict();
+const Locator = z.discriminatedUnion("kind", [
+    z
+        .object({
+        kind: z.literal("span_attr"),
+        key: z.string(),
+        value_path: z.string().optional(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("resource_attr"),
+        key: z.string(),
+        value_path: z.string().optional(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("event_attr"),
+        key: z.string(),
+        event_name: z.string(),
+        value_path: z.string().optional(),
+    })
+        .strict(),
+]);
+const TextRule = z
+    .object({
+    emit: z.literal("text"),
+    match: Match.optional(),
+    role: z.enum(["user", "assistant"]),
+    locator: Locator,
+    tool_name: z.string().optional(),
+})
+    .strict();
+const RequestResponseRule = z
+    .object({
+    emit: z.literal("request_response"),
+    match: Match.optional(),
+    input_locator: Locator,
+    output_locator: Locator,
+})
+    .strict();
+const ToolPairRule = z
+    .object({
+    emit: z.literal("tool_pair"),
+    match: Match.optional(),
+    input_locator: Locator,
+    output_locator: Locator,
+    tool_name: z.string().optional(),
+    tool_name_locator: Locator.optional(),
+})
+    .strict();
+const GenAiMessagesRule = z
+    .object({
+    emit: z.literal("genai_messages"),
+    match: Match.optional(),
+    input_locator: Locator,
+    output_locator: Locator,
+})
+    .strict();
+const ExtractorRule = z.discriminatedUnion("emit", [
+    TextRule,
+    RequestResponseRule,
+    ToolPairRule,
+    GenAiMessagesRule,
+]);
+export const FilterYamlSchema = z
+    .object({
+    name: z.string(),
+    evaluator_id: z.string().optional(),
+    judge_id: z.string().optional(),
+    filter_criteria: Match.default({ conditions: [] }),
+    sampling_rate: z.number().min(0).max(1).optional(),
+    delay_seconds: z.number().int().min(0).optional(),
+    is_active: z.boolean().optional(),
+    extractor_rules: z.array(ExtractorRule).default([]),
+})
+    .strict()
+    .refine((v) => Boolean(v.evaluator_id) !== Boolean(v.judge_id), {
+    message: "exactly one of evaluator_id or judge_id is required",
+});
+export function loadFilterYaml(source) {
+    let raw;
+    try {
+        raw = yaml.load(source);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        throw new Error(`Invalid YAML: ${msg}`);
+    }
+    const result = FilterYamlSchema.safeParse(raw);
+    if (!result.success) {
+        const issues = result.error.issues.map((i) => `${i.path.join(".") || "<root>"}: ${i.message}`);
+        throw new Error(`Invalid filter YAML:\n${issues.join("\n")}`);
+    }
+    return result.data;
+}

package/examples/otel-filters/claude-code.yaml

@@ -0,0 +1,31 @@
+name: claude-code-conversation-eval
+judge_id: 0193b6a0-e75d-7a47-9c6f-2f3e3b8f7c91
+sampling_rate: 1.0
+delay_seconds: 60
+is_active: true
+
+filter_criteria:
+  conditions:
+    - column: span_name
+      operator: starts_with
+      value: claude_code.
+
+extractor_rules:
+  - emit: text
+    match:
+      conditions:
+        - column: span_name
+          operator: "="
+          value: claude_code.interaction
+    role: user
+    locator: { kind: span_attr, key: user_prompt }
+
+  - emit: tool_pair
+    match:
+      conditions:
+        - column: span_name
+          operator: "="
+          value: claude_code.tool
+    tool_name: Bash
+    input_locator: { kind: event_attr, event_name: tool.output, key: bash_command }
+    output_locator: { kind: event_attr, event_name: tool.output, key: output }

package/examples/otel-filters/genai-explicit.yaml

@@ -0,0 +1,16 @@
+name: custom-genai-keys-eval
+judge_id: 0193b6a0-e75d-7a47-9c6f-2f3e3b8f7c91
+sampling_rate: 1.0
+delay_seconds: 60
+is_active: true
+
+filter_criteria:
+  conditions:
+    - column: span_name
+      operator: starts_with
+      value: my_framework.
+
+extractor_rules:
+  - emit: genai_messages
+    input_locator: { kind: span_attr, key: my_framework.input.json }
+    output_locator: { kind: span_attr, key: my_framework.output.json }

package/examples/otel-filters/openinference-agent.yaml

@@ -0,0 +1,17 @@
+name: openinference-agent-eval
+judge_id: 0193b6a0-e75d-7a47-9c6f-2f3e3b8f7c91
+sampling_rate: 1.0
+delay_seconds: 60
+is_active: true
+
+filter_criteria:
+  conditions:
+    - column: attribute
+      key: openinference.span.kind
+      operator: eq
+      value: AGENT
+
+extractor_rules:
+  - emit: request_response
+    input_locator: { kind: span_attr, key: input.value }
+    output_locator: { kind: span_attr, key: output.value }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@root-signals/scorable-cli",
-  "version": "0.8.0",
+  "version": "0.9.1",
   "description": "CLI for Scorable",
   "license": "Apache-2.0",
   "author": "Scorable",
@@ -13,6 +13,7 @@
     "dist/**/*.js.map",
     "dist/**/*.d.ts",
     "dist/**/*.d.ts.map",
+    "examples/**/*.yaml",
     "README.md",
     "LICENSE"
   ],
@@ -31,12 +32,13 @@
   },
   "dependencies": {
     "@inquirer/prompts": "^8.3.2",
-    "@root-signals/scorable": "^0.6.0",
+    "@root-signals/scorable": "^0.6.1",
     "chalk": "^5.6.2",
     "cli-table3": "^0.6.5",
     "commander": "^14.0.3",
     "js-yaml": "^4.1.1",
-    "ora": "^9.3.0"
+    "ora": "^9.3.0",
+    "zod": "^4.4.3"
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.9",