@root-signals/scorable-cli 0.7.0 → 0.9.0

package/dist/lib/filter-yaml.d.ts

@@ -0,0 +1,179 @@
+import { z } from "zod";
+export declare const FilterYamlSchema: z.ZodObject<{
+    name: z.ZodString;
+    evaluator_id: z.ZodOptional<z.ZodString>;
+    judge_id: z.ZodOptional<z.ZodString>;
+    filter_criteria: z.ZodDefault<z.ZodObject<{
+        conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+            column: z.ZodString;
+            operator: z.ZodString;
+            value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+            key: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>>;
+    }, z.core.$strict>>;
+    sampling_rate: z.ZodOptional<z.ZodNumber>;
+    delay_seconds: z.ZodOptional<z.ZodNumber>;
+    is_active: z.ZodOptional<z.ZodBoolean>;
+    extractor_rules: z.ZodDefault<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        emit: z.ZodLiteral<"text">;
+        match: z.ZodOptional<z.ZodObject<{
+            conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                column: z.ZodString;
+                operator: z.ZodString;
+                value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+                key: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>>;
+        role: z.ZodEnum<{
+            user: "user";
+            assistant: "assistant";
+        }>;
+        locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+        tool_name: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>, z.ZodObject<{
+        emit: z.ZodLiteral<"request_response">;
+        match: z.ZodOptional<z.ZodObject<{
+            conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                column: z.ZodString;
+                operator: z.ZodString;
+                value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+                key: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>>;
+        input_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+        output_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+    }, z.core.$strict>, z.ZodObject<{
+        emit: z.ZodLiteral<"tool_pair">;
+        match: z.ZodOptional<z.ZodObject<{
+            conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                column: z.ZodString;
+                operator: z.ZodString;
+                value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+                key: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>>;
+        input_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+        output_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+        tool_name: z.ZodOptional<z.ZodString>;
+        tool_name_locator: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">>;
+    }, z.core.$strict>, z.ZodObject<{
+        emit: z.ZodLiteral<"genai_messages">;
+        match: z.ZodOptional<z.ZodObject<{
+            conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                column: z.ZodString;
+                operator: z.ZodString;
+                value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+                key: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>>;
+        input_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+        output_locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"span_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"resource_attr">;
+            key: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"event_attr">;
+            key: z.ZodString;
+            event_name: z.ZodString;
+            value_path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "kind">;
+    }, z.core.$strict>], "emit">>>;
+}, z.core.$strict>;
+export type FilterYaml = z.infer<typeof FilterYamlSchema>;
+export declare function loadFilterYaml(source: string): FilterYaml;
+//# sourceMappingURL=filter-yaml.d.ts.map

package/dist/lib/filter-yaml.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter-yaml.d.ts","sourceRoot":"","sources":["../../src/lib/filter-yaml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyFxB,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAczB,CAAC;AAEL,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAczD"}

package/dist/lib/filter-yaml.js

@@ -0,0 +1,111 @@
+import { z } from "zod";
+import yaml from "js-yaml";
+const MatchCondition = z
+    .object({
+    column: z.string(),
+    operator: z.string(),
+    value: z.union([z.string(), z.number(), z.boolean()]).optional(),
+    key: z.string().optional(),
+})
+    .strict();
+const Match = z
+    .object({
+    conditions: z.array(MatchCondition).default([]),
+})
+    .strict();
+const Locator = z.discriminatedUnion("kind", [
+    z
+        .object({
+        kind: z.literal("span_attr"),
+        key: z.string(),
+        value_path: z.string().optional(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("resource_attr"),
+        key: z.string(),
+        value_path: z.string().optional(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("event_attr"),
+        key: z.string(),
+        event_name: z.string(),
+        value_path: z.string().optional(),
+    })
+        .strict(),
+]);
+const TextRule = z
+    .object({
+    emit: z.literal("text"),
+    match: Match.optional(),
+    role: z.enum(["user", "assistant"]),
+    locator: Locator,
+    tool_name: z.string().optional(),
+})
+    .strict();
+const RequestResponseRule = z
+    .object({
+    emit: z.literal("request_response"),
+    match: Match.optional(),
+    input_locator: Locator,
+    output_locator: Locator,
+})
+    .strict();
+const ToolPairRule = z
+    .object({
+    emit: z.literal("tool_pair"),
+    match: Match.optional(),
+    input_locator: Locator,
+    output_locator: Locator,
+    tool_name: z.string().optional(),
+    tool_name_locator: Locator.optional(),
+})
+    .strict();
+const GenAiMessagesRule = z
+    .object({
+    emit: z.literal("genai_messages"),
+    match: Match.optional(),
+    input_locator: Locator,
+    output_locator: Locator,
+})
+    .strict();
+const ExtractorRule = z.discriminatedUnion("emit", [
+    TextRule,
+    RequestResponseRule,
+    ToolPairRule,
+    GenAiMessagesRule,
+]);
+export const FilterYamlSchema = z
+    .object({
+    name: z.string(),
+    evaluator_id: z.string().optional(),
+    judge_id: z.string().optional(),
+    filter_criteria: Match.default({ conditions: [] }),
+    sampling_rate: z.number().min(0).max(1).optional(),
+    delay_seconds: z.number().int().min(0).optional(),
+    is_active: z.boolean().optional(),
+    extractor_rules: z.array(ExtractorRule).default([]),
+})
+    .strict()
+    .refine((v) => Boolean(v.evaluator_id) !== Boolean(v.judge_id), {
+    message: "exactly one of evaluator_id or judge_id is required",
+});
+export function loadFilterYaml(source) {
+    let raw;
+    try {
+        raw = yaml.load(source);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        throw new Error(`Invalid YAML: ${msg}`);
+    }
+    const result = FilterYamlSchema.safeParse(raw);
+    if (!result.success) {
+        const issues = result.error.issues.map((i) => `${i.path.join(".") || "<root>"}: ${i.message}`);
+        throw new Error(`Invalid filter YAML:\n${issues.join("\n")}`);
+    }
+    return result.data;
+}

package/examples/otel-filters/claude-code.yaml

@@ -0,0 +1,31 @@
+name: claude-code-conversation-eval
+judge_id: 0193b6a0-e75d-7a47-9c6f-2f3e3b8f7c91
+sampling_rate: 1.0
+delay_seconds: 60
+is_active: true
+
+filter_criteria:
+  conditions:
+    - column: span_name
+      operator: starts_with
+      value: claude_code.
+
+extractor_rules:
+  - emit: text
+    match:
+      conditions:
+        - column: span_name
+          operator: "="
+          value: claude_code.interaction
+    role: user
+    locator: { kind: span_attr, key: user_prompt }
+
+  - emit: tool_pair
+    match:
+      conditions:
+        - column: span_name
+          operator: "="
+          value: claude_code.tool
+    tool_name: Bash
+    input_locator: { kind: event_attr, event_name: tool.output, key: bash_command }
+    output_locator: { kind: event_attr, event_name: tool.output, key: output }

package/examples/otel-filters/genai-explicit.yaml

@@ -0,0 +1,16 @@
+name: custom-genai-keys-eval
+judge_id: 0193b6a0-e75d-7a47-9c6f-2f3e3b8f7c91
+sampling_rate: 1.0
+delay_seconds: 60
+is_active: true
+
+filter_criteria:
+  conditions:
+    - column: span_name
+      operator: starts_with
+      value: my_framework.
+
+extractor_rules:
+  - emit: genai_messages
+    input_locator: { kind: span_attr, key: my_framework.input.json }
+    output_locator: { kind: span_attr, key: my_framework.output.json }

package/examples/otel-filters/openinference-agent.yaml

@@ -0,0 +1,17 @@
+name: openinference-agent-eval
+judge_id: 0193b6a0-e75d-7a47-9c6f-2f3e3b8f7c91
+sampling_rate: 1.0
+delay_seconds: 60
+is_active: true
+
+filter_criteria:
+  conditions:
+    - column: attribute
+      key: openinference.span.kind
+      operator: eq
+      value: AGENT
+
+extractor_rules:
+  - emit: request_response
+    input_locator: { kind: span_attr, key: input.value }
+    output_locator: { kind: span_attr, key: output.value }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@root-signals/scorable-cli",
-  "version": "0.7.0",
+  "version": "0.9.0",
   "description": "CLI for Scorable",
   "license": "Apache-2.0",
   "author": "Scorable",
@@ -13,6 +13,7 @@
     "dist/**/*.js.map",
     "dist/**/*.d.ts",
     "dist/**/*.d.ts.map",
+    "examples/**/*.yaml",
     "README.md",
     "LICENSE"
   ],
@@ -36,7 +37,8 @@
     "cli-table3": "^0.6.5",
     "commander": "^14.0.3",
     "js-yaml": "^4.1.1",
-    "ora": "^9.3.0"
+    "ora": "^9.3.0",
+    "zod": "^4.4.3"
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.9",