@smthrs/evals 0.0.0-stage → 1.0.0-rc.3

package/dist/esm/internal/scorerLabel.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorerLabel.js","sourceRoot":"","sources":["../../../src/internal/scorerLabel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,QAA+E,EACvE,EAAE;IACV,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAA;IAChC,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAA;AAC/G,CAAC,CAAA","sourcesContent":["/**\n * The one scorer label rule every rendering in this package shares.\n *\n * A scorer has two identities: `scorer` is the key, a digest of the scorer's\n * own declaration and the only thing a baseline matches on, and `scorerName`\n * is the readable name carried beside it. A digest is unreadable in a report\n * cell, a CI log line, or a diagnostic, so each of them prints the name with\n * the first eight key characters instead. Without a name there is nothing to\n * abbreviate the key down to, so the label is the key itself: a truncated\n * digest and no name would leave a reader nothing to match a baseline record\n * on. One rule is what keeps a report from naming one scorer two ways, an\n * inconclusive reason built by the runner and a Missing row built by the\n * reporter agree.\n *\n * @since 0.1.0\n */\n\n/**\n * Renders `name (first 8 of the key)`, or the bare key when the scorer carries\n * no name.\n *\n * Every flow carries a name, so a scorer a run holds always has one. A stored\n * observation may not: a baseline recorded before a scorer was named, or one\n * written by another tool, carries the key alone, and it labels as the key\n * rather than as a blank followed by a digest.\n *\n * @since 0.1.0\n * @private\n */\nexport const scorerLabel = (\n  identity: { readonly scorer: string; readonly scorerName?: string | undefined }\n): string => {\n  const name = identity.scorerName\n  return name === undefined || name.length === 0 ? identity.scorer : `${name} (${identity.scorer.slice(0, 8)})`\n}\n"]}

package/dist/esm/internal/tupleKey.d.ts

@@ -0,0 +1,20 @@
+/**
+ * The one injective tuple encoder every key in this package is built with.
+ *
+ * Joining caller-supplied strings on a delimiter is not injective: under a NUL
+ * separator the tuples `["a", "b" + NUL + "c"]` and `["a" + NUL + "b", "c"]`
+ * produce one key, so two distinct score jobs could share an identity and two
+ * distinct baseline records could sort as one. Encoding the tuple as a JSON
+ * array keeps the components separated by the encoding itself, whatever they
+ * contain.
+ *
+ * @since 0.1.0
+ */
+/**
+ * Encodes a tuple of strings as one key.
+ *
+ * @since 0.1.0
+ * @private
+ */
+export declare const tupleKey: (...parts: ReadonlyArray<string>) => string;
+//# sourceMappingURL=tupleKey.d.ts.map

package/dist/esm/internal/tupleKey.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tupleKey.d.ts","sourceRoot":"","sources":["../../../src/internal/tupleKey.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH;;;;;GAKG;AACH,eAAO,MAAM,QAAQ,GAAI,GAAG,OAAO,aAAa,CAAC,MAAM,CAAC,KAAG,MAA+B,CAAA"}

package/dist/esm/internal/tupleKey.js

@@ -0,0 +1,20 @@
+/**
+ * The one injective tuple encoder every key in this package is built with.
+ *
+ * Joining caller-supplied strings on a delimiter is not injective: under a NUL
+ * separator the tuples `["a", "b" + NUL + "c"]` and `["a" + NUL + "b", "c"]`
+ * produce one key, so two distinct score jobs could share an identity and two
+ * distinct baseline records could sort as one. Encoding the tuple as a JSON
+ * array keeps the components separated by the encoding itself, whatever they
+ * contain.
+ *
+ * @since 0.1.0
+ */
+/**
+ * Encodes a tuple of strings as one key.
+ *
+ * @since 0.1.0
+ * @private
+ */
+export const tupleKey = (...parts) => JSON.stringify(parts);
+//# sourceMappingURL=tupleKey.js.map

package/dist/esm/internal/tupleKey.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tupleKey.js","sourceRoot":"","sources":["../../../src/internal/tupleKey.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,GAAG,KAA4B,EAAU,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA","sourcesContent":["/**\n * The one injective tuple encoder every key in this package is built with.\n *\n * Joining caller-supplied strings on a delimiter is not injective: under a NUL\n * separator the tuples `[\"a\", \"b\" + NUL + \"c\"]` and `[\"a\" + NUL + \"b\", \"c\"]`\n * produce one key, so two distinct score jobs could share an identity and two\n * distinct baseline records could sort as one. Encoding the tuple as a JSON\n * array keeps the components separated by the encoding itself, whatever they\n * contain.\n *\n * @since 0.1.0\n */\n\n/**\n * Encodes a tuple of strings as one key.\n *\n * @since 0.1.0\n * @private\n */\nexport const tupleKey = (...parts: ReadonlyArray<string>): string => JSON.stringify(parts)\n"]}

package/docs/README.md

@@ -0,0 +1,130 @@
+---
+title: "@smthrs/evals"
+description: "Fixed-suite evaluation for flows: score a target against committed cases, compare the run with a baseline, and gate CI on the result."
+---
+
+`@smthrs/evals` tells you whether a flow still behaves. A flow is a declaration
+of agent work built with [@smthrs/core](/api/core), and the thing a suite grades
+is one execution of it. You declare a fixed set of cases and the scorers that
+grade them, run the suite, and commit what it scored as a baseline file. Every
+later run is compared with that file and graded into a CI exit code.
+
+## Why you would reach for it
+
+A unit test asserts equality. A flow that calls a model produces an answer that
+is rarely equal to anything, only better or worse than the last one, so the
+usual assertion has nothing to hold. Scoring the answer gives you a number, and
+a number on its own decides nothing until there is another number to compare it
+with.
+
+This package supplies that half of the loop:
+
+- **Fixed suites.** `Suite.make` validates a declaration once and freezes it, so
+  the input half of the measurement cannot drift between the run that recorded a
+  baseline and the run being judged.
+- **A committed baseline.** `Baseline.write` emits byte-stable JSON, so the
+  record of what the target used to score is a file you commit and diff.
+- **A comparison that names the fault.** A score that dropped at a changed step
+  key is a regression in the target. A score that moved at an unchanged step key
+  is nondeterminism in the target or the scorer. Both are red, and they send you
+  to different places.
+- **A grade CI can act on.** `Gate.ciGrade` returns 0 for a pass, 1 for a real
+  red, and 5 for a harness that could not decide, so a broken scorer never reads
+  as a failing target.
+
+Every stage is an `Effect` value, and nothing in a run reads a clock or
+generates an identifier: `runId` and `at` come from the caller, so two runs over
+the same inputs produce byte-identical observations.
+
+The package is at 1.0.0-rc.0 and is not on the npm registry yet, so you get it
+by working in a clone of the
+[Smithers repository](https://github.com/smithersai/smithers).
+[Installation](./installation.md) has the steps.
+
+## A working example
+
+This script scores one case, compares the run with a committed
+`baseline.json`, prints the Markdown report, and exits with the gate's grade.
+
+```ts
+import { Baseline, CaseExecutor, Gate, Regression, Report, Runner, Suite } from "@smthrs/evals"
+import { Binding, Scorer } from "@smthrs/scorers"
+import { Effect, Layer } from "effect"
+import { readFile } from "node:fs/promises"
+
+const greet = ({ name: "greet" })
+
+const polite = Scorer.make({
+  id: "example/polite",
+  version: "1",
+  name: "polite",
+  score: ({ output }) => Effect.succeed({ score: String(output).startsWith("Hello") ? 1 : 0 })
+})
+
+const executor = CaseExecutor.make((suiteCase) =>
+  Effect.succeed({
+    output: `Hello, ${(suiteCase.input as { readonly name: string }).name}`,
+    stepKey: suiteCase.name,
+    latencyMs: 0,
+    target: greet
+  })
+)
+
+const program = Effect.gen(function*() {
+  const suite = yield* Suite.make({
+    name: "greetings",
+    cases: [{ name: "ada", input: { name: "Ada" } }],
+    bindings: [Binding.make({ scorer: polite, appliesTo: greet })],
+    concurrency: 1
+  })
+  const run = yield* Runner.run(suite, { runId: "ci-1", at: "2026-01-01T00:00:00.000Z" })
+  const committed = yield* Baseline.load(
+    yield* Effect.promise(() => readFile("baseline.json", "utf8"))
+  )
+  const comparison = yield* Regression.compare(committed, run)
+  yield* Effect.sync(() => process.stdout.write(Report.markdown(comparison)))
+  return Gate.ciGrade(yield* Gate.check(comparison, { mean: 1 })).exitCode
+}).pipe(Effect.provide(Layer.succeed(CaseExecutor.CaseExecutor)(executor)))
+
+process.exitCode = await Effect.runPromise(program)
+```
+
+`Runner.run` needs only the `CaseExecutor` service: scoring runs in process by
+default. Write `baseline.json` once with `Baseline.fromRun` and `Baseline.write`
+before the comparison has anything to read;
+[Quickstart](./quickstart.md) walks both halves of the loop end to end.
+
+## How this relates to @smthrs/agent
+
+What a suite grades is a flow execution, and a `CaseExecutor` is one callback
+from a case to that execution. Anything can sit behind the callback: a plain
+function, a flow, or a whole agent run. That keeps `@smthrs/evals` independent
+of what it measures, and it is why the examples on this site grade a two-line
+greeting.
+
+The parent package, [@smthrs/agent](/api/agent), is what usually sits behind the
+callback in practice. An `AgentAction` or `AgentSession` runs the model-backed
+work, the executor reduces that run to an output and a step key, and the suite
+grades it like any other execution. The evaluation suite that ships with
+Smithers does exactly this: it runs the agent loop against a scripted model with
+no network access, so a score moves only when the agent's behavior moves.
+
+`@smthrs/agent` is in turn one package of the smithers command-line tool,
+[@smthrs/cli](/api/cli), which is where the whole system is driven from.
+
+## Where to go next
+
+- [Installation](./installation.md): what the package needs to run, and its
+  entry points.
+- [Quickstart](./quickstart.md): one complete loop, from suite to gate.
+- How-to guides: [author a suite](./guides/author-a-suite.md),
+  [run a suite](./guides/run-a-suite.md),
+  [maintain a baseline](./guides/maintain-a-baseline.md),
+  [gate a run in CI](./guides/gate-a-run-in-ci.md).
+- Concepts: [fixed suites](./concepts/fixed-suites.md),
+  [step keys and comparison](./concepts/step-keys.md),
+  [determinism and canonical artifacts](./concepts/determinism.md).
+- The [API reference](./api.md) documents every export, the failure codes, and
+  the batch protocol.
+- [Troubleshooting](./troubleshooting.md) maps every failure code to its cause
+  and its fix.