@siftline/core 0.0.3 → 0.1.1

package/dist/jsonl-CzTtnvyn.mjs

@@ -0,0 +1,139 @@
+import { z } from "zod";
+//#region src/recipe.ts
+function choice(instructions, criteria) {
+	return {
+		type: "choice",
+		instructions,
+		criteria
+	};
+}
+function noul(instructions, criteria) {
+	return criteria ? {
+		type: "noul",
+		instructions,
+		criteria
+	} : {
+		type: "noul",
+		instructions
+	};
+}
+function score(instructions, criteria) {
+	return {
+		type: "score",
+		instructions,
+		criteria
+	};
+}
+const jsonValue = z.lazy(() => z.union([
+	z.string(),
+	z.number(),
+	z.boolean(),
+	z.null(),
+	z.array(jsonValue),
+	z.record(z.string(), jsonValue)
+]));
+const entryType = z.union([
+	z.string(),
+	z.record(z.string(), jsonValue),
+	z.array(jsonValue),
+	z.null()
+]);
+const entry = z.union([
+	z.string().min(1),
+	z.record(z.string(), jsonValue),
+	z.array(jsonValue)
+]);
+const questionName = z.string().regex(/^[a-z][a-z0-9_]*$/);
+const choiceQuestion = z.object({
+	type: z.literal("choice"),
+	instructions: entry,
+	criteria: z.record(z.string().min(1), entry).refine((c) => Object.keys(c).length >= 2, "at least two labels")
+}).strict();
+const noulQuestion = z.object({
+	type: z.literal("noul"),
+	instructions: entry,
+	criteria: z.object({
+		true: entry.optional(),
+		false: entry.optional()
+	}).strict().optional()
+}).strict();
+const scoreQuestion = z.object({
+	type: z.literal("score"),
+	instructions: entry,
+	criteria: z.tuple([entry, entry], entry)
+}).strict();
+const questionSchema = z.discriminatedUnion("type", [
+	choiceQuestion,
+	noulQuestion,
+	scoreQuestion
+]);
+const recipeSchema = z.object({
+	format: z.literal(1),
+	name: z.string().min(1),
+	version: z.number().int().min(1),
+	model: z.string().min(1).refine((m) => m !== "jev-latest" && m !== "jev-preview", "pin a versioned model"),
+	reviewThreshold: z.number().min(0).max(1),
+	questions: z.record(questionName, questionSchema).refine((q) => Object.keys(q).length >= 1, "at least one question")
+}).strict();
+/** Returns the object it built, not `parse`'s copy, whose type is the erased `Recipe`. */
+function defineRecipe(input) {
+	const recipe = {
+		format: 1,
+		name: input.name,
+		version: input.version,
+		model: input.model,
+		reviewThreshold: input.reviewThreshold ?? .7,
+		questions: input.questions
+	};
+	recipeSchema.parse(recipe);
+	return recipe;
+}
+function parseRecipe(text) {
+	return recipeSchema.parse(JSON.parse(text));
+}
+function serializeRecipe(recipe) {
+	const { format, name, version, model, reviewThreshold, questions } = recipe;
+	return `${JSON.stringify({
+		format,
+		name,
+		version,
+		model,
+		reviewThreshold,
+		questions
+	}, null, 2)}\n`;
+}
+//#endregion
+//#region src/thrown.ts
+const thrownSchema = z.object({
+	name: z.string().optional().catch(void 0),
+	message: z.string().optional().catch(void 0),
+	status: z.number().optional().catch(void 0),
+	retryAfterMs: z.number().optional().catch(void 0),
+	requestId: z.string().optional().catch(void 0),
+	body: z.unknown().optional()
+});
+/** Anything that is not an object carries no fields. */
+function decodeThrown(cause) {
+	const decoded = thrownSchema.safeParse(cause);
+	return decoded.success ? decoded.data : {};
+}
+//#endregion
+//#region src/jsonl.ts
+/**
+* One value per non-blank line. A line that `parseLine` throws on is handed to `fail` with
+* its 1-based number, and whatever `fail` returns is thrown.
+*/
+function parseJsonLines(text, parseLine, fail) {
+	const values = [];
+	for (const [offset, line] of text.split("\n").entries()) {
+		if (line.trim() === "") continue;
+		try {
+			values.push(parseLine(line));
+		} catch (cause) {
+			throw fail(offset + 1, cause);
+		}
+	}
+	return values;
+}
+//#endregion
+export { entryType as a, parseRecipe as c, recipeSchema as d, score as f, defineRecipe as i, questionName as l, decodeThrown as n, jsonValue as o, serializeRecipe as p, choice as r, noul as s, parseJsonLines as t, questionSchema as u };

package/dist/testing.d.mts

@@ -1,14 +1,133 @@
+import { c as SystemOneRequest, h as JsonValue, l as SystemOneResult, s as SystemOneClient } from "./client-B_yz9_JS.mjs";
+import { z } from "zod";
 //#region src/testing.d.ts
-/**
- * A stand-in for the TypeSafe client the Engine will judge through. Test code depends
- * on this shape so that swapping the real client in never reaches a test file.
- */
-export interface FakeTypeSafeClient {
-  readonly kind: "fake";
+export declare const replayLineSchema: z.ZodUnion<readonly [z.ZodObject<{
+  format: z.ZodLiteral<1>;
+  id: z.ZodString;
+  recordedAt: z.ZodISODateTime;
+  requestId: z.ZodOptional<z.ZodString>;
+  durationMs: z.ZodNumber;
+  request: z.ZodObject<{
+    state: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodNull]>;
+    questions: z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
+      type: z.ZodLiteral<"choice">;
+      instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>;
+      criteria: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>>;
+    }, z.core.$strict>, z.ZodObject<{
+      type: z.ZodLiteral<"noul">;
+      instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>;
+      criteria: z.ZodOptional<z.ZodObject<{
+        true: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>>;
+        false: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>>;
+      }, z.core.$strict>>;
+    }, z.core.$strict>, z.ZodObject<{
+      type: z.ZodLiteral<"score">;
+      instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>;
+      criteria: z.ZodTuple<[z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>, z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>], z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>>;
+    }, z.core.$strict>], "type">>;
+    model: z.ZodString;
+  }, z.core.$strict>;
+  response: z.ZodObject<{
+    model: z.ZodString;
+    answers: z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
+      type: z.ZodLiteral<"choice">;
+      choice: z.ZodString;
+      confidence: z.ZodNumber;
+      probabilities: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    }, z.core.$strict>, z.ZodObject<{
+      type: z.ZodLiteral<"noul">;
+      noul: z.ZodNumber;
+    }, z.core.$strict>, z.ZodObject<{
+      type: z.ZodLiteral<"score">;
+      score: z.ZodNumber;
+      confidence: z.ZodNumber;
+      legend: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
+      probabilities: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    }, z.core.$strict>], "type">>;
+    usage: z.ZodObject<{
+      input_tokens: z.ZodNumber;
+      output_tokens: z.ZodNumber;
+    }, z.core.$strict>;
+  }, z.core.$strict>;
+}, z.core.$strict>, z.ZodObject<{
+  format: z.ZodLiteral<1>;
+  id: z.ZodString;
+  recordedAt: z.ZodISODateTime;
+  requestId: z.ZodOptional<z.ZodString>;
+  durationMs: z.ZodNumber;
+  request: z.ZodObject<{
+    state: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodNull]>;
+    questions: z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
+      type: z.ZodLiteral<"choice">;
+      instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>;
+      criteria: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>>;
+    }, z.core.$strict>, z.ZodObject<{
+      type: z.ZodLiteral<"noul">;
+      instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>;
+      criteria: z.ZodOptional<z.ZodObject<{
+        true: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>>;
+        false: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>>;
+      }, z.core.$strict>>;
+    }, z.core.$strict>, z.ZodObject<{
+      type: z.ZodLiteral<"score">;
+      instructions: z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>;
+      criteria: z.ZodTuple<[z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>, z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>], z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>, z.ZodArray<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>]>>;
+    }, z.core.$strict>], "type">>;
+    model: z.ZodString;
+  }, z.core.$strict>;
+  error: z.ZodObject<{
+    name: z.ZodString;
+    message: z.ZodString;
+    status: z.ZodNullable<z.ZodNumber>;
+    retryAfterMs: z.ZodNullable<z.ZodNumber>;
+    requestId: z.ZodOptional<z.ZodString>;
+    body: z.ZodUnknown;
+  }, z.core.$strict>;
+}, z.core.$strict>]>;
+export interface ReplayError {
+  name: string;
+  message: string;
+  status: number | null;
+  retryAfterMs: number | null;
+  requestId?: string;
+  body?: unknown;
+}
+interface ReplayLineHead {
+  format: 1;
+  id: string;
+  recordedAt: string;
+  requestId?: string;
+  durationMs: number;
+  request: SystemOneRequest;
+}
+export type ReplayLine = (ReplayLineHead & {
+  response: SystemOneResult;
+}) | (ReplayLineHead & {
+  error: ReplayError;
+});
+/** Parses JSONL, skipping blank lines. A bad line throws naming its 1-based number. */
+export declare function parseReplayLines(text: string): ReplayLine[];
+export type ScriptStep = {
+  response: SystemOneResult;
+} | {
+  error: unknown;
+};
+export interface ScriptedClient extends SystemOneClient {
+  /** The requests received so far, in call order. */
+  readonly calls: readonly SystemOneRequest[];
 }
 /**
- * Walking-skeleton factory for the `./testing` entry. It proves the second export
- * condition resolves for consumers; the real fake arrives with the Engine.
+ * Answers in call order. The gate is FIFO, so a script and a judged batch line up.
+ * Failures reject rather than throw synchronously, as a real client does.
  */
-export declare function createFakeTypeSafeClient(): FakeTypeSafeClient;
+export declare function createScriptedClient(script: readonly ScriptStep[]): ScriptedClient;
+/** Answers the line whose `request` is deep-equal to the incoming one. Key order is free. */
+export declare function createReplayClient(lines: readonly ReplayLine[]): SystemOneClient;
+export interface RecordingOptions {
+  /** The `id` written on each line. Defaults to the 1-based call index. */
+  id?: (request: SystemOneRequest, index: number) => string;
+  now?: () => Date;
+}
+/** Wraps a live client and hands one replay line per call to `sink`. The live test's half. */
+export declare function createRecordingClient(inner: SystemOneClient, sink: (line: ReplayLine) => void, options?: RecordingOptions): SystemOneClient;
 //#endregion

package/dist/testing.mjs

@@ -1,10 +1,163 @@
+import { a as entryType, n as decodeThrown, o as jsonValue, t as parseJsonLines, u as questionSchema } from "./jsonl-CzTtnvyn.mjs";
+import { z } from "zod";
 //#region src/testing.ts
+const probability = z.number().min(0).max(1);
+const probabilities = z.record(z.string().min(1), z.number());
+const answerSchema = z.discriminatedUnion("type", [
+	z.object({
+		type: z.literal("choice"),
+		choice: z.string().min(1),
+		confidence: probability,
+		probabilities
+	}).strict(),
+	z.object({
+		type: z.literal("noul"),
+		noul: probability
+	}).strict(),
+	z.object({
+		type: z.literal("score"),
+		score: z.number(),
+		confidence: probability,
+		legend: z.record(z.string().min(1), jsonValue).optional(),
+		probabilities
+	}).strict()
+]);
+const systemOneRequestSchema = z.object({
+	state: entryType,
+	questions: z.record(z.string().min(1), questionSchema),
+	model: z.string().min(1)
+}).strict();
+const systemOneResultSchema = z.object({
+	model: z.string().min(1),
+	answers: z.record(z.string().min(1), answerSchema),
+	usage: z.object({
+		input_tokens: z.number().int().nonnegative(),
+		output_tokens: z.number().int().nonnegative()
+	}).strict()
+}).strict();
+const replayErrorSchema = z.object({
+	name: z.string().min(1),
+	message: z.string(),
+	status: z.number().int().nullable(),
+	retryAfterMs: z.number().int().nonnegative().nullable(),
+	requestId: z.string().min(1).optional(),
+	body: z.unknown()
+}).strict();
+const replayLineFields = {
+	format: z.literal(1),
+	id: z.string().min(1),
+	recordedAt: z.iso.datetime(),
+	requestId: z.string().min(1).optional(),
+	durationMs: z.number().int().nonnegative(),
+	request: systemOneRequestSchema
+};
+const replayLineSchema = z.union([z.object({
+	...replayLineFields,
+	response: systemOneResultSchema
+}).strict(), z.object({
+	...replayLineFields,
+	error: replayErrorSchema
+}).strict()]);
+/** Parses JSONL, skipping blank lines. A bad line throws naming its 1-based number. */
+function parseReplayLines(text) {
+	return parseJsonLines(text, (line) => replayLineSchema.parse(JSON.parse(line)), (line, cause) => cause instanceof SyntaxError ? new Error(`replay line ${line} is not JSON`, { cause }) : new Error(`replay line ${line} is invalid: ${messageOf(cause)}`, { cause }));
+}
+function messageOf(cause) {
+	return cause instanceof Error ? cause.message : String(cause);
+}
 /**
-* Walking-skeleton factory for the `./testing` entry. It proves the second export
-* condition resolves for consumers; the real fake arrives with the Engine.
+* Answers in call order. The gate is FIFO, so a script and a judged batch line up.
+* Failures reject rather than throw synchronously, as a real client does.
 */
-function createFakeTypeSafeClient() {
-	return { kind: "fake" };
+function createScriptedClient(script) {
+	const calls = [];
+	return {
+		calls,
+		systemOne: async (request) => {
+			const step = script[calls.length];
+			calls.push(request);
+			if (!step) throw new Error(`scripted client exhausted after ${script.length} calls`);
+			if ("error" in step) throw step.error;
+			return step.response;
+		}
+	};
+}
+/** Objects write their keys sorted, so two values that differ only in key order match. */
+function canonicalJson(value) {
+	if (value === null || !(value instanceof Object)) return JSON.stringify(value);
+	if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+	return `{${Object.entries(value).toSorted(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([key, member]) => `${JSON.stringify(key)}:${canonicalJson(member)}`).join(",")}}`;
+}
+function requestKey(request) {
+	return canonicalJson(jsonValue.parse(JSON.parse(JSON.stringify(request))));
+}
+/** Rebuilds what the SDK threw closely enough for the Judge's duck typing to see it. */
+function replayError(recorded) {
+	const error = new Error(recorded.message);
+	error.name = recorded.name;
+	return Object.assign(error, {
+		status: recorded.status,
+		retryAfterMs: recorded.retryAfterMs,
+		requestId: recorded.requestId,
+		body: recorded.body
+	});
+}
+/** Answers the line whose `request` is deep-equal to the incoming one. Key order is free. */
+function createReplayClient(lines) {
+	const keyed = lines.map((line) => ({
+		key: requestKey(line.request),
+		line
+	}));
+	return { systemOne: async (request) => {
+		const key = requestKey(request);
+		const line = keyed.find((candidate) => candidate.key === key)?.line;
+		if (!line) throw new Error(`no replay line matches the request for model ${request.model} and questions ${Object.keys(request.questions).join(", ")}`);
+		if ("error" in line) throw replayError(line.error);
+		return line.response;
+	} };
+}
+/** Core cannot name the SDK's error classes, so it decodes their fields. */
+function recordError(cause) {
+	const thrown = decodeThrown(cause);
+	return {
+		name: thrown.name ?? "Error",
+		message: thrown.message ?? String(cause),
+		status: thrown.status ?? null,
+		retryAfterMs: thrown.retryAfterMs ?? null,
+		requestId: thrown.requestId,
+		body: thrown.body
+	};
+}
+/** Wraps a live client and hands one replay line per call to `sink`. The live test's half. */
+function createRecordingClient(inner, sink, options = {}) {
+	const now = options.now ?? (() => /* @__PURE__ */ new Date());
+	const nextId = options.id ?? ((_request, index) => `${index}`);
+	let calls = 0;
+	return { systemOne: async (request, callOptions) => {
+		const startedAt = now();
+		const head = {
+			format: 1,
+			id: nextId(request, ++calls),
+			recordedAt: startedAt.toISOString(),
+			request
+		};
+		try {
+			const response = await inner.systemOne(request, callOptions);
+			sink({
+				...head,
+				durationMs: now().getTime() - startedAt.getTime(),
+				response
+			});
+			return response;
+		} catch (cause) {
+			sink({
+				...head,
+				durationMs: now().getTime() - startedAt.getTime(),
+				error: recordError(cause)
+			});
+			throw cause;
+		}
+	} };
 }
 //#endregion
-export { createFakeTypeSafeClient };
+export { createRecordingClient, createReplayClient, createScriptedClient, parseReplayLines, replayLineSchema };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@siftline/core",
-  "version": "0.0.3",
+  "version": "0.1.1",
   "description": "The Siftline engine: recipes, rules, fixtures and the judge wrapper.",
   "keywords": [
     "classification",
@@ -51,6 +51,7 @@
   "devDependencies": {
     "@arethetypeswrong/core": "0.18.5",
     "@siftline/config": "workspace:*",
+    "@typesafe-ai/sdk": "0.6.0",
     "publint": "0.3.24",
     "tsdown": "0.23.0",
     "typescript": "7.0.2",