@siftline/core 0.0.2 → 0.1.0

package/dist/index.mjs

@@ -1,16 +1,672 @@
+import { a as entryType, c as parseRecipe, d as recipeSchema, f as score, i as defineRecipe, l as questionName, n as isObjectLike, p as serializeRecipe, r as choice, s as noul, t as parseJsonLines, u as questionSchema } from "./jsonl-Cyqrt9-a.mjs";
+import { z } from "zod";
+//#region package.json
+var version = "0.1.0";
 //#endregion
-//#region src/index.ts
+//#region src/decision.ts
+/** Strict: an unknown key is a bad Record, not a field to ignore. */
+const recordSchema = z.object({
+	id: z.string().min(1),
+	state: entryType,
+	trimmed: z.boolean().optional()
+}).strict();
+const answerValue = z.union([
+	z.string(),
+	z.boolean(),
+	z.number()
+]);
 /**
-* The published version of `@siftline/core`, baked in at build time.
-*
-* Changesets bumps `package.json`; this constant follows it without a second edit.
+* Why `value` is not an answer to `asked`, or `null` when it is. Rules and Fixtures both
+* validate against the Recipe with it, so the two report the same words.
 */
-const VERSION = "0.0.2";
+function answerValueProblem(asked, value) {
+	const shown = JSON.stringify(value);
+	if (asked.type === "choice") {
+		if (typeof value !== "string") return `value ${shown} is not a label`;
+		return Object.hasOwn(asked.criteria, value) ? null : `unknown label ${shown}`;
+	}
+	if (asked.type === "noul") return typeof value === "boolean" ? null : `value ${shown} is not a boolean`;
+	const last = asked.criteria.length - 1;
+	if (typeof value !== "number" || !Number.isInteger(value) || value < 0 || value > last) return `level index ${shown} is out of range (0-${last})`;
+	return null;
+}
+const unit = z.number().min(0).max(1);
+const levelIndex = z.string().regex(/^\d+$/);
+const choiceEvidence = z.object({
+	confidence: unit,
+	probabilities: z.record(z.string().min(1), unit)
+}).strict();
+const noulEvidence = z.object({
+	probability: unit,
+	confidence: unit
+}).strict();
+const scoreEvidence = z.object({
+	score: z.number(),
+	confidence: unit,
+	probabilities: z.record(levelIndex, unit)
+}).strict();
+const evidence = z.union([
+	noulEvidence,
+	scoreEvidence,
+	choiceEvidence
+]);
+const decisionSchema = z.object({
+	format: z.literal(1),
+	id: z.string().min(1),
+	recordId: z.string().min(1),
+	recipe: z.object({
+		name: z.string().min(1),
+		version: z.number().int().min(1)
+	}).strict(),
+	model: z.string().min(1),
+	judgedAt: z.iso.datetime(),
+	trimmed: z.boolean(),
+	answers: z.record(questionName, answerValue),
+	questions: z.record(questionName, evidence),
+	confidence: unit,
+	review: z.boolean(),
+	rule: z.string().min(1).nullable(),
+	action: z.string().min(1).nullable(),
+	usage: z.object({
+		inputTokens: z.number().int().min(0),
+		outputTokens: z.number().int().min(0)
+	}).strict()
+}).strict();
+function parseDecision(line) {
+	return decisionSchema.parse(JSON.parse(line));
+}
+function orderEvidence(block) {
+	if ("probability" in block) return {
+		probability: block.probability,
+		confidence: block.confidence
+	};
+	if ("score" in block) return {
+		score: block.score,
+		confidence: block.confidence,
+		probabilities: block.probabilities
+	};
+	return {
+		confidence: block.confidence,
+		probabilities: block.probabilities
+	};
+}
+/** The only writer. One compact line, no trailing newline, and nothing is rounded. */
+function serializeDecision(decision) {
+	const questions = {};
+	for (const [name, block] of Object.entries(decision.questions)) questions[name] = orderEvidence(block);
+	return JSON.stringify({
+		format: decision.format,
+		id: decision.id,
+		recordId: decision.recordId,
+		recipe: {
+			name: decision.recipe.name,
+			version: decision.recipe.version
+		},
+		model: decision.model,
+		judgedAt: decision.judgedAt,
+		trimmed: decision.trimmed,
+		answers: decision.answers,
+		questions,
+		confidence: decision.confidence,
+		review: decision.review,
+		rule: decision.rule,
+		action: decision.action,
+		usage: {
+			inputTokens: decision.usage.inputTokens,
+			outputTokens: decision.usage.outputTokens
+		}
+	});
+}
+//#endregion
+//#region src/errors.ts
+/** The base every toolkit error extends, in core and in `@siftline/actions`. */
+var SiftlineError = class extends Error {
+	code;
+	retryable;
+	constructor(message, code, retryable, options) {
+		super(message, options);
+		this.name = new.target.name;
+		this.code = code;
+		this.retryable = retryable;
+	}
+};
+//#endregion
+//#region src/rules.ts
+const ruleConditionSchema = z.discriminatedUnion("comparator", [
+	z.object({
+		question: questionName,
+		comparator: z.literal("is"),
+		value: z.union([
+			z.string().min(1),
+			z.boolean(),
+			z.number().int().min(0)
+		])
+	}).strict(),
+	z.object({
+		question: questionName,
+		comparator: z.literal("isOneOf"),
+		value: z.array(z.string().min(1)).min(1)
+	}).strict(),
+	z.object({
+		question: questionName,
+		comparator: z.literal("atLeast"),
+		value: z.number().int().min(0)
+	}).strict(),
+	z.object({
+		question: questionName,
+		comparator: z.literal("atMost"),
+		value: z.number().int().min(0)
+	}).strict()
+]);
+const ruleSchema = z.object({
+	id: z.string().min(1),
+	condition: ruleConditionSchema,
+	action: z.string().min(1).nullable()
+}).strict();
+function matches(answers, condition) {
+	const actual = answers[condition.question];
+	if (actual === void 0) return false;
+	if (condition.comparator === "is") return actual === condition.value;
+	if (condition.comparator === "isOneOf") return typeof actual === "string" && condition.value.includes(actual);
+	if (typeof actual !== "number") return false;
+	return condition.comparator === "atLeast" ? actual >= condition.value : actual <= condition.value;
+}
+/** Pure, first match wins, no review gate. Cloud runs it again on corrected answers. */
+function evaluateRules(answers, rules) {
+	for (const rule of rules) if (matches(answers, rule.condition)) return {
+		rule: rule.id,
+		action: rule.action
+	};
+	return {
+		rule: null,
+		action: null
+	};
+}
+function routeDecision(decision, rules) {
+	if (decision.review) return {
+		...decision,
+		rule: null,
+		action: null
+	};
+	return {
+		...decision,
+		...evaluateRules(decision.answers, rules)
+	};
+}
+const comparatorsFor = {
+	choice: ["is", "isOneOf"],
+	noul: ["is"],
+	score: [
+		"is",
+		"atLeast",
+		"atMost"
+	]
+};
+/** The portal's stale-Rule warning. Never runs inside `evaluateRules`. */
+function validateRules(rules, recipe) {
+	const problems = [];
+	const seen = /* @__PURE__ */ new Set();
+	for (const rule of rules) {
+		const report = (problem) => {
+			problems.push({
+				rule: rule.id,
+				problem
+			});
+		};
+		if (seen.has(rule.id)) report(`duplicate id "${rule.id}"`);
+		seen.add(rule.id);
+		const { question, comparator, value } = rule.condition;
+		const asked = recipe.questions[question];
+		if (!asked) {
+			report(`unknown question "${question}"`);
+			continue;
+		}
+		if (!comparatorsFor[asked.type].includes(comparator)) {
+			report(`comparator "${comparator}" is not valid for a ${asked.type} question`);
+			continue;
+		}
+		for (const candidate of Array.isArray(value) ? value : [value]) {
+			const problem = answerValueProblem(asked, candidate);
+			if (problem !== null) report(problem);
+		}
+	}
+	return problems;
+}
+//#endregion
+//#region src/judge.ts
+/** The gate's width when the caller sets none (cloud ADR 0005). */
+const DEFAULT_MAX_IN_FLIGHT = 8;
 /**
-* Walking-skeleton export. It exists so the build, type, test, pack and publish path
-* can be proven before any Engine behaviour is written, and it will be deleted when
-* the first recipe lands.
+* The client gave up on a 429 or a 529. The fallback delay when `retryAfterMs` is `null`
+* belongs to the caller.
 */
-const PLACEHOLDER = "siftline-core-walking-skeleton";
+var JudgeExhaustedError = class extends SiftlineError {
+	retryAfterMs;
+	constructor(message, retryAfterMs, options) {
+		super(message, "jev_exhausted", true, options);
+		this.retryAfterMs = retryAfterMs;
+	}
+};
+/** Everything else a judged call can fail on. The thrown value stays on `cause`. */
+var JudgeError = class extends SiftlineError {
+	status;
+	reason;
+	constructor(message, reason, status, options) {
+		super(message, "jev_error", reason === "network" || reason === "timeout", options);
+		this.status = status;
+		this.reason = reason;
+	}
+};
+const RETRY_MODES = {
+	prompt: {
+		retry: {
+			maxRetries: 1,
+			backoffMaxMs: 2e3,
+			maxRetryAfterMs: 5e3
+		},
+		timeout: 1e4
+	},
+	patient: {
+		retry: {
+			maxRetries: 5,
+			backoffMaxMs: 3e4,
+			maxRetryAfterMs: 6e4
+		},
+		timeout: 3e4
+	}
+};
+/**
+* FIFO, unbounded wait. A slot is held for the whole `systemOne` call, the SDK's internal
+* retries included, and an aborted waiter leaves the queue without ever taking one.
+*/
+function createGate(limit) {
+	let inFlight = 0;
+	const queue = [];
+	function release() {
+		const next = queue.shift();
+		if (next) next.admit();
+		else inFlight -= 1;
+	}
+	return async (signal) => {
+		signal?.throwIfAborted();
+		if (inFlight < limit) {
+			inFlight += 1;
+			return release;
+		}
+		await new Promise((resolve, reject) => {
+			let onAbort;
+			const waiter = { admit: () => {
+				if (onAbort) signal?.removeEventListener("abort", onAbort);
+				resolve();
+			} };
+			if (signal) {
+				onAbort = () => {
+					const index = queue.indexOf(waiter);
+					if (index >= 0) queue.splice(index, 1);
+					reject(signal.reason);
+				};
+				signal.addEventListener("abort", onAbort, { once: true });
+			}
+			queue.push(waiter);
+		});
+		return release;
+	};
+}
+function errorType(thrown) {
+	const body = thrown["body"];
+	if (!isObjectLike(body)) return void 0;
+	const detail = body["detail"];
+	if (!isObjectLike(detail)) return void 0;
+	const type = detail["error_type"];
+	return typeof type === "string" ? type : void 0;
+}
+function reasonOf(thrown, status) {
+	if (status === null) {
+		const name = thrown["name"];
+		return typeof name === "string" && /timeout/i.test(name) ? "timeout" : "network";
+	}
+	const type = errorType(thrown);
+	if (type === "api_usage_error" || type === "max_tokens_exceeded") return type;
+	return "unknown";
+}
+function isAbort(cause, thrown, signal) {
+	if (signal?.aborted === true && cause === signal.reason) return true;
+	const name = thrown["name"];
+	return name === "AbortError" || name === "APIUserAbortError";
+}
+/** Duck typing, because core cannot import the SDK's error classes. Never returns. */
+function mapClientError(cause, signal) {
+	const thrown = isObjectLike(cause) ? cause : {};
+	if (isAbort(cause, thrown, signal)) throw cause;
+	const rawStatus = thrown["status"];
+	const status = typeof rawStatus === "number" ? rawStatus : null;
+	const rawMessage = thrown["message"];
+	const message = typeof rawMessage === "string" && rawMessage !== "" ? rawMessage : String(cause);
+	if (status === 429 || status === 529) {
+		const retryAfterMs = thrown["retryAfterMs"];
+		throw new JudgeExhaustedError(message, typeof retryAfterMs === "number" ? retryAfterMs : null, { cause });
+	}
+	throw new JudgeError(message, reasonOf(thrown, status), status, { cause });
+}
+function invalidAnswers(message) {
+	return new JudgeError(message, "invalid_answers", null);
+}
+/** Lowest key wins a tie, so the caller reads insertion order as the tie-break. */
+function argmax(probabilities) {
+	let best = "";
+	let top = Number.NEGATIVE_INFINITY;
+	for (const [key, probability] of Object.entries(probabilities)) if (probability > top) {
+		top = probability;
+		best = key;
+	}
+	return best;
+}
+function rebuild(name, keys, source) {
+	const probabilities = {};
+	for (const key of keys) {
+		const probability = source[key];
+		if (typeof probability !== "number") throw invalidAnswers(`question ${name} has no probability for ${key}`);
+		probabilities[key] = probability;
+	}
+	return probabilities;
+}
+function mismatch(name, question, answer) {
+	return invalidAnswers(`question ${name} is a ${question.type}, answered as a ${answer.type}`);
+}
+function mapAnswer(name, question, answer) {
+	if (answer.type === "choice") {
+		if (question.type !== "choice") throw mismatch(name, question, answer);
+		const probabilities = rebuild(name, Object.keys(question.criteria), answer.probabilities);
+		return {
+			answer: argmax(probabilities),
+			evidence: {
+				confidence: answer.confidence,
+				probabilities
+			},
+			confidence: answer.confidence
+		};
+	}
+	if (answer.type === "score") {
+		if (question.type !== "score") throw mismatch(name, question, answer);
+		const probabilities = rebuild(name, question.criteria.map((_criterion, index) => String(index)), answer.probabilities);
+		return {
+			answer: Number(argmax(probabilities)),
+			evidence: {
+				score: answer.score,
+				confidence: answer.confidence,
+				probabilities
+			},
+			confidence: answer.confidence
+		};
+	}
+	if (question.type !== "noul") throw mismatch(name, question, answer);
+	const probability = answer.noul;
+	const confidence = Math.round(Math.abs(2 * probability - 1) * 100) / 100;
+	return {
+		answer: probability >= .5,
+		evidence: {
+			probability,
+			confidence
+		},
+		confidence
+	};
+}
+/** Rebuilt in Recipe insertion order, which is what `serializeDecision` writes out. */
+function mapAnswers(recipe, result) {
+	const names = Object.keys(recipe.questions);
+	const answered = Object.keys(result.answers);
+	if (answered.length !== names.length || !names.every((name) => name in result.answers)) throw invalidAnswers(`the model answered [${answered.join(", ")}], the Recipe asks [${names.join(", ")}]`);
+	const answers = {};
+	const questions = {};
+	const confidences = [];
+	for (const [name, question] of Object.entries(recipe.questions)) {
+		const response = result.answers[name];
+		if (!response) throw invalidAnswers(`question ${name} went unanswered`);
+		const mapped = mapAnswer(name, question, response);
+		answers[name] = mapped.answer;
+		questions[name] = mapped.evidence;
+		confidences.push(mapped.confidence);
+	}
+	return {
+		answers,
+		questions,
+		confidence: Math.min(...confidences)
+	};
+}
+function createJudge(options) {
+	const { client } = options;
+	const { retry, timeout } = RETRY_MODES[options.retry];
+	const now = options.now ?? (() => /* @__PURE__ */ new Date());
+	const acquire = createGate(options.maxInFlight ?? 8);
+	return async (record, recipe, callOptions = {}) => {
+		const { signal } = callOptions;
+		const release = await acquire(signal);
+		let result;
+		try {
+			result = await client.systemOne({
+				state: record.state,
+				questions: recipe.questions,
+				model: recipe.model
+			}, {
+				retry,
+				signal,
+				timeout
+			});
+		} catch (cause) {
+			mapClientError(cause, signal);
+		} finally {
+			release();
+		}
+		const mapped = mapAnswers(recipe, result);
+		const { answers, questions } = mapped;
+		return {
+			format: 1,
+			id: callOptions.id ?? crypto.randomUUID(),
+			recordId: record.id,
+			recipe: {
+				name: recipe.name,
+				version: recipe.version
+			},
+			model: result.model,
+			judgedAt: now().toISOString(),
+			trimmed: record.trimmed ?? false,
+			answers,
+			questions,
+			confidence: mapped.confidence,
+			review: mapped.confidence < recipe.reviewThreshold,
+			rule: null,
+			action: null,
+			usage: {
+				inputTokens: result.usage.input_tokens,
+				outputTokens: result.usage.output_tokens
+			}
+		};
+	};
+}
+//#endregion
+//#region src/fixtures.ts
+const fixtureSchema = z.object({
+	id: z.string().min(1).optional(),
+	origin: z.string().min(1).optional(),
+	by: z.string().min(1).optional(),
+	state: entryType,
+	expect: z.record(questionName, answerValue).refine((e) => Object.keys(e).length >= 1, "at least one expectation")
+}).strict();
+/** A bad line, named by its 1-based position. Core never reads files, so there is no path. */
+var FixtureParseError = class extends SiftlineError {
+	line;
+	constructor(message, line, options) {
+		super(message, "fixture_invalid", false, options);
+		this.line = line;
+	}
+};
+function parseFixture(line) {
+	return fixtureSchema.parse(JSON.parse(line));
+}
+/** The only writer. One compact line, no trailing newline, absent optionals omitted. */
+function serializeFixture(fixture) {
+	const line = {};
+	if (fixture.id !== void 0) line["id"] = fixture.id;
+	if (fixture.origin !== void 0) line["origin"] = fixture.origin;
+	if (fixture.by !== void 0) line["by"] = fixture.by;
+	line["state"] = fixture.state;
+	line["expect"] = fixture.expect;
+	return JSON.stringify(line);
+}
+function parseFixtures(text) {
+	return parseJsonLines(text, parseFixture, (line, cause) => new FixtureParseError(`fixture line ${line} is not a valid Fixture`, line, { cause }));
+}
+var FixtureValidationError = class extends SiftlineError {
+	problems;
+	constructor(problems, options) {
+		super(summarize(problems), "fixture_invalid", false, options);
+		this.problems = problems;
+	}
+};
+/** The first problem, and how many stand behind it. */
+function summarize(problems) {
+	const first = problems[0];
+	if (!first) return "the Fixtures do not fit the Recipe";
+	const head = `${first.fixture}: ${first.problem}`;
+	return problems.length === 1 ? head : `${head} (and ${problems.length - 1} more)`;
+}
+/** The portal's stale-Fixture warning, and what the runner refuses to start on. */
+function validateFixtures(fixtures, recipe) {
+	const problems = [];
+	const seen = /* @__PURE__ */ new Set();
+	for (const [offset, fixture] of fixtures.entries()) {
+		const named = fixture.id ?? String(offset + 1);
+		const report = (problem) => {
+			problems.push({
+				fixture: named,
+				problem
+			});
+		};
+		if (fixture.id !== void 0) {
+			if (seen.has(fixture.id)) report(`duplicate id "${fixture.id}"`);
+			seen.add(fixture.id);
+		}
+		for (const [question, expected] of Object.entries(fixture.expect)) {
+			const asked = recipe.questions[question];
+			if (!asked) {
+				report(`unknown question "${question}"`);
+				continue;
+			}
+			const problem = answerValueProblem(asked, expected);
+			if (problem !== null) report(`${problem} for question "${question}"`);
+		}
+	}
+	return problems;
+}
+function defineFixtures(recipe, fixtures) {
+	for (const fixture of fixtures) fixtureSchema.parse(fixture);
+	const problems = validateFixtures(fixtures, recipe);
+	if (problems.length > 0) throw new FixtureValidationError(problems);
+	return fixtures.map((fixture) => ({
+		...fixture,
+		expect: inRecipeOrder(fixture.expect, recipe)
+	}));
+}
+function inRecipeOrder(expect, recipe) {
+	const ordered = {};
+	for (const question of Object.keys(recipe.questions)) {
+		const expected = expect[question];
+		if (expected !== void 0) ordered[question] = expected;
+	}
+	return ordered;
+}
+/** Pure. Strict equality per type, no tolerance, in `expect` insertion order. */
+function compareAnswers(expect, answers) {
+	const mismatches = [];
+	for (const [question, expected] of Object.entries(expect)) {
+		if (expected === void 0) continue;
+		const actual = answers[question];
+		if (actual !== expected) mismatches.push({
+			question,
+			expected,
+			actual
+		});
+	}
+	return mismatches;
+}
+/**
+* The pure half. A Question absent from a Fixture's `expect` leaves that Question's
+* denominator; an unsure Decision counts on its answers, because the threshold is a routing
+* knob and must not hide errors.
+*/
+function scoreResults(results, recipe) {
+	const questions = {};
+	for (const question of Object.keys(recipe.questions)) questions[question] = {
+		asserted: 0,
+		matched: 0,
+		accuracy: null
+	};
+	for (const result of results) {
+		const missed = new Set(result.mismatches.map((mismatch) => mismatch.question));
+		for (const [question, expected] of Object.entries(result.expect)) {
+			if (expected === void 0) continue;
+			const tally = questions[question];
+			if (!tally) continue;
+			tally.asserted += 1;
+			if (!missed.has(question)) tally.matched += 1;
+		}
+	}
+	const scored = [];
+	for (const tally of Object.values(questions)) {
+		if (tally.asserted === 0) continue;
+		tally.accuracy = tally.matched / tally.asserted;
+		scored.push(tally.accuracy);
+	}
+	return {
+		recipe: {
+			name: recipe.name,
+			version: recipe.version
+		},
+		model: results[0]?.decision.model ?? "",
+		fixtures: results,
+		questions,
+		accuracy: scored.length === 0 ? null : Math.min(...scored)
+	};
+}
+/**
+* Fires every Fixture through the Judge at once and lets the Judge's gate meter concurrency.
+* The first `JudgeError` or `JudgeExhaustedError` aborts the rest and is rethrown: a partial
+* accuracy is worse than none.
+*/
+async function testRecipe(judge, recipe, fixtures, options = {}) {
+	const problems = validateFixtures(fixtures, recipe);
+	if (problems.length > 0) throw new FixtureValidationError(problems);
+	const { onResult } = options;
+	const controller = new AbortController();
+	const signal = options.signal ? AbortSignal.any([options.signal, controller.signal]) : controller.signal;
+	return scoreResults(await Promise.all(fixtures.map(async (fixture, offset) => {
+		const index = offset + 1;
+		const id = fixture.id ?? `fixture:${index}`;
+		let decision;
+		try {
+			decision = await judge({
+				id,
+				state: fixture.state
+			}, recipe, { signal });
+		} catch (cause) {
+			controller.abort(cause);
+			throw cause;
+		}
+		const result = {
+			index,
+			id,
+			decision,
+			expect: fixture.expect,
+			mismatches: compareAnswers(fixture.expect, decision.answers),
+			review: decision.review
+		};
+		onResult?.(result);
+		return result;
+	})), recipe);
+}
+//#endregion
+//#region src/index.ts
+/** The published version of `@siftline/core`, baked in at build time. */
+const VERSION = version;
 //#endregion
-export { PLACEHOLDER, VERSION };
+export { DEFAULT_MAX_IN_FLIGHT, FixtureParseError, FixtureValidationError, JudgeError, JudgeExhaustedError, SiftlineError, VERSION, choice, compareAnswers, createJudge, decisionSchema, defineFixtures, defineRecipe, evaluateRules, fixtureSchema, noul, parseDecision, parseFixture, parseFixtures, parseRecipe, questionSchema, recipeSchema, recordSchema, routeDecision, ruleConditionSchema, ruleSchema, score, scoreResults, serializeDecision, serializeFixture, serializeRecipe, testRecipe, validateFixtures, validateRules };