jev-recipes 0.9.5 → 0.9.7

package/dist/adapters/ai-sdk/index.d.ts

@@ -1,8 +1,8 @@
-import type { LanguageModel, PrepareStepFunction, StopCondition, Tool, ToolSet } from 'ai';
+import type { FlexibleSchema, LanguageModel, PrepareStepFunction, StopCondition, Tool, ToolSet } from 'ai';
 import type { RecipeName } from '../../catalog/schema.js';
 import type { ModelRouteResult } from '../../recipes/model-route/index.js';
 import type { RecipeOptions } from '../../src/schema.js';
-import type { CompletionOptions, CompletionToolOutput, GuardOptions, RecipeToolOptions } from '../shared.js';
+import type { BlockedToolOutput, CompletionOptions, CompletionToolOutput, GuardOptions, RecipeToolOptions } from '../shared.js';
 export type { BlockedToolOutput, CompletionEvent, CompletionOptions, CompletionToolOutput, GuardEvent, GuardOptions, RecipeToolOptions, } from '../shared.js';
 export type RecipeTool = Tool<Record<string, unknown>, Record<string, unknown>>;
 export declare function recipeTool(name: RecipeName, options?: RecipeToolOptions): RecipeTool;
@@ -10,7 +10,20 @@ export declare function recipeTools(names: readonly RecipeName[], options?: Reci
 export type GuardToolsOptions = GuardOptions & {
     request?: string;
 };
-export declare function guardTools<TOOLS extends ToolSet>(tools: TOOLS, options?: GuardToolsOptions): TOOLS;
+type ToolExecution<DEFINITION extends ToolSet[string]> = NonNullable<DEFINITION['execute']>;
+type ToolOutput<RESULT> = RESULT extends AsyncIterable<infer OUTPUT> ? OUTPUT : Awaited<RESULT>;
+type GuardedOutput<DEFINITION extends ToolSet[string]> = ToolOutput<ReturnType<ToolExecution<DEFINITION>>> | BlockedToolOutput;
+type GuardedExecution<DEFINITION extends ToolSet[string]> = {
+    [KEY in keyof Pick<DEFINITION, 'execute'>]: (...args: Parameters<ToolExecution<DEFINITION>>) => AsyncGenerator<GuardedOutput<DEFINITION>>;
+};
+type GuardedTool<DEFINITION extends ToolSet[string]> = DEFINITION extends unknown ? Omit<DEFINITION, 'execute' | 'toModelOutput' | 'outputSchema'> & GuardedExecution<DEFINITION> & {
+    outputSchema?: FlexibleSchema<GuardedOutput<DEFINITION>>;
+    toModelOutput?: NonNullable<Tool<Parameters<ToolExecution<DEFINITION>>[0], GuardedOutput<DEFINITION>>['toModelOutput']>;
+} : never;
+export type GuardedTools<TOOLS extends ToolSet> = {
+    [NAME in keyof TOOLS]: 'execute' extends keyof TOOLS[NAME] ? ToolExecution<TOOLS[NAME]> extends never ? TOOLS[NAME] : GuardedTool<TOOLS[NAME]> : TOOLS[NAME];
+};
+export declare function guardTools<TOOLS extends ToolSet>(tools: TOOLS, options?: GuardToolsOptions): GuardedTools<TOOLS>;
 export type ModelCandidate = {
     id: string;
     text: string;

package/dist/adapters/ai-sdk/index.js

@@ -1,6 +1,8 @@
-import { jsonSchema, tool } from 'ai';
+import { asSchema, jsonSchema, tool } from 'ai';
+import { z } from 'zod';
 import { modelRoute } from '../../recipes/model-route/index.js';
-import { COMPLETION_TOOL_DESCRIPTION, COMPLETION_TOOL_INPUT_SCHEMA, DEFAULT_COMPLETION_TOOL_NAME, blockedOutput, defineRecipeTool, messageText, requireText, reviewCompletion, reviewToolCall, } from '../shared.js';
+import { toolCallGateResultSchema } from '../../recipes/tool-call-gate/index.js';
+import { BLOCKED_TOOL_OUTPUT_KIND, COMPLETION_TOOL_DESCRIPTION, COMPLETION_TOOL_INPUT_SCHEMA, DEFAULT_COMPLETION_TOOL_NAME, blockedOutput, defineRecipeTool, messageText, requireText, reviewCompletion, reviewToolCall, } from '../shared.js';
 export function recipeTool(name, options = {}) {
     const definition = defineRecipeTool(name, options);
     return tool({
@@ -13,27 +15,75 @@ export function recipeTools(names, options = {}) {
     return Object.fromEntries(names.map((name) => [name, recipeTool(name, options)]));
 }
 export function guardTools(tools, options = {}) {
+    return Object.fromEntries(Object.entries(tools).map(([name, definition]) => [name, guardTool(name, definition, options)]));
+}
+function guardTool(toolName, definition, options) {
+    const execute = definition.execute;
+    if (typeof execute !== 'function')
+        return definition;
     const { request, ...guardOptions } = options;
-    const guarded = Object.entries(tools).map(([toolName, definition]) => {
-        if (!('execute' in definition) || typeof definition.execute !== 'function')
-            return [toolName, definition];
-        const execute = definition.execute;
-        const wrapped = {
-            ...definition,
-            execute: async (input, executionOptions) => {
-                const decision = await reviewToolCall({
-                    toolName,
-                    input,
-                    request: request ?? requireText(lastUserText(executionOptions.messages), 'request'),
-                }, guardOptions, executionOptions.abortSignal);
-                if (decision.action !== 'allow')
-                    return blockedOutput(toolName, decision);
-                return execute(input, executionOptions);
-            },
-        };
-        return [toolName, wrapped];
+    const toModelOutput = definition.toModelOutput;
+    return {
+        ...definition,
+        execute: async function* (input, executionOptions) {
+            guardOptions.signal?.throwIfAborted();
+            executionOptions.abortSignal?.throwIfAborted();
+            const decision = await reviewToolCall({
+                toolName,
+                input,
+                request: request ?? requireText(lastUserText(executionOptions.messages), 'request'),
+            }, guardOptions, executionOptions.abortSignal);
+            guardOptions.signal?.throwIfAborted();
+            executionOptions.abortSignal?.throwIfAborted();
+            if (decision.action !== 'allow') {
+                yield blockedOutput(toolName, decision);
+                return;
+            }
+            const output = execute(input, executionOptions);
+            if (isAsyncIterable(output))
+                yield* output;
+            else
+                yield await output;
+        },
+        ...(toModelOutput === undefined
+            ? {}
+            : {
+                toModelOutput: (result) => blockedToolOutputSchema.safeParse(result.output).success
+                    ? { type: 'json', value: result.output }
+                    : toModelOutput(result),
+            }),
+        ...(definition.outputSchema === undefined
+            ? {}
+            : { outputSchema: includeBlockedOutput(definition.outputSchema) }),
+    };
+}
+function isAsyncIterable(value) {
+    return (typeof value?.[Symbol.asyncIterator] ===
+        'function');
+}
+const blockedToolOutputSchema = toolCallGateResultSchema
+    .pick({ verdict: true, status: true, detected: true, confidence: true })
+    .extend({
+    kind: z.literal(BLOCKED_TOOL_OUTPUT_KIND),
+    blocked: z.literal(true),
+    action: z.enum(['ask', 'deny']),
+    message: z.string(),
+});
+function includeBlockedOutput(outputSchema) {
+    const original = asSchema(outputSchema);
+    return jsonSchema(async () => ({
+        anyOf: [
+            { $id: 'urn:jev-recipes:tool-output', ...(await original.jsonSchema) },
+            z.toJSONSchema(blockedToolOutputSchema),
+        ],
+    }), {
+        validate: async (value) => {
+            const blocked = blockedToolOutputSchema.safeParse(value);
+            if (blocked.success)
+                return { success: true, value: blocked.data };
+            return original.validate ? original.validate(value) : { success: true, value };
+        },
     });
-    return Object.fromEntries(guarded);
 }
 export function routeModelStep(options) {
     const { candidates, request, context, minConfidence, fallback, onDecision, ...recipeOptions } = options;

package/dist/adapters/langchain/index.d.ts

@@ -1,11 +1,4 @@
-import type { DynamicStructuredTool } from '@langchain/core/tools';
-export type GuardableTool = {
-    name: string;
-    description: string;
-    schema: unknown;
-    returnDirect?: boolean;
-    invoke(input: any, config?: any): Promise<unknown>;
-};
+import { DynamicStructuredTool } from '@langchain/core/tools';
 import type { RecipeName } from '../../catalog/schema.js';
 import { COMPLETION_TOOL_INPUT_SCHEMA } from '../shared.js';
 import type { CompletionOptions, GuardOptions, RecipeToolOptions } from '../shared.js';
@@ -18,6 +11,13 @@ export type GuardRequest = string | ((input: unknown, config: unknown) => string
 export type GuardToolsOptions = GuardOptions & {
     request: GuardRequest;
 };
+export type GuardableTool = {
+    name: string;
+    description: string;
+    schema: unknown;
+    returnDirect?: boolean;
+    invoke(input: any, config?: any): Promise<unknown>;
+};
 export declare function guardTools(tools: readonly GuardableTool[], options: GuardToolsOptions): DynamicStructuredTool[];
 export declare function completionTool(options: CompletionOptions & {
     task: string;

package/dist/adapters/langchain/index.js

@@ -1,4 +1,5 @@
-import { tool } from '@langchain/core/tools';
+import { DynamicStructuredTool, tool } from '@langchain/core/tools';
+import { ToolMessage } from '@langchain/core/messages';
 import { COMPLETION_TOOL_DESCRIPTION, COMPLETION_TOOL_INPUT_SCHEMA, DEFAULT_COMPLETION_TOOL_NAME, blockedOutput, defineRecipeTool, requireText, reviewCompletion, reviewToolCall, } from '../shared.js';
 export function recipeTool(name, options = {}) {
     const definition = defineRecipeTool(name, options);
@@ -12,26 +13,53 @@ export function recipeTools(names, options = {}) {
     return names.map((name) => recipeTool(name, options));
 }
 export function guardTools(tools, options) {
-    const { request, ...guardOptions } = options;
-    return tools.map((original) => {
-        const schema = original.schema;
-        return tool(async (input, config) => {
-            const decision = await reviewToolCall({
-                toolName: original.name,
-                input,
-                request: typeof request === 'function' ? request(input, config) : request,
-            }, guardOptions, config?.signal);
-            if (decision.action !== 'allow')
-                return blockedOutput(original.name, decision);
-            const { toolCall: _toolCall, ...delegatedConfig } = config ?? {};
-            return original.invoke(input, delegatedConfig);
-        }, {
+    return tools.map((original) => new GuardedTool(original, options));
+}
+class GuardedTool extends DynamicStructuredTool {
+    original;
+    guardOptions;
+    constructor(original, guardOptions) {
+        super({
             name: original.name,
             description: original.description,
-            schema,
-            ...(original.returnDirect === undefined ? {} : { returnDirect: original.returnDirect }),
+            schema: original.schema,
+            returnDirect: original.returnDirect ?? false,
+            func: (input, _runManager, config) => this.invoke(input, config),
         });
-    });
+        this.original = original;
+        this.guardOptions = guardOptions;
+    }
+    async invoke(input, config) {
+        const { request, ...options } = this.guardOptions;
+        const args = isToolCall(input) ? input.args : input;
+        const requestConfig = isToolCall(input) ? { ...config, toolCall: input } : config;
+        options.signal?.throwIfAborted();
+        config?.signal?.throwIfAborted();
+        const decision = await reviewToolCall({
+            toolName: this.name,
+            input: args,
+            request: typeof request === 'function' ? request(args, requestConfig) : request,
+        }, options, config?.signal);
+        options.signal?.throwIfAborted();
+        config?.signal?.throwIfAborted();
+        if (decision.action === 'allow')
+            return this.original.invoke(input, config);
+        const output = blockedOutput(this.name, decision);
+        const toolCallId = requestConfig?.toolCall?.id;
+        return toolCallId === undefined
+            ? output
+            : new ToolMessage({
+                name: this.name,
+                tool_call_id: toolCallId,
+                content: JSON.stringify(output),
+            });
+    }
+    call(input, config, tags) {
+        return this.invoke(input, tags === undefined ? config : { ...config, tags: config?.tags ?? tags });
+    }
+}
+function isToolCall(input) {
+    return (typeof input === 'object' && input !== null && 'type' in input && input.type === 'tool_call');
 }
 export function completionTool(options) {
     return tool((input, config) => reviewCompletion({ task: requireText(options.task, 'task'), report: input.report, evidence: input.evidence }, options, config?.signal), {

package/dist/adapters/shared.d.ts

@@ -22,7 +22,9 @@ export type GuardEvent = {
     input: unknown;
     decision: ToolCallGateResult;
 };
+export declare const BLOCKED_TOOL_OUTPUT_KIND = "jev-recipes/blocked-tool-output";
 export type BlockedToolOutput = {
+    kind: typeof BLOCKED_TOOL_OUTPUT_KIND;
     blocked: true;
     action: 'ask' | 'deny';
     verdict: ToolCallGateResult['verdict'];

package/dist/adapters/shared.js

@@ -2,6 +2,7 @@ import { describeRecipe } from '../catalog/index.js';
 import { loadRecipe } from '../catalog/runner.js';
 import { completionGate } from '../recipes/completion-gate/index.js';
 import { toolCallGate } from '../recipes/tool-call-gate/index.js';
+export const BLOCKED_TOOL_OUTPUT_KIND = 'jev-recipes/blocked-tool-output';
 export const DEFAULT_COMPLETION_TOOL_NAME = 'report_completion';
 export const COMPLETION_TOOL_DESCRIPTION = 'Report that the task is complete. State what was done in report, and put the evidence that demonstrates it, such as test output, command results, or observed behavior, in evidence. A claim without evidence is not accepted.';
 export const COMPLETION_TOOL_INPUT_SCHEMA = {
@@ -54,6 +55,7 @@ export function blockedOutput(toolName, decision) {
         ? `The call to ${toolName} was refused by policy and did not run.${risks} Do not retry it. Explain what you intended and continue with permitted work.`
         : `The call to ${toolName} did not run because a person must confirm it first.${risks} Do not retry it unattended. Explain what you intended and continue with other permitted work.`;
     return {
+        kind: BLOCKED_TOOL_OUTPUT_KIND,
         blocked: true,
         action,
         verdict: decision.verdict,

package/dist/evaluation/archive.d.ts

@@ -142,9 +142,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;
@@ -225,9 +225,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;
@@ -328,9 +328,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;
@@ -411,9 +411,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;

package/dist/evaluation/report.d.ts

@@ -29,9 +29,9 @@ export declare function buildReport(id: string, rows: EvaluationRow[], confidenc
     }[];
     thresholdEvaluation: string;
     thresholds: {
+        readyAccuracy: number | null;
         minConfidence: number;
         deferRate: number | null;
-        readyAccuracy: number | null;
         readyCases: number;
         reviewCases: number;
         failedCases: number;
@@ -112,9 +112,9 @@ export declare function buildReport(id: string, rows: EvaluationRow[], confidenc
     }[];
     thresholdEvaluation: string;
     thresholds: {
+        readyAccuracy: number | null;
         minConfidence: number;
         deferRate: number | null;
-        readyAccuracy: number | null;
         readyCases: number;
         reviewCases: number;
         failedCases: number;

package/dist/evaluation/report.js

@@ -1,26 +1,34 @@
 import { isRecord, roundedTo } from './decisions.js';
 import { matchesExpected } from './comparison.js';
+import { resultMetadataSchema } from '../src/schema.js';
 export function buildReport(id, rows, confidenceThresholds, price, evidence) {
     const completed = rows.filter((row) => row.error === undefined);
     const ready = completed.filter((row) => row.ready);
-    const thresholds = confidenceThresholds.map((minConfidence) => {
+    const thresholdEvaluations = confidenceThresholds.map((minConfidence) => {
         const decisions = completed
             .flatMap((row) => row.decisions)
             .filter((decision) => decision.minConfidence === minConfidence);
         const accepted = decisions.filter((decision) => decision.ready);
+        const correctReadyCases = accepted.filter((decision) => decision.correct).length;
         return {
             minConfidence,
             deferRate: completed.length
                 ? roundedTo(3, (completed.length - accepted.length) / completed.length)
                 : null,
-            readyAccuracy: accuracyOf(accepted),
+            readyAccuracy: accepted.length ? correctReadyCases / accepted.length : null,
             readyCases: accepted.length,
             reviewCases: completed.length - accepted.length,
             failedCases: rows.length - completed.length,
-            readyAccuracyInterval95: wilsonInterval(accepted.filter((entry) => entry.correct).length, accepted.length),
+            readyAccuracyInterval95: wilsonInterval(correctReadyCases, accepted.length),
         };
     });
-    const usage = usageOf(rows);
+    const suggestedMinConfidence = thresholdEvaluations.find((entry) => entry.readyAccuracy !== null && entry.readyAccuracy >= 0.95)?.minConfidence ?? null;
+    const thresholds = thresholdEvaluations.map((threshold) => ({
+        ...threshold,
+        readyAccuracy: threshold.readyAccuracy === null ? null : roundedTo(3, threshold.readyAccuracy),
+    }));
+    const usage = summarizeUsage(rows);
+    const cost = estimateKnownCost(usage, price);
     const models = [...new Set(rows.flatMap((row) => (row.model ? [row.model] : [])))].sort();
     return {
         ...(evidence ? { evidence } : {}),
@@ -60,21 +68,10 @@ export function buildReport(id, rows, confidenceThresholds, price, evidence) {
                 ? 'recipe-replay'
                 : 'not-applicable',
         thresholds,
-        suggestedMinConfidence: thresholds.find((entry) => entry.readyAccuracy !== null && entry.readyAccuracy >= 0.95)
-            ?.minConfidence ?? null,
+        suggestedMinConfidence,
         latencyMs: latencyOf(rows.map((row) => row.durationMs)),
         usage,
-        cost: price &&
-            usage.models.every((model) => model === price.model) &&
-            usage.unknownResponses === 0 &&
-            usage.failedRequests === 0
-            ? {
-                ...price,
-                estimated: roundedTo(8, (usage.input_tokens * price.inputPerMillion +
-                    usage.output_tokens * price.outputPerMillion) /
-                    1_000_000),
-            }
-            : null,
+        cost,
         confusion: confusionOf(rows),
         failures: rows
             .filter((row) => !row.correct)
@@ -126,7 +123,7 @@ function latencyOf(values) {
         totalCaseTime: roundedTo(2, values.reduce((sum, value) => sum + value, 0)),
     };
 }
-function usageOf(rows) {
+function summarizeUsage(rows) {
     let input_tokens = 0, output_tokens = 0, unknownResponses = 0, failedRequests = 0;
     const models = new Set();
     const exchanges = rows.flatMap((row) => row.exchanges);
@@ -135,18 +132,17 @@ function usageOf(rows) {
             failedRequests++;
             continue;
         }
-        const response = exchange.response;
-        if (isRecord(response) && typeof response.model === 'string')
-            models.add(response.model);
-        if (!isRecord(response) ||
-            !isRecord(response.usage) ||
-            typeof response.usage.input_tokens !== 'number' ||
-            typeof response.usage.output_tokens !== 'number') {
+        const response = isRecord(exchange.response) ? exchange.response : {};
+        const usage = resultMetadataSchema.shape.usage.safeParse(response.usage);
+        const model = resultMetadataSchema.shape.model.safeParse(response.model);
+        if (!usage.success || !model.success)
             unknownResponses++;
+        if (model.success)
+            models.add(model.data);
+        if (!usage.success)
             continue;
-        }
-        input_tokens += response.usage.input_tokens;
-        output_tokens += response.usage.output_tokens;
+        input_tokens += usage.data.input_tokens;
+        output_tokens += usage.data.output_tokens;
     }
     return {
         requests: exchanges.length,
@@ -157,6 +153,18 @@ function usageOf(rows) {
         models: [...models].sort(),
     };
 }
+function estimateKnownCost(usage, price) {
+    if (!price ||
+        usage.unknownResponses > 0 ||
+        usage.failedRequests > 0 ||
+        usage.models.some((model) => model !== price.model))
+        return null;
+    return {
+        ...price,
+        estimated: roundedTo(8, (usage.input_tokens * price.inputPerMillion + usage.output_tokens * price.outputPerMillion) /
+            1_000_000),
+    };
+}
 function confusionOf(rows) {
     const byExpected = new Map();
     for (const row of rows) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jev-recipes",
-  "version": "0.9.5",
+  "version": "0.9.7",
   "description": "248 small Jev decisions for JavaScript and TypeScript: route, rerank, gate, grade, compare, and label text for agents, RAG, support, code review, and music. Each returns a typed result with explicit uncertainty. Plug into Vercel AI SDK or LangChain agents, import one recipe, or run the CLI with JSON from any language.",
   "type": "module",
   "license": "MIT",