jev-recipes 0.9.7 → 0.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -90,8 +90,9 @@ export function routeModelStep(options) {
|
|
|
90
90
|
if (!candidates.length)
|
|
91
91
|
throw new Error('Supply at least one model candidate.');
|
|
92
92
|
const decisions = new WeakMap();
|
|
93
|
-
return async ({ initialMessages }) => {
|
|
94
|
-
|
|
93
|
+
return async ({ initialMessages, steps }) => {
|
|
94
|
+
// The SDK owns one steps array per generation, even when callers reuse messages.
|
|
95
|
+
if (!decisions.has(steps)) {
|
|
95
96
|
const decision = await modelRoute({
|
|
96
97
|
request: request ?? requireText(firstUserText(initialMessages), 'request'),
|
|
97
98
|
models: candidates.map(({ id, text }) => ({ id, text })),
|
|
@@ -102,9 +103,9 @@ export function routeModelStep(options) {
|
|
|
102
103
|
? candidates.find((candidate) => candidate.id === decision.selection)?.model
|
|
103
104
|
: fallback;
|
|
104
105
|
onDecision?.({ decision, model: selected });
|
|
105
|
-
decisions.set(
|
|
106
|
+
decisions.set(steps, selected);
|
|
106
107
|
}
|
|
107
|
-
const model = decisions.get(
|
|
108
|
+
const model = decisions.get(steps);
|
|
108
109
|
return model === undefined ? {} : { model };
|
|
109
110
|
};
|
|
110
111
|
}
|
|
@@ -3,6 +3,7 @@ type Schema = Record<string, unknown>;
|
|
|
3
3
|
export declare function resultSchemaFor(recipe: string): Schema;
|
|
4
4
|
export declare function comparableDecision(value: unknown, recipe: string): unknown;
|
|
5
5
|
export declare function matchesExpected(actual: unknown, expected: unknown, recipe: string): boolean;
|
|
6
|
+
export declare function isReady(result: unknown, recipe: string): boolean;
|
|
6
7
|
export declare function isMetadataPath(schema: Schema, segments: string[]): boolean;
|
|
7
8
|
export declare function schemaAtPath(schema: Schema, segments: string[]): Schema | undefined;
|
|
8
9
|
export {};
|
|
@@ -33,6 +33,25 @@ export function comparableDecision(value, recipe) {
|
|
|
33
33
|
export function matchesExpected(actual, expected, recipe) {
|
|
34
34
|
return isDeepStrictEqual(comparableDecision(actual, recipe), comparableDecision(expected, recipe));
|
|
35
35
|
}
|
|
36
|
+
export function isReady(result, recipe) {
|
|
37
|
+
if (!isRecord(result))
|
|
38
|
+
return false;
|
|
39
|
+
const schema = resultSchemaFor(recipe);
|
|
40
|
+
if (isMetadataPath(schema, ['status']) && result.status !== undefined)
|
|
41
|
+
return result.status === 'ready';
|
|
42
|
+
return !hasReview(result, schema);
|
|
43
|
+
}
|
|
44
|
+
function hasReview(value, schema) {
|
|
45
|
+
if (!schema)
|
|
46
|
+
return false;
|
|
47
|
+
if (Array.isArray(value))
|
|
48
|
+
return value.some((item, index) => hasReview(item, schemaAtPath(schema, [String(index)])));
|
|
49
|
+
if (!isRecord(value))
|
|
50
|
+
return false;
|
|
51
|
+
return Object.entries(value).some(([name, field]) => name === 'status' && isMetadataPath(schema, [name])
|
|
52
|
+
? field === 'review'
|
|
53
|
+
: hasReview(field, schemaAtPath(schema, [name])));
|
|
54
|
+
}
|
|
36
55
|
function normalize(value, schema) {
|
|
37
56
|
if (!schema)
|
|
38
57
|
return value;
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
export declare function isRecord(value: unknown): value is Record<string, unknown>;
|
|
2
2
|
export declare function valueAtPath(record: unknown, path: string): unknown;
|
|
3
3
|
export declare function pickPaths(record: unknown, paths: string[]): Record<string, unknown>;
|
|
4
|
-
export declare function hasReviewAnywhere(value: unknown): boolean;
|
|
5
|
-
export declare function isReady(result: unknown): boolean;
|
|
6
4
|
export declare function caseConfidence(result: unknown, expectedPaths?: string[]): number | undefined;
|
|
7
5
|
export declare function mapWithConcurrencyLimit<T, R>(items: T[], limit: number, worker: (item: T, index: number) => Promise<R>): Promise<R[]>;
|
|
8
6
|
export declare function roundedTo(places: number, value: number): number;
|
|
@@ -11,20 +11,6 @@ export function valueAtPath(record, path) {
|
|
|
11
11
|
export function pickPaths(record, paths) {
|
|
12
12
|
return Object.fromEntries(paths.map((path) => [path, valueAtPath(record, path)]));
|
|
13
13
|
}
|
|
14
|
-
export function hasReviewAnywhere(value) {
|
|
15
|
-
if (Array.isArray(value))
|
|
16
|
-
return value.some(hasReviewAnywhere);
|
|
17
|
-
if (isRecord(value))
|
|
18
|
-
return Object.entries(value).some(([name, field]) => name === 'status' ? field === 'review' : hasReviewAnywhere(field));
|
|
19
|
-
return false;
|
|
20
|
-
}
|
|
21
|
-
export function isReady(result) {
|
|
22
|
-
if (!isRecord(result))
|
|
23
|
-
return false;
|
|
24
|
-
if (result.status !== undefined)
|
|
25
|
-
return result.status === 'ready';
|
|
26
|
-
return !hasReviewAnywhere(result);
|
|
27
|
-
}
|
|
28
14
|
export function caseConfidence(result, expectedPaths = []) {
|
|
29
15
|
if (isRecord(result) && typeof result.confidence === 'number')
|
|
30
16
|
return result.confidence;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { isDeepStrictEqual } from 'node:util';
|
|
2
2
|
import { createClient } from '../src/client.js';
|
|
3
|
-
import { caseConfidence,
|
|
4
|
-
import { matchesExpected } from './comparison.js';
|
|
3
|
+
import { caseConfidence, mapWithConcurrencyLimit, pickPaths } from './decisions.js';
|
|
4
|
+
import { isReady, matchesExpected } from './comparison.js';
|
|
5
5
|
export const confidenceThresholds = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95];
|
|
6
6
|
export async function evaluateCases(recipe, cases, options = {}) {
|
|
7
7
|
let requests = 0;
|
|
@@ -52,7 +52,7 @@ async function evaluateCase(recipe, goldenCase, client, options) {
|
|
|
52
52
|
row.result = await recipe.run(goldenCase.input, recipeOptions);
|
|
53
53
|
row.actual = pickPaths(row.result, Object.keys(goldenCase.expected));
|
|
54
54
|
row.correct = matchesExpected(row.actual, row.expected, recipe.id);
|
|
55
|
-
row.ready = isReady(row.result);
|
|
55
|
+
row.ready = isReady(row.result, recipe.id);
|
|
56
56
|
row.confidence = caseConfidence(row.result, Object.keys(goldenCase.expected));
|
|
57
57
|
const model = pickPaths(row.result, ['model']).model;
|
|
58
58
|
if (typeof model === 'string')
|
|
@@ -72,7 +72,7 @@ async function evaluateCase(recipe, goldenCase, client, options) {
|
|
|
72
72
|
playback.assertComplete();
|
|
73
73
|
row.decisions.push({
|
|
74
74
|
minConfidence,
|
|
75
|
-
ready: isReady(result),
|
|
75
|
+
ready: isReady(result, recipe.id),
|
|
76
76
|
correct: matchesExpected(pickPaths(result, Object.keys(row.expected)), row.expected, recipe.id),
|
|
77
77
|
});
|
|
78
78
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "jev-recipes",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.9",
|
|
4
4
|
"description": "248 small Jev decisions for JavaScript and TypeScript: route, rerank, gate, grade, compare, and label text for agents, RAG, support, code review, and music. Each returns a typed result with explicit uncertainty. Plug into Vercel AI SDK or LangChain agents, import one recipe, or run the CLI with JSON from any language.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|