@sebastiantuyu/agest 0.3.2 → 0.3.3-next.10

package/dist/adapters/tracing.js

@@ -0,0 +1,338 @@
+import { computeCost } from "../pricing";
+import { logger } from "../logger";
+/**
+ * Creates a LangChain callback handler that records every LLM and tool
+ * invocation as a `TimelineEvent`. Returns a handle whose `drain()` method
+ * yields the captured events with `startMs` / `endMs` relative to the
+ * provided baseline.
+ *
+ * Designed to fail open: any unexpected callback shape is ignored rather
+ * than throwing — the underlying agent run must not be broken by tracing.
+ */
+export async function createTracingHandle(baselineMs) {
+    // Import lazily so the adapter still works when @langchain/core is not present.
+    // BaseCallbackHandler is the runtime contract LangChain checks for.
+    let BaseCallbackHandler;
+    try {
+        ({ BaseCallbackHandler } = await import("@langchain/core/callbacks/base"));
+    }
+    catch (err) {
+        logger.debug(`[agest] tracing disabled: could not load @langchain/core/callbacks/base — ` +
+            `install @langchain/core as a peer to capture per-scene cost/timeline. (${err.message})`);
+        return { callbacks: [], drain: () => ({ events: [] }) };
+    }
+    const events = [];
+    const openLLMs = new Map();
+    const openTools = new Map();
+    let lastModelName;
+    class AgestTracer extends BaseCallbackHandler {
+        name = "AgestTracer";
+        awaitHandlers = true;
+        handleLLMStart(llm, _prompts, runId, _parentRunId, extraParams) {
+            openLLMs.set(runId, {
+                startMs: now() - baselineMs,
+                name: extractModelName(llm, extraParams),
+            });
+        }
+        handleChatModelStart(llm, _messages, runId, _parentRunId, extraParams) {
+            openLLMs.set(runId, {
+                startMs: now() - baselineMs,
+                name: extractModelName(llm, extraParams),
+            });
+        }
+        handleLLMEnd(output, runId) {
+            const open = openLLMs.get(runId);
+            if (!open)
+                return;
+            openLLMs.delete(runId);
+            const endMs = now() - baselineMs;
+            const tokens = extractTokensFromLLMOutput(output);
+            const providerCost = extractProviderCost(output);
+            const cachedInputTokens = extractCachedTokens(output);
+            const name = open.name ?? extractModelNameFromOutput(output) ?? "model";
+            if (name && name !== "model")
+                lastModelName = name;
+            const cost = computeCost({
+                model: name,
+                inputTokens: tokens?.input,
+                outputTokens: tokens?.output,
+                cachedInputTokens,
+                providerCost,
+            });
+            events.push({
+                kind: "model",
+                name,
+                startMs: open.startMs,
+                endMs,
+                durationMs: Math.max(0, endMs - open.startMs),
+                tokens,
+                cachedInputTokens,
+                cost: stripCostIfEmpty(cost),
+            });
+        }
+        handleLLMError(err, runId) {
+            const open = openLLMs.get(runId);
+            if (!open)
+                return;
+            openLLMs.delete(runId);
+            const endMs = now() - baselineMs;
+            events.push({
+                kind: "model",
+                name: open.name ?? "model",
+                startMs: open.startMs,
+                endMs,
+                durationMs: Math.max(0, endMs - open.startMs),
+                error: err?.message ?? String(err),
+            });
+        }
+        handleToolStart(tool, _input, runId, _parentRunId, _tags, _metadata, runName) {
+            openTools.set(runId, {
+                startMs: now() - baselineMs,
+                name: extractToolName(tool, runName) ?? "tool",
+            });
+        }
+        handleToolEnd(_output, runId) {
+            const open = openTools.get(runId);
+            if (!open)
+                return;
+            openTools.delete(runId);
+            const endMs = now() - baselineMs;
+            events.push({
+                kind: "tool",
+                name: open.name,
+                startMs: open.startMs,
+                endMs,
+                durationMs: Math.max(0, endMs - open.startMs),
+            });
+        }
+        handleToolError(err, runId) {
+            const open = openTools.get(runId);
+            if (!open)
+                return;
+            openTools.delete(runId);
+            const endMs = now() - baselineMs;
+            events.push({
+                kind: "tool",
+                name: open.name,
+                startMs: open.startMs,
+                endMs,
+                durationMs: Math.max(0, endMs - open.startMs),
+                error: err?.message ?? String(err),
+            });
+        }
+    }
+    const handler = new AgestTracer();
+    return {
+        callbacks: [handler],
+        drain: () => {
+            const ordered = [...events].sort((a, b) => a.startMs - b.startMs);
+            events.length = 0;
+            return { events: ordered, modelName: lastModelName };
+        },
+    };
+}
+/**
+ * Public tracing helper for custom executors (i.e. agents not wired through
+ * the `langchain()` adapter). Create one per scene run, hand its `callbacks`
+ * to your LangChain/LangGraph invocation, then spread `collect()` into the
+ * response metadata.
+ *
+ * @example
+ * ```ts
+ * const trace = await createTrace({ model: env.OPENROUTER_MODEL });
+ * const plan = await generatePlan(input, { callbacks: trace.callbacks });
+ * return { text: render(plan), metadata: { model, tools, ...trace.collect() } };
+ * ```
+ */
+export async function createTrace(opts) {
+    const baseline = performance.now();
+    const handle = await createTracingHandle(baseline);
+    let collected;
+    return {
+        callbacks: handle.callbacks,
+        collect() {
+            if (collected)
+                return collected;
+            const drained = handle.drain();
+            const { tokens, cost } = summarizeEvents(drained.events, opts?.model ?? drained.modelName);
+            collected = { events: drained.events, tokens, cost };
+            return collected;
+        },
+    };
+}
+/**
+ * Aggregate token counts and cost across a timeline's model events.
+ * Provider-reported cost wins; otherwise the table-derived cost; otherwise
+ * cost is recomputed from `model` and the summed tokens. `fallbackTokens` is
+ * used only when no model event carried usage.
+ */
+export function summarizeEvents(events, model, fallbackTokens) {
+    const modelEvents = events.filter((e) => e.kind === "model");
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let providerCost = 0;
+    let hasProviderCost = false;
+    let hasTableCost = false;
+    let tableCost = 0;
+    let hasTokens = false;
+    for (const e of modelEvents) {
+        if (e.tokens) {
+            hasTokens = true;
+            inputTokens += e.tokens.input;
+            outputTokens += e.tokens.output;
+        }
+        if (e.cost?.source === "provider" && e.cost.totalUsd != null) {
+            hasProviderCost = true;
+            providerCost += e.cost.totalUsd;
+        }
+        else if (e.cost?.source === "table" && e.cost.totalUsd != null) {
+            hasTableCost = true;
+            tableCost += e.cost.totalUsd;
+        }
+    }
+    let tokens = hasTokens ? { input: inputTokens, output: outputTokens } : undefined;
+    if (!tokens && fallbackTokens)
+        tokens = fallbackTokens;
+    let cost;
+    if (hasProviderCost) {
+        cost = { totalUsd: providerCost, source: "provider" };
+    }
+    else if (hasTableCost) {
+        cost = { totalUsd: tableCost, source: "table" };
+    }
+    else if (tokens && model) {
+        const computed = computeCost({
+            model,
+            inputTokens: tokens.input,
+            outputTokens: tokens.output,
+        });
+        if (computed.source !== "unavailable")
+            cost = computed;
+    }
+    return { tokens, cost };
+}
+function now() {
+    return performance.now();
+}
+function extractModelName(llm, extraParams) {
+    const invocation = (extraParams?.invocation_params ?? {});
+    if (typeof invocation.model === "string")
+        return invocation.model;
+    if (typeof invocation.model_name === "string")
+        return invocation.model_name;
+    const kwargs = llm?.kwargs;
+    if (kwargs) {
+        if (typeof kwargs.model === "string")
+            return kwargs.model;
+        if (typeof kwargs.model_name === "string")
+            return kwargs.model_name;
+        if (typeof kwargs.modelName === "string")
+            return kwargs.modelName;
+    }
+    const id = llm?.id;
+    if (Array.isArray(id) && id.length > 0 && typeof id[id.length - 1] === "string") {
+        return id[id.length - 1];
+    }
+    return undefined;
+}
+function extractModelNameFromOutput(output) {
+    const gen = output?.generations?.[0]?.[0];
+    return (gen?.message?.response_metadata?.model_name ??
+        gen?.message?.response_metadata?.model ??
+        output?.llmOutput?.modelName ??
+        output?.llmOutput?.model);
+}
+function extractTokensFromLLMOutput(output) {
+    const usage = output?.llmOutput?.tokenUsage ??
+        output?.llmOutput?.usage ??
+        output?.generations?.[0]?.[0]?.message?.usage_metadata ??
+        output?.generations?.[0]?.[0]?.message?.response_metadata?.usage;
+    if (!usage)
+        return undefined;
+    const input = usage.input_tokens ?? usage.prompt_tokens ?? usage.promptTokens ?? 0;
+    const out = usage.output_tokens ?? usage.completion_tokens ?? usage.completionTokens ?? 0;
+    if (!input && !out)
+        return undefined;
+    return { input, output: out };
+}
+/** Collect the usage-bearing objects LangChain/OpenRouter may attach to an LLM result. */
+function usageObjects(output) {
+    const msg = output?.generations?.[0]?.[0]?.message;
+    return [
+        output?.llmOutput?.usage,
+        output?.llmOutput?.tokenUsage,
+        output?.llmOutput?.estimatedTokenUsage,
+        output?.llmOutput,
+        msg?.usage_metadata,
+        msg?.response_metadata?.usage,
+        msg?.response_metadata?.tokenUsage,
+        msg?.response_metadata?.estimatedTokenUsage,
+        msg?.response_metadata,
+        msg?.additional_kwargs?.usage,
+    ].filter((u) => u && typeof u === "object");
+}
+/**
+ * OpenRouter (with `usage: { include: true }`) reports real USD cost. LangChain
+ * surfaces it inconsistently across versions, so scan the known usage objects
+ * for a numeric `cost` / `total_cost`.
+ */
+function extractProviderCost(output) {
+    for (const u of usageObjects(output)) {
+        const c = (typeof u.cost === "number" ? u.cost : undefined) ??
+            (typeof u.total_cost === "number" ? u.total_cost : undefined) ??
+            (typeof u.cost_usd === "number" ? u.cost_usd : undefined) ??
+            (typeof u.cost_details?.upstream_inference_cost === "number"
+                ? u.cost_details.upstream_inference_cost
+                : undefined);
+        if (typeof c === "number" && Number.isFinite(c))
+            return c;
+    }
+    return undefined;
+}
+/**
+ * Cached (prompt-cache hit) input tokens, when the provider reports them.
+ * Charged at a fraction of the normal input rate, so surfacing them lets the
+ * report explain why provider cost is below the flat-table estimate.
+ */
+function extractCachedTokens(output) {
+    for (const u of usageObjects(output)) {
+        const cached = u.input_token_details?.cache_read ??
+            u.prompt_tokens_details?.cached_tokens ??
+            u.cache_read_input_tokens ??
+            u.cached_tokens;
+        if (typeof cached === "number" && cached > 0)
+            return cached;
+    }
+    return undefined;
+}
+const TOOL_CLASS_NAMES = new Set([
+    "DynamicStructuredTool",
+    "DynamicTool",
+    "StructuredTool",
+    "Tool",
+]);
+function extractToolName(tool, runName) {
+    // `runName` is the actual tool name LangChain assigns the run (e.g.
+    // "search_recipes"); prefer it over the serialized class name.
+    if (runName && !TOOL_CLASS_NAMES.has(runName))
+        return runName;
+    if (tool) {
+        if (typeof tool.name === "string" && !TOOL_CLASS_NAMES.has(tool.name))
+            return tool.name;
+        if (typeof tool.kwargs?.name === "string")
+            return tool.kwargs.name;
+        if (Array.isArray(tool.id) && tool.id.length > 0) {
+            const last = String(tool.id[tool.id.length - 1]);
+            if (!TOOL_CLASS_NAMES.has(last))
+                return last;
+        }
+        if (typeof tool.name === "string")
+            return tool.name;
+    }
+    return runName;
+}
+function stripCostIfEmpty(cost) {
+    if (cost.source === "unavailable" && cost.totalUsd == null)
+        return undefined;
+    return cost;
+}

package/dist/assertions.d.ts

@@ -1,3 +1,4 @@
+import { type StandardSchemaV1 } from "./schema";
 import type { JudgeCriteria } from "./judge";
 export interface PendingJudgement {
     value: unknown;
@@ -5,10 +6,64 @@ export interface PendingJudgement {
 }
 export declare function collectPendingJudgements(): PendingJudgement[];
 export interface AgentMatchers {
+    /** Assert the agent refused. */
     refusal(): void;
+    /** Assert the agent did NOT refuse. */
     notRefusal(): void;
-    containing(text: string): void;
-    matchingPattern(regex: RegExp): void;
+    /**
+     * Text containment: `text` appears as a substring. For a non-string value the
+     * serialized form is searched. Case-INsensitive by default; pass
+     * `{ caseSensitive: true }` for an exact substring.
+     */
+    containingText(text: string | number, opts?: {
+        caseSensitive?: boolean;
+    }): void;
+    /** Assert text containment does NOT hold. See {@link containingText}. */
+    notContainingText(text: string | number, opts?: {
+        caseSensitive?: boolean;
+    }): void;
+    /**
+     * Array membership: the value is an array containing `item` as an EXACT
+     * (deep-equal) element. Throws if the value is not an array. Use
+     * {@link containingSubset} when you want partial element matching.
+     */
+    containingItem(item: unknown): void;
+    /**
+     * Structural subset: `subset` is recursively contained in the value.
+     * - object value + object `subset` → every key in `subset` is present with a
+     *   recursively-contained value (extra keys allowed).
+     * - array value + array `subset` → every `subset` element matches a distinct
+     *   element of the value (partial element matching, order-independent).
+     *
+     * Exact at the leaves (case-sensitive). Throws if the value is not an
+     * object/array, or `subset` is not an object/array.
+     */
+    containingSubset(subset: object): void;
+    /** Assert the serialized text view matches `pattern`. */
+    matchingPattern(pattern: RegExp): void;
+    /** Deep structural equality against the native value. */
+    equalTo(expected: unknown): void;
+    /** Assert deep structural INequality against the native value. */
+    notEqualTo(expected: unknown): void;
+    /** Assert the value (array/string) has length `n`. */
+    ofLength(n: number): void;
+    /**
+     * Validate the native value against a Standard Schema (zod 4, valibot,
+     * arktype, …). Throws with the schema's formatted issues on failure.
+     * Synchronous — for async (`refine`-style) schemas, declare the schema at the
+     * agent() or scene().expectSchema() level instead.
+     */
+    matchingSchema(schema: StandardSchemaV1): void;
+    /**
+     * Escape hatch for anything not covered by a named matcher: a predicate over
+     * the native value. Stays deterministic — use it to express negatives too,
+     * e.g. `satisfying((v) => !v.includes("secret"))`.
+     */
+    satisfying(predicate: (value: any) => boolean, message?: string): void;
+    /**
+     * Queue an LLM-judged assertion, resolved asynchronously by the runner.
+     * Fuzzy + paid (express the negative in `failWhen`).
+     */
     judgedBy(criteria: JudgeCriteria): void;
 }
 export interface AgentExpectation {

package/dist/assertions.js

@@ -1,46 +1,132 @@
+import { isDeepStrictEqual } from "node:util";
 import { isRefusal } from "./refusal";
+import { serializeValue } from "./resolve";
+import { isObjectLike, isPlainObject, structuralContains } from "./match";
+import { validateSync } from "./schema";
 let pendingJudgements = [];
 export function collectPendingJudgements() {
     const collected = pendingJudgements;
     pendingJudgements = [];
     return collected;
 }
+/**
+ * 100-char preview for error messages. Uses COMPACT JSON for objects (the
+ * judge-facing `serializeValue` pretty-prints; error previews stay terse and
+ * match the library's original contract).
+ */
+function preview(value) {
+    let s;
+    if (typeof value === "string") {
+        s = value;
+    }
+    else {
+        try {
+            s = JSON.stringify(value);
+        }
+        catch {
+            s = String(value);
+        }
+    }
+    return s.slice(0, 100);
+}
+/** Compact one-line form for an inline needle/expected in an error message. */
+function compact(value) {
+    try {
+        return typeof value === "string" ? value : JSON.stringify(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+/** Human-readable type label for diagnostics (e.g. "a number", "an array"). */
+function describeType(value) {
+    if (value === null)
+        return "null";
+    if (Array.isArray(value))
+        return "an array";
+    return `a ${typeof value}`;
+}
+/**
+ * Substring search shared by `containingText` / `notContainingText`. A string
+ * value is searched directly; anything else via its serialized form.
+ * Case-insensitive unless `caseSensitive` is set.
+ */
+function textContains(value, text, opts) {
+    const actual = typeof value === "string" ? value : serializeValue(value);
+    const needle = String(text);
+    const hit = opts?.caseSensitive
+        ? actual.includes(needle)
+        : actual.toLowerCase().includes(needle.toLowerCase());
+    return { actual, hit };
+}
+function makeMatchers(value) {
+    const assert = (cond, message) => {
+        if (!cond)
+            throw new Error(message);
+    };
+    return {
+        refusal() {
+            assert(isRefusal(value), `Expected a refusal but got: "${preview(value)}"`);
+        },
+        notRefusal() {
+            assert(!isRefusal(value), `Expected a non-refusal response but got: "${preview(value)}"`);
+        },
+        containingText(text, opts) {
+            const { actual, hit } = textContains(value, text, opts);
+            assert(hit, `Expected response to contain "${text}" but got: "${actual.slice(0, 100)}"`);
+        },
+        notContainingText(text, opts) {
+            const { actual, hit } = textContains(value, text, opts);
+            assert(!hit, `Expected response NOT to contain "${text}" but got: "${actual.slice(0, 100)}"`);
+        },
+        containingItem(item) {
+            if (!Array.isArray(value)) {
+                throw new Error(`containingItem() expects an array value but got ${describeType(value)}. ` +
+                    `Use containingText() for substrings or containingSubset() for objects.`);
+            }
+            assert(value.some((el) => isDeepStrictEqual(el, item)), `Expected array to contain item ${compact(item)} but it did not (got ${preview(value)})`);
+        },
+        containingSubset(subset) {
+            if (!Array.isArray(value) && !isObjectLike(value)) {
+                throw new Error(`containingSubset() expects an object or array value but got ${describeType(value)}.`);
+            }
+            if (!Array.isArray(subset) && !isPlainObject(subset)) {
+                throw new Error(`containingSubset() expects an object or array subset but got ${describeType(subset)}.`);
+            }
+            assert(structuralContains(value, subset), `Expected value to contain subset ${compact(subset)} but it did not (got ${preview(value)})`);
+        },
+        matchingPattern(pattern) {
+            const actual = typeof value === "string" ? value : serializeValue(value);
+            assert(pattern.test(actual), `Expected response to match ${pattern} but got: "${actual.slice(0, 100)}"`);
+        },
+        equalTo(expected) {
+            assert(isDeepStrictEqual(value, expected), `Expected value to equal ${compact(expected)} but got ${preview(value)}`);
+        },
+        notEqualTo(expected) {
+            assert(!isDeepStrictEqual(value, expected), `Expected value NOT to equal ${compact(expected)} but it did`);
+        },
+        ofLength(n) {
+            const len = typeof value === "string" || Array.isArray(value)
+                ? value.length
+                : NaN;
+            assert(len === n, `Expected length ${n} but got ${Number.isNaN(len) ? "a non-measurable value" : len}`);
+        },
+        matchingSchema(schema) {
+            const outcome = validateSync(schema, value);
+            assert(outcome.ok, `Schema validation failed for value "${preview(value)}" — ${outcome.ok ? "" : outcome.message}`);
+        },
+        satisfying(predicate, message) {
+            assert(Boolean(predicate(value)), message ?? `Predicate failed for value: "${preview(value)}"`);
+        },
+        judgedBy(criteria) {
+            pendingJudgements.push({ value, criteria });
+        },
+    };
+}
 export function expect(value) {
     return {
         get toBe() {
-            return {
-                refusal() {
-                    if (!isRefusal(value)) {
-                        const preview = typeof value === "string"
-                            ? value.slice(0, 100)
-                            : JSON.stringify(value).slice(0, 100);
-                        throw new Error(`Expected a refusal but got: "${preview}"`);
-                    }
-                },
-                notRefusal() {
-                    if (isRefusal(value)) {
-                        const preview = typeof value === "string"
-                            ? value.slice(0, 100)
-                            : JSON.stringify(value).slice(0, 100);
-                        throw new Error(`Expected a non-refusal response but got: "${preview}"`);
-                    }
-                },
-                containing(text) {
-                    const actual = typeof value === "string" ? value : String(value);
-                    if (!actual.toLowerCase().includes(text.toLowerCase())) {
-                        throw new Error(`Expected response to contain "${text}" but got: "${actual.slice(0, 100)}"`);
-                    }
-                },
-                matchingPattern(regex) {
-                    const actual = typeof value === "string" ? value : String(value);
-                    if (!regex.test(actual)) {
-                        throw new Error(`Expected response to match ${regex} but got: "${actual.slice(0, 100)}"`);
-                    }
-                },
-                judgedBy(criteria) {
-                    pendingJudgements.push({ value, criteria });
-                },
-            };
+            return makeMatchers(value);
         },
     };
 }

package/dist/cli.d.ts

@@ -1,2 +1,16 @@
 #!/usr/bin/env node
-export {};
+export interface ParsedRunArgs {
+    pattern?: string;
+    targets: string[];
+    full: boolean;
+}
+/**
+ * Extract the args that follow the command word from a full `process.argv`.
+ * `argv = [execPath, scriptPath, command, ...commandArgs]`, so the command's
+ * args always start at index 3. Capturing them here (once, from the original
+ * argv) avoids re-slicing a mutated argv downstream — the double-shift that
+ * silently dropped a lone `run` target and made discovery scan the whole cwd.
+ */
+export declare function getCommandArgs(argv: string[]): string[];
+export declare function parseRunArgs(args: string[]): ParsedRunArgs;
+export declare function main(argv: string[]): Promise<void>;