@agnt5/sdk 0.3.6 → 0.3.10

package/dist/prompt-executor.js

@@ -0,0 +1,213 @@
+import { LM } from './lm.js';
+export const PROMPT_EXECUTOR_COMPONENT_NAME = 'agnt5_prompt_executor';
+export const PROMPT_EXECUTOR_ALIASES = new Set([PROMPT_EXECUTOR_COMPONENT_NAME, 'run_prompt']);
+export const PROMPT_WORKER_INPUT_SCHEMA_VERSION = 'agnt5.eval.prompt_worker_input.v1';
+export const PROMPT_EXECUTOR_METADATA = {
+    source: 'agnt5_builtin',
+    agnt5_builtin: 'prompt_executor',
+    schema_version: PROMPT_WORKER_INPUT_SCHEMA_VERSION,
+};
+const PLACEHOLDER_RE = /\{\{\s*([A-Za-z_][A-Za-z0-9_.-]*)\s*\}\}/g;
+export function isPromptExecutorComponent(name) {
+    return PROMPT_EXECUTOR_ALIASES.has(name);
+}
+export async function executePromptWorkerInput(payload, generateFn = defaultGenerate) {
+    validatePayload(payload);
+    const variables = resolveVariables(payload);
+    const messages = renderMessages(payload.prompt.messages, variables);
+    const model = normalizePromptModel(payload.prompt.model);
+    const parameters = payload.prompt.parameters ?? {};
+    const responseFormat = buildResponseFormat(payload.prompt);
+    const response = await generateFn({
+        model,
+        messages,
+        temperature: optionalNumber(parameters.temperature),
+        maxTokens: optionalNumber(parameters.max_tokens),
+        topP: optionalNumber(parameters.top_p),
+        responseFormat,
+    });
+    if (typeof response === 'string') {
+        return parseJsonOutput(response);
+    }
+    if (response && typeof response === 'object' && 'text' in response) {
+        const text = response.text;
+        return parseJsonOutput(text);
+    }
+    return response;
+}
+function validatePayload(payload) {
+    if (!payload || typeof payload !== 'object') {
+        throw new Error('Prompt executor input must be a JSON object');
+    }
+    if (payload.schema_version !== PROMPT_WORKER_INPUT_SCHEMA_VERSION) {
+        throw new Error(`Unsupported prompt executor input schema ${JSON.stringify(payload.schema_version)}; ` +
+            `expected ${JSON.stringify(PROMPT_WORKER_INPUT_SCHEMA_VERSION)}`);
+    }
+    if (!payload.prompt || typeof payload.prompt !== 'object') {
+        throw new Error('Prompt executor input must include a prompt object');
+    }
+    if (!payload.prompt.model) {
+        throw new Error('Prompt executor prompt.model is required');
+    }
+    if (!payload.prompt.messages?.length) {
+        throw new Error('Prompt executor prompt.messages is required');
+    }
+}
+function resolveVariables(payload) {
+    if (payload.variables && typeof payload.variables === 'object') {
+        return payload.variables;
+    }
+    if (payload.input && typeof payload.input === 'object' && !Array.isArray(payload.input)) {
+        return payload.input;
+    }
+    return { input: payload.input };
+}
+function renderMessages(messages, variables) {
+    return messages.map((message) => ({
+        role: message.role,
+        content: renderValue(message.content, variables),
+    }));
+}
+function renderValue(value, variables) {
+    if (typeof value === 'string') {
+        return value.replace(PLACEHOLDER_RE, (_match, expression) => stringifyValue(lookupVariable(variables, expression, _match)));
+    }
+    if (value && typeof value === 'object') {
+        return JSON.stringify(renderNested(value, variables));
+    }
+    if (value == null) {
+        return '';
+    }
+    return String(value);
+}
+function renderNested(value, variables) {
+    if (typeof value === 'string') {
+        return value.replace(PLACEHOLDER_RE, (_match, expression) => stringifyValue(lookupVariable(variables, expression, _match)));
+    }
+    if (Array.isArray(value)) {
+        return value.map((item) => renderNested(item, variables));
+    }
+    if (value && typeof value === 'object') {
+        return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, renderNested(item, variables)]));
+    }
+    return value;
+}
+function lookupVariable(variables, expression, fallback) {
+    let current = variables;
+    for (const part of expression.split('.')) {
+        if (current && typeof current === 'object' && part in current) {
+            current = current[part];
+            continue;
+        }
+        return fallback;
+    }
+    return current;
+}
+function stringifyValue(value) {
+    if (typeof value === 'string') {
+        return value;
+    }
+    if (value == null) {
+        return '';
+    }
+    if (typeof value === 'object') {
+        return JSON.stringify(value);
+    }
+    return String(value);
+}
+export function normalizePromptModel(model) {
+    const normalized = model.trim();
+    if (!normalized) {
+        throw new Error('Prompt executor prompt.model must be a non-empty string');
+    }
+    if (normalized.includes('/')) {
+        return normalized;
+    }
+    const lowered = normalized.toLowerCase();
+    if (lowered.startsWith('gpt-') ||
+        lowered.startsWith('chatgpt-') ||
+        /^o[1-9](?:-|$)/.test(lowered)) {
+        return `openai/${normalized}`;
+    }
+    if (lowered.startsWith('claude-')) {
+        return `anthropic/${normalized}`;
+    }
+    if (lowered.startsWith('gemini-')) {
+        return `google/${normalized}`;
+    }
+    if (lowered.startsWith('mistral-')) {
+        return `mistral/${normalized}`;
+    }
+    return normalized;
+}
+function buildResponseFormat(prompt) {
+    if (prompt.response_format === 'json_schema' &&
+        prompt.response_schema &&
+        typeof prompt.response_schema === 'object') {
+        return {
+            formatType: 'json_schema',
+            schemaName: 'agnt5_prompt_output',
+            schema: JSON.stringify(prompt.response_schema),
+            strict: true,
+        };
+    }
+    return undefined;
+}
+function parseJsonOutput(text) {
+    const trimmed = text.trim();
+    if (!trimmed) {
+        return text;
+    }
+    try {
+        return JSON.parse(trimmed);
+    }
+    catch {
+        return text;
+    }
+}
+function optionalNumber(value) {
+    return value == null ? undefined : Number(value);
+}
+async function defaultGenerate(request) {
+    const provider = request.model.split('/', 1)[0];
+    const client = createLMClient(provider);
+    return await client.generate({
+        model: request.model,
+        messages: request.messages,
+        config: {
+            temperature: request.temperature,
+            maxOutputTokens: request.maxTokens,
+            topP: request.topP,
+            responseFormat: request.responseFormat,
+        },
+    });
+}
+function createLMClient(provider) {
+    switch (provider) {
+        case 'openai':
+            return LM.openai();
+        case 'anthropic':
+            return LM.anthropic();
+        case 'groq':
+            return LM.groq();
+        case 'openrouter':
+            return LM.openrouter();
+        case 'deepseek':
+            return LM.deepseek();
+        case 'google':
+        case 'gemini':
+            return LM.google();
+        case 'mistral':
+            return LM.mistral();
+        case 'ollama':
+            return LM.ollama();
+        case 'xai':
+            return LM.xai();
+        case 'huggingface':
+        case 'hf':
+            return LM.huggingface();
+        default:
+            throw new Error(`Unsupported prompt executor model provider: ${provider}`);
+    }
+}
+//# sourceMappingURL=prompt-executor.js.map

package/dist/prompt-executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-executor.js","sourceRoot":"","sources":["../src/prompt-executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAkE,MAAM,SAAS,CAAC;AAE7F,MAAM,CAAC,MAAM,8BAA8B,GAAG,uBAAuB,CAAC;AACtE,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,8BAA8B,EAAE,YAAY,CAAC,CAAC,CAAC;AAC/F,MAAM,CAAC,MAAM,kCAAkC,GAAG,mCAAmC,CAAC;AAEtF,MAAM,CAAC,MAAM,wBAAwB,GAA2B;IAC9D,MAAM,EAAE,eAAe;IACvB,aAAa,EAAE,iBAAiB;IAChC,cAAc,EAAE,kCAAkC;CACnD,CAAC;AA4BF,MAAM,cAAc,GAAG,2CAA2C,CAAC;AAEnE,MAAM,UAAU,yBAAyB,CAAC,IAAY;IACpD,OAAO,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,OAA0B,EAC1B,aAA+B,eAAe;IAE9C,eAAe,CAAC,OAAO,CAAC,CAAC;IAEzB,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;IACnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAE3D,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC;QAChC,KAAK;QACL,QAAQ;QACR,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,WAAW,CAAC;QACnD,SAAS,EAAE,cAAc,CAAC,UAAU,CAAC,UAAU,CAAC;QAChD,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,KAAK,CAAC;QACtC,cAAc;KACf,CAAC,CAAC;IAEH,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAC;QACnE,MAAM,IAAI,GAAI,QAA6B,CAAC,IAAI,CAAC;QACjD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,eAAe,CAAC,OAA0B;IACjD,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,OAAO,CAAC,cAAc,KAAK,kCAAkC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,4CAA4C,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI;YACpF,YAAY,IAAI,CAAC,SAAS,CAAC,kCAAkC,CAAC,EAAE,CACnE,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,OAA0B;IAClD,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QAC/D,OAAO,OAAO,CAAC,SAAS,CAAC;IAC3B,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxF,OAAO,OAAO,CAAC,KAAgC,CAAC;IAClD,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,cAAc,CACrB,QAAmD,EACnD,SAAkC;IAElC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAChC,IAAI,EAAE,OAAO,CAAC,IAAuB;QACrC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC;KACjD,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,WAAW,CAAC,KAAc,EAAE,SAAkC;IACrE,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,UAAkB,EAAE,EAAE,CAClE,cAAc,CAAC,cAAc,CAAC,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC,CAC9D,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,OAAO,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,YAAY,CAAC,KAAc,EAAE,SAAkC;IACtE,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,UAAkB,EAAE,EAAE,CAClE,cAAc,CAAC,cAAc,CAAC,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC,CAC9D,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CACjF,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CACrB,SAAkC,EAClC,UAAkB,EAClB,QAAgB;IAEhB,IAAI,OAAO,GAAY,SAAS,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACzC,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;YAC9D,OAAO,GAAI,OAAmC,CAAC,IAAI,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,MAAM,OAAO,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;IACzC,IACE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC;QAC1B,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC;QAC9B,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAC9B,CAAC;QACD,OAAO,UAAU,UAAU,EAAE,CAAC;IAChC,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAClC,OAAO,aAAa,UAAU,EAAE,CAAC;IACnC,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAClC,OAAO,UAAU,UAAU,EAAE,CAAC;IAChC,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACnC,OAAO,WAAW,UAAU,EAAE,CAAC;IACjC,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAmC;IAC9D,IACE,MAAM,CAAC,eAAe,KAAK,aAAa;QACxC,MAAM,CAAC,eAAe;QACtB,OAAO,MAAM,CAAC,eAAe,KAAK,QAAQ,EAC1C,CAAC;QACD,OAAO;YACL,UAAU,EAAE,aAAa;YACzB,UAAU,EAAE,qBAAqB;YACjC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC;YAC9C,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,KAAgC;IACtD,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,OAO9B;IACC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACxC,OAAO,MAAM,MAAM,CAAC,QAAQ,CAAC;QAC3B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,MAAM,EAAE;YACN,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,eAAe,EAAE,OAAO,CAAC,SAAS;YAClC,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,cAAc,EAAE,OAAO,CAAC,cAAc;SACvC;KACF,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB;IACtC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ;YACX,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC;QACrB,KAAK,WAAW;YACd,OAAO,EAAE,CAAC,SAAS,EAAE,CAAC;QACxB,KAAK,MAAM;YACT,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,YAAY;YACf,OAAO,EAAE,CAAC,UAAU,EAAE,CAAC;QACzB,KAAK,UAAU;YACb,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;QACvB,KAAK,QAAQ,CAAC;QACd,KAAK,QAAQ;YACX,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC;QACrB,KAAK,SAAS;YACZ,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;QACtB,KAAK,QAAQ;YACX,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC;QACrB,KAAK,KAAK;YACR,OAAO,EAAE,CAAC,GAAG,EAAE,CAAC;QAClB,KAAK,aAAa,CAAC;QACnB,KAAK,IAAI;YACP,OAAO,EAAE,CAAC,WAAW,EAAE,CAAC;QAC1B;YACE,MAAM,IAAI,KAAK,CAAC,+CAA+C,QAAQ,EAAE,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC"}

package/dist/scorer.d.ts

@@ -27,6 +27,18 @@ export interface TraceEvent {
     data: Record<string, any>;
     name?: string;
 }
+/** Tool call extracted from journal events or normalized session payloads */
+export interface ToolCall {
+    name: string;
+    arguments?: any;
+    callId?: string;
+    spanId?: string;
+    timestampNs?: number;
+    startedAt?: number;
+    endedAt?: number;
+    status?: string;
+    metadata: Record<string, any>;
+}
 /** Result of a scorer evaluation */
 export declare class ScorerResult {
     /** Score between 0.0 and 1.0 */
@@ -70,11 +82,13 @@ export interface ScorerContext {
 }
 /** Scorer handler function signature */
 export type ScorerHandler = (ctx: ScorerContext, request: ScorerRequest) => ScorerResult | Promise<ScorerResult>;
+export type ScorerScope = 'item' | 'run' | 'trace' | 'span' | 'session' | 'fleet_run';
 /** Configuration for a registered scorer */
 export interface ScorerConfig {
     name: string;
     handler: ScorerHandler;
     description: string;
+    scope: ScorerScope;
     isAsync: boolean;
     inputSchema?: Record<string, any>;
 }
@@ -91,7 +105,7 @@ export interface ScorerConfig {
  * );
  * ```
  */
-export declare function scorer(name?: string, description?: string): <F extends ScorerHandler>(handler: F) => F;
+export declare function scorer(name?: string, description?: string, scope?: ScorerScope): <F extends ScorerHandler>(handler: F) => F;
 /** Check if a function is a scorer */
 export declare function isScorer(fn: any): boolean;
 /** Get scorer config from a decorated function */
@@ -173,6 +187,8 @@ export declare function levenshtein(request: ScorerRequest): ScorerResult;
  * parsing rules, same parse_error label on bad output.
  */
 export declare function llmJudge(request: ScorerRequest, ctx?: ScorerContext): Promise<ScorerResult>;
+export declare function correctness(request: ScorerRequest, ctx?: ScorerContext): Promise<ScorerResult>;
+export declare function faithfulness(request: ScorerRequest, ctx?: ScorerContext): Promise<ScorerResult>;
 /**
  * Run a scorer by name against a request.
  */
@@ -183,4 +199,20 @@ export declare function getRequestConfig(request: ScorerRequest, key: string, de
 export declare function getTraceEvents(request: ScorerRequest, eventType: string): TraceEvent[];
 /** Get total tokens from trace events */
 export declare function getTotalTokens(request: ScorerRequest): number;
+/** Extract typed tool calls from ScorerRequest journal events */
+export declare function getToolCalls(request: ScorerRequest): ToolCall[];
+/** Extract typed tool calls from journal events */
+export declare function extractToolCallsFromEvents(events?: TraceEvent[]): ToolCall[];
+/** Return tool names in observed call order */
+export declare function getToolCallNames(request: ScorerRequest): string[];
+/** Return tool-call names from typed tool calls */
+export declare function toolCallNames(calls: ToolCall[]): string[];
+/** Return true when the observed trajectory exactly matches expected */
+export declare function toolTrajectoryExact(actual: string[], expected: string[]): boolean;
+/** Return true when expected appears as an ordered subsequence */
+export declare function toolTrajectoryInOrder(actual: string[], expected: string[]): boolean;
+/** Return true when actual contains expected names with matching counts */
+export declare function toolTrajectoryAnyOrder(actual: string[], expected: string[]): boolean;
+/** Compare a tool trajectory using exact, in_order, or any_order semantics */
+export declare function toolTrajectoryMatches(actual: string[], expected: string[], mode?: 'exact' | 'in_order' | 'any_order'): boolean;
 //# sourceMappingURL=scorer.d.ts.map

package/dist/scorer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../src/scorer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B,kCAAkC;IAClC,MAAM,EAAE,GAAG,CAAC;IACZ,iCAAiC;IACjC,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,gCAAgC;IAChC,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,6CAA6C;IAC7C,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IACrB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B;AAED,6CAA6C;AAC7C,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,oCAAoC;AACpC,qBAAa,YAAY;IACvB,gCAAgC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,2BAA2B;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,0CAA0C;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,0BAA0B;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAE5B,IAAI,EAAE;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAQD,8BAA8B;IAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY;IAI/C,8BAA8B;IAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY;CAGhD;AAED,wDAAwD;AACxD,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,2CAA2C;AAC3C,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC;CAC7D;AAED,wCAAwC;AACxC,MAAM,MAAM,aAAa,GAAG,CAC1B,GAAG,EAAE,aAAa,EAClB,OAAO,EAAE,aAAa,KACnB,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;AAE1C,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,aAAa,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACnC;AAMD;;;;;;;;;;;;GAYG;AACH,wBAAgB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,IACvC,CAAC,SAAS,aAAa,EAAE,SAAS,CAAC,KAAG,CAAC,CAazD;AAED,sCAAsC;AACtC,wBAAgB,QAAQ,CAAC,EAAE,EAAE,GAAG,GAAG,OAAO,CAEzC;AAED,kDAAkD;AAClD,wBAAgB,eAAe,CAAC,EAAE,EAAE,GAAG,GAAG,YAAY,GAAG,SAAS,CAEjE;AAID;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAmC;IAE1D,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIlD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC;IAIvC,MAAM,CAAC,SAAS,IAAI,MAAM,EAAE;IAI5B,MAAM,CAAC,KAAK,IAAI,IAAI;CAGrB;AAID,uCAAuC;AACvC,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAS/D;AAED;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAc7D;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAY9D;AAED;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAoB/D;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAkE/D;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAgDjE;AAED,wDAAwD;AACxD,wBAAgB,WAAW,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAuChE;AAkBD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CAyEvB;AA0GD;;GAEG;AACH,wBAAsB,SAAS,CAC7B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CAcvB;AAID,uCAAuC;AACvC,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,GAAG,GAAG,GAAG,CAE7F;AAED,oDAAoD;AACpD,wBAAgB,cAAc,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,UAAU,EAAE,CAEtF;AAED,yCAAyC;AACzC,wBAAgB,cAAc,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,CAI7D"}
+{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../src/scorer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B,kCAAkC;IAClC,MAAM,EAAE,GAAG,CAAC;IACZ,iCAAiC;IACjC,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,gCAAgC;IAChC,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,6CAA6C;IAC7C,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IACrB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B;AAED,6CAA6C;AAC7C,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,6EAA6E;AAC7E,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,oCAAoC;AACpC,qBAAa,YAAY;IACvB,gCAAgC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,2BAA2B;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,0CAA0C;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,0BAA0B;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAE5B,IAAI,EAAE;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAQD,8BAA8B;IAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY;IAI/C,8BAA8B;IAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY;CAGhD;AAED,wDAAwD;AACxD,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,2CAA2C;AAC3C,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC;CAC7D;AAED,wCAAwC;AACxC,MAAM,MAAM,aAAa,GAAG,CAC1B,GAAG,EAAE,aAAa,EAClB,OAAO,EAAE,aAAa,KACnB,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;AAE1C,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,WAAW,CAAC;AAEtF,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,aAAa,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACnC;AAMD;;;;;;;;;;;;GAYG;AACH,wBAAgB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,WAAoB,IACpE,CAAC,SAAS,aAAa,EAAE,SAAS,CAAC,KAAG,CAAC,CAczD;AAED,sCAAsC;AACtC,wBAAgB,QAAQ,CAAC,EAAE,EAAE,GAAG,GAAG,OAAO,CAEzC;AAED,kDAAkD;AAClD,wBAAgB,eAAe,CAAC,EAAE,EAAE,GAAG,GAAG,YAAY,GAAG,SAAS,CAEjE;AAID;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAmC;IAE1D,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIlD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC;IAIvC,MAAM,CAAC,SAAS,IAAI,MAAM,EAAE;IAI5B,MAAM,CAAC,KAAK,IAAI,IAAI;CAGrB;AAID,uCAAuC;AACvC,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAS/D;AAED;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAc7D;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAY9D;AAED;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAoB/D;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAkE/D;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAgDjE;AAED,wDAAwD;AACxD,wBAAgB,WAAW,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAuChE;AAwBD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CAyHvB;AAmND,wBAAsB,WAAW,CAC/B,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CAyBvB;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CAgCvB;AAwFD;;GAEG;AACH,wBAAsB,SAAS,CAC7B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CAwBvB;AA6ID,uCAAuC;AACvC,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,GAAG,GAAG,GAAG,CAE7F;AAED,oDAAoD;AACpD,wBAAgB,cAAc,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,UAAU,EAAE,CAEtF;AAED,yCAAyC;AACzC,wBAAgB,cAAc,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,CAI7D;AAED,iEAAiE;AACjE,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,QAAQ,EAAE,CAE/D;AAED,mDAAmD;AACnD,wBAAgB,0BAA0B,CAAC,MAAM,GAAE,UAAU,EAAO,GAAG,QAAQ,EAAE,CA0BhF;AAED,+CAA+C;AAC/C,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,EAAE,CAEjE;AAED,mDAAmD;AACnD,wBAAgB,aAAa,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,EAAE,CAEzD;AAED,wEAAwE;AACxE,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAEjF;AAED,kEAAkE;AAClE,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAUnF;AAED,2EAA2E;AAC3E,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CASpF;AAED,8EAA8E;AAC9E,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,EAAE,MAAM,EAAE,EAClB,IAAI,GAAE,OAAO,GAAG,UAAU,GAAG,WAAqB,GACjD,OAAO,CAIT"}