@n8n/agents 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +88 -0
- package/dist/build.tsbuildinfo +1 -0
- package/dist/evals/categorization.d.ts +2 -0
- package/dist/evals/categorization.js +23 -0
- package/dist/evals/categorization.js.map +1 -0
- package/dist/evals/contains-keywords.d.ts +2 -0
- package/dist/evals/contains-keywords.js +29 -0
- package/dist/evals/contains-keywords.js.map +1 -0
- package/dist/evals/correctness.d.ts +2 -0
- package/dist/evals/correctness.js +28 -0
- package/dist/evals/correctness.js.map +1 -0
- package/dist/evals/helpfulness.d.ts +2 -0
- package/dist/evals/helpfulness.js +26 -0
- package/dist/evals/helpfulness.js.map +1 -0
- package/dist/evals/index.d.ts +7 -0
- package/dist/evals/index.js +18 -0
- package/dist/evals/index.js.map +1 -0
- package/dist/evals/json-validity.d.ts +2 -0
- package/dist/evals/json-validity.js +21 -0
- package/dist/evals/json-validity.js.map +1 -0
- package/dist/evals/parse-judge-response.d.ts +2 -0
- package/dist/evals/parse-judge-response.js +25 -0
- package/dist/evals/parse-judge-response.js.map +1 -0
- package/dist/evals/string-similarity.d.ts +2 -0
- package/dist/evals/string-similarity.js +42 -0
- package/dist/evals/string-similarity.js.map +1 -0
- package/dist/evals/tool-call-accuracy.d.ts +2 -0
- package/dist/evals/tool-call-accuracy.js +29 -0
- package/dist/evals/tool-call-accuracy.js.map +1 -0
- package/dist/index.d.ts +34 -0
- package/dist/index.js +85 -0
- package/dist/index.js.map +1 -0
- package/dist/integrations/langsmith.d.ts +13 -0
- package/dist/integrations/langsmith.js +77 -0
- package/dist/integrations/langsmith.js.map +1 -0
- package/dist/runtime/agent-runtime.d.ts +80 -0
- package/dist/runtime/agent-runtime.js +1246 -0
- package/dist/runtime/agent-runtime.js.map +1 -0
- package/dist/runtime/event-bus.d.ts +14 -0
- package/dist/runtime/event-bus.js +53 -0
- package/dist/runtime/event-bus.js.map +1 -0
- package/dist/runtime/logger.d.ts +22 -0
- package/dist/runtime/logger.js +40 -0
- package/dist/runtime/logger.js.map +1 -0
- package/dist/runtime/mcp-connection.d.ts +23 -0
- package/dist/runtime/mcp-connection.js +135 -0
- package/dist/runtime/mcp-connection.js.map +1 -0
- package/dist/runtime/mcp-tool-resolver.d.ts +7 -0
- package/dist/runtime/mcp-tool-resolver.js +64 -0
- package/dist/runtime/mcp-tool-resolver.js.map +1 -0
- package/dist/runtime/memory-store.d.ts +29 -0
- package/dist/runtime/memory-store.js +68 -0
- package/dist/runtime/memory-store.js.map +1 -0
- package/dist/runtime/message-list.d.ts +25 -0
- package/dist/runtime/message-list.js +79 -0
- package/dist/runtime/message-list.js.map +1 -0
- package/dist/runtime/messages.d.ts +8 -0
- package/dist/runtime/messages.js +239 -0
- package/dist/runtime/messages.js.map +1 -0
- package/dist/runtime/model-factory.d.ts +33 -0
- package/dist/runtime/model-factory.js +68 -0
- package/dist/runtime/model-factory.js.map +1 -0
- package/dist/runtime/run-state.d.ts +9 -0
- package/dist/runtime/run-state.js +49 -0
- package/dist/runtime/run-state.js.map +1 -0
- package/dist/runtime/runtime-helpers.d.ts +21 -0
- package/dist/runtime/runtime-helpers.js +96 -0
- package/dist/runtime/runtime-helpers.js.map +1 -0
- package/dist/runtime/stream.d.ts +15 -0
- package/dist/runtime/stream.js +94 -0
- package/dist/runtime/stream.js.map +1 -0
- package/dist/runtime/strip-orphaned-tool-messages.d.ts +2 -0
- package/dist/runtime/strip-orphaned-tool-messages.js +48 -0
- package/dist/runtime/strip-orphaned-tool-messages.js.map +1 -0
- package/dist/runtime/title-generation.d.ts +11 -0
- package/dist/runtime/title-generation.js +56 -0
- package/dist/runtime/title-generation.js.map +1 -0
- package/dist/runtime/tool-adapter.d.ts +23 -0
- package/dist/runtime/tool-adapter.js +104 -0
- package/dist/runtime/tool-adapter.js.map +1 -0
- package/dist/runtime/working-memory.d.ts +21 -0
- package/dist/runtime/working-memory.js +138 -0
- package/dist/runtime/working-memory.js.map +1 -0
- package/dist/sdk/agent.d.ts +88 -0
- package/dist/sdk/agent.js +327 -0
- package/dist/sdk/agent.js.map +1 -0
- package/dist/sdk/catalog.d.ts +31 -0
- package/dist/sdk/catalog.js +80 -0
- package/dist/sdk/catalog.js.map +1 -0
- package/dist/sdk/eval.d.ts +23 -0
- package/dist/sdk/eval.js +99 -0
- package/dist/sdk/eval.js.map +1 -0
- package/dist/sdk/evaluate.d.ts +13 -0
- package/dist/sdk/evaluate.js +84 -0
- package/dist/sdk/evaluate.js.map +1 -0
- package/dist/sdk/guardrail.d.ts +14 -0
- package/dist/sdk/guardrail.js +41 -0
- package/dist/sdk/guardrail.js.map +1 -0
- package/dist/sdk/mcp-client.d.ts +16 -0
- package/dist/sdk/mcp-client.js +128 -0
- package/dist/sdk/mcp-client.js.map +1 -0
- package/dist/sdk/memory.d.ts +22 -0
- package/dist/sdk/memory.js +101 -0
- package/dist/sdk/memory.js.map +1 -0
- package/dist/sdk/message.d.ts +4 -0
- package/dist/sdk/message.js +18 -0
- package/dist/sdk/message.js.map +1 -0
- package/dist/sdk/network.d.ts +20 -0
- package/dist/sdk/network.js +46 -0
- package/dist/sdk/network.js.map +1 -0
- package/dist/sdk/provider-tools.d.ts +18 -0
- package/dist/sdk/provider-tools.js +31 -0
- package/dist/sdk/provider-tools.js.map +1 -0
- package/dist/sdk/telemetry.d.ts +31 -0
- package/dist/sdk/telemetry.js +186 -0
- package/dist/sdk/telemetry.js.map +1 -0
- package/dist/sdk/tool.d.ts +38 -0
- package/dist/sdk/tool.js +143 -0
- package/dist/sdk/tool.js.map +1 -0
- package/dist/sdk/verify.d.ts +5 -0
- package/dist/sdk/verify.js +37 -0
- package/dist/sdk/verify.js.map +1 -0
- package/dist/storage/postgres-memory.d.ts +79 -0
- package/dist/storage/postgres-memory.js +411 -0
- package/dist/storage/postgres-memory.js.map +1 -0
- package/dist/storage/sqlite-memory.d.ts +60 -0
- package/dist/storage/sqlite-memory.js +349 -0
- package/dist/storage/sqlite-memory.js.map +1 -0
- package/dist/types/index.d.ts +13 -0
- package/dist/types/index.js +6 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/runtime/event.d.ts +41 -0
- package/dist/types/runtime/event.js +14 -0
- package/dist/types/runtime/event.js.map +1 -0
- package/dist/types/runtime/message-list.d.ts +7 -0
- package/dist/types/runtime/message-list.js +3 -0
- package/dist/types/runtime/message-list.js.map +1 -0
- package/dist/types/sdk/agent.d.ts +174 -0
- package/dist/types/sdk/agent.js +3 -0
- package/dist/types/sdk/agent.js.map +1 -0
- package/dist/types/sdk/eval.d.ts +41 -0
- package/dist/types/sdk/eval.js +3 -0
- package/dist/types/sdk/eval.js.map +1 -0
- package/dist/types/sdk/guardrail.d.ts +9 -0
- package/dist/types/sdk/guardrail.js +3 -0
- package/dist/types/sdk/guardrail.js.map +1 -0
- package/dist/types/sdk/mcp.d.ts +24 -0
- package/dist/types/sdk/mcp.js +3 -0
- package/dist/types/sdk/mcp.js.map +1 -0
- package/dist/types/sdk/memory.d.ts +99 -0
- package/dist/types/sdk/memory.js +3 -0
- package/dist/types/sdk/memory.js.map +1 -0
- package/dist/types/sdk/message.d.ts +79 -0
- package/dist/types/sdk/message.js +3 -0
- package/dist/types/sdk/message.js.map +1 -0
- package/dist/types/sdk/provider.d.ts +16 -0
- package/dist/types/sdk/provider.js +3 -0
- package/dist/types/sdk/provider.js.map +1 -0
- package/dist/types/sdk/tool.d.ts +32 -0
- package/dist/types/sdk/tool.js +3 -0
- package/dist/types/sdk/tool.js.map +1 -0
- package/dist/types/telemetry.d.ts +18 -0
- package/dist/types/telemetry.js +3 -0
- package/dist/types/telemetry.js.map +1 -0
- package/dist/types/utils/json.d.ts +5 -0
- package/dist/types/utils/json.js +3 -0
- package/dist/types/utils/json.js.map +1 -0
- package/dist/utils/zod.d.ts +3 -0
- package/dist/utils/zod.js +7 -0
- package/dist/utils/zod.js.map +1 -0
- package/dist/workspace/filesystem/base-filesystem.d.ts +40 -0
- package/dist/workspace/filesystem/base-filesystem.js +99 -0
- package/dist/workspace/filesystem/base-filesystem.js.map +1 -0
- package/dist/workspace/index.d.ts +8 -0
- package/dist/workspace/index.js +17 -0
- package/dist/workspace/index.js.map +1 -0
- package/dist/workspace/lifecycle.d.ts +13 -0
- package/dist/workspace/lifecycle.js +11 -0
- package/dist/workspace/lifecycle.js.map +1 -0
- package/dist/workspace/process.d.ts +36 -0
- package/dist/workspace/process.js +76 -0
- package/dist/workspace/process.js.map +1 -0
- package/dist/workspace/sandbox/base-sandbox.d.ts +28 -0
- package/dist/workspace/sandbox/base-sandbox.js +163 -0
- package/dist/workspace/sandbox/base-sandbox.js.map +1 -0
- package/dist/workspace/tools/append-file.d.ts +3 -0
- package/dist/workspace/tools/append-file.js +22 -0
- package/dist/workspace/tools/append-file.js.map +1 -0
- package/dist/workspace/tools/copy-file.d.ts +3 -0
- package/dist/workspace/tools/copy-file.js +26 -0
- package/dist/workspace/tools/copy-file.js.map +1 -0
- package/dist/workspace/tools/delete-file.d.ts +3 -0
- package/dist/workspace/tools/delete-file.js +26 -0
- package/dist/workspace/tools/delete-file.js.map +1 -0
- package/dist/workspace/tools/execute-command.d.ts +3 -0
- package/dist/workspace/tools/execute-command.js +39 -0
- package/dist/workspace/tools/execute-command.js.map +1 -0
- package/dist/workspace/tools/file-stat.d.ts +3 -0
- package/dist/workspace/tools/file-stat.js +33 -0
- package/dist/workspace/tools/file-stat.js.map +1 -0
- package/dist/workspace/tools/list-files.d.ts +3 -0
- package/dist/workspace/tools/list-files.js +28 -0
- package/dist/workspace/tools/list-files.js.map +1 -0
- package/dist/workspace/tools/mkdir.d.ts +3 -0
- package/dist/workspace/tools/mkdir.js +22 -0
- package/dist/workspace/tools/mkdir.js.map +1 -0
- package/dist/workspace/tools/move-file.d.ts +3 -0
- package/dist/workspace/tools/move-file.js +26 -0
- package/dist/workspace/tools/move-file.js.map +1 -0
- package/dist/workspace/tools/process-tools.d.ts +4 -0
- package/dist/workspace/tools/process-tools.js +37 -0
- package/dist/workspace/tools/process-tools.js.map +1 -0
- package/dist/workspace/tools/read-file.d.ts +3 -0
- package/dist/workspace/tools/read-file.js +24 -0
- package/dist/workspace/tools/read-file.js.map +1 -0
- package/dist/workspace/tools/rmdir.d.ts +3 -0
- package/dist/workspace/tools/rmdir.js +26 -0
- package/dist/workspace/tools/rmdir.js.map +1 -0
- package/dist/workspace/tools/workspace-tools.d.ts +8 -0
- package/dist/workspace/tools/workspace-tools.js +39 -0
- package/dist/workspace/tools/workspace-tools.js.map +1 -0
- package/dist/workspace/tools/write-file.d.ts +3 -0
- package/dist/workspace/tools/write-file.js +26 -0
- package/dist/workspace/tools/write-file.js.map +1 -0
- package/dist/workspace/types.d.ts +178 -0
- package/dist/workspace/types.js +7 -0
- package/dist/workspace/types.js.map +1 -0
- package/dist/workspace/workspace.d.ts +21 -0
- package/dist/workspace/workspace.js +131 -0
- package/dist/workspace/workspace.js.map +1 -0
- package/package.json +65 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.categorization = categorization;
|
|
4
|
+
const eval_1 = require("../sdk/eval");
|
|
5
|
+
function categorization() {
|
|
6
|
+
return new eval_1.Eval('categorization')
|
|
7
|
+
.description('Checks if output matches the expected category label')
|
|
8
|
+
.check(({ output, expected }) => {
|
|
9
|
+
if (!expected) {
|
|
10
|
+
return { pass: false, reasoning: 'No expected category provided' };
|
|
11
|
+
}
|
|
12
|
+
const normalOutput = output.toLowerCase().trim();
|
|
13
|
+
const normalExpected = expected.toLowerCase().trim();
|
|
14
|
+
if (normalOutput === normalExpected) {
|
|
15
|
+
return { pass: true, reasoning: 'Exact match' };
|
|
16
|
+
}
|
|
17
|
+
if (normalOutput.includes(normalExpected)) {
|
|
18
|
+
return { pass: true, reasoning: `Output contains expected label "${expected}"` };
|
|
19
|
+
}
|
|
20
|
+
return { pass: false, reasoning: `Expected "${expected}", got "${output}"` };
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=categorization.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"categorization.js","sourceRoot":"","sources":["../../src/evals/categorization.ts"],"names":[],"mappings":";;AAGA,wCAqBC;AAxBD,sCAAmC;AAGnC,SAAgB,cAAc;IAC7B,OAAO,IAAI,WAAI,CAAC,gBAAgB,CAAC;SAC/B,WAAW,CAAC,sDAAsD,CAAC;SACnE,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC/B,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,+BAA+B,EAAE,CAAC;QACpE,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QACjD,MAAM,cAAc,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAErD,IAAI,YAAY,KAAK,cAAc,EAAE,CAAC;YACrC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC;QACjD,CAAC;QAED,IAAI,YAAY,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YAC3C,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,mCAAmC,QAAQ,GAAG,EAAE,CAAC;QAClF,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,QAAQ,WAAW,MAAM,GAAG,EAAE,CAAC;IAC9E,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.containsKeywords = containsKeywords;
|
|
4
|
+
const eval_1 = require("../sdk/eval");
|
|
5
|
+
function containsKeywords() {
|
|
6
|
+
return new eval_1.Eval('contains-keywords')
|
|
7
|
+
.description('Checks if output contains all expected keywords')
|
|
8
|
+
.check(({ output, expected }) => {
|
|
9
|
+
if (!expected) {
|
|
10
|
+
return { pass: false, reasoning: 'No expected keywords provided' };
|
|
11
|
+
}
|
|
12
|
+
const keywords = expected
|
|
13
|
+
.split(',')
|
|
14
|
+
.map((k) => k.trim().toLowerCase())
|
|
15
|
+
.filter(Boolean);
|
|
16
|
+
if (keywords.length === 0) {
|
|
17
|
+
return { pass: false, reasoning: 'No keywords to check' };
|
|
18
|
+
}
|
|
19
|
+
const normalOutput = output.toLowerCase();
|
|
20
|
+
const missing = keywords.filter((k) => !normalOutput.includes(k));
|
|
21
|
+
return {
|
|
22
|
+
pass: missing.length === 0,
|
|
23
|
+
reasoning: missing.length === 0
|
|
24
|
+
? `All ${keywords.length} keywords found`
|
|
25
|
+
: `Missing ${missing.length}/${keywords.length} keywords: ${missing.join(', ')}`,
|
|
26
|
+
};
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=contains-keywords.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"contains-keywords.js","sourceRoot":"","sources":["../../src/evals/contains-keywords.ts"],"names":[],"mappings":";;AAOA,4CA2BC;AAlCD,sCAAmC;AAOnC,SAAgB,gBAAgB;IAC/B,OAAO,IAAI,WAAI,CAAC,mBAAmB,CAAC;SAClC,WAAW,CAAC,iDAAiD,CAAC;SAC9D,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC/B,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,+BAA+B,EAAE,CAAC;QACpE,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ;aACvB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;aAClC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,sBAAsB,EAAE,CAAC;QAC3D,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAElE,OAAO;YACN,IAAI,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC;YAC1B,SAAS,EACR,OAAO,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,OAAO,QAAQ,CAAC,MAAM,iBAAiB;gBACzC,CAAC,CAAC,WAAW,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,cAAc,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;SAClF,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.correctness = correctness;
|
|
4
|
+
const parse_judge_response_1 = require("./parse-judge-response");
|
|
5
|
+
const eval_1 = require("../sdk/eval");
|
|
6
|
+
function correctness() {
|
|
7
|
+
return new eval_1.Eval('correctness')
|
|
8
|
+
.description('Judges if the output is factually correct compared to the expected answer')
|
|
9
|
+
.judge(async ({ input, output, expected, llm }) => {
|
|
10
|
+
const prompt = [
|
|
11
|
+
'You are evaluating an AI assistant response for factual correctness.',
|
|
12
|
+
'',
|
|
13
|
+
`User question: ${input}`,
|
|
14
|
+
`Expected answer: ${expected ?? '(none provided)'}`,
|
|
15
|
+
`Actual answer: ${output}`,
|
|
16
|
+
'',
|
|
17
|
+
'Does the actual answer correctly address the question and match the expected answer?',
|
|
18
|
+
'Answer with pass or fail:',
|
|
19
|
+
'- pass = the answer is correct and addresses the question',
|
|
20
|
+
'- fail = the answer is incorrect, incomplete, or irrelevant',
|
|
21
|
+
'',
|
|
22
|
+
'Respond with ONLY a JSON object (no markdown fences): {"pass": true/false, "reasoning": "<explanation>"}',
|
|
23
|
+
].join('\n');
|
|
24
|
+
const result = await llm(prompt);
|
|
25
|
+
return (0, parse_judge_response_1.parseJudgeResponse)(result.text);
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=correctness.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"correctness.js","sourceRoot":"","sources":["../../src/evals/correctness.ts"],"names":[],"mappings":";;AAOA,kCAsBC;AA7BD,iEAA4D;AAC5D,sCAAmC;AAMnC,SAAgB,WAAW;IAC1B,OAAO,IAAI,WAAI,CAAC,aAAa,CAAC;SAC5B,WAAW,CAAC,2EAA2E,CAAC;SACxF,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE;QACjD,MAAM,MAAM,GAAG;YACd,sEAAsE;YACtE,EAAE;YACF,kBAAkB,KAAK,EAAE;YACzB,oBAAoB,QAAQ,IAAI,iBAAiB,EAAE;YACnD,kBAAkB,MAAM,EAAE;YAC1B,EAAE;YACF,sFAAsF;YACtF,2BAA2B;YAC3B,2DAA2D;YAC3D,6DAA6D;YAC7D,EAAE;YACF,0GAA0G;SAC1G,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,IAAA,yCAAkB,EAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.helpfulness = helpfulness;
|
|
4
|
+
const parse_judge_response_1 = require("./parse-judge-response");
|
|
5
|
+
const eval_1 = require("../sdk/eval");
|
|
6
|
+
function helpfulness() {
|
|
7
|
+
return new eval_1.Eval('helpfulness')
|
|
8
|
+
.description('Judges whether the response is helpful for the user query')
|
|
9
|
+
.judge(async ({ input, output, llm }) => {
|
|
10
|
+
const prompt = [
|
|
11
|
+
'You are evaluating an AI assistant response for helpfulness.',
|
|
12
|
+
'',
|
|
13
|
+
`User question: ${input}`,
|
|
14
|
+
`Assistant response: ${output}`,
|
|
15
|
+
'',
|
|
16
|
+
'Is this response helpful to the user?',
|
|
17
|
+
'- pass = the response is helpful, addresses the question, and provides useful information',
|
|
18
|
+
'- fail = the response is unhelpful, off-topic, or lacks useful information',
|
|
19
|
+
'',
|
|
20
|
+
'Respond with ONLY a JSON object (no markdown fences): {"pass": true/false, "reasoning": "<explanation>"}',
|
|
21
|
+
].join('\n');
|
|
22
|
+
const result = await llm(prompt);
|
|
23
|
+
return (0, parse_judge_response_1.parseJudgeResponse)(result.text);
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=helpfulness.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"helpfulness.js","sourceRoot":"","sources":["../../src/evals/helpfulness.ts"],"names":[],"mappings":";;AAOA,kCAoBC;AA3BD,iEAA4D;AAC5D,sCAAmC;AAMnC,SAAgB,WAAW;IAC1B,OAAO,IAAI,WAAI,CAAC,aAAa,CAAC;SAC5B,WAAW,CAAC,2DAA2D,CAAC;SACxE,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE;QACvC,MAAM,MAAM,GAAG;YACd,8DAA8D;YAC9D,EAAE;YACF,kBAAkB,KAAK,EAAE;YACzB,uBAAuB,MAAM,EAAE;YAC/B,EAAE;YACF,uCAAuC;YACvC,2FAA2F;YAC3F,4EAA4E;YAC5E,EAAE;YACF,0GAA0G;SAC1G,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,IAAA,yCAAkB,EAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { correctness } from './correctness';
|
|
2
|
+
export { helpfulness } from './helpfulness';
|
|
3
|
+
export { stringSimilarity } from './string-similarity';
|
|
4
|
+
export { categorization } from './categorization';
|
|
5
|
+
export { containsKeywords } from './contains-keywords';
|
|
6
|
+
export { jsonValidity } from './json-validity';
|
|
7
|
+
export { toolCallAccuracy } from './tool-call-accuracy';
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.toolCallAccuracy = exports.jsonValidity = exports.containsKeywords = exports.categorization = exports.stringSimilarity = exports.helpfulness = exports.correctness = void 0;
|
|
4
|
+
var correctness_1 = require("./correctness");
|
|
5
|
+
Object.defineProperty(exports, "correctness", { enumerable: true, get: function () { return correctness_1.correctness; } });
|
|
6
|
+
var helpfulness_1 = require("./helpfulness");
|
|
7
|
+
Object.defineProperty(exports, "helpfulness", { enumerable: true, get: function () { return helpfulness_1.helpfulness; } });
|
|
8
|
+
var string_similarity_1 = require("./string-similarity");
|
|
9
|
+
Object.defineProperty(exports, "stringSimilarity", { enumerable: true, get: function () { return string_similarity_1.stringSimilarity; } });
|
|
10
|
+
var categorization_1 = require("./categorization");
|
|
11
|
+
Object.defineProperty(exports, "categorization", { enumerable: true, get: function () { return categorization_1.categorization; } });
|
|
12
|
+
var contains_keywords_1 = require("./contains-keywords");
|
|
13
|
+
Object.defineProperty(exports, "containsKeywords", { enumerable: true, get: function () { return contains_keywords_1.containsKeywords; } });
|
|
14
|
+
var json_validity_1 = require("./json-validity");
|
|
15
|
+
Object.defineProperty(exports, "jsonValidity", { enumerable: true, get: function () { return json_validity_1.jsonValidity; } });
|
|
16
|
+
var tool_call_accuracy_1 = require("./tool-call-accuracy");
|
|
17
|
+
Object.defineProperty(exports, "toolCallAccuracy", { enumerable: true, get: function () { return tool_call_accuracy_1.toolCallAccuracy; } });
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":";;;AAAA,6CAA4C;AAAnC,0GAAA,WAAW,OAAA;AACpB,6CAA4C;AAAnC,0GAAA,WAAW,OAAA;AACpB,yDAAuD;AAA9C,qHAAA,gBAAgB,OAAA;AACzB,mDAAkD;AAAzC,gHAAA,cAAc,OAAA;AACvB,yDAAuD;AAA9C,qHAAA,gBAAgB,OAAA;AACzB,iDAA+C;AAAtC,6GAAA,YAAY,OAAA;AACrB,2DAAwD;AAA/C,sHAAA,gBAAgB,OAAA"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.jsonValidity = jsonValidity;
|
|
4
|
+
const eval_1 = require("../sdk/eval");
|
|
5
|
+
function jsonValidity() {
|
|
6
|
+
return new eval_1.Eval('json-validity')
|
|
7
|
+
.description('Checks if output is valid JSON')
|
|
8
|
+
.check(({ output }) => {
|
|
9
|
+
try {
|
|
10
|
+
JSON.parse(output);
|
|
11
|
+
return { pass: true, reasoning: 'Valid JSON' };
|
|
12
|
+
}
|
|
13
|
+
catch (e) {
|
|
14
|
+
return {
|
|
15
|
+
pass: false,
|
|
16
|
+
reasoning: `Invalid JSON: ${e instanceof Error ? e.message : 'parse error'}`,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=json-validity.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-validity.js","sourceRoot":"","sources":["../../src/evals/json-validity.ts"],"names":[],"mappings":";;AAGA,oCAcC;AAjBD,sCAAmC;AAGnC,SAAgB,YAAY;IAC3B,OAAO,IAAI,WAAI,CAAC,eAAe,CAAC;SAC9B,WAAW,CAAC,gCAAgC,CAAC;SAC7C,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QACrB,IAAI,CAAC;YACJ,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACnB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,OAAO;gBACN,IAAI,EAAE,KAAK;gBACX,SAAS,EAAE,iBAAiB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,EAAE;aAC5E,CAAC;QACH,CAAC;IACF,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseJudgeResponse = parseJudgeResponse;
|
|
4
|
+
function parseJudgeResponse(text) {
|
|
5
|
+
const stripped = text
|
|
6
|
+
.replace(/^```(?:json)?\s*\n?/i, '')
|
|
7
|
+
.replace(/\n?```\s*$/i, '')
|
|
8
|
+
.trim();
|
|
9
|
+
try {
|
|
10
|
+
const parsed = JSON.parse(stripped);
|
|
11
|
+
const pass = parsed.pass ?? (parsed.score !== undefined ? parsed.score >= 0.7 : false);
|
|
12
|
+
return {
|
|
13
|
+
pass,
|
|
14
|
+
reasoning: parsed.reasoning ?? stripped,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
const lowerText = stripped.toLowerCase();
|
|
19
|
+
const hasPassTrue = lowerText.includes('"pass": true') || lowerText.includes('"pass":true');
|
|
20
|
+
const hasFailFalse = lowerText.includes('"pass": false') || lowerText.includes('"pass":false');
|
|
21
|
+
const pass = hasPassTrue || (!hasFailFalse && /\bpass\b/i.test(stripped) && !/\bfail\b/i.test(stripped));
|
|
22
|
+
return { pass, reasoning: stripped };
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=parse-judge-response.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-judge-response.js","sourceRoot":"","sources":["../../src/evals/parse-judge-response.ts"],"names":[],"mappings":";;AAMA,gDAyBC;AAzBD,SAAgB,kBAAkB,CAAC,IAAY;IAE9C,MAAM,QAAQ,GAAG,IAAI;SACnB,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC;SACnC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;SAC1B,IAAI,EAAE,CAAC;IAET,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA2D,CAAC;QAE9F,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvF,OAAO;YACN,IAAI;YACJ,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,QAAQ;SACvC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QAER,MAAM,SAAS,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC5F,MAAM,YAAY,GAAG,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QAE/F,MAAM,IAAI,GACT,WAAW,IAAI,CAAC,CAAC,YAAY,IAAI,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7F,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtC,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.stringSimilarity = stringSimilarity;
|
|
4
|
+
const eval_1 = require("../sdk/eval");
|
|
5
|
+
function diceSimilarity(a, b) {
|
|
6
|
+
const normalA = a.toLowerCase().trim();
|
|
7
|
+
const normalB = b.toLowerCase().trim();
|
|
8
|
+
if (normalA === normalB)
|
|
9
|
+
return 1;
|
|
10
|
+
if (normalA.length < 2 || normalB.length < 2)
|
|
11
|
+
return 0;
|
|
12
|
+
const bigrams = (s) => {
|
|
13
|
+
const set = new Set();
|
|
14
|
+
for (let i = 0; i < s.length - 1; i++) {
|
|
15
|
+
set.add(s.slice(i, i + 2));
|
|
16
|
+
}
|
|
17
|
+
return set;
|
|
18
|
+
};
|
|
19
|
+
const aBigrams = bigrams(normalA);
|
|
20
|
+
const bBigrams = bigrams(normalB);
|
|
21
|
+
let intersection = 0;
|
|
22
|
+
for (const bg of aBigrams) {
|
|
23
|
+
if (bBigrams.has(bg))
|
|
24
|
+
intersection++;
|
|
25
|
+
}
|
|
26
|
+
return (2 * intersection) / (aBigrams.size + bBigrams.size);
|
|
27
|
+
}
|
|
28
|
+
function stringSimilarity() {
|
|
29
|
+
return new eval_1.Eval('string-similarity')
|
|
30
|
+
.description('Measures string similarity between output and expected answer')
|
|
31
|
+
.check(({ output, expected }) => {
|
|
32
|
+
if (expected === undefined) {
|
|
33
|
+
return { pass: false, reasoning: 'No expected value provided' };
|
|
34
|
+
}
|
|
35
|
+
const similarity = diceSimilarity(output, expected);
|
|
36
|
+
return {
|
|
37
|
+
pass: similarity >= 0.7,
|
|
38
|
+
reasoning: `Dice similarity: ${(similarity * 100).toFixed(1)}%`,
|
|
39
|
+
};
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=string-similarity.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"string-similarity.js","sourceRoot":"","sources":["../../src/evals/string-similarity.ts"],"names":[],"mappings":";;AAgCA,4CAaC;AA7CD,sCAAmC;AAMnC,SAAS,cAAc,CAAC,CAAS,EAAE,CAAS;IAC3C,MAAM,OAAO,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACvC,MAAM,OAAO,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IAEvC,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,CAAC,CAAC;IAClC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAEvD,MAAM,OAAO,GAAG,CAAC,CAAS,EAAe,EAAE;QAC1C,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,GAAG,CAAC;IACZ,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAClC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,YAAY,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,CAAC,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;AAC7D,CAAC;AAGD,SAAgB,gBAAgB;IAC/B,OAAO,IAAI,WAAI,CAAC,mBAAmB,CAAC;SAClC,WAAW,CAAC,+DAA+D,CAAC;SAC5E,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC/B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC5B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,4BAA4B,EAAE,CAAC;QACjE,CAAC;QACD,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,OAAO;YACN,IAAI,EAAE,UAAU,IAAI,GAAG;YACvB,SAAS,EAAE,oBAAoB,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;SAC/D,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.toolCallAccuracy = toolCallAccuracy;
|
|
4
|
+
const eval_1 = require("../sdk/eval");
|
|
5
|
+
function toolCallAccuracy() {
|
|
6
|
+
return new eval_1.Eval('tool-call-accuracy')
|
|
7
|
+
.description('Checks if the agent called all expected tools')
|
|
8
|
+
.check(({ expected, toolCalls }) => {
|
|
9
|
+
if (!expected) {
|
|
10
|
+
return { pass: false, reasoning: 'No expected tool names provided' };
|
|
11
|
+
}
|
|
12
|
+
const expectedTools = expected
|
|
13
|
+
.split(',')
|
|
14
|
+
.map((t) => t.trim().toLowerCase())
|
|
15
|
+
.filter(Boolean);
|
|
16
|
+
if (expectedTools.length === 0) {
|
|
17
|
+
return { pass: false, reasoning: 'No expected tools to check' };
|
|
18
|
+
}
|
|
19
|
+
const calledTools = new Set((toolCalls ?? []).map((tc) => tc.tool.toLowerCase()));
|
|
20
|
+
const missing = expectedTools.filter((t) => !calledTools.has(t));
|
|
21
|
+
return {
|
|
22
|
+
pass: missing.length === 0,
|
|
23
|
+
reasoning: missing.length === 0
|
|
24
|
+
? `All ${expectedTools.length} expected tools were called`
|
|
25
|
+
: `Missing tools: ${missing.join(', ')}. Called: [${[...calledTools].join(', ') || 'none'}]`,
|
|
26
|
+
};
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=tool-call-accuracy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-call-accuracy.js","sourceRoot":"","sources":["../../src/evals/tool-call-accuracy.ts"],"names":[],"mappings":";;AAOA,4CA2BC;AAlCD,sCAAmC;AAOnC,SAAgB,gBAAgB;IAC/B,OAAO,IAAI,WAAI,CAAC,oBAAoB,CAAC;SACnC,WAAW,CAAC,+CAA+C,CAAC;SAC5D,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE;QAClC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,iCAAiC,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,aAAa,GAAG,QAAQ;aAC5B,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;aAClC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClB,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,4BAA4B,EAAE,CAAC;QACjE,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAClF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEjE,OAAO;YACN,IAAI,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC;YAC1B,SAAS,EACR,OAAO,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,OAAO,aAAa,CAAC,MAAM,6BAA6B;gBAC1D,CAAC,CAAC,kBAAkB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,GAAG;SAC9F,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export type { BuiltTool, BuiltProviderTool, BuiltAgent, BuiltMemory, BuiltGuardrail, BuiltEval, RunOptions, AgentResult, GenerateResult, StreamResult, EvalInput, EvalScore, EvalRunResult, EvalResults, ToolContext, InterruptibleToolContext, CheckpointStore, StreamChunk, SubAgentUsage, Provider, ThinkingConfig, ThinkingConfigFor, AnthropicThinkingConfig, OpenAIThinkingConfig, GoogleThinkingConfig, XaiThinkingConfig, SerializableAgentState, AgentRunState, MemoryConfig, TitleGenerationConfig, Thread, SemanticRecallConfig, ResumeOptions, McpServerConfig, McpVerifyResult, ModelConfig, ExecutionOptions, PersistedExecutionOptions, BuiltTelemetry, AttributeValue, } from './types';
|
|
2
|
+
export type { ProviderOptions } from '@ai-sdk/provider-utils';
|
|
3
|
+
export { AgentEvent } from './types';
|
|
4
|
+
export type { AgentEventData, AgentEventHandler } from './types';
|
|
5
|
+
export { Tool } from './sdk/tool';
|
|
6
|
+
export { Memory } from './sdk/memory';
|
|
7
|
+
export { Guardrail } from './sdk/guardrail';
|
|
8
|
+
export { Eval } from './sdk/eval';
|
|
9
|
+
export { evaluate } from './sdk/evaluate';
|
|
10
|
+
export type { DatasetRow, EvaluateConfig } from './sdk/evaluate';
|
|
11
|
+
export * as evals from './evals/index';
|
|
12
|
+
export { Telemetry } from './sdk/telemetry';
|
|
13
|
+
export { LangSmithTelemetry } from './integrations/langsmith';
|
|
14
|
+
export type { LangSmithTelemetryConfig } from './integrations/langsmith';
|
|
15
|
+
export { Agent } from './sdk/agent';
|
|
16
|
+
export { McpClient } from './sdk/mcp-client';
|
|
17
|
+
export { Network } from './sdk/network';
|
|
18
|
+
export { providerTools } from './sdk/provider-tools';
|
|
19
|
+
export { verify } from './sdk/verify';
|
|
20
|
+
export type { VerifyResult } from './sdk/verify';
|
|
21
|
+
export type { ContentCitation, ContentFile, ContentMetadata, ContentReasoning, ContentText, ContentToolCall, ContentToolResult, Message, MessageContent, MessageRole, AgentMessage, CustomAgentMessages, AgentDbMessage, } from './types/sdk/message';
|
|
22
|
+
export { toDbMessage, filterLlmMessages, isLlmMessage, } from './sdk/message';
|
|
23
|
+
export { fetchProviderCatalog } from './sdk/catalog';
|
|
24
|
+
export type { ProviderCatalog, ProviderInfo, ModelInfo, ModelCost, ModelLimits, } from './sdk/catalog';
|
|
25
|
+
export { SqliteMemory } from './storage/sqlite-memory';
|
|
26
|
+
export type { SqliteMemoryConfig } from './storage/sqlite-memory';
|
|
27
|
+
export { PostgresMemory } from './storage/postgres-memory';
|
|
28
|
+
export type { PostgresMemoryConfig } from './storage/postgres-memory';
|
|
29
|
+
export { Workspace } from './workspace';
|
|
30
|
+
export { BaseFilesystem } from './workspace';
|
|
31
|
+
export { BaseSandbox } from './workspace';
|
|
32
|
+
export { createWorkspaceTools } from './workspace';
|
|
33
|
+
export { SandboxProcessManager, ProcessHandle } from './workspace';
|
|
34
|
+
export type { BaseFilesystemOptions, FilesystemLifecycleHook, WorkspaceFilesystem, WorkspaceSandbox, WorkspaceConfig, CommandResult, CommandOptions, ExecuteCommandOptions, FileContent, FileStat, FileEntry, ReadOptions, WriteOptions, ListOptions, RemoveOptions, CopyOptions, ProviderStatus, SandboxInfo, LocalFilesystemOptions, LocalSandboxOptions, DaytonaSandboxOptions, BaseSandboxOptions, MountConfig, MountResult, SpawnProcessOptions, ProcessInfo, } from './workspace';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.ProcessHandle = exports.SandboxProcessManager = exports.createWorkspaceTools = exports.BaseSandbox = exports.BaseFilesystem = exports.Workspace = exports.PostgresMemory = exports.SqliteMemory = exports.fetchProviderCatalog = exports.isLlmMessage = exports.filterLlmMessages = exports.toDbMessage = exports.verify = exports.providerTools = exports.Network = exports.McpClient = exports.Agent = exports.LangSmithTelemetry = exports.Telemetry = exports.evals = exports.evaluate = exports.Eval = exports.Guardrail = exports.Memory = exports.Tool = exports.AgentEvent = void 0;
|
|
37
|
+
var types_1 = require("./types");
|
|
38
|
+
Object.defineProperty(exports, "AgentEvent", { enumerable: true, get: function () { return types_1.AgentEvent; } });
|
|
39
|
+
var tool_1 = require("./sdk/tool");
|
|
40
|
+
Object.defineProperty(exports, "Tool", { enumerable: true, get: function () { return tool_1.Tool; } });
|
|
41
|
+
var memory_1 = require("./sdk/memory");
|
|
42
|
+
Object.defineProperty(exports, "Memory", { enumerable: true, get: function () { return memory_1.Memory; } });
|
|
43
|
+
var guardrail_1 = require("./sdk/guardrail");
|
|
44
|
+
Object.defineProperty(exports, "Guardrail", { enumerable: true, get: function () { return guardrail_1.Guardrail; } });
|
|
45
|
+
var eval_1 = require("./sdk/eval");
|
|
46
|
+
Object.defineProperty(exports, "Eval", { enumerable: true, get: function () { return eval_1.Eval; } });
|
|
47
|
+
var evaluate_1 = require("./sdk/evaluate");
|
|
48
|
+
Object.defineProperty(exports, "evaluate", { enumerable: true, get: function () { return evaluate_1.evaluate; } });
|
|
49
|
+
exports.evals = __importStar(require("./evals/index"));
|
|
50
|
+
var telemetry_1 = require("./sdk/telemetry");
|
|
51
|
+
Object.defineProperty(exports, "Telemetry", { enumerable: true, get: function () { return telemetry_1.Telemetry; } });
|
|
52
|
+
var langsmith_1 = require("./integrations/langsmith");
|
|
53
|
+
Object.defineProperty(exports, "LangSmithTelemetry", { enumerable: true, get: function () { return langsmith_1.LangSmithTelemetry; } });
|
|
54
|
+
var agent_1 = require("./sdk/agent");
|
|
55
|
+
Object.defineProperty(exports, "Agent", { enumerable: true, get: function () { return agent_1.Agent; } });
|
|
56
|
+
var mcp_client_1 = require("./sdk/mcp-client");
|
|
57
|
+
Object.defineProperty(exports, "McpClient", { enumerable: true, get: function () { return mcp_client_1.McpClient; } });
|
|
58
|
+
var network_1 = require("./sdk/network");
|
|
59
|
+
Object.defineProperty(exports, "Network", { enumerable: true, get: function () { return network_1.Network; } });
|
|
60
|
+
var provider_tools_1 = require("./sdk/provider-tools");
|
|
61
|
+
Object.defineProperty(exports, "providerTools", { enumerable: true, get: function () { return provider_tools_1.providerTools; } });
|
|
62
|
+
var verify_1 = require("./sdk/verify");
|
|
63
|
+
Object.defineProperty(exports, "verify", { enumerable: true, get: function () { return verify_1.verify; } });
|
|
64
|
+
var message_1 = require("./sdk/message");
|
|
65
|
+
Object.defineProperty(exports, "toDbMessage", { enumerable: true, get: function () { return message_1.toDbMessage; } });
|
|
66
|
+
Object.defineProperty(exports, "filterLlmMessages", { enumerable: true, get: function () { return message_1.filterLlmMessages; } });
|
|
67
|
+
Object.defineProperty(exports, "isLlmMessage", { enumerable: true, get: function () { return message_1.isLlmMessage; } });
|
|
68
|
+
var catalog_1 = require("./sdk/catalog");
|
|
69
|
+
Object.defineProperty(exports, "fetchProviderCatalog", { enumerable: true, get: function () { return catalog_1.fetchProviderCatalog; } });
|
|
70
|
+
var sqlite_memory_1 = require("./storage/sqlite-memory");
|
|
71
|
+
Object.defineProperty(exports, "SqliteMemory", { enumerable: true, get: function () { return sqlite_memory_1.SqliteMemory; } });
|
|
72
|
+
var postgres_memory_1 = require("./storage/postgres-memory");
|
|
73
|
+
Object.defineProperty(exports, "PostgresMemory", { enumerable: true, get: function () { return postgres_memory_1.PostgresMemory; } });
|
|
74
|
+
var workspace_1 = require("./workspace");
|
|
75
|
+
Object.defineProperty(exports, "Workspace", { enumerable: true, get: function () { return workspace_1.Workspace; } });
|
|
76
|
+
var workspace_2 = require("./workspace");
|
|
77
|
+
Object.defineProperty(exports, "BaseFilesystem", { enumerable: true, get: function () { return workspace_2.BaseFilesystem; } });
|
|
78
|
+
var workspace_3 = require("./workspace");
|
|
79
|
+
Object.defineProperty(exports, "BaseSandbox", { enumerable: true, get: function () { return workspace_3.BaseSandbox; } });
|
|
80
|
+
var workspace_4 = require("./workspace");
|
|
81
|
+
Object.defineProperty(exports, "createWorkspaceTools", { enumerable: true, get: function () { return workspace_4.createWorkspaceTools; } });
|
|
82
|
+
var workspace_5 = require("./workspace");
|
|
83
|
+
Object.defineProperty(exports, "SandboxProcessManager", { enumerable: true, get: function () { return workspace_5.SandboxProcessManager; } });
|
|
84
|
+
Object.defineProperty(exports, "ProcessHandle", { enumerable: true, get: function () { return workspace_5.ProcessHandle; } });
|
|
85
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2CA,iCAAqC;AAA5B,mGAAA,UAAU,OAAA;AAGnB,mCAAkC;AAAzB,4FAAA,IAAI,OAAA;AACb,uCAAsC;AAA7B,gGAAA,MAAM,OAAA;AACf,6CAA4C;AAAnC,sGAAA,SAAS,OAAA;AAClB,mCAAkC;AAAzB,4FAAA,IAAI,OAAA;AACb,2CAA0C;AAAjC,oGAAA,QAAQ,OAAA;AAEjB,uDAAuC;AACvC,6CAA4C;AAAnC,sGAAA,SAAS,OAAA;AAClB,sDAA8D;AAArD,+GAAA,kBAAkB,OAAA;AAE3B,qCAAoC;AAA3B,8FAAA,KAAK,OAAA;AACd,+CAA6C;AAApC,uGAAA,SAAS,OAAA;AAClB,yCAAwC;AAA/B,kGAAA,OAAO,OAAA;AAChB,uDAAqD;AAA5C,+GAAA,aAAa,OAAA;AACtB,uCAAsC;AAA7B,gGAAA,MAAM,OAAA;AAiBf,yCAIuB;AAHtB,sGAAA,WAAW,OAAA;AACX,4GAAA,iBAAiB,OAAA;AACjB,uGAAA,YAAY,OAAA;AAEb,yCAAqD;AAA5C,+GAAA,oBAAoB,OAAA;AAQ7B,yDAAuD;AAA9C,6GAAA,YAAY,OAAA;AAErB,6DAA2D;AAAlD,iHAAA,cAAc,OAAA;AAGvB,yCAAwC;AAA/B,sGAAA,SAAS,OAAA;AAClB,yCAA6C;AAApC,2GAAA,cAAc,OAAA;AACvB,yCAA0C;AAAjC,wGAAA,WAAW,OAAA;AACpB,yCAAmD;AAA1C,iHAAA,oBAAoB,OAAA;AAC7B,yCAAmE;AAA1D,kHAAA,qBAAqB,OAAA;AAAE,0GAAA,aAAa,OAAA"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Telemetry } from '../sdk/telemetry';
|
|
2
|
+
import type { BuiltTelemetry } from '../types/telemetry';
|
|
3
|
+
export interface LangSmithTelemetryConfig {
|
|
4
|
+
apiKey?: string;
|
|
5
|
+
project?: string;
|
|
6
|
+
endpoint?: string;
|
|
7
|
+
url?: string;
|
|
8
|
+
}
|
|
9
|
+
export declare class LangSmithTelemetry extends Telemetry {
|
|
10
|
+
private langsmithConfig?;
|
|
11
|
+
constructor(config?: LangSmithTelemetryConfig);
|
|
12
|
+
build(): Promise<BuiltTelemetry>;
|
|
13
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.LangSmithTelemetry = void 0;
|
|
37
|
+
const telemetry_1 = require("../sdk/telemetry");
|
|
38
|
+
async function createLangSmithTracer(config, resolvedApiKey) {
|
|
39
|
+
const { NodeTracerProvider } = (await Promise.resolve().then(() => __importStar(require('@opentelemetry/sdk-trace-node'))));
|
|
40
|
+
const { LangSmithOTLPTraceExporter } = (await Promise.resolve().then(() => __importStar(require('langsmith/experimental/otel/exporter'))));
|
|
41
|
+
const { LangSmithOTLPSpanProcessor } = (await Promise.resolve().then(() => __importStar(require('langsmith/experimental/otel/processor'))));
|
|
42
|
+
const apiKey = config?.apiKey ?? resolvedApiKey;
|
|
43
|
+
const usingResolvedKey = !config?.apiKey && resolvedApiKey !== undefined;
|
|
44
|
+
const url = usingResolvedKey
|
|
45
|
+
? undefined
|
|
46
|
+
: (config?.url ??
|
|
47
|
+
(config?.endpoint ? `${config.endpoint.replace(/\/$/, '')}/otel/v1/traces` : undefined));
|
|
48
|
+
const exporter = new LangSmithOTLPTraceExporter({
|
|
49
|
+
apiKey,
|
|
50
|
+
projectName: config?.project,
|
|
51
|
+
...(url ? { url } : {}),
|
|
52
|
+
});
|
|
53
|
+
const processor = new LangSmithOTLPSpanProcessor(exporter);
|
|
54
|
+
const provider = new NodeTracerProvider({
|
|
55
|
+
spanProcessors: [processor],
|
|
56
|
+
});
|
|
57
|
+
return { tracer: provider.getTracer('@n8n/agents'), provider };
|
|
58
|
+
}
|
|
59
|
+
class LangSmithTelemetry extends telemetry_1.Telemetry {
|
|
60
|
+
constructor(config) {
|
|
61
|
+
super();
|
|
62
|
+
this.langsmithConfig = config;
|
|
63
|
+
}
|
|
64
|
+
async build() {
|
|
65
|
+
if (this.otlpEndpointValue !== undefined) {
|
|
66
|
+
throw new Error('LangSmithTelemetry creates its own tracer — do not use .otlpEndpoint().');
|
|
67
|
+
}
|
|
68
|
+
this.tracerValue = undefined;
|
|
69
|
+
process.env.LANGCHAIN_TRACING_V2 ??= 'true';
|
|
70
|
+
const { tracer, provider } = await createLangSmithTracer(this.langsmithConfig, this.resolvedKey);
|
|
71
|
+
this.tracerValue = tracer;
|
|
72
|
+
const built = await super.build();
|
|
73
|
+
return { ...built, provider };
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
exports.LangSmithTelemetry = LangSmithTelemetry;
|
|
77
|
+
//# sourceMappingURL=langsmith.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"langsmith.js","sourceRoot":"","sources":["../../src/integrations/langsmith.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAA6C;AAsB7C,KAAK,UAAU,qBAAqB,CACnC,MAAiC,EACjC,cAAuB;IAEvB,MAAM,EAAE,kBAAkB,EAAE,GAAG,CAAC,wDAAa,+BAA+B,GAAC,CAM5E,CAAC;IAEF,MAAM,EAAE,0BAA0B,EAAE,GAAG,CAAC,wDAAa,sCAAsC,GAAC,CAM3F,CAAC;IAEF,MAAM,EAAE,0BAA0B,EAAE,GAAG,CAAC,wDACvC,uCAAuC,GACvC,CAEA,CAAC;IAKF,MAAM,MAAM,GAAG,MAAM,EAAE,MAAM,IAAI,cAAc,CAAC;IAChD,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,MAAM,IAAI,cAAc,KAAK,SAAS,CAAC;IACzE,MAAM,GAAG,GAAG,gBAAgB;QAC3B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG;YACb,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAE3F,MAAM,QAAQ,GAAG,IAAI,0BAA0B,CAAC;QAC/C,MAAM;QACN,WAAW,EAAE,MAAM,EAAE,OAAO;QAC5B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvB,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,IAAI,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IAE3D,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC;QACvC,cAAc,EAAE,CAAC,SAAS,CAAC;KAC3B,CAAC,CAAC;IAGH,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,QAAQ,EAAE,CAAC;AAChE,CAAC;AAuBD,MAAa,kBAAmB,SAAQ,qBAAS;IAGhD,YAAY,MAAiC;QAC5C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;IAC/B,CAAC;IAGQ,KAAK,CAAC,KAAK;QACnB,IAAI,IAAI,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;QAC5F,CAAC;QAID,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;QAI7B,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,MAAM,CAAC;QAE5C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,qBAAqB,CACvD,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,CAChB,CAAC;QACF,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAG1B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QAIlC,OAAO,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;CACD;AAnCD,gDAmCC"}
|