@executioncontrolprotocol/runtime 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/context-loader.d.ts +31 -0
- package/dist/engine/context-loader.d.ts.map +1 -0
- package/dist/engine/context-loader.js +90 -0
- package/dist/engine/context-loader.js.map +1 -0
- package/dist/engine/index.d.ts +6 -0
- package/dist/engine/index.d.ts.map +1 -0
- package/dist/engine/index.js +6 -0
- package/dist/engine/index.js.map +1 -0
- package/dist/engine/runner.d.ts +92 -0
- package/dist/engine/runner.d.ts.map +1 -0
- package/dist/engine/runner.js +852 -0
- package/dist/engine/runner.js.map +1 -0
- package/dist/engine/schema-validator.d.ts +32 -0
- package/dist/engine/schema-validator.d.ts.map +1 -0
- package/dist/engine/schema-validator.js +69 -0
- package/dist/engine/schema-validator.js.map +1 -0
- package/dist/engine/system-config-loader.d.ts +39 -0
- package/dist/engine/system-config-loader.d.ts.map +1 -0
- package/dist/engine/system-config-loader.js +80 -0
- package/dist/engine/system-config-loader.js.map +1 -0
- package/dist/engine/types.d.ts +324 -0
- package/dist/engine/types.d.ts.map +1 -0
- package/dist/engine/types.js +10 -0
- package/dist/engine/types.js.map +1 -0
- package/dist/evals/index.d.ts +3 -0
- package/dist/evals/index.d.ts.map +1 -0
- package/dist/evals/index.js +3 -0
- package/dist/evals/index.js.map +1 -0
- package/dist/evals/scorer.d.ts +23 -0
- package/dist/evals/scorer.d.ts.map +1 -0
- package/dist/evals/scorer.js +133 -0
- package/dist/evals/scorer.js.map +1 -0
- package/dist/evals/types.d.ts +128 -0
- package/dist/evals/types.d.ts.map +1 -0
- package/dist/evals/types.js +11 -0
- package/dist/evals/types.js.map +1 -0
- package/dist/extensions/builtin.d.ts +44 -0
- package/dist/extensions/builtin.d.ts.map +1 -0
- package/dist/extensions/builtin.js +66 -0
- package/dist/extensions/builtin.js.map +1 -0
- package/dist/extensions/index.d.ts +4 -0
- package/dist/extensions/index.d.ts.map +1 -0
- package/dist/extensions/index.js +4 -0
- package/dist/extensions/index.js.map +1 -0
- package/dist/extensions/progress-loggers/file-logger.d.ts +27 -0
- package/dist/extensions/progress-loggers/file-logger.d.ts.map +1 -0
- package/dist/extensions/progress-loggers/file-logger.js +54 -0
- package/dist/extensions/progress-loggers/file-logger.js.map +1 -0
- package/dist/extensions/registry.d.ts +74 -0
- package/dist/extensions/registry.d.ts.map +1 -0
- package/dist/extensions/registry.js +126 -0
- package/dist/extensions/registry.js.map +1 -0
- package/dist/extensions/types.d.ts +78 -0
- package/dist/extensions/types.d.ts.map +1 -0
- package/dist/extensions/types.js +7 -0
- package/dist/extensions/types.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/mounts/hydrator.d.ts +46 -0
- package/dist/mounts/hydrator.d.ts.map +1 -0
- package/dist/mounts/hydrator.js +142 -0
- package/dist/mounts/hydrator.js.map +1 -0
- package/dist/mounts/index.d.ts +4 -0
- package/dist/mounts/index.d.ts.map +1 -0
- package/dist/mounts/index.js +4 -0
- package/dist/mounts/index.js.map +1 -0
- package/dist/mounts/interpolation.d.ts +33 -0
- package/dist/mounts/interpolation.d.ts.map +1 -0
- package/dist/mounts/interpolation.js +59 -0
- package/dist/mounts/interpolation.js.map +1 -0
- package/dist/mounts/types.d.ts +80 -0
- package/dist/mounts/types.d.ts.map +1 -0
- package/dist/mounts/types.js +10 -0
- package/dist/mounts/types.js.map +1 -0
- package/dist/policies/enforcer.d.ts +23 -0
- package/dist/policies/enforcer.d.ts.map +1 -0
- package/dist/policies/enforcer.js +111 -0
- package/dist/policies/enforcer.js.map +1 -0
- package/dist/policies/index.d.ts +3 -0
- package/dist/policies/index.d.ts.map +1 -0
- package/dist/policies/index.js +3 -0
- package/dist/policies/index.js.map +1 -0
- package/dist/policies/types.d.ts +87 -0
- package/dist/policies/types.d.ts.map +1 -0
- package/dist/policies/types.js +11 -0
- package/dist/policies/types.js.map +1 -0
- package/dist/protocols/a2a/a2a-transport.d.ts +40 -0
- package/dist/protocols/a2a/a2a-transport.d.ts.map +1 -0
- package/dist/protocols/a2a/a2a-transport.js +212 -0
- package/dist/protocols/a2a/a2a-transport.js.map +1 -0
- package/dist/protocols/a2a/index.d.ts +3 -0
- package/dist/protocols/a2a/index.d.ts.map +1 -0
- package/dist/protocols/a2a/index.js +2 -0
- package/dist/protocols/a2a/index.js.map +1 -0
- package/dist/protocols/agent-transport.d.ts +101 -0
- package/dist/protocols/agent-transport.d.ts.map +1 -0
- package/dist/protocols/agent-transport.js +11 -0
- package/dist/protocols/agent-transport.js.map +1 -0
- package/dist/protocols/index.d.ts +5 -0
- package/dist/protocols/index.d.ts.map +1 -0
- package/dist/protocols/index.js +5 -0
- package/dist/protocols/index.js.map +1 -0
- package/dist/protocols/mcp/index.d.ts +2 -0
- package/dist/protocols/mcp/index.d.ts.map +1 -0
- package/dist/protocols/mcp/index.js +2 -0
- package/dist/protocols/mcp/index.js.map +1 -0
- package/dist/protocols/mcp/mcp-tool-invoker.d.ts +30 -0
- package/dist/protocols/mcp/mcp-tool-invoker.d.ts.map +1 -0
- package/dist/protocols/mcp/mcp-tool-invoker.js +121 -0
- package/dist/protocols/mcp/mcp-tool-invoker.js.map +1 -0
- package/dist/protocols/tool-invoker.d.ts +91 -0
- package/dist/protocols/tool-invoker.d.ts.map +1 -0
- package/dist/protocols/tool-invoker.js +11 -0
- package/dist/protocols/tool-invoker.js.map +1 -0
- package/dist/providers/index.d.ts +4 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +4 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/model-provider.d.ts +132 -0
- package/dist/providers/model-provider.d.ts.map +1 -0
- package/dist/providers/model-provider.js +10 -0
- package/dist/providers/model-provider.js.map +1 -0
- package/dist/providers/ollama/index.d.ts +3 -0
- package/dist/providers/ollama/index.d.ts.map +1 -0
- package/dist/providers/ollama/index.js +2 -0
- package/dist/providers/ollama/index.js.map +1 -0
- package/dist/providers/ollama/ollama-provider.d.ts +41 -0
- package/dist/providers/ollama/ollama-provider.d.ts.map +1 -0
- package/dist/providers/ollama/ollama-provider.js +113 -0
- package/dist/providers/ollama/ollama-provider.js.map +1 -0
- package/dist/providers/openai/index.d.ts +3 -0
- package/dist/providers/openai/index.d.ts.map +1 -0
- package/dist/providers/openai/index.js +2 -0
- package/dist/providers/openai/index.js.map +1 -0
- package/dist/providers/openai/openai-provider.d.ts +41 -0
- package/dist/providers/openai/openai-provider.d.ts.map +1 -0
- package/dist/providers/openai/openai-provider.js +150 -0
- package/dist/providers/openai/openai-provider.js.map +1 -0
- package/dist/testing/cassette.d.ts +88 -0
- package/dist/testing/cassette.d.ts.map +1 -0
- package/dist/testing/cassette.js +123 -0
- package/dist/testing/cassette.js.map +1 -0
- package/dist/testing/index.d.ts +5 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +5 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/mock-agent-transport.d.ts +49 -0
- package/dist/testing/mock-agent-transport.d.ts.map +1 -0
- package/dist/testing/mock-agent-transport.js +71 -0
- package/dist/testing/mock-agent-transport.js.map +1 -0
- package/dist/testing/mock-model-provider.d.ts +69 -0
- package/dist/testing/mock-model-provider.d.ts.map +1 -0
- package/dist/testing/mock-model-provider.js +92 -0
- package/dist/testing/mock-model-provider.js.map +1 -0
- package/dist/testing/mock-tool-invoker.d.ts +65 -0
- package/dist/testing/mock-tool-invoker.d.ts.map +1 -0
- package/dist/testing/mock-tool-invoker.js +85 -0
- package/dist/testing/mock-tool-invoker.js.map +1 -0
- package/dist/tracing/collector.d.ts +75 -0
- package/dist/tracing/collector.d.ts.map +1 -0
- package/dist/tracing/collector.js +106 -0
- package/dist/tracing/collector.js.map +1 -0
- package/dist/tracing/exporters/console.d.ts +17 -0
- package/dist/tracing/exporters/console.d.ts.map +1 -0
- package/dist/tracing/exporters/console.js +76 -0
- package/dist/tracing/exporters/console.js.map +1 -0
- package/dist/tracing/exporters/index.d.ts +4 -0
- package/dist/tracing/exporters/index.d.ts.map +1 -0
- package/dist/tracing/exporters/index.js +3 -0
- package/dist/tracing/exporters/index.js.map +1 -0
- package/dist/tracing/exporters/json-file.d.ts +30 -0
- package/dist/tracing/exporters/json-file.d.ts.map +1 -0
- package/dist/tracing/exporters/json-file.js +28 -0
- package/dist/tracing/exporters/json-file.js.map +1 -0
- package/dist/tracing/formatter.d.ts +16 -0
- package/dist/tracing/formatter.d.ts.map +1 -0
- package/dist/tracing/formatter.js +81 -0
- package/dist/tracing/formatter.js.map +1 -0
- package/dist/tracing/graph.d.ts +17 -0
- package/dist/tracing/graph.d.ts.map +1 -0
- package/dist/tracing/graph.js +116 -0
- package/dist/tracing/graph.js.map +1 -0
- package/dist/tracing/index.d.ts +6 -0
- package/dist/tracing/index.d.ts.map +1 -0
- package/dist/tracing/index.js +6 -0
- package/dist/tracing/index.js.map +1 -0
- package/dist/tracing/types.d.ts +124 -0
- package/dist/tracing/types.d.ts.map +1 -0
- package/dist/tracing/types.js +11 -0
- package/dist/tracing/types.js.map +1 -0
- package/package.json +42 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rubric-based scorer for eval outputs.
|
|
3
|
+
*
|
|
4
|
+
* Evaluates engine outputs against a rubric of weighted criteria.
|
|
5
|
+
* Supports field presence, pattern matching, array length checks,
|
|
6
|
+
* tool usage validation, and schema conformance.
|
|
7
|
+
*
|
|
8
|
+
* @category Evals
|
|
9
|
+
*/
|
|
10
|
+
import type { ExecutionResult } from "../engine/types.js";
|
|
11
|
+
import type { EvalExpected, EvalResult } from "./types.js";
|
|
12
|
+
/**
|
|
13
|
+
* Score an execution result against an eval rubric.
|
|
14
|
+
*
|
|
15
|
+
* @param caseId - The eval case ID.
|
|
16
|
+
* @param expected - The expected outcome with rubric.
|
|
17
|
+
* @param executionResult - The engine's execution result.
|
|
18
|
+
* @returns The eval result with per-criterion scores.
|
|
19
|
+
*
|
|
20
|
+
* @category Evals
|
|
21
|
+
*/
|
|
22
|
+
export declare function scoreExecution(caseId: string, expected: EvalExpected, executionResult: ExecutionResult): EvalResult;
|
|
23
|
+
//# sourceMappingURL=scorer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../../src/evals/scorer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAoC,YAAY,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAmH7F;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,YAAY,EACtB,eAAe,EAAE,eAAe,GAC/B,UAAU,CAsBZ"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rubric-based scorer for eval outputs.
|
|
3
|
+
*
|
|
4
|
+
* Evaluates engine outputs against a rubric of weighted criteria.
|
|
5
|
+
* Supports field presence, pattern matching, array length checks,
|
|
6
|
+
* tool usage validation, and schema conformance.
|
|
7
|
+
*
|
|
8
|
+
* @category Evals
|
|
9
|
+
*/
|
|
10
|
+
function getNestedValue(obj, path) {
|
|
11
|
+
const parts = path.split(".");
|
|
12
|
+
let current = obj;
|
|
13
|
+
for (const part of parts) {
|
|
14
|
+
if (current === null || current === undefined || typeof current !== "object") {
|
|
15
|
+
return undefined;
|
|
16
|
+
}
|
|
17
|
+
current = current[part];
|
|
18
|
+
}
|
|
19
|
+
return current;
|
|
20
|
+
}
|
|
21
|
+
function evaluateCriterion(criterion, output, executionResult) {
|
|
22
|
+
const { check } = criterion;
|
|
23
|
+
switch (check.type) {
|
|
24
|
+
case "schema-valid": {
|
|
25
|
+
const passed = executionResult.success && output !== undefined;
|
|
26
|
+
return {
|
|
27
|
+
criterion,
|
|
28
|
+
passed,
|
|
29
|
+
score: passed ? criterion.weight : 0,
|
|
30
|
+
message: passed ? "Output is schema-valid" : "Output missing or execution failed",
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
case "field-exists": {
|
|
34
|
+
const value = output ? getNestedValue(output, check.path) : undefined;
|
|
35
|
+
const passed = value !== undefined && value !== null;
|
|
36
|
+
return {
|
|
37
|
+
criterion,
|
|
38
|
+
passed,
|
|
39
|
+
score: passed ? criterion.weight : 0,
|
|
40
|
+
message: passed ? `Field "${check.path}" exists` : `Field "${check.path}" is missing`,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
case "field-matches": {
|
|
44
|
+
const value = output ? getNestedValue(output, check.path) : undefined;
|
|
45
|
+
const regex = new RegExp(check.pattern, "i");
|
|
46
|
+
const passed = typeof value === "string" && regex.test(value);
|
|
47
|
+
return {
|
|
48
|
+
criterion,
|
|
49
|
+
passed,
|
|
50
|
+
score: passed ? criterion.weight : 0,
|
|
51
|
+
message: passed
|
|
52
|
+
? `Field "${check.path}" matches pattern`
|
|
53
|
+
: `Field "${check.path}" does not match /${check.pattern}/`,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
case "array-min-length": {
|
|
57
|
+
const value = output ? getNestedValue(output, check.path) : undefined;
|
|
58
|
+
const arr = Array.isArray(value) ? value : [];
|
|
59
|
+
const passed = arr.length >= check.min;
|
|
60
|
+
return {
|
|
61
|
+
criterion,
|
|
62
|
+
passed,
|
|
63
|
+
score: passed ? criterion.weight : 0,
|
|
64
|
+
message: passed
|
|
65
|
+
? `Array "${check.path}" has ${arr.length} items (>= ${check.min})`
|
|
66
|
+
: `Array "${check.path}" has ${arr.length} items (need >= ${check.min})`,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
case "no-forbidden-tools": {
|
|
70
|
+
const usedTools = executionResult.log
|
|
71
|
+
.filter((e) => e.message.includes("Tool ") && !e.message.includes("denied"))
|
|
72
|
+
.map((e) => e.message);
|
|
73
|
+
const violations = check.forbidden.filter((tool) => usedTools.some((msg) => msg.includes(tool)));
|
|
74
|
+
const passed = violations.length === 0;
|
|
75
|
+
return {
|
|
76
|
+
criterion,
|
|
77
|
+
passed,
|
|
78
|
+
score: passed ? criterion.weight : 0,
|
|
79
|
+
message: passed
|
|
80
|
+
? "No forbidden tools used"
|
|
81
|
+
: `Forbidden tools used: ${violations.join(", ")}`,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
case "tool-count-within": {
|
|
85
|
+
const count = executionResult.totalBudgetUsage.toolCalls;
|
|
86
|
+
const passed = count <= check.max;
|
|
87
|
+
return {
|
|
88
|
+
criterion,
|
|
89
|
+
passed,
|
|
90
|
+
score: passed ? criterion.weight : 0,
|
|
91
|
+
message: passed
|
|
92
|
+
? `Tool calls (${count}) within limit (${check.max})`
|
|
93
|
+
: `Tool calls (${count}) exceeded limit (${check.max})`,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
case "custom": {
|
|
97
|
+
return {
|
|
98
|
+
criterion,
|
|
99
|
+
passed: false,
|
|
100
|
+
score: 0,
|
|
101
|
+
message: `Custom check "${check.fn}" not implemented in scorer`,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Score an execution result against an eval rubric.
|
|
108
|
+
*
|
|
109
|
+
* @param caseId - The eval case ID.
|
|
110
|
+
* @param expected - The expected outcome with rubric.
|
|
111
|
+
* @param executionResult - The engine's execution result.
|
|
112
|
+
* @returns The eval result with per-criterion scores.
|
|
113
|
+
*
|
|
114
|
+
* @category Evals
|
|
115
|
+
*/
|
|
116
|
+
export function scoreExecution(caseId, expected, executionResult) {
|
|
117
|
+
const output = executionResult.output;
|
|
118
|
+
const criteria = expected.rubric.map((criterion) => evaluateCriterion(criterion, output, executionResult));
|
|
119
|
+
const totalScore = criteria.reduce((sum, c) => sum + c.score, 0);
|
|
120
|
+
const hardGatesPassed = (expected.hardGates ?? []).every((gateName) => {
|
|
121
|
+
const criterion = criteria.find((c) => c.criterion.name === gateName);
|
|
122
|
+
return criterion?.passed ?? false;
|
|
123
|
+
});
|
|
124
|
+
const passed = totalScore >= expected.minScore && hardGatesPassed;
|
|
125
|
+
return {
|
|
126
|
+
caseId,
|
|
127
|
+
passed,
|
|
128
|
+
score: totalScore,
|
|
129
|
+
criteria,
|
|
130
|
+
executionDurationMs: executionResult.durationMs,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
//# sourceMappingURL=scorer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scorer.js","sourceRoot":"","sources":["../../src/evals/scorer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,SAAS,cAAc,CAAC,GAAY,EAAE,IAAY;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,OAAO,GAAG,GAAG,CAAC;IAElB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC7E,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,GAAI,OAAmC,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,iBAAiB,CACxB,SAA0B,EAC1B,MAA2C,EAC3C,eAAgC;IAEhC,MAAM,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC;IAE5B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,cAAc,CAAC,CAAC,CAAC;YACpB,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,IAAI,MAAM,KAAK,SAAS,CAAC;YAC/D,OAAO;gBACL,SAAS;gBACT,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACpC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,oCAAoC;aAClF,CAAC;QACJ,CAAC;QAED,KAAK,cAAc,CAAC,CAAC,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACtE,MAAM,MAAM,GAAG,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,CAAC;YACrD,OAAO;gBACL,SAAS;gBACT,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACpC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,cAAc;aACtF,CAAC;QACJ,CAAC;QAED,KAAK,eAAe,CAAC,CAAC,CAAC;YACrB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACtE,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9D,OAAO;gBACL,SAAS;gBACT,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACpC,OAAO,EAAE,MAAM;oBACb,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,mBAAmB;oBACzC,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,qBAAqB,KAAK,CAAC,OAAO,GAAG;aAC9D,CAAC;QACJ,CAAC;QAED,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACxB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACtE,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC;YACvC,OAAO;gBACL,SAAS;gBACT,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACpC,OAAO,EAAE,MAAM;oBACb,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,SAAS,GAAG,CAAC,MAAM,cAAc,KAAK,CAAC,GAAG,GAAG;oBACnE,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,SAAS,GAAG,CAAC,MAAM,mBAAmB,KAAK,CAAC,GAAG,GAAG;aAC3E,CAAC;QACJ,CAAC;QAED,KAAK,oBAAoB,CAAC,CAAC,CAAC;YAC1B,MAAM,SAAS,GAAG,eAAe,CAAC,GAAG;iBAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;iBAC3E,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACzB,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACjD,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAC5C,CAAC;YACF,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC;YACvC,OAAO;gBACL,SAAS;gBACT,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACpC,OAAO,EAAE,MAAM;oBACb,CAAC,CAAC,yBAAyB;oBAC3B,CAAC,CAAC,yBAAyB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aACrD,CAAC;QACJ,CAAC;QAED,KAAK,mBAAmB,CAAC,CAAC,CAAC;YACzB,MAAM,KAAK,GAAG,eAAe,CAAC,gBAAgB,CAAC,SAAS,CAAC;YACzD,MAAM,MAAM,GAAG,KAAK,IAAI,KAAK,CAAC,GAAG,CAAC;YAClC,OAAO;gBACL,SAAS;gBACT,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACpC,OAAO,EAAE,MAAM;oBACb,CAAC,CAAC,eAAe,KAAK,mBAAmB,KAAK,CAAC,GAAG,GAAG;oBACrD,CAAC,CAAC,eAAe,KAAK,qBAAqB,KAAK,CAAC,GAAG,GAAG;aAC1D,CAAC;QACJ,CAAC;QAED,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,OAAO;gBACL,SAAS;gBACT,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAiB,KAAK,CAAC,EAAE,6BAA6B;aAChE,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAc,EACd,QAAsB,EACtB,eAAgC;IAEhC,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC;IACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CACjD,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,eAAe,CAAC,CACtD,CAAC;IAEF,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAEjE,MAAM,eAAe,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE;QACpE,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACtE,OAAO,SAAS,EAAE,MAAM,IAAI,KAAK,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,UAAU,IAAI,QAAQ,CAAC,QAAQ,IAAI,eAAe,CAAC;IAElE,OAAO;QACL,MAAM;QACN,MAAM;QACN,KAAK,EAAE,UAAU;QACjB,QAAQ;QACR,mBAAmB,EAAE,eAAe,CAAC,UAAU;KAChD,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for the ECP evaluation framework.
|
|
3
|
+
*
|
|
4
|
+
* Eval cases define expected behavior and rubric-based scoring
|
|
5
|
+
* criteria for agent outputs. Used for regression testing and
|
|
6
|
+
* quality assurance of LLM-driven execution.
|
|
7
|
+
*
|
|
8
|
+
* @category Evals
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* A single criterion in a scoring rubric.
|
|
12
|
+
*
|
|
13
|
+
* @category Evals
|
|
14
|
+
*/
|
|
15
|
+
export interface RubricCriterion {
|
|
16
|
+
/** Human-readable name for this criterion. */
|
|
17
|
+
name: string;
|
|
18
|
+
/** Description of what this criterion checks. */
|
|
19
|
+
description: string;
|
|
20
|
+
/** Weight of this criterion (0-1). All weights should sum to 1. */
|
|
21
|
+
weight: number;
|
|
22
|
+
/** The type of check to perform. */
|
|
23
|
+
check: {
|
|
24
|
+
type: "schema-valid";
|
|
25
|
+
} | {
|
|
26
|
+
type: "field-exists";
|
|
27
|
+
path: string;
|
|
28
|
+
} | {
|
|
29
|
+
type: "field-matches";
|
|
30
|
+
path: string;
|
|
31
|
+
pattern: string;
|
|
32
|
+
} | {
|
|
33
|
+
type: "array-min-length";
|
|
34
|
+
path: string;
|
|
35
|
+
min: number;
|
|
36
|
+
} | {
|
|
37
|
+
type: "no-forbidden-tools";
|
|
38
|
+
forbidden: string[];
|
|
39
|
+
} | {
|
|
40
|
+
type: "tool-count-within";
|
|
41
|
+
max: number;
|
|
42
|
+
} | {
|
|
43
|
+
type: "custom";
|
|
44
|
+
fn: string;
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Expected outcome definition for an eval case.
|
|
49
|
+
*
|
|
50
|
+
* @category Evals
|
|
51
|
+
*/
|
|
52
|
+
export interface EvalExpected {
|
|
53
|
+
/** Rubric criteria the output must satisfy. */
|
|
54
|
+
rubric: RubricCriterion[];
|
|
55
|
+
/** Minimum overall score (0-1) to pass. */
|
|
56
|
+
minScore: number;
|
|
57
|
+
/** Hard gates that must pass regardless of score. */
|
|
58
|
+
hardGates?: string[];
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* An eval case definition.
|
|
62
|
+
*
|
|
63
|
+
* @category Evals
|
|
64
|
+
*/
|
|
65
|
+
export interface EvalCase {
|
|
66
|
+
/** Unique case ID. */
|
|
67
|
+
id: string;
|
|
68
|
+
/** Human-readable description. */
|
|
69
|
+
description: string;
|
|
70
|
+
/** Path to the Context manifest for this case. */
|
|
71
|
+
contextPath: string;
|
|
72
|
+
/** Inputs to supply to the Context. */
|
|
73
|
+
inputs: Record<string, string | number | boolean>;
|
|
74
|
+
/** Expected outcome and scoring rubric. */
|
|
75
|
+
expected: EvalExpected;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* The result of evaluating a single rubric criterion.
|
|
79
|
+
*
|
|
80
|
+
* @category Evals
|
|
81
|
+
*/
|
|
82
|
+
export interface CriterionResult {
|
|
83
|
+
/** The criterion that was evaluated. */
|
|
84
|
+
criterion: RubricCriterion;
|
|
85
|
+
/** Whether the criterion passed. */
|
|
86
|
+
passed: boolean;
|
|
87
|
+
/** The score for this criterion (0 or weight). */
|
|
88
|
+
score: number;
|
|
89
|
+
/** Human-readable explanation. */
|
|
90
|
+
message: string;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* The result of running an eval case.
|
|
94
|
+
*
|
|
95
|
+
* @category Evals
|
|
96
|
+
*/
|
|
97
|
+
export interface EvalResult {
|
|
98
|
+
/** The case that was evaluated. */
|
|
99
|
+
caseId: string;
|
|
100
|
+
/** Whether the eval passed (score >= minScore and all hard gates pass). */
|
|
101
|
+
passed: boolean;
|
|
102
|
+
/** Overall score (0-1). */
|
|
103
|
+
score: number;
|
|
104
|
+
/** Per-criterion results. */
|
|
105
|
+
criteria: CriterionResult[];
|
|
106
|
+
/** The execution result from the engine. */
|
|
107
|
+
executionDurationMs: number;
|
|
108
|
+
/** Error if the run itself failed. */
|
|
109
|
+
error?: string;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Summary of an eval suite run.
|
|
113
|
+
*
|
|
114
|
+
* @category Evals
|
|
115
|
+
*/
|
|
116
|
+
export interface EvalSuiteResult {
|
|
117
|
+
/** Total cases run. */
|
|
118
|
+
totalCases: number;
|
|
119
|
+
/** Cases that passed. */
|
|
120
|
+
passedCases: number;
|
|
121
|
+
/** Overall pass rate (0-1). */
|
|
122
|
+
passRate: number;
|
|
123
|
+
/** Per-case results. */
|
|
124
|
+
results: EvalResult[];
|
|
125
|
+
/** Total wall-clock time. */
|
|
126
|
+
totalDurationMs: number;
|
|
127
|
+
}
|
|
128
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evals/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAC;IAEb,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IAEpB,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;IAEf,oCAAoC;IACpC,KAAK,EACD;QAAE,IAAI,EAAE,cAAc,CAAA;KAAE,GACxB;QAAE,IAAI,EAAE,cAAc,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GACtC;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GACxD;QAAE,IAAI,EAAE,kBAAkB,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GACvD;QAAE,IAAI,EAAE,oBAAoB,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,GACnD;QAAE,IAAI,EAAE,mBAAmB,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAC1C;QAAE,IAAI,EAAE,QAAQ,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;CACpC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,+CAA+C;IAC/C,MAAM,EAAE,eAAe,EAAE,CAAC;IAE1B,2CAA2C;IAC3C,QAAQ,EAAE,MAAM,CAAC;IAEjB,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,sBAAsB;IACtB,EAAE,EAAE,MAAM,CAAC;IAEX,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IAEpB,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;IAEpB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;IAElD,2CAA2C;IAC3C,QAAQ,EAAE,YAAY,CAAC;CACxB;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,wCAAwC;IACxC,SAAS,EAAE,eAAe,CAAC;IAE3B,oCAAoC;IACpC,MAAM,EAAE,OAAO,CAAC;IAEhB,kDAAkD;IAClD,KAAK,EAAE,MAAM,CAAC;IAEd,kCAAkC;IAClC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,mCAAmC;IACnC,MAAM,EAAE,MAAM,CAAC;IAEf,2EAA2E;IAC3E,MAAM,EAAE,OAAO,CAAC;IAEhB,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAC;IAEd,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,EAAE,CAAC;IAE5B,4CAA4C;IAC5C,mBAAmB,EAAE,MAAM,CAAC;IAE5B,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,uBAAuB;IACvB,UAAU,EAAE,MAAM,CAAC;IAEnB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IAEpB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IAEjB,wBAAwB;IACxB,OAAO,EAAE,UAAU,EAAE,CAAC;IAEtB,6BAA6B;IAC7B,eAAe,EAAE,MAAM,CAAC;CACzB"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for the ECP evaluation framework.
|
|
3
|
+
*
|
|
4
|
+
* Eval cases define expected behavior and rubric-based scoring
|
|
5
|
+
* criteria for agent outputs. Used for regression testing and
|
|
6
|
+
* quality assurance of LLM-driven execution.
|
|
7
|
+
*
|
|
8
|
+
* @category Evals
|
|
9
|
+
*/
|
|
10
|
+
export {};
|
|
11
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/evals/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in extension registrations for core runtime providers.
|
|
3
|
+
*
|
|
4
|
+
* @category Extensions
|
|
5
|
+
*/
|
|
6
|
+
import type { ExtensionVersion } from "@executioncontrolprotocol/spec";
|
|
7
|
+
import type { OpenAIProviderConfig } from "../providers/openai/openai-provider.js";
|
|
8
|
+
import type { OllamaProviderConfig } from "../providers/ollama/ollama-provider.js";
|
|
9
|
+
import type { FileProgressLoggerConfig } from "./progress-loggers/file-logger.js";
|
|
10
|
+
import type { ExtensionRegistry } from "./registry.js";
|
|
11
|
+
/**
|
|
12
|
+
* Configuration for registering built-in model providers.
|
|
13
|
+
*
|
|
14
|
+
* @category Extensions
|
|
15
|
+
*/
|
|
16
|
+
export interface BuiltinModelProviderConfig {
|
|
17
|
+
/** Built-in extension version to report for registrations. */
|
|
18
|
+
version?: ExtensionVersion;
|
|
19
|
+
/** Default OpenAI provider configuration. */
|
|
20
|
+
openai?: OpenAIProviderConfig;
|
|
21
|
+
/** Default Ollama provider configuration. */
|
|
22
|
+
ollama?: OllamaProviderConfig;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Register built-in model providers in a registry.
|
|
26
|
+
*/
|
|
27
|
+
export declare function registerBuiltinModelProviders(registry: ExtensionRegistry, config?: BuiltinModelProviderConfig): void;
|
|
28
|
+
/**
|
|
29
|
+
* Configuration for registering built-in progress loggers.
|
|
30
|
+
*
|
|
31
|
+
* @category Extensions
|
|
32
|
+
*/
|
|
33
|
+
export interface BuiltinProgressLoggerConfig {
|
|
34
|
+
/** Built-in extension version to report for registrations. */
|
|
35
|
+
version?: ExtensionVersion;
|
|
36
|
+
/** Configuration for the file progress logger (log dir, file name). */
|
|
37
|
+
file?: FileProgressLoggerConfig;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Register built-in progress loggers in a registry.
|
|
41
|
+
* Includes a file logger that writes to the user's ECP directory (~/.ecp/logs).
|
|
42
|
+
*/
|
|
43
|
+
export declare function registerBuiltinProgressLoggers(registry: ExtensionRegistry, config?: BuiltinProgressLoggerConfig): void;
|
|
44
|
+
//# sourceMappingURL=builtin.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"builtin.d.ts","sourceRoot":"","sources":["../../src/extensions/builtin.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAEvE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wCAAwC,CAAC;AAEnF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wCAAwC,CAAC;AAEnF,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAClF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEvD;;;;GAIG;AACH,MAAM,WAAW,0BAA0B;IACzC,8DAA8D;IAC9D,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAE3B,6CAA6C;IAC7C,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAE9B,6CAA6C;IAC7C,MAAM,CAAC,EAAE,oBAAoB,CAAC;CAC/B;AAOD;;GAEG;AACH,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,GAAE,0BAA+B,GACtC,IAAI,CA8BN;AAED;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IAC1C,8DAA8D;IAC9D,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAE3B,uEAAuE;IACvE,IAAI,CAAC,EAAE,wBAAwB,CAAC;CACjC;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,GAAE,2BAAgC,GACvC,IAAI,CAgBN"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in extension registrations for core runtime providers.
|
|
3
|
+
*
|
|
4
|
+
* @category Extensions
|
|
5
|
+
*/
|
|
6
|
+
import { OpenAIProvider } from "../providers/openai/openai-provider.js";
|
|
7
|
+
import { OllamaProvider } from "../providers/ollama/ollama-provider.js";
|
|
8
|
+
import { createFileProgressLogger } from "./progress-loggers/file-logger.js";
|
|
9
|
+
function asRecord(value) {
|
|
10
|
+
if (!value || typeof value !== "object")
|
|
11
|
+
return {};
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Register built-in model providers in a registry.
|
|
16
|
+
*/
|
|
17
|
+
export function registerBuiltinModelProviders(registry, config = {}) {
|
|
18
|
+
const version = config.version ?? "0.3.0";
|
|
19
|
+
registry.registerModelProvider({
|
|
20
|
+
id: "openai",
|
|
21
|
+
kind: "model-provider",
|
|
22
|
+
sourceType: "builtin",
|
|
23
|
+
version,
|
|
24
|
+
description: "Built-in OpenAI model provider extension.",
|
|
25
|
+
create(overrides) {
|
|
26
|
+
return new OpenAIProvider({
|
|
27
|
+
...config.openai,
|
|
28
|
+
...asRecord(overrides),
|
|
29
|
+
});
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
registry.registerModelProvider({
|
|
33
|
+
id: "ollama",
|
|
34
|
+
kind: "model-provider",
|
|
35
|
+
sourceType: "builtin",
|
|
36
|
+
version,
|
|
37
|
+
description: "Built-in Ollama model provider extension.",
|
|
38
|
+
create(overrides) {
|
|
39
|
+
return new OllamaProvider({
|
|
40
|
+
...config.ollama,
|
|
41
|
+
...asRecord(overrides),
|
|
42
|
+
});
|
|
43
|
+
},
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Register built-in progress loggers in a registry.
|
|
48
|
+
* Includes a file logger that writes to the user's ECP directory (~/.ecp/logs).
|
|
49
|
+
*/
|
|
50
|
+
export function registerBuiltinProgressLoggers(registry, config = {}) {
|
|
51
|
+
const version = config.version ?? "0.3.0";
|
|
52
|
+
registry.registerProgressLogger({
|
|
53
|
+
id: "file",
|
|
54
|
+
kind: "progress-logger",
|
|
55
|
+
sourceType: "builtin",
|
|
56
|
+
version,
|
|
57
|
+
description: "Appends execution progress to a log file in the user ECP directory (~/.ecp/logs).",
|
|
58
|
+
create(overrides) {
|
|
59
|
+
return createFileProgressLogger({
|
|
60
|
+
...config.file,
|
|
61
|
+
...overrides,
|
|
62
|
+
});
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=builtin.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"builtin.js","sourceRoot":"","sources":["../../src/extensions/builtin.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,wCAAwC,CAAC;AAExE,OAAO,EAAE,cAAc,EAAE,MAAM,wCAAwC,CAAC;AAExE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAoB7E,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACnD,OAAO,KAAgC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,6BAA6B,CAC3C,QAA2B,EAC3B,SAAqC,EAAE;IAEvC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC;IAE1C,QAAQ,CAAC,qBAAqB,CAAC;QAC7B,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,SAAS;QACrB,OAAO;QACP,WAAW,EAAE,2CAA2C;QACxD,MAAM,CAAC,SAAS;YACd,OAAO,IAAI,cAAc,CAAC;gBACxB,GAAG,MAAM,CAAC,MAAM;gBAChB,GAAG,QAAQ,CAAC,SAAS,CAAC;aACvB,CAAC,CAAC;QACL,CAAC;KACF,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,CAAC;QAC7B,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,SAAS;QACrB,OAAO;QACP,WAAW,EAAE,2CAA2C;QACxD,MAAM,CAAC,SAAS;YACd,OAAO,IAAI,cAAc,CAAC;gBACxB,GAAG,MAAM,CAAC,MAAM;gBAChB,GAAG,QAAQ,CAAC,SAAS,CAAC;aACvB,CAAC,CAAC;QACL,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAeD;;;GAGG;AACH,MAAM,UAAU,8BAA8B,CAC5C,QAA2B,EAC3B,SAAsC,EAAE;IAExC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC;IAE1C,QAAQ,CAAC,sBAAsB,CAAC;QAC9B,EAAE,EAAE,MAAM;QACV,IAAI,EAAE,iBAAiB;QACvB,UAAU,EAAE,SAAS;QACrB,OAAO;QACP,WAAW,EAAE,mFAAmF;QAChG,MAAM,CAAC,SAAS;YACd,OAAO,wBAAwB,CAAC;gBAC9B,GAAG,MAAM,CAAC,IAAI;gBACd,GAAI,SAAsC;aAC3C,CAAC,CAAC;QACL,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/extensions/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/extensions/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in file progress logger — appends execution progress events to a log file
|
|
3
|
+
* in the user's ECP config directory (~/.ecp/logs by default).
|
|
4
|
+
*
|
|
5
|
+
* @category Extensions
|
|
6
|
+
*/
|
|
7
|
+
import type { ProgressCallback } from "../../engine/types.js";
|
|
8
|
+
/**
|
|
9
|
+
* Configuration for the file progress logger.
|
|
10
|
+
*
|
|
11
|
+
* @category Extensions
|
|
12
|
+
*/
|
|
13
|
+
export interface FileProgressLoggerConfig {
|
|
14
|
+
/**
|
|
15
|
+
* Directory for log files. Defaults to ~/.ecp/logs.
|
|
16
|
+
*/
|
|
17
|
+
logDir?: string;
|
|
18
|
+
/**
|
|
19
|
+
* Log file name. Defaults to ecp.log.
|
|
20
|
+
*/
|
|
21
|
+
logFile?: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Create a progress callback that appends each event to a log file in the user directory.
|
|
25
|
+
*/
|
|
26
|
+
export declare function createFileProgressLogger(config?: FileProgressLoggerConfig): ProgressCallback;
|
|
27
|
+
//# sourceMappingURL=file-logger.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-logger.d.ts","sourceRoot":"","sources":["../../../src/extensions/progress-loggers/file-logger.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAA0B,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEtF;;;;GAIG;AACH,MAAM,WAAW,wBAAwB;IACvC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AA+BD;;GAEG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,GAAE,wBAA6B,GACpC,gBAAgB,CAclB"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in file progress logger — appends execution progress events to a log file
|
|
3
|
+
* in the user's ECP config directory (~/.ecp/logs by default).
|
|
4
|
+
*
|
|
5
|
+
* @category Extensions
|
|
6
|
+
*/
|
|
7
|
+
import { appendFileSync, mkdirSync, existsSync } from "node:fs";
|
|
8
|
+
import { dirname, resolve } from "node:path";
|
|
9
|
+
import { homedir } from "node:os";
|
|
10
|
+
const DEFAULT_LOG_DIR = "logs";
|
|
11
|
+
const DEFAULT_LOG_FILE = "ecp.log";
|
|
12
|
+
function getLogPath(config = {}) {
|
|
13
|
+
const base = config.logDir ?? resolve(homedir(), ".ecp", DEFAULT_LOG_DIR);
|
|
14
|
+
const file = config.logFile ?? DEFAULT_LOG_FILE;
|
|
15
|
+
return resolve(base, file);
|
|
16
|
+
}
|
|
17
|
+
function formatEvent(event) {
|
|
18
|
+
const ts = new Date().toISOString();
|
|
19
|
+
switch (event.type) {
|
|
20
|
+
case "phase":
|
|
21
|
+
return `${ts} [phase] ${event.status}\n`;
|
|
22
|
+
case "step_start":
|
|
23
|
+
return `${ts} [step_start] step=${event.step} kind=${event.kind} ${event.executorName ?? ""} ${event.description}\n`;
|
|
24
|
+
case "step_complete":
|
|
25
|
+
const tokens = event.tokens
|
|
26
|
+
? ` tokens=${event.tokens.prompt}+${event.tokens.completion}=${event.tokens.total}`
|
|
27
|
+
: "";
|
|
28
|
+
const out = event.output !== undefined ? " output=true" : "";
|
|
29
|
+
return `${ts} [step_complete] step=${event.step} kind=${event.kind} ${event.executorName ?? ""} ${event.durationMs}ms${tokens}${out}\n`;
|
|
30
|
+
case "executor_reasoning":
|
|
31
|
+
return `${ts} [reasoning] ${event.executorName} ${event.reasoning.slice(0, 80).replace(/\n/g, " ")}...\n`;
|
|
32
|
+
default:
|
|
33
|
+
return `${ts} [${event.type}]\n`;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Create a progress callback that appends each event to a log file in the user directory.
|
|
38
|
+
*/
|
|
39
|
+
export function createFileProgressLogger(config = {}) {
|
|
40
|
+
const logPath = getLogPath(config);
|
|
41
|
+
const dir = dirname(logPath);
|
|
42
|
+
return (event) => {
|
|
43
|
+
try {
|
|
44
|
+
if (!existsSync(dir)) {
|
|
45
|
+
mkdirSync(dir, { recursive: true });
|
|
46
|
+
}
|
|
47
|
+
appendFileSync(logPath, formatEvent(event), "utf-8");
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
// Ignore write errors so the run is not broken.
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=file-logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-logger.js","sourceRoot":"","sources":["../../../src/extensions/progress-loggers/file-logger.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAoBlC,MAAM,eAAe,GAAG,MAAM,CAAC;AAC/B,MAAM,gBAAgB,GAAG,SAAS,CAAC;AAEnC,SAAS,UAAU,CAAC,SAAmC,EAAE;IACvD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;IAC1E,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,IAAI,gBAAgB,CAAC;IAChD,OAAO,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAAC,KAA6B;IAChD,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpC,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,OAAO;YACV,OAAO,GAAG,EAAE,YAAY,KAAK,CAAC,MAAM,IAAI,CAAC;QAC3C,KAAK,YAAY;YACf,OAAO,GAAG,EAAE,sBAAsB,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,YAAY,IAAI,EAAE,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC;QACvH,KAAK,eAAe;YAClB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM;gBACzB,CAAC,CAAC,WAAW,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE;gBACnF,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,OAAO,GAAG,EAAE,yBAAyB,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,YAAY,IAAI,EAAE,IAAI,KAAK,CAAC,UAAU,KAAK,MAAM,GAAG,GAAG,IAAI,CAAC;QAC1I,KAAK,oBAAoB;YACvB,OAAO,GAAG,EAAE,gBAAgB,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC;QAC5G;YACE,OAAO,GAAG,EAAE,KAAM,KAA0B,CAAC,IAAI,KAAK,CAAC;IAC3D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACtC,SAAmC,EAAE;IAErC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7B,OAAO,CAAC,KAA6B,EAAE,EAAE;QACvC,IAAI,CAAC;YACH,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC;YACD,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,gDAAgD;QAClD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runtime extension registry for model providers, executors, and plugins.
|
|
3
|
+
*
|
|
4
|
+
* @category Extensions
|
|
5
|
+
*/
|
|
6
|
+
import type { ExecutorRegistration, ModelProviderRegistration, PluginRegistration, ProgressLoggerRegistration } from "./types.js";
|
|
7
|
+
import type { ModelProvider } from "../providers/model-provider.js";
|
|
8
|
+
/**
|
|
9
|
+
* In-memory registry of runtime extension factories.
|
|
10
|
+
*
|
|
11
|
+
* @category Extensions
|
|
12
|
+
*/
|
|
13
|
+
export declare class ExtensionRegistry {
|
|
14
|
+
private readonly modelProviders;
|
|
15
|
+
private readonly executors;
|
|
16
|
+
private readonly plugins;
|
|
17
|
+
private readonly progressLoggers;
|
|
18
|
+
private locked;
|
|
19
|
+
/**
|
|
20
|
+
* Prevent further registration changes.
|
|
21
|
+
*/
|
|
22
|
+
lock(): void;
|
|
23
|
+
/**
|
|
24
|
+
* Register a model provider extension.
|
|
25
|
+
*/
|
|
26
|
+
registerModelProvider(registration: ModelProviderRegistration): void;
|
|
27
|
+
/**
|
|
28
|
+
* Register an executor extension.
|
|
29
|
+
*/
|
|
30
|
+
registerExecutor(registration: ExecutorRegistration): void;
|
|
31
|
+
/**
|
|
32
|
+
* Register a progress logger extension.
|
|
33
|
+
*/
|
|
34
|
+
registerProgressLogger(registration: ProgressLoggerRegistration): void;
|
|
35
|
+
/**
|
|
36
|
+
* Register a plugin extension.
|
|
37
|
+
*/
|
|
38
|
+
registerPlugin(registration: PluginRegistration): void;
|
|
39
|
+
/**
|
|
40
|
+
* Resolve and instantiate a model provider by ID.
|
|
41
|
+
*/
|
|
42
|
+
createModelProvider(id: string, config?: Record<string, unknown>): ModelProvider;
|
|
43
|
+
/**
|
|
44
|
+
* Get a model provider registration by ID.
|
|
45
|
+
*/
|
|
46
|
+
getModelProviderRegistration(id: string): ModelProviderRegistration | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* List registered model provider extensions.
|
|
49
|
+
*/
|
|
50
|
+
listModelProviders(): ModelProviderRegistration[];
|
|
51
|
+
/**
|
|
52
|
+
* List registered executor extensions.
|
|
53
|
+
*/
|
|
54
|
+
listExecutors(): ExecutorRegistration[];
|
|
55
|
+
/**
|
|
56
|
+
* List registered plugin extensions.
|
|
57
|
+
*/
|
|
58
|
+
listPlugins(): PluginRegistration[];
|
|
59
|
+
/**
|
|
60
|
+
* Create a progress logger by ID.
|
|
61
|
+
*/
|
|
62
|
+
createProgressLogger(id: string, config?: Record<string, unknown>): ReturnType<ProgressLoggerRegistration["create"]>;
|
|
63
|
+
/**
|
|
64
|
+
* Get a progress logger registration by ID.
|
|
65
|
+
*/
|
|
66
|
+
getProgressLoggerRegistration(id: string): ProgressLoggerRegistration | undefined;
|
|
67
|
+
/**
|
|
68
|
+
* List registered progress logger extensions.
|
|
69
|
+
*/
|
|
70
|
+
listProgressLoggers(): ProgressLoggerRegistration[];
|
|
71
|
+
private assertUnlocked;
|
|
72
|
+
private assertNotRegistered;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=registry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/extensions/registry.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,0BAA0B,EAC3B,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAEpE;;;;GAIG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAgD;IAC/E,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA2C;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyC;IACjE,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAiD;IACjF,OAAO,CAAC,MAAM,CAAS;IAEvB;;OAEG;IACH,IAAI,IAAI,IAAI;IAIZ;;OAEG;IACH,qBAAqB,CAAC,YAAY,EAAE,yBAAyB,GAAG,IAAI;IAMpE;;OAEG;IACH,gBAAgB,CAAC,YAAY,EAAE,oBAAoB,GAAG,IAAI;IAM1D;;OAEG;IACH,sBAAsB,CAAC,YAAY,EAAE,0BAA0B,GAAG,IAAI;IAMtE;;OAEG;IACH,cAAc,CAAC,YAAY,EAAE,kBAAkB,GAAG,IAAI;IAMtD;;OAEG;IACH,mBAAmB,CACjB,EAAE,EAAE,MAAM,EACV,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,aAAa;IAQhB;;OAEG;IACH,4BAA4B,CAAC,EAAE,EAAE,MAAM,GAAG,yBAAyB,GAAG,SAAS;IAI/E;;OAEG;IACH,kBAAkB,IAAI,yBAAyB,EAAE;IAIjD;;OAEG;IACH,aAAa,IAAI,oBAAoB,EAAE;IAIvC;;OAEG;IACH,WAAW,IAAI,kBAAkB,EAAE;IAInC;;OAEG;IACH,oBAAoB,CAClB,EAAE,EAAE,MAAM,EACV,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,UAAU,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IAQnD;;OAEG;IACH,6BAA6B,CAAC,EAAE,EAAE,MAAM,GAAG,0BAA0B,GAAG,SAAS;IAIjF;;OAEG;IACH,mBAAmB,IAAI,0BAA0B,EAAE;IAInD,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,mBAAmB;CAa5B"}
|