@modular-prompt/experiment 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/cli/args.d.ts.map +1 -1
- package/dist/cli/args.js +4 -0
- package/dist/cli/args.js.map +1 -1
- package/dist/config/dynamic-loader.d.ts.map +1 -1
- package/dist/config/dynamic-loader.js +6 -4
- package/dist/config/dynamic-loader.js.map +1 -1
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +3 -1
- package/dist/config/loader.js.map +1 -1
- package/dist/logger.d.ts +9 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +9 -0
- package/dist/logger.js.map +1 -0
- package/dist/run-comparison.js +10 -0
- package/dist/run-comparison.js.map +1 -1
- package/dist/runner/driver-manager.d.ts.map +1 -1
- package/dist/runner/driver-manager.js +8 -6
- package/dist/runner/driver-manager.js.map +1 -1
- package/dist/runner/evaluator.d.ts.map +1 -1
- package/dist/runner/evaluator.js +6 -4
- package/dist/runner/evaluator.js.map +1 -1
- package/dist/runner/experiment.d.ts.map +1 -1
- package/dist/runner/experiment.js +18 -17
- package/dist/runner/experiment.js.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -168,6 +168,15 @@ npx modular-experiment examples/experiment.yaml --evaluate
|
|
|
168
168
|
|
|
169
169
|
# Run multiple times for statistics
|
|
170
170
|
npx modular-experiment examples/experiment.yaml --repeat 10
|
|
171
|
+
|
|
172
|
+
# Run with detailed logging to JSONL file
|
|
173
|
+
npx modular-experiment examples/experiment.yaml --log-file experiment.jsonl
|
|
174
|
+
|
|
175
|
+
# Run with verbose output (show internal operations)
|
|
176
|
+
npx modular-experiment examples/experiment.yaml --verbose
|
|
177
|
+
|
|
178
|
+
# Combine options
|
|
179
|
+
npx modular-experiment examples/experiment.yaml --evaluate --log-file experiment.jsonl --verbose
|
|
171
180
|
```
|
|
172
181
|
|
|
173
182
|
## Configuration
|
|
@@ -393,6 +402,8 @@ Options:
|
|
|
393
402
|
--evaluate Enable evaluation phase
|
|
394
403
|
--evaluators <names> Comma-separated evaluator names (default: all)
|
|
395
404
|
--dry-run Display execution plan without running the experiment
|
|
405
|
+
--log-file <path> Log file path for JSONL output (detailed logs)
|
|
406
|
+
--verbose Enable verbose output (show detailed internal operations)
|
|
396
407
|
```
|
|
397
408
|
|
|
398
409
|
**Note**: All paths specified in the config file are resolved relative to the config file's directory.
|
package/dist/cli/args.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAgB,SAAS,IAAI,yBAAyB,
|
|
1
|
+
{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAgB,SAAS,IAAI,yBAAyB,CAiCrD"}
|
package/dist/cli/args.js
CHANGED
|
@@ -16,6 +16,8 @@ export function parseArgs() {
|
|
|
16
16
|
.option('--evaluate', 'Enable AI-based evaluation of outputs', false)
|
|
17
17
|
.option('--evaluators <names>', 'Comma-separated evaluator names (default: all)')
|
|
18
18
|
.option('--dry-run', 'Display execution plan without running the experiment', false)
|
|
19
|
+
.option('--log-file <path>', 'Log file path for JSONL output (detailed logs)')
|
|
20
|
+
.option('--verbose', 'Enable verbose output (show detailed internal operations)', false)
|
|
19
21
|
.parse();
|
|
20
22
|
const config = program.args[0];
|
|
21
23
|
const options = program.opts();
|
|
@@ -28,6 +30,8 @@ export function parseArgs() {
|
|
|
28
30
|
enableEvaluation: options.evaluate,
|
|
29
31
|
evaluatorFilter: options.evaluators?.split(',').map((s) => s.trim()),
|
|
30
32
|
dryRun: options.dryRun,
|
|
33
|
+
logFile: options.logFile,
|
|
34
|
+
verbose: options.verbose,
|
|
31
35
|
};
|
|
32
36
|
}
|
|
33
37
|
//# sourceMappingURL=args.js.map
|
package/dist/cli/args.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAG/B,MAAM,UAAU,SAAS;IACvB,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO;SACJ,IAAI,CAAC,oBAAoB,CAAC;SAC1B,WAAW,CAAC,2CAA2C,CAAC;SACxD,QAAQ,CAAC,UAAU,EAAE,oDAAoD,CAAC;SAC1E,MAAM,CAAC,oBAAoB,EAAE,uBAAuB,CAAC;SACrD,MAAM,CAAC,oBAAoB,EAAE,oDAAoD,CAAC;SAClF,MAAM,CAAC,mBAAmB,EAAE,qDAAqD,CAAC;SAClF,MAAM,CAAC,kBAAkB,EAAE,uBAAuB,EAAE,GAAG,CAAC;SACxD,MAAM,CAAC,YAAY,EAAE,uCAAuC,EAAE,KAAK,CAAC;SACpE,MAAM,CAAC,sBAAsB,EAAE,gDAAgD,CAAC;SAChF,MAAM,CAAC,WAAW,EAAE,uDAAuD,EAAE,KAAK,CAAC;SACnF,KAAK,EAAE,CAAC;IAEX,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAE/B,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;QAC1C,cAAc,EAAE,OAAO,CAAC,QAAQ;QAChC,WAAW,EAAE,OAAO,CAAC,KAAK;QAC1B,YAAY,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACzC,gBAAgB,EAAE,OAAO,CAAC,QAAQ;QAClC,eAAe,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5E,MAAM,EAAE,OAAO,CAAC,MAAM;
|
|
1
|
+
{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAG/B,MAAM,UAAU,SAAS;IACvB,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO;SACJ,IAAI,CAAC,oBAAoB,CAAC;SAC1B,WAAW,CAAC,2CAA2C,CAAC;SACxD,QAAQ,CAAC,UAAU,EAAE,oDAAoD,CAAC;SAC1E,MAAM,CAAC,oBAAoB,EAAE,uBAAuB,CAAC;SACrD,MAAM,CAAC,oBAAoB,EAAE,oDAAoD,CAAC;SAClF,MAAM,CAAC,mBAAmB,EAAE,qDAAqD,CAAC;SAClF,MAAM,CAAC,kBAAkB,EAAE,uBAAuB,EAAE,GAAG,CAAC;SACxD,MAAM,CAAC,YAAY,EAAE,uCAAuC,EAAE,KAAK,CAAC;SACpE,MAAM,CAAC,sBAAsB,EAAE,gDAAgD,CAAC;SAChF,MAAM,CAAC,WAAW,EAAE,uDAAuD,EAAE,KAAK,CAAC;SACnF,MAAM,CAAC,mBAAmB,EAAE,gDAAgD,CAAC;SAC7E,MAAM,CAAC,WAAW,EAAE,2DAA2D,EAAE,KAAK,CAAC;SACvF,KAAK,EAAE,CAAC;IAEX,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAE/B,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;QAC1C,cAAc,EAAE,OAAO,CAAC,QAAQ;QAChC,WAAW,EAAE,OAAO,CAAC,KAAK;QAC1B,YAAY,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACzC,gBAAgB,EAAE,OAAO,CAAC,QAAQ;QAClC,eAAe,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5E,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dynamic-loader.d.ts","sourceRoot":"","sources":["../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,KAAK,EACV,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,gBAAgB,EACjB,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"dynamic-loader.d.ts","sourceRoot":"","sources":["../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,KAAK,EACV,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,gBAAgB,EACjB,MAAM,aAAa,CAAC;AAOrB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,kBAAkB,EAAE,EAC1B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,EAAE,CAAC,CAqE5B;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,eAAe,EAAE,EACvB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAsB7B"}
|
|
@@ -8,6 +8,8 @@ import { pathToFileURL } from 'url';
|
|
|
8
8
|
import { resolve } from 'path';
|
|
9
9
|
import { baseEvaluationModule } from '../base-evaluation-module.js';
|
|
10
10
|
import { getBuiltinEvaluator } from '../evaluators/index.js';
|
|
11
|
+
import { logger as baseLogger } from '../logger.js';
|
|
12
|
+
const logger = baseLogger.context('dynamic-loader');
|
|
11
13
|
/**
|
|
12
14
|
* Load evaluators from references
|
|
13
15
|
*
|
|
@@ -26,7 +28,7 @@ export async function loadEvaluators(refs, basePath) {
|
|
|
26
28
|
const imported = await import(fileUrl);
|
|
27
29
|
evaluator = imported.default;
|
|
28
30
|
if (!evaluator) {
|
|
29
|
-
|
|
31
|
+
logger.warn(`No default export in ${ref.path}`);
|
|
30
32
|
continue;
|
|
31
33
|
}
|
|
32
34
|
}
|
|
@@ -49,7 +51,7 @@ export async function loadEvaluators(refs, basePath) {
|
|
|
49
51
|
// Builtin evaluator (name only)
|
|
50
52
|
evaluator = getBuiltinEvaluator(ref.name);
|
|
51
53
|
if (!evaluator) {
|
|
52
|
-
|
|
54
|
+
logger.warn(`Builtin evaluator not found: ${ref.name}`);
|
|
53
55
|
continue;
|
|
54
56
|
}
|
|
55
57
|
}
|
|
@@ -78,7 +80,7 @@ export async function loadEvaluators(refs, basePath) {
|
|
|
78
80
|
});
|
|
79
81
|
}
|
|
80
82
|
else {
|
|
81
|
-
|
|
83
|
+
logger.warn(`Unknown evaluator type: ${ref.name}`);
|
|
82
84
|
}
|
|
83
85
|
}
|
|
84
86
|
return evaluators;
|
|
@@ -98,7 +100,7 @@ export async function loadModules(refs, basePath) {
|
|
|
98
100
|
const imported = await import(fileUrl);
|
|
99
101
|
const module = imported.default;
|
|
100
102
|
if (!module) {
|
|
101
|
-
|
|
103
|
+
logger.warn(`No default export in ${ref.path}`);
|
|
102
104
|
continue;
|
|
103
105
|
}
|
|
104
106
|
modules.push({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dynamic-loader.js","sourceRoot":"","sources":["../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAO/B,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"dynamic-loader.js","sourceRoot":"","sources":["../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAO/B,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAapD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAA0B,EAC1B,QAAgB;IAEhB,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,SAAsD,CAAC;QAE3D,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YAClB,gBAAgB;YAChB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;YACvC,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC;YAE7B,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;YAC3B,oDAAoD;YACpD,MAAM,YAAY,GAAG,KAAK,CAAC,oBAAoB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC7D,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;gBAClC,IAAI,EAAE,QAAQ;gBACd,eAAe,EAAE;oBACf,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;oBAClC,MAAM,EAAE,YAAY;iBACrB;aACF,CAAC,CAAC;YACH,SAAS;QACX,CAAC;aAAM,CAAC;YACN,gCAAgC;YAChC,SAAS,GAAG,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAE1C,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,gCAAgC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBACxD,SAAS;YACX,CAAC;QACH,CAAC;QAED,qCAAqC;QACrC,IAAI,UAAU,IAAI,SAAS,IAAI,OAAO,SAAS,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;YACxE,iBAAiB;YACjB,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,IAAI,EAAE;gBAC3D,IAAI,EAAE,MAAM;gBACZ,aAAa,EAAE,SAA0B;aAC1C,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,4CAA4C;YAC5C,MAAM,YAAY,GAAG,KAAK,CAAC,oBAAoB,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACnE,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,IAAI,EAAE;gBAC3D,IAAI,EAAE,QAAQ;gBACd,eAAe,EAAE;oBACf,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,MAAM,EAAE,YAAY;iBACrB;aACF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAuB,EACvB,QAAgB;IAEhB,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;QAC7C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAChD,SAAS;QACX,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,IAAI,EAAE;YACxD,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAC3E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,KAAK,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAC3E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,KAAK,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAKhE,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAGD,MAAM,MAAM,gBAAgB,GAAG,YAAY,CAAC;AAsB5C;;;;;GAKG;AACH,wBAAsB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAsGpF"}
|
package/dist/config/loader.js
CHANGED
|
@@ -6,6 +6,8 @@ import { parse as parseYaml } from 'yaml';
|
|
|
6
6
|
import { resolve, dirname, extname } from 'path';
|
|
7
7
|
import { createJiti } from 'jiti';
|
|
8
8
|
import { AIService } from '@modular-prompt/driver';
|
|
9
|
+
import { logger as baseLogger } from '../logger.js';
|
|
10
|
+
const logger = baseLogger.context('config-loader');
|
|
9
11
|
/**
|
|
10
12
|
* Resolve path relative to config file directory
|
|
11
13
|
*
|
|
@@ -82,7 +84,7 @@ export async function loadExperimentConfig(configPath) {
|
|
|
82
84
|
if (serverConfig.credentials?.googleApplicationCredentials) {
|
|
83
85
|
const resolvedPath = resolveConfigPath(configDir, serverConfig.credentials.googleApplicationCredentials);
|
|
84
86
|
process.env.GOOGLE_APPLICATION_CREDENTIALS = resolvedPath;
|
|
85
|
-
|
|
87
|
+
logger.verbose(`Setting GOOGLE_APPLICATION_CREDENTIALS=${resolvedPath}`);
|
|
86
88
|
}
|
|
87
89
|
// Validation
|
|
88
90
|
if (!serverConfig.models || Object.keys(serverConfig.models).length === 0) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAClC,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAClC,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAG3E,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;AAcnD;;;;;;GAMG;AACH,SAAS,iBAAiB,CAAC,SAAiB,EAAE,IAAY;IACxD,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;IACpD,CAAC;IACD,oCAAoC;IACpC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,kDAAkD;IAClD,OAAO,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAClC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,UAAkB;IAC3D,0CAA0C;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAEhC,iCAAiC;IACjC,IAAI,MAAW,CAAC;IAEhB,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACtC,cAAc;QACd,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;SAAM,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QAC9E,0DAA0D;QAC1D,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACvC,cAAc,EAAE,IAAI,EAAG,mCAAmC;YAC1D,KAAK,EAAE,IAAI,EAAY,wCAAwC;YAC/D,YAAY,EAAE,KAAK,EAAI,0BAA0B;SAClD,CAAC,CAAC;QAEH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAEvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,0BAA0B,UAAU,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,4CAA4C,CAAC,CAAC;IACxG,CAAC;IAED,qBAAqB;IACrB,MAAM,OAAO,GAAsB,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACxD,MAAM,SAAS,GAAe,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;IACrD,MAAM,UAAU,GAAyB,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;IAEjE,oDAAoD;IACpD,MAAM,YAAY,GAAG;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC;IAEF,iEAAiE;IACjE,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;QACzB,KAAK,MAAM,UAAU,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAEtD,8CAA8C;YAC9C,IAAI,YAAY,CAAC,eAAe,EAAE,CAAC;gBACjC,YAAY,CAAC,eAAe,GAAG,iBAAiB,CAAC,SAAS,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,YAAY,CAAC,WAAW,EAAE,4BAA4B,EAAE,CAAC;QAC3D,MAAM,YAAY,GAAG,iBAAiB,CAAC,SAAS,EAAE,YAAY,CAAC,WAAW,CAAC,4BAA4B,CAAC,CAAC;QACzG,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,YAAY,CAAC;QAC1D,MAAM,CAAC,OAAO,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,aAAa;IACb,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1E,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,mCAAmC;IACnC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAErE,qCAAqC;IACrC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpB,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,IAAI,+BAA+B,SAAS,GAAG,CAAC,CAAC;gBAC3F,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,oDAAoD;IACpD,iFAAiF;IACjF,MAAM,eAAe,GAAsB;QACzC,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,OAAO,EAAE,YAAY,CAAC,OAAO,IAAI,EAAE;QACnC,cAAc,EAAE;YACd,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;SAChB;KACF,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,eAAe,CAAC,CAAC;IAEjD,OAAO;QACL,YAAY;QACZ,OAAO;QACP,SAAS;QACT,UAAU;QACV,SAAS;QACT,SAAS;KACV,CAAC;AACJ,CAAC"}
|
package/dist/logger.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C;;GAEG;AACH,eAAO,MAAM,MAAM,QAAwD,CAAC"}
|
package/dist/logger.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Experiment package logger
|
|
3
|
+
*/
|
|
4
|
+
import { Logger } from '@modular-prompt/utils';
|
|
5
|
+
/**
|
|
6
|
+
* Experiment package logger with 'experiment' prefix
|
|
7
|
+
*/
|
|
8
|
+
export const logger = new Logger({ prefix: 'experiment', context: 'main' });
|
|
9
|
+
//# sourceMappingURL=logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C;;GAEG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC"}
|
package/dist/run-comparison.js
CHANGED
|
@@ -25,8 +25,18 @@ import { loadModules, loadEvaluators } from './config/dynamic-loader.js';
|
|
|
25
25
|
import { DriverManager } from './runner/driver-manager.js';
|
|
26
26
|
import { ExperimentRunner } from './runner/experiment.js';
|
|
27
27
|
import { StatisticsReporter } from './reporter/statistics.js';
|
|
28
|
+
import { Logger } from '@modular-prompt/utils';
|
|
28
29
|
// Parse CLI arguments
|
|
29
30
|
const options = parseArgs();
|
|
31
|
+
// Configure logger
|
|
32
|
+
Logger.configure({
|
|
33
|
+
level: options.verbose ? 'debug' : 'info',
|
|
34
|
+
accumulateLevel: 'debug',
|
|
35
|
+
isMcpMode: false,
|
|
36
|
+
accumulate: !!options.logFile,
|
|
37
|
+
maxEntries: 10000,
|
|
38
|
+
logFile: options.logFile,
|
|
39
|
+
});
|
|
30
40
|
// Display header
|
|
31
41
|
console.log('='.repeat(80));
|
|
32
42
|
console.log('Module Comparison Experiment');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-comparison.js","sourceRoot":"","sources":["../src/run-comparison.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"run-comparison.js","sourceRoot":"","sources":["../src/run-comparison.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,sBAAsB;AACtB,MAAM,OAAO,GAAG,SAAS,EAAE,CAAC;AAE5B,mBAAmB;AACnB,MAAM,CAAC,SAAS,CAAC;IACf,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;IACzC,eAAe,EAAE,OAAO;IACxB,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;IAC7B,UAAU,EAAE,KAAK;IACjB,OAAO,EAAE,OAAO,CAAC,OAAO;CACzB,CAAC,CAAC;AAEH,iBAAiB;AACjB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;AAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;AAC7C,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,CAAC,cAAc,IAAI,KAAK,EAAE,CAAC,CAAC;AACpE,OAAO,CAAC,GAAG,CAAC,iBAAiB,OAAO,CAAC,WAAW,IAAI,oBAAoB,EAAE,CAAC,CAAC;AAC5E,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;AACrE,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,WAAW,UAAU,CAAC,CAAC;AACtD,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;AAChF,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;AAC7E,CAAC;AACD,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;AAC/E,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,qBAAqB;AACrB,MAAM,EACJ,YAAY,EACZ,OAAO,EAAE,aAAa,EACtB,SAAS,EAAE,eAAe,EAC1B,UAAU,EAAE,gBAAgB,EAC5B,SAAS,EACT,SAAS,EACV,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAEnD,8CAA8C;AAC9C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAEnC,uCAAuC;AACvC,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAgB,EAAE,EAAE,CAC9E,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAC/D,CAAC;AAEF,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IACxB,MAAM,eAAe,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAgB,EAAE,EAAE,CACvE,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,WAAW,CACtC,CAAC;IACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,gDAAgD,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,mBAAmB,eAAe,CAAC,MAAM,0BAA0B,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IACxG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAgB,EAAE,EAAE,CACtD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAC7D,CAAC;AACJ,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,GAAG,CAAC,mBAAmB,YAAY,CAAC,MAAM,YAAY,CAAC,CAAC;IAChE,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAgB,EAAE,EAAE,CACnD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAC7D,CAAC;AACJ,CAAC;AAED,gCAAgC;AAChC,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAgB,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC;AACxF,IAAI,MAAM,EAAE,CAAC;IACX,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,4GAA4G,CAAC,CAAC;AAC5H,CAAC;AACD,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,kBAAkB;AAClB,MAAM,YAAY,GAAG,eAAe,CAAC;AACrC,MAAM,SAAS,GAAG,OAAO,CAAC,cAAc;IACtC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,OAAO,CAAC,cAAc,CAAC;IACtE,CAAC,CAAC,YAAY,CAAC;AAEjB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IAC3B,OAAO,CAAC,KAAK,CAAC,wBAAwB,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9G,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,eAAe,CAAC,CAAC;AAC3D,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,wCAAwC;AACxC,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;AAC/D,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY;IAClC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,YAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAChE,CAAC,CAAC,UAAU,CAAC;AAEf,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IACzB,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACtC,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,MAAM,aAAa,CAAC,CAAC;AACvD,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACrE,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,8DAA8D;AAC9D,IAAI,UAAU,CAAC;AACf,IAAI,cAAc,CAAC;AACnB,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAC7B,8CAA8C;IAC9C,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC;IACxE,UAAU,GAAG,OAAO,CAAC,eAAe;QAClC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,eAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtE,CAAC,CAAC,aAAa,CAAC;IAElB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,8CAA8C;IAC9C,IAAI,CAAC,YAAY,CAAC,UAAU,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACjE,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAC/D,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;QACvE,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC1C,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,gBAAgB,GAAG,YAAY,CAAC,UAAU,CAAC;IAEjD,mCAAmC;IACnC,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC;IACzC,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAEjD,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC9C,OAAO,CAAC,KAAK,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,KAAK,CAAC,yEAAyE,CAAC,CAAC;QACzF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,cAAc,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAEtD,OAAO,CAAC,GAAG,CAAC,8BAA8B,UAAU,CAAC,MAAM,gBAAgB,CAAC,CAAC;IAC7E,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACpF,OAAO,CAAC,GAAG,CAAC,uBAAuB,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC;AAED,kBAAkB;AAClB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,iBAAiB;AACjB,MAAM,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;AAC1C,MAAM,MAAM,GAAG,IAAI,gBAAgB,CACjC,SAAS,EACT,aAAa,EACb,OAAO,EACP,SAAS,EACT,MAAM,EACN,OAAO,CAAC,WAAW,EACnB,UAAU,EACV,cAAc,CACf,CAAC;AAEF,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;AAEnC,qBAAqB;AACrB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;AACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAE5B,kBAAkB;AAClB,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;AAE9B,iCAAiC;AACjC,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACjD,QAAQ,CAAC,MAAM,EAAE,CAAC;AACpB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver-manager.d.ts","sourceRoot":"","sources":["../../src/runner/driver-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"driver-manager.d.ts","sourceRoot":"","sources":["../../src/runner/driver-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAKnE,qBAAa,aAAa;IACxB,OAAO,CAAC,KAAK,CAA0B;IAEvC;;;;;;;;;;OAUG;IACG,WAAW,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC;IAY9F;;;;OAIG;IACG,KAAK,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB7C;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAiB/B"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Driver manager for caching and cleanup
|
|
3
3
|
*/
|
|
4
|
+
import { logger as baseLogger } from '../logger.js';
|
|
5
|
+
const logger = baseLogger.context('driver-manager');
|
|
4
6
|
export class DriverManager {
|
|
5
7
|
cache = new Map();
|
|
6
8
|
/**
|
|
@@ -16,10 +18,10 @@ export class DriverManager {
|
|
|
16
18
|
*/
|
|
17
19
|
async getOrCreate(aiService, modelName, modelSpec) {
|
|
18
20
|
if (this.cache.has(modelName)) {
|
|
19
|
-
|
|
21
|
+
logger.verbose(`Using cached driver for ${modelName}`);
|
|
20
22
|
return this.cache.get(modelName);
|
|
21
23
|
}
|
|
22
|
-
|
|
24
|
+
logger.info(`Creating new driver for ${modelName} (${modelSpec.provider}:${modelSpec.model})`);
|
|
23
25
|
const driver = await aiService.createDriver(modelSpec);
|
|
24
26
|
this.cache.set(modelName, driver);
|
|
25
27
|
return driver;
|
|
@@ -37,12 +39,12 @@ export class DriverManager {
|
|
|
37
39
|
try {
|
|
38
40
|
if (typeof driver.close === 'function') {
|
|
39
41
|
await driver.close();
|
|
40
|
-
|
|
42
|
+
logger.verbose(`Closed driver: ${modelName}`);
|
|
41
43
|
}
|
|
42
44
|
this.cache.delete(modelName);
|
|
43
45
|
}
|
|
44
46
|
catch (error) {
|
|
45
|
-
|
|
47
|
+
logger.warn(`Failed to close driver ${modelName}: ${error instanceof Error ? error.message : String(error)}`);
|
|
46
48
|
}
|
|
47
49
|
}
|
|
48
50
|
/**
|
|
@@ -55,11 +57,11 @@ export class DriverManager {
|
|
|
55
57
|
try {
|
|
56
58
|
if (driver && typeof driver.close === 'function') {
|
|
57
59
|
await driver.close();
|
|
58
|
-
|
|
60
|
+
logger.verbose(`Closed driver: ${key}`);
|
|
59
61
|
}
|
|
60
62
|
}
|
|
61
63
|
catch (error) {
|
|
62
|
-
|
|
64
|
+
logger.warn(`Failed to close driver ${key}: ${error instanceof Error ? error.message : String(error)}`);
|
|
63
65
|
}
|
|
64
66
|
}
|
|
65
67
|
console.log('✅ Cleanup completed');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver-manager.js","sourceRoot":"","sources":["../../src/runner/driver-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"driver-manager.js","sourceRoot":"","sources":["../../src/runner/driver-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAEpD,MAAM,OAAO,aAAa;IAChB,KAAK,GAAG,IAAI,GAAG,EAAe,CAAC;IAEvC;;;;;;;;;;OAUG;IACH,KAAK,CAAC,WAAW,CAAC,SAAoB,EAAE,SAAiB,EAAE,SAAoB;QAC7E,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,CAAC,OAAO,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACnC,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,2BAA2B,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;QAC/F,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK,CAAC,SAAiB;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;gBACvC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM,CAAC,OAAO,CAAC,kBAAkB,SAAS,EAAE,CAAC,CAAC;YAChD,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,0BAA0B,SAAS,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAEjC,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;oBACjD,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;oBACrB,MAAM,CAAC,OAAO,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,0BAA0B,GAAG,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC1G,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IACrC,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/runner/evaluator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/runner/evaluator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAKnE,qBAAa,eAAe;IAExB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,cAAc;gBADd,SAAS,EAAE,SAAS,EACpB,cAAc,EAAE,SAAS;IAGnC;;;;;;OAMG;IACG,QAAQ,CACZ,SAAS,EAAE,eAAe,EAC1B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,gBAAgB,CAAC;IAuB5B;;OAEG;YACW,kBAAkB;IAiEhC;;;;;OAKG;IACH,cAAc,CAAC,OAAO,EAAE,gBAAgB,EAAE,EAAE,UAAU,CAAC,EAAE,eAAe,EAAE,GAAG,IAAI;CAmDlF"}
|
package/dist/runner/evaluator.js
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
* Runs evaluation (code or prompt-based) for a single module
|
|
5
5
|
*/
|
|
6
6
|
import { compile } from '@modular-prompt/core';
|
|
7
|
+
import { logger as baseLogger } from '../logger.js';
|
|
8
|
+
const logger = baseLogger.context('evaluator');
|
|
7
9
|
export class EvaluatorRunner {
|
|
8
10
|
aiService;
|
|
9
11
|
evaluatorModel;
|
|
@@ -19,7 +21,7 @@ export class EvaluatorRunner {
|
|
|
19
21
|
* @returns Evaluation result
|
|
20
22
|
*/
|
|
21
23
|
async evaluate(evaluator, context) {
|
|
22
|
-
|
|
24
|
+
logger.verbose(`Evaluating ${context.moduleName} with evaluator: ${evaluator.name}`);
|
|
23
25
|
try {
|
|
24
26
|
if (evaluator.type === 'code') {
|
|
25
27
|
// Code evaluator - direct execution
|
|
@@ -32,7 +34,7 @@ export class EvaluatorRunner {
|
|
|
32
34
|
}
|
|
33
35
|
catch (error) {
|
|
34
36
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
35
|
-
|
|
37
|
+
logger.error(`Evaluation error for ${context.moduleName} (${evaluator.name}): ${errorMessage}`);
|
|
36
38
|
return {
|
|
37
39
|
evaluator: evaluator.name,
|
|
38
40
|
moduleName: context.moduleName,
|
|
@@ -55,7 +57,7 @@ export class EvaluatorRunner {
|
|
|
55
57
|
maxTokens: 4096,
|
|
56
58
|
});
|
|
57
59
|
const elapsed = Date.now() - startTime;
|
|
58
|
-
|
|
60
|
+
logger.verbose(`Evaluation completed for ${context.moduleName} (${evaluator.name}): ${elapsed}ms`);
|
|
59
61
|
// Close driver
|
|
60
62
|
if (driver && typeof driver.close === 'function') {
|
|
61
63
|
await driver.close();
|
|
@@ -88,7 +90,7 @@ export class EvaluatorRunner {
|
|
|
88
90
|
}
|
|
89
91
|
}
|
|
90
92
|
catch {
|
|
91
|
-
|
|
93
|
+
logger.warn(`Failed to parse JSON response for evaluator: ${evaluator.name}`);
|
|
92
94
|
}
|
|
93
95
|
// Fallback: return raw response
|
|
94
96
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../../src/runner/evaluator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../../src/runner/evaluator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAI/C,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;AAE/C,MAAM,OAAO,eAAe;IAEhB;IACA;IAFV,YACU,SAAoB,EACpB,cAAyB;QADzB,cAAS,GAAT,SAAS,CAAW;QACpB,mBAAc,GAAd,cAAc,CAAW;IAChC,CAAC;IAEJ;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CACZ,SAA0B,EAC1B,OAA0B;QAE1B,MAAM,CAAC,OAAO,CAAC,cAAc,OAAO,CAAC,UAAU,oBAAoB,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QAErF,IAAI,CAAC;YACH,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC9B,oCAAoC;gBACpC,OAAO,MAAM,SAAS,CAAC,aAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACN,mCAAmC;gBACnC,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,KAAK,CAAC,wBAAwB,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,MAAM,YAAY,EAAE,CAAC,CAAC;YAEhG,OAAO;gBACL,SAAS,EAAE,SAAS,CAAC,IAAI;gBACzB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,KAAK,EAAE,YAAY;aACpB,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAC9B,SAA0B,EAC1B,OAA0B;QAE1B,4BAA4B;QAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,eAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAErE,oCAAoC;QACpC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAEtE,iBAAiB;QACjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE;YAC1C,WAAW,EAAE,GAAG,EAAE,8CAA8C;YAChE,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAEvC,MAAM,CAAC,OAAO,CAAC,4BAA4B,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,MAAM,OAAO,IAAI,CAAC,CAAC;QAEnG,eAAe;QACf,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACjD,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QAED,qCAAqC;QACrC,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAuB,CAAC;YAClD,OAAO;gBACL,SAAS,EAAE,SAAS,CAAC,IAAI;gBACzB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,GAAG,EAAE,MAAM,CAAC,OAAO;aACpB,CAAC;QACJ,CAAC;QAED,2CAA2C;QAC3C,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YACtE,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,OAAO;oBACL,SAAS,EAAE,SAAS,CAAC,IAAI;oBACzB,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,GAAG,EAAE,MAAM,CAAC,OAAO;iBACpB,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC,gDAAgD,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,gCAAgC;QAChC,OAAO;YACL,SAAS,EAAE,SAAS,CAAC,IAAI;YACzB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,SAAS,EAAE,MAAM,CAAC,OAAO;YACzB,GAAG,EAAE,MAAM,CAAC,OAAO;SACpB,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,cAAc,CAAC,OAA2B,EAAE,UAA8B;QACxE,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,kBAAkB;QAClB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA8B,CAAC;QACvD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;gBACrC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACtC,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;QAED,KAAK,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC,IAAI,QAAQ,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,MAAM,UAAU,EAAE,CAAC,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEzC,8CAA8C;gBAC9C,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,SAAS,CAAC,CAAC;oBACpE,IAAI,SAAS,EAAE,WAAW,EAAE,CAAC;wBAC3B,OAAO,CAAC,GAAG,CAAC,SAAS,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;oBAChD,CAAC;gBACH,CAAC;gBAED,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;oBACjB,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;gBAChD,CAAC;qBAAM,CAAC;oBACN,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;wBAC/B,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBAC5D,CAAC;oBACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;wBACrB,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;oBACtD,CAAC;oBACD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBACnB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;wBAC9B,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;oBAC/F,CAAC;gBACH,CAAC;gBACD,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9B,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAe,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAChF,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAAuC,MAAM,aAAa,CAAC;AAC/G,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAe,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAChF,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAAuC,MAAM,aAAa,CAAC;AAC/G,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAMnE,qBAAa,gBAAgB;IAEzB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,cAAc,CAAC;gBAPf,SAAS,EAAE,SAAS,EACpB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,gBAAgB,EAAE,EAC3B,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EACjC,WAAW,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,eAAe,EAAE,YAAA,EAC9B,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,SAAS,CAAA;KAAE,YAAA;IAG5D;;;;OAIG;IACG,GAAG,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;IA4GlC;;OAEG;YACW,aAAa;IAyC3B;;OAEG;YACW,kBAAkB;IA2BhC;;OAEG;IACH,OAAO,CAAC,cAAc;CAiCvB"}
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { formatCompletionPrompt } from '@modular-prompt/driver';
|
|
5
5
|
import { EvaluatorRunner } from './evaluator.js';
|
|
6
|
+
import { logger as baseLogger } from '../logger.js';
|
|
7
|
+
const logger = baseLogger.context('runner');
|
|
6
8
|
export class ExperimentRunner {
|
|
7
9
|
aiService;
|
|
8
10
|
driverManager;
|
|
@@ -40,11 +42,10 @@ export class ExperimentRunner {
|
|
|
40
42
|
console.log();
|
|
41
43
|
// Compile all modules with testCase.input as context
|
|
42
44
|
const compiledModules = this.modules.map(module => {
|
|
43
|
-
|
|
45
|
+
logger.verbose(`Compiling prompt for module: ${module.name}`);
|
|
44
46
|
const compiled = module.compile(testCase.input);
|
|
45
47
|
const prompt = formatCompletionPrompt(compiled);
|
|
46
|
-
|
|
47
|
-
console.log();
|
|
48
|
+
logger.verbose(`Prompt length for ${module.name}: ${prompt.length} chars`);
|
|
48
49
|
return {
|
|
49
50
|
name: module.name,
|
|
50
51
|
compiled,
|
|
@@ -80,7 +81,7 @@ export class ExperimentRunner {
|
|
|
80
81
|
console.log(`🤖 Testing with ${modelName} (${modelSpec.provider}:${modelSpec.model})`);
|
|
81
82
|
// Close previous driver if switching models
|
|
82
83
|
if (previousDriver && previousModelName && previousModelName !== modelName) {
|
|
83
|
-
|
|
84
|
+
logger.verbose(`Switching from ${previousModelName} to ${modelName}, closing previous driver...`);
|
|
84
85
|
await this.driverManager.close(previousModelName);
|
|
85
86
|
previousDriver = null;
|
|
86
87
|
}
|
|
@@ -124,10 +125,10 @@ export class ExperimentRunner {
|
|
|
124
125
|
* Run module test with multiple repetitions
|
|
125
126
|
*/
|
|
126
127
|
async runModuleTest(moduleName, compiled, driver) {
|
|
127
|
-
|
|
128
|
+
logger.verbose(`Running ${this.repeatCount} time(s) for module: ${moduleName}`);
|
|
128
129
|
const runs = [];
|
|
129
130
|
for (let i = 0; i < this.repeatCount; i++) {
|
|
130
|
-
|
|
131
|
+
logger.verbose(`Run ${i + 1}/${this.repeatCount} for module: ${moduleName}`);
|
|
131
132
|
const startTime = Date.now();
|
|
132
133
|
try {
|
|
133
134
|
const result = await driver.query(compiled, {
|
|
@@ -135,7 +136,7 @@ export class ExperimentRunner {
|
|
|
135
136
|
maxTokens: 2048,
|
|
136
137
|
});
|
|
137
138
|
const elapsed = Date.now() - startTime;
|
|
138
|
-
|
|
139
|
+
logger.verbose(`Module ${moduleName} run ${i + 1}: Success (${elapsed}ms)`);
|
|
139
140
|
runs.push({
|
|
140
141
|
success: true,
|
|
141
142
|
elapsed,
|
|
@@ -145,7 +146,7 @@ export class ExperimentRunner {
|
|
|
145
146
|
catch (error) {
|
|
146
147
|
const elapsed = Date.now() - startTime;
|
|
147
148
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
148
|
-
|
|
149
|
+
logger.error(`Module ${moduleName} run ${i + 1}: Error (${elapsed}ms): ${errorMessage}`);
|
|
149
150
|
runs.push({
|
|
150
151
|
success: false,
|
|
151
152
|
elapsed,
|
|
@@ -153,7 +154,6 @@ export class ExperimentRunner {
|
|
|
153
154
|
});
|
|
154
155
|
}
|
|
155
156
|
}
|
|
156
|
-
console.log();
|
|
157
157
|
return runs;
|
|
158
158
|
}
|
|
159
159
|
/**
|
|
@@ -192,16 +192,17 @@ export class ExperimentRunner {
|
|
|
192
192
|
console.log(` ✅ [${module1.name}] and [${module2.name}] are identical`);
|
|
193
193
|
}
|
|
194
194
|
else {
|
|
195
|
-
console.log(` ⚠️ [${module1.name}] and [${module2.name}] differ
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
195
|
+
console.log(` ⚠️ [${module1.name}] and [${module2.name}] differ`);
|
|
196
|
+
logger.verbose(`Prompt comparison details:`);
|
|
197
|
+
logger.verbose(` ${module1.name}: ${module1.prompt.length} chars`);
|
|
198
|
+
logger.verbose(` ${module2.name}: ${module2.prompt.length} chars`);
|
|
199
|
+
logger.verbose(` Diff: ${module2.prompt.length - module1.prompt.length} chars`);
|
|
200
|
+
// Find first difference (verbose only)
|
|
200
201
|
for (let k = 0; k < Math.max(module1.prompt.length, module2.prompt.length); k++) {
|
|
201
202
|
if (module1.prompt[k] !== module2.prompt[k]) {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
203
|
+
logger.verbose(` First diff at position ${k}:`);
|
|
204
|
+
logger.verbose(` ${module1.name}: ${JSON.stringify(module1.prompt.substring(k, k + 50))}`);
|
|
205
|
+
logger.verbose(` ${module2.name}: ${JSON.stringify(module2.prompt.substring(k, k + 50))}`);
|
|
205
206
|
break;
|
|
206
207
|
}
|
|
207
208
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAKhE,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAKhE,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAE5C,MAAM,OAAO,gBAAgB;IAEjB;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IARV,YACU,SAAoB,EACpB,aAA4B,EAC5B,OAA2B,EAC3B,SAAqB,EACrB,MAAiC,EACjC,WAAmB,EACnB,UAA8B,EAC9B,cAAkD;QAPlD,cAAS,GAAT,SAAS,CAAW;QACpB,kBAAa,GAAb,aAAa,CAAe;QAC5B,YAAO,GAAP,OAAO,CAAoB;QAC3B,cAAS,GAAT,SAAS,CAAY;QACrB,WAAM,GAAN,MAAM,CAA2B;QACjC,gBAAW,GAAX,WAAW,CAAQ;QACnB,eAAU,GAAV,UAAU,CAAoB;QAC9B,mBAAc,GAAd,cAAc,CAAoC;IACzD,CAAC;IAEJ;;;;OAIG;IACH,KAAK,CAAC,GAAG;QACP,MAAM,UAAU,GAAiB,EAAE,CAAC;QACpC,MAAM,kBAAkB,GAAwB,EAAE,CAAC;QAEnD,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3C,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,gBAAgB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;YAEd,qDAAqD;YACrD,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBAChD,MAAM,CAAC,OAAO,CAAC,gCAAgC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAChD,MAAM,MAAM,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;gBAChD,MAAM,CAAC,OAAO,CAAC,qBAAqB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAE3E,OAAO;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,QAAQ;oBACR,MAAM;iBACP,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,sCAAsC;YACtC,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;YACvC,CAAC;YAED,oDAAoD;YACpD,MAAM,YAAY,GAA6C,QAAQ,CAAC,MAAM;gBAC5E,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;oBACzB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;wBACV,OAAO,CAAC,IAAI,CAAC,cAAc,IAAI,wCAAwC,CAAC,CAAC;wBACzE,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACxB,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAA6C;gBAChE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;qBACxB,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC;qBAC7C,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAE/C,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;gBAClE,OAAO,CAAC,GAAG,EAAE,CAAC;gBACd,SAAS;YACX,CAAC;YAED,uBAAuB;YACvB,IAAI,cAAc,GAAQ,IAAI,CAAC;YAC/B,IAAI,iBAAiB,GAAkB,IAAI,CAAC;YAE5C,KAAK,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,YAAY,EAAE,CAAC;gBAChE,OAAO,CAAC,GAAG,CAAC,mBAAmB,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;gBAEvF,4CAA4C;gBAC5C,IAAI,cAAc,IAAI,iBAAiB,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBAC3E,MAAM,CAAC,OAAO,CAAC,kBAAkB,iBAAiB,OAAO,SAAS,8BAA8B,CAAC,CAAC;oBAClG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAClD,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;gBAED,sCAAsC;gBACtC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;gBAC1F,cAAc,GAAG,MAAM,CAAC;gBACxB,iBAAiB,GAAG,SAAS,CAAC;gBAE9B,mBAAmB;gBACnB,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC;oBACzD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;oBAE9D,UAAU,CAAC,IAAI,CAAC;wBACd,QAAQ,EAAE,QAAQ,CAAC,IAAI;wBACvB,KAAK,EAAE,SAAS;wBAChB,MAAM,EAAE,IAAI;wBACZ,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;4BACnB,OAAO,EAAE,CAAC,CAAC,OAAO;4BAClB,OAAO,EAAE,CAAC,CAAC,OAAO;4BAClB,OAAO,EAAE,CAAC,CAAC,WAAW,EAAE,OAAO,IAAI,EAAE;4BACrC,KAAK,EAAE,CAAC,CAAC,KAAK;yBACf,CAAC,CAAC;qBACJ,CAAC,CAAC;oBAEH,iDAAiD;oBACjD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;oBACnD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC9B,kBAAkB,CAAC,IAAI,CAAC;4BACtB,UAAU,EAAE,IAAI;4BAChB,MAAM;4BACN,IAAI,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAY,EAAE,CAAC,CAAC;yBACjE,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,kDAAkD;QAClD,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzE,MAAM,IAAI,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,UAAkB,EAClB,QAAa,EACb,MAAW;QAEX,MAAM,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,WAAW,wBAAwB,UAAU,EAAE,CAAC,CAAC;QAEhF,MAAM,IAAI,GAA4F,EAAE,CAAC;QAEzG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,gBAAgB,UAAU,EAAE,CAAC,CAAC;YAE7E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE;oBAC1C,WAAW,EAAE,GAAG;oBAChB,SAAS,EAAE,IAAI;iBAChB,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBAEvC,MAAM,CAAC,OAAO,CAAC,UAAU,UAAU,QAAQ,CAAC,GAAG,CAAC,cAAc,OAAO,KAAK,CAAC,CAAC;gBAC5E,IAAI,CAAC,IAAI,CAAC;oBACR,OAAO,EAAE,IAAI;oBACb,OAAO;oBACP,WAAW,EAAE,MAAM;iBACpB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBACvC,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5E,MAAM,CAAC,KAAK,CAAC,UAAU,UAAU,QAAQ,CAAC,GAAG,CAAC,YAAY,OAAO,QAAQ,YAAY,EAAE,CAAC,CAAC;gBACzF,IAAI,CAAC,IAAI,CAAC;oBACR,OAAO,EAAE,KAAK;oBACd,OAAO;oBACP,KAAK,EAAE,YAAY;iBACpB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAC9B,kBAAuC;QAEvC,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,cAAe,CAAC,IAAI,CAAC,CAAC;QACvF,MAAM,cAAc,GAAuB,EAAE,CAAC;QAE9C,2CAA2C;QAC3C,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,EAAE,CAAC;YAEd,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAW,EAAE,CAAC;gBACzC,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAClE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,eAAe,CAAC,cAAc,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,eAAwD;QAC7E,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,OAAO,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBAEnC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;oBACtC,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,IAAI,UAAU,OAAO,CAAC,IAAI,iBAAiB,CAAC,CAAC;gBAC5E,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,IAAI,UAAU,OAAO,CAAC,IAAI,UAAU,CAAC,CAAC;oBACrE,MAAM,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC;oBAC7C,MAAM,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;oBACpE,MAAM,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;oBACpE,MAAM,CAAC,OAAO,CAAC,WAAW,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;oBAEjF,uCAAuC;oBACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;wBAChF,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;4BAC5C,MAAM,CAAC,OAAO,CAAC,4BAA4B,CAAC,GAAG,CAAC,CAAC;4BACjD,MAAM,CAAC,OAAO,CAAC,OAAO,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;4BAC9F,MAAM,CAAC,OAAO,CAAC,OAAO,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;4BAC9F,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;CACF"}
|
package/dist/types.d.ts
CHANGED
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,KAAK,EAAE,GAAG,CAAC;IACX,iGAAiG;IACjG,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,GAAG,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,KAAK,CAAC;QACV,WAAW,EAAE,WAAW,CAAC;KAC1B,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;;OASG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACrE;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;;OASG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/E;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3C;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,iBAAiB;IAClE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,KAAK,EAAE,GAAG,CAAC;IACX,iGAAiG;IACjG,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,GAAG,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,KAAK,CAAC;QACV,WAAW,EAAE,WAAW,CAAC;KAC1B,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;;OASG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACrE;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;;OASG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/E;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3C;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,iBAAiB;IAClE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modular-prompt/experiment",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "Experiment framework for comparing and evaluating prompt modules",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
"yaml": "^2.3.4",
|
|
26
26
|
"zod": "^3.22.4",
|
|
27
27
|
"@modular-prompt/core": "0.1.10",
|
|
28
|
-
"@modular-prompt/driver": "0.
|
|
28
|
+
"@modular-prompt/driver": "0.5.0",
|
|
29
|
+
"@modular-prompt/utils": "0.2.0"
|
|
29
30
|
},
|
|
30
31
|
"devDependencies": {
|
|
31
32
|
"@eslint/js": "^9.34.0",
|