@modular-prompt/experiment 0.3.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/args.d.ts.map +1 -1
- package/dist/cli/args.js +2 -0
- package/dist/cli/args.js.map +1 -1
- package/dist/config/dynamic-loader.d.ts +7 -0
- package/dist/config/dynamic-loader.d.ts.map +1 -1
- package/dist/config/dynamic-loader.js +30 -7
- package/dist/config/dynamic-loader.js.map +1 -1
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +13 -3
- package/dist/config/loader.js.map +1 -1
- package/dist/run-comparison.js +74 -17
- package/dist/run-comparison.js.map +1 -1
- package/dist/runner/driver-manager.d.ts +12 -2
- package/dist/runner/driver-manager.d.ts.map +1 -1
- package/dist/runner/driver-manager.js +16 -0
- package/dist/runner/driver-manager.js.map +1 -1
- package/dist/runner/experiment.d.ts.map +1 -1
- package/dist/runner/experiment.js +82 -31
- package/dist/runner/experiment.js.map +1 -1
- package/dist/trace/writer.d.ts +2 -0
- package/dist/trace/writer.d.ts.map +1 -0
- package/dist/trace/writer.js +66 -0
- package/dist/trace/writer.js.map +1 -0
- package/dist/types.d.ts +26 -2
- package/dist/types.d.ts.map +1 -1
- package/examples/tools-experiment.yaml +4 -4
- package/package.json +5 -5
- package/skills/experiment/SKILL.md +49 -0
package/dist/cli/args.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAgB,SAAS,IAAI,yBAAyB,
|
|
1
|
+
{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAgB,SAAS,IAAI,yBAAyB,CAmCrD"}
|
package/dist/cli/args.js
CHANGED
|
@@ -18,6 +18,7 @@ export function parseArgs() {
|
|
|
18
18
|
.option('--dry-run', 'Display execution plan without running the experiment', false)
|
|
19
19
|
.option('--log-file <path>', 'Log file path for JSONL output (detailed logs)')
|
|
20
20
|
.option('--verbose', 'Enable verbose output (show detailed internal operations)', false)
|
|
21
|
+
.option('--trace-dir <dir>', 'Trace output directory (structured execution logs)')
|
|
21
22
|
.parse();
|
|
22
23
|
const config = program.args[0];
|
|
23
24
|
const options = program.opts();
|
|
@@ -32,6 +33,7 @@ export function parseArgs() {
|
|
|
32
33
|
dryRun: options.dryRun,
|
|
33
34
|
logFile: options.logFile,
|
|
34
35
|
verbose: options.verbose,
|
|
36
|
+
traceDir: options.traceDir,
|
|
35
37
|
};
|
|
36
38
|
}
|
|
37
39
|
//# sourceMappingURL=args.js.map
|
package/dist/cli/args.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAG/B,MAAM,UAAU,SAAS;IACvB,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO;SACJ,IAAI,CAAC,oBAAoB,CAAC;SAC1B,WAAW,CAAC,2CAA2C,CAAC;SACxD,QAAQ,CAAC,UAAU,EAAE,oDAAoD,CAAC;SAC1E,MAAM,CAAC,oBAAoB,EAAE,uBAAuB,CAAC;SACrD,MAAM,CAAC,oBAAoB,EAAE,oDAAoD,CAAC;SAClF,MAAM,CAAC,mBAAmB,EAAE,qDAAqD,CAAC;SAClF,MAAM,CAAC,kBAAkB,EAAE,uBAAuB,EAAE,GAAG,CAAC;SACxD,MAAM,CAAC,YAAY,EAAE,uCAAuC,EAAE,KAAK,CAAC;SACpE,MAAM,CAAC,sBAAsB,EAAE,gDAAgD,CAAC;SAChF,MAAM,CAAC,WAAW,EAAE,uDAAuD,EAAE,KAAK,CAAC;SACnF,MAAM,CAAC,mBAAmB,EAAE,gDAAgD,CAAC;SAC7E,MAAM,CAAC,WAAW,EAAE,2DAA2D,EAAE,KAAK,CAAC;SACvF,KAAK,EAAE,CAAC;IAEX,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAE/B,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;QAC1C,cAAc,EAAE,OAAO,CAAC,QAAQ;QAChC,WAAW,EAAE,OAAO,CAAC,KAAK;QAC1B,YAAY,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACzC,gBAAgB,EAAE,OAAO,CAAC,QAAQ;QAClC,eAAe,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5E,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;
|
|
1
|
+
{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAG/B,MAAM,UAAU,SAAS;IACvB,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO;SACJ,IAAI,CAAC,oBAAoB,CAAC;SAC1B,WAAW,CAAC,2CAA2C,CAAC;SACxD,QAAQ,CAAC,UAAU,EAAE,oDAAoD,CAAC;SAC1E,MAAM,CAAC,oBAAoB,EAAE,uBAAuB,CAAC;SACrD,MAAM,CAAC,oBAAoB,EAAE,oDAAoD,CAAC;SAClF,MAAM,CAAC,mBAAmB,EAAE,qDAAqD,CAAC;SAClF,MAAM,CAAC,kBAAkB,EAAE,uBAAuB,EAAE,GAAG,CAAC;SACxD,MAAM,CAAC,YAAY,EAAE,uCAAuC,EAAE,KAAK,CAAC;SACpE,MAAM,CAAC,sBAAsB,EAAE,gDAAgD,CAAC;SAChF,MAAM,CAAC,WAAW,EAAE,uDAAuD,EAAE,KAAK,CAAC;SACnF,MAAM,CAAC,mBAAmB,EAAE,gDAAgD,CAAC;SAC7E,MAAM,CAAC,WAAW,EAAE,2DAA2D,EAAE,KAAK,CAAC;SACvF,MAAM,CAAC,mBAAmB,EAAE,oDAAoD,CAAC;SACjF,KAAK,EAAE,CAAC;IAEX,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAE/B,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;QAC1C,cAAc,EAAE,OAAO,CAAC,QAAQ;QAChC,WAAW,EAAE,OAAO,CAAC,KAAK;QAC1B,YAAY,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACzC,gBAAgB,EAAE,OAAO,CAAC,QAAQ;QAClC,eAAe,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5E,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC;AACJ,CAAC"}
|
|
@@ -24,10 +24,17 @@ export interface LoadedEvaluator {
|
|
|
24
24
|
export declare function loadEvaluators(refs: EvaluatorReference[], basePath: string): Promise<LoadedEvaluator[]>;
|
|
25
25
|
/**
|
|
26
26
|
* Module reference in config file
|
|
27
|
+
* - path: external file reference
|
|
28
|
+
* - module: inline PromptModule definition (static content only)
|
|
27
29
|
*/
|
|
28
30
|
export type ModuleReference = {
|
|
29
31
|
name: string;
|
|
30
32
|
path: string;
|
|
33
|
+
export?: string;
|
|
34
|
+
description?: string;
|
|
35
|
+
} | {
|
|
36
|
+
name: string;
|
|
37
|
+
module: Record<string, any>;
|
|
31
38
|
description?: string;
|
|
32
39
|
};
|
|
33
40
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dynamic-loader.d.ts","sourceRoot":"","sources":["../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"dynamic-loader.d.ts","sourceRoot":"","sources":["../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,KAAK,EACV,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,gBAAgB,EACjB,MAAM,aAAa,CAAC;AAsBrB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,kBAAkB,EAAE,EAC1B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,EAAE,CAAC,CAoE5B;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,GACvB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAExE;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,eAAe,EAAE,EACvB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAgC7B"}
|
|
@@ -4,12 +4,27 @@
|
|
|
4
4
|
* Loads evaluators and modules from external files or inline definitions
|
|
5
5
|
*/
|
|
6
6
|
import { merge } from '@modular-prompt/core';
|
|
7
|
+
import { createJiti } from 'jiti';
|
|
7
8
|
import { pathToFileURL } from 'url';
|
|
8
9
|
import { resolve } from 'path';
|
|
9
10
|
import { baseEvaluationModule } from '../base-evaluation-module.js';
|
|
10
11
|
import { getBuiltinEvaluator } from '../evaluators/index.js';
|
|
11
12
|
import { logger as baseLogger } from '../logger.js';
|
|
12
13
|
const logger = baseLogger.context('dynamic-loader');
|
|
14
|
+
/**
|
|
15
|
+
* Import a file, using jiti for .ts files
|
|
16
|
+
*/
|
|
17
|
+
async function importFile(filePath) {
|
|
18
|
+
if (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) {
|
|
19
|
+
const jiti = createJiti(import.meta.url, {
|
|
20
|
+
interopDefault: true,
|
|
21
|
+
moduleCache: false,
|
|
22
|
+
});
|
|
23
|
+
return jiti.import(filePath);
|
|
24
|
+
}
|
|
25
|
+
const fileUrl = pathToFileURL(filePath).href;
|
|
26
|
+
return import(fileUrl);
|
|
27
|
+
}
|
|
13
28
|
/**
|
|
14
29
|
* Load evaluators from references
|
|
15
30
|
*
|
|
@@ -22,10 +37,9 @@ export async function loadEvaluators(refs, basePath) {
|
|
|
22
37
|
for (const ref of refs) {
|
|
23
38
|
let evaluator;
|
|
24
39
|
if ('path' in ref) {
|
|
25
|
-
// External file
|
|
40
|
+
// External file (supports .ts via jiti)
|
|
26
41
|
const filePath = resolve(basePath, ref.path);
|
|
27
|
-
const
|
|
28
|
-
const imported = await import(fileUrl);
|
|
42
|
+
const imported = await importFile(filePath);
|
|
29
43
|
evaluator = imported.default;
|
|
30
44
|
if (!evaluator) {
|
|
31
45
|
logger.warn(`No default export in ${ref.path}`);
|
|
@@ -95,12 +109,21 @@ export async function loadEvaluators(refs, basePath) {
|
|
|
95
109
|
export async function loadModules(refs, basePath) {
|
|
96
110
|
const modules = [];
|
|
97
111
|
for (const ref of refs) {
|
|
112
|
+
if ('module' in ref) {
|
|
113
|
+
// Inline module definition
|
|
114
|
+
modules.push({
|
|
115
|
+
name: ref.name,
|
|
116
|
+
description: ref.description || '',
|
|
117
|
+
module: ref.module,
|
|
118
|
+
});
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
98
121
|
const filePath = resolve(basePath, ref.path);
|
|
99
|
-
const
|
|
100
|
-
const
|
|
101
|
-
const module = imported
|
|
122
|
+
const imported = await importFile(filePath);
|
|
123
|
+
const exportName = ref.export || 'default';
|
|
124
|
+
const module = imported[exportName];
|
|
102
125
|
if (!module) {
|
|
103
|
-
logger.warn(`No
|
|
126
|
+
logger.warn(`No export '${exportName}' in ${ref.path}`);
|
|
104
127
|
continue;
|
|
105
128
|
}
|
|
106
129
|
modules.push({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dynamic-loader.js","sourceRoot":"","sources":["../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAO/B,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"dynamic-loader.js","sourceRoot":"","sources":["../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAO/B,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAEpD;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACvC,cAAc,EAAE,IAAI;YACpB,WAAW,EAAE,KAAK;SACnB,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;IAC7C,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC;AACzB,CAAC;AAaD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAA0B,EAC1B,QAAgB;IAEhB,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,SAAsD,CAAC;QAE3D,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YAClB,wCAAwC;YACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC5C,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC;YAE7B,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;YAC3B,oDAAoD;YACpD,MAAM,YAAY,GAAG,KAAK,CAAC,oBAAoB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC7D,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;gBAClC,IAAI,EAAE,QAAQ;gBACd,eAAe,EAAE;oBACf,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;oBAClC,MAAM,EAAE,YAAY;iBACrB;aACF,CAAC,CAAC;YACH,SAAS;QACX,CAAC;aAAM,CAAC;YACN,gCAAgC;YAChC,SAAS,GAAG,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAE1C,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,gCAAgC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBACxD,SAAS;YACX,CAAC;QACH,CAAC;QAED,qCAAqC;QACrC,IAAI,UAAU,IAAI,SAAS,IAAI,OAAO,SAAS,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;YACxE,iBAAiB;YACjB,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,IAAI,EAAE;gBAC3D,IAAI,EAAE,MAAM;gBACZ,aAAa,EAAE,SAA0B;aAC1C,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,4CAA4C;YAC5C,MAAM,YAAY,GAAG,KAAK,CAAC,oBAAoB,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACnE,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,IAAI,EAAE;gBAC3D,IAAI,EAAE,QAAQ;gBACd,eAAe,EAAE;oBACf,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,MAAM,EAAE,YAAY;iBACrB;aACF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAWD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAuB,EACvB,QAAgB;IAEhB,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;YACpB,2BAA2B;YAC3B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;gBAClC,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC;QAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QAEpC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,cAAc,UAAU,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YACxD,SAAS;QACX,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;YAClC,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAC3E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,KAAK,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAKhE,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAGD,MAAM,MAAM,gBAAgB,GAAG,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAC3E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,KAAK,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAKhE,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAGD,MAAM,MAAM,gBAAgB,GAAG,YAAY,CAAC;AAuB5C;;;;;GAKG;AACH,wBAAsB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CA+GpF"}
|
package/dist/config/loader.js
CHANGED
|
@@ -95,9 +95,19 @@ export async function loadExperimentConfig(configPath) {
|
|
|
95
95
|
// Validate testCase model references
|
|
96
96
|
for (const testCase of testCases) {
|
|
97
97
|
if (testCase.models) {
|
|
98
|
-
for (const
|
|
99
|
-
if (
|
|
100
|
-
|
|
98
|
+
for (const entry of testCase.models) {
|
|
99
|
+
if (typeof entry === 'string') {
|
|
100
|
+
if (!modelNames.has(entry)) {
|
|
101
|
+
throw new Error(`❌ TestCase '${testCase.name}' references unknown model '${entry}'`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
// Inline DriverSet: check all referenced models exist
|
|
106
|
+
for (const [role, refName] of Object.entries(entry)) {
|
|
107
|
+
if (!modelNames.has(refName)) {
|
|
108
|
+
throw new Error(`❌ TestCase '${testCase.name}' references unknown model '${refName}' for role '${role}'`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
101
111
|
}
|
|
102
112
|
}
|
|
103
113
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAClC,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAG3E,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;AAcnD;;;;;;GAMG;AACH,SAAS,iBAAiB,CAAC,SAAiB,EAAE,IAAY;IACxD,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;IACpD,CAAC;IACD,oCAAoC;IACpC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,kDAAkD;IAClD,OAAO,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAClC,CAAC;
|
|
1
|
+
{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAClC,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAG3E,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;AAcnD;;;;;;GAMG;AACH,SAAS,iBAAiB,CAAC,SAAiB,EAAE,IAAY;IACxD,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;IACpD,CAAC;IACD,oCAAoC;IACpC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,kDAAkD;IAClD,OAAO,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAClC,CAAC;AAGD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,UAAkB;IAC3D,0CAA0C;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAEhC,iCAAiC;IACjC,IAAI,MAAW,CAAC;IAEhB,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACtC,cAAc;QACd,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;SAAM,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QAC9E,0DAA0D;QAC1D,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACvC,cAAc,EAAE,IAAI,EAAG,mCAAmC;YAC1D,KAAK,EAAE,IAAI,EAAY,wCAAwC;YAC/D,YAAY,EAAE,KAAK,EAAI,0BAA0B;SAClD,CAAC,CAAC;QAEH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAEvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,0BAA0B,UAAU,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,4CAA4C,CAAC,CAAC;IACxG,CAAC;IAED,qBAAqB;IACrB,MAAM,OAAO,GAAsB,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACxD,MAAM,SAAS,GAAe,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;IACrD,MAAM,UAAU,GAAyB,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;IAEjE,oDAAoD;IACpD,MAAM,YAAY,GAAG;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC;IAEF,iEAAiE;IACjE,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;QACzB,KAAK,MAAM,UAAU,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAEtD,8CAA8C;YAC9C,IAAI,YAAY,CAAC,eAAe,EAAE,CAAC;gBACjC,YAAY,CAAC,eAAe,GAAG,iBAAiB,CAAC,SAAS,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,YAAY,CAAC,WAAW,EAAE,4BAA4B,EAAE,CAAC;QAC3D,MAAM,YAAY,GAAG,iBAAiB,CAAC,SAAS,EAAE,YAAY,CAAC,WAAW,CAAC,4BAA4B,CAAC,CAAC;QACzG,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,YAAY,CAAC;QAC1D,MAAM,CAAC,OAAO,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,aAAa;IACb,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1E,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,mCAAmC;IACnC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAErE,qCAAqC;IACrC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC3B,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,IAAI,+BAA+B,KAAK,GAAG,CAAC,CAAC;oBACvF,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,sDAAsD;oBACtD,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;wBACpD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC7B,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,IAAI,+BAA+B,OAAO,eAAe,IAAI,GAAG,CAAC,CAAC;wBAC5G,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,oDAAoD;IACpD,iFAAiF;IACjF,MAAM,eAAe,GAAsB;QACzC,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,OAAO,EAAE,YAAY,CAAC,OAAO,IAAI,EAAE;QACnC,cAAc,EAAE;YACd,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;SAChB;KACF,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,eAAe,CAAC,CAAC;IAEjD,OAAO;QACL,YAAY;QACZ,OAAO;QACP,SAAS;QACT,UAAU;QACV,SAAS;QACT,SAAS;KACV,CAAC;AACJ,CAAC"}
|
package/dist/run-comparison.js
CHANGED
|
@@ -33,10 +33,47 @@ Logger.configure({
|
|
|
33
33
|
level: options.verbose ? 'debug' : 'info',
|
|
34
34
|
accumulateLevel: 'debug',
|
|
35
35
|
isMcpMode: false,
|
|
36
|
-
accumulate: !!options.logFile,
|
|
36
|
+
accumulate: !!options.logFile || !!options.traceDir,
|
|
37
37
|
maxEntries: 10000,
|
|
38
38
|
logFile: options.logFile,
|
|
39
39
|
});
|
|
40
|
+
// 異常終了時のログ書き出し
|
|
41
|
+
async function flushLogsOnExit(signal) {
|
|
42
|
+
console.error(`\n⚠️ ${signal} received, flushing logs...`);
|
|
43
|
+
try {
|
|
44
|
+
if (options.logFile) {
|
|
45
|
+
const { logger } = await import('./logger.js');
|
|
46
|
+
await logger.flush();
|
|
47
|
+
console.error(`📄 Log file written: ${options.logFile}`);
|
|
48
|
+
}
|
|
49
|
+
if (options.traceDir) {
|
|
50
|
+
const { writeTrace } = await import('./trace/writer.js');
|
|
51
|
+
await writeTrace(options.traceDir);
|
|
52
|
+
console.error(`📂 Trace written: ${options.traceDir}`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
catch (e) {
|
|
56
|
+
console.error('Failed to flush logs:', e);
|
|
57
|
+
}
|
|
58
|
+
process.exit(1);
|
|
59
|
+
}
|
|
60
|
+
let logsFlushed = false;
|
|
61
|
+
process.on('SIGINT', () => flushLogsOnExit('SIGINT'));
|
|
62
|
+
process.on('SIGTERM', () => flushLogsOnExit('SIGTERM'));
|
|
63
|
+
process.on('uncaughtException', (err) => {
|
|
64
|
+
console.error('❌ Uncaught exception:', err.message);
|
|
65
|
+
flushLogsOnExit('uncaughtException');
|
|
66
|
+
});
|
|
67
|
+
process.on('unhandledRejection', (reason) => {
|
|
68
|
+
console.error('❌ Unhandled rejection:', reason);
|
|
69
|
+
flushLogsOnExit('unhandledRejection');
|
|
70
|
+
});
|
|
71
|
+
// Promise が settle せずにイベントループが空になった場合
|
|
72
|
+
process.on('beforeExit', (code) => {
|
|
73
|
+
if (!logsFlushed) {
|
|
74
|
+
flushLogsOnExit(`beforeExit (code=${code})`);
|
|
75
|
+
}
|
|
76
|
+
});
|
|
40
77
|
// Display header
|
|
41
78
|
console.log('='.repeat(80));
|
|
42
79
|
console.log('Module Comparison Experiment');
|
|
@@ -51,6 +88,9 @@ if (options.enableEvaluation) {
|
|
|
51
88
|
console.log(`Evaluators: ${options.evaluatorFilter?.join(', ') || 'all'}`);
|
|
52
89
|
}
|
|
53
90
|
console.log(`Dry run: ${options.dryRun ? 'enabled (plan only)' : 'disabled'}`);
|
|
91
|
+
if (options.traceDir) {
|
|
92
|
+
console.log(`Trace dir: ${options.traceDir}`);
|
|
93
|
+
}
|
|
54
94
|
console.log('='.repeat(80));
|
|
55
95
|
console.log();
|
|
56
96
|
// Load configuration
|
|
@@ -154,22 +194,39 @@ if (options.dryRun) {
|
|
|
154
194
|
// Run experiment
|
|
155
195
|
const driverManager = new DriverManager();
|
|
156
196
|
const runner = new ExperimentRunner(aiService, driverManager, modules, testCases, models, options.repeatCount, evaluators, evaluatorModel);
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
console.log('
|
|
161
|
-
console.log('
|
|
162
|
-
|
|
163
|
-
if
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
197
|
+
try {
|
|
198
|
+
const results = await runner.run();
|
|
199
|
+
// Display completion
|
|
200
|
+
console.log('='.repeat(80));
|
|
201
|
+
console.log('✨ Experiment completed');
|
|
202
|
+
console.log('='.repeat(80));
|
|
203
|
+
// Display statistics if repeated
|
|
204
|
+
if (options.repeatCount > 1) {
|
|
205
|
+
const reporter = new StatisticsReporter(results);
|
|
206
|
+
reporter.report();
|
|
207
|
+
}
|
|
167
208
|
}
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
209
|
+
catch (error) {
|
|
210
|
+
console.error('❌ Experiment failed:', error instanceof Error ? error.message : String(error));
|
|
211
|
+
if (error instanceof Error && error.stack) {
|
|
212
|
+
console.error(error.stack);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
finally {
|
|
216
|
+
// Flush log file if configured
|
|
217
|
+
if (options.logFile) {
|
|
218
|
+
const { logger } = await import('./logger.js');
|
|
219
|
+
await logger.flush();
|
|
220
|
+
console.log(`📄 Log file written: ${options.logFile}`);
|
|
221
|
+
}
|
|
222
|
+
// Write trace if configured
|
|
223
|
+
if (options.traceDir) {
|
|
224
|
+
const { writeTrace } = await import('./trace/writer.js');
|
|
225
|
+
await writeTrace(options.traceDir);
|
|
226
|
+
console.log(`📂 Trace written: ${options.traceDir}`);
|
|
227
|
+
}
|
|
228
|
+
// Cleanup drivers
|
|
229
|
+
await driverManager.cleanup();
|
|
230
|
+
logsFlushed = true;
|
|
174
231
|
}
|
|
175
232
|
//# sourceMappingURL=run-comparison.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-comparison.js","sourceRoot":"","sources":["../src/run-comparison.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,sBAAsB;AACtB,MAAM,OAAO,GAAG,SAAS,EAAE,CAAC;AAE5B,mBAAmB;AACnB,MAAM,CAAC,SAAS,CAAC;IACf,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;IACzC,eAAe,EAAE,OAAO;IACxB,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;
|
|
1
|
+
{"version":3,"file":"run-comparison.js","sourceRoot":"","sources":["../src/run-comparison.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,sBAAsB;AACtB,MAAM,OAAO,GAAG,SAAS,EAAE,CAAC;AAE5B,mBAAmB;AACnB,MAAM,CAAC,SAAS,CAAC;IACf,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;IACzC,eAAe,EAAE,OAAO;IACxB,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ;IACnD,UAAU,EAAE,KAAK;IACjB,OAAO,EAAE,OAAO,CAAC,OAAO;CACzB,CAAC,CAAC;AAEH,eAAe;AACf,KAAK,UAAU,eAAe,CAAC,MAAc;IAC3C,OAAO,CAAC,KAAK,CAAC,SAAS,MAAM,6BAA6B,CAAC,CAAC;IAC5D,IAAI,CAAC;QACH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YAC/C,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,OAAO,CAAC,KAAK,CAAC,wBAAwB,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACzD,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnC,OAAO,CAAC,KAAK,CAAC,qBAAqB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,IAAI,WAAW,GAAG,KAAK,CAAC;AAExB,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;AACtD,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC;AACxD,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,GAAG,EAAE,EAAE;IACtC,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IACpD,eAAe,CAAC,mBAAmB,CAAC,CAAC;AACvC,CAAC,CAAC,CAAC;AACH,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,MAAM,EAAE,EAAE;IAC1C,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,MAAM,CAAC,CAAC;IAChD,eAAe,CAAC,oBAAoB,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AACH,sCAAsC;AACtC,OAAO,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;IAChC,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,eAAe,CAAC,oBAAoB,IAAI,GAAG,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,iBAAiB;AACjB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;AAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;AAC7C,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,CAAC,cAAc,IAAI,KAAK,EAAE,CAAC,CAAC;AACpE,OAAO,CAAC,GAAG,CAAC,iBAAiB,OAAO,CAAC,WAAW,IAAI,oBAAoB,EAAE,CAAC,CAAC;AAC5E,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;AACrE,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,WAAW,UAAU,CAAC,CAAC;AACtD,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;AAChF,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;AAC7E,CAAC;AACD,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;AAC/E,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IACrB,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;AAChD,CAAC;AACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,qBAAqB;AACrB,MAAM,EACJ,YAAY,EACZ,OAAO,EAAE,aAAa,EACtB,SAAS,EAAE,eAAe,EAC1B,UAAU,EAAE,gBAAgB,EAC5B,SAAS,EACT,SAAS,EACV,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAEnD,8CAA8C;AAC9C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAEnC,uCAAuC;AACvC,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAgB,EAAE,EAAE,CAC9E,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CACvD,CAAC;AAEF,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IACxB,MAAM,eAAe,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAgB,EAAE,EAAE,CACvE,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,WAAW,CACtC,CAAC;IACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,gDAAgD,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,mBAAmB,eAAe,CAAC,MAAM,0BAA0B,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IACxG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAgB,EAAE,EAAE,CACtD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAC7D,CAAC;AACJ,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,GAAG,CAAC,mBAAmB,YAAY,CAAC,MAAM,YAAY,CAAC,CAAC;IAChE,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAgB,EAAE,EAAE,CACnD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAC7D,CAAC;AACJ,CAAC;AAED,gCAAgC;AAChC,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAgB,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC;AACxF,IAAI,MAAM,EAAE,CAAC;IACX,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,4GAA4G,CAAC,CAAC;AAC5H,CAAC;AACD,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,kBAAkB;AAClB,MAAM,YAAY,GAAG,eAAe,CAAC;AACrC,MAAM,SAAS,GAAG,OAAO,CAAC,cAAc;IACtC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,OAAO,CAAC,cAAc,CAAC;IACtE,CAAC,CAAC,YAAY,CAAC;AAEjB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IAC3B,OAAO,CAAC,KAAK,CAAC,wBAAwB,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9G,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,eAAe,CAAC,CAAC;AAC3D,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,wCAAwC;AACxC,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;AAC/D,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY;IAClC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,YAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAChE,CAAC,CAAC,UAAU,CAAC;AAEf,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IACzB,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACtC,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,MAAM,aAAa,CAAC,CAAC;AACvD,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACrE,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,8DAA8D;AAC9D,IAAI,UAAU,CAAC;AACf,IAAI,cAAc,CAAC;AACnB,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAC7B,8CAA8C;IAC9C,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC;IACxE,UAAU,GAAG,OAAO,CAAC,eAAe;QAClC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,eAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtE,CAAC,CAAC,aAAa,CAAC;IAElB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,8CAA8C;IAC9C,IAAI,CAAC,YAAY,CAAC,UAAU,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACjE,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAC/D,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;QACvE,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC1C,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,gBAAgB,GAAG,YAAY,CAAC,UAAU,CAAC;IAEjD,mCAAmC;IACnC,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC;IACzC,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAEjD,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QACrC,OAAO,CAAC,KAAK,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,KAAK,CAAC,yEAAyE,CAAC,CAAC;QACzF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,cAAc,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAEtD,OAAO,CAAC,GAAG,CAAC,8BAA8B,UAAU,CAAC,MAAM,gBAAgB,CAAC,CAAC;IAC7E,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACpF,OAAO,CAAC,GAAG,CAAC,uBAAuB,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC;AAED,kBAAkB;AAClB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,iBAAiB;AACjB,MAAM,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;AAC1C,MAAM,MAAM,GAAG,IAAI,gBAAgB,CACjC,SAAS,EACT,aAAa,EACb,OAAO,EACP,SAAS,EACT,MAAM,EACN,OAAO,CAAC,WAAW,EACnB,UAAU,EACV,cAAc,CACf,CAAC;AAEF,IAAI,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;IAEnC,qBAAqB;IACrB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,iCAAiC;IACjC,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACjD,QAAQ,CAAC,MAAM,EAAE,CAAC;IACpB,CAAC;AACH,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9F,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC1C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;QAAS,CAAC;IACT,+BAA+B;IAC/B,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAC/C,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,4BAA4B;IAC5B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;QACzD,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,kBAAkB;IAClB,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;IAE9B,WAAW,GAAG,IAAI,CAAC;AACrB,CAAC"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Driver manager for caching and cleanup
|
|
3
3
|
*/
|
|
4
|
-
import type { AIService, ModelSpec } from '@modular-prompt/driver';
|
|
4
|
+
import type { AIService, ModelSpec, AIDriver } from '@modular-prompt/driver';
|
|
5
|
+
import type { DriverSet } from '@modular-prompt/process';
|
|
5
6
|
export declare class DriverManager {
|
|
6
7
|
private cache;
|
|
7
8
|
/**
|
|
@@ -15,7 +16,16 @@ export declare class DriverManager {
|
|
|
15
16
|
* @param modelSpec - Model spec
|
|
16
17
|
* @returns Driver instance
|
|
17
18
|
*/
|
|
18
|
-
getOrCreate(aiService: AIService, modelName: string, modelSpec: ModelSpec): Promise<
|
|
19
|
+
getOrCreate(aiService: AIService, modelName: string, modelSpec: ModelSpec): Promise<AIDriver>;
|
|
20
|
+
/**
|
|
21
|
+
* Get or create DriverSet
|
|
22
|
+
*
|
|
23
|
+
* @param aiService - AIService instance
|
|
24
|
+
* @param roleMapping - Role to model name mapping
|
|
25
|
+
* @param allModels - All model configurations
|
|
26
|
+
* @returns DriverSet instance
|
|
27
|
+
*/
|
|
28
|
+
getOrCreateDriverSet(aiService: AIService, roleMapping: Record<string, string>, allModels: Record<string, ModelSpec>): Promise<DriverSet>;
|
|
19
29
|
/**
|
|
20
30
|
* Close and remove a specific driver from cache
|
|
21
31
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver-manager.d.ts","sourceRoot":"","sources":["../../src/runner/driver-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"driver-manager.d.ts","sourceRoot":"","sources":["../../src/runner/driver-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC7E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAKzD,qBAAa,aAAa;IACxB,OAAO,CAAC,KAAK,CAA+B;IAE5C;;;;;;;;;;OAUG;IACG,WAAW,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;IAYnG;;;;;;;OAOG;IACG,oBAAoB,CACxB,SAAS,EAAE,SAAS,EACpB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACnC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GACnC,OAAO,CAAC,SAAS,CAAC;IASrB;;;;OAIG;IACG,KAAK,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB7C;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAiB/B"}
|
|
@@ -26,6 +26,22 @@ export class DriverManager {
|
|
|
26
26
|
this.cache.set(modelName, driver);
|
|
27
27
|
return driver;
|
|
28
28
|
}
|
|
29
|
+
/**
|
|
30
|
+
* Get or create DriverSet
|
|
31
|
+
*
|
|
32
|
+
* @param aiService - AIService instance
|
|
33
|
+
* @param roleMapping - Role to model name mapping
|
|
34
|
+
* @param allModels - All model configurations
|
|
35
|
+
* @returns DriverSet instance
|
|
36
|
+
*/
|
|
37
|
+
async getOrCreateDriverSet(aiService, roleMapping, allModels) {
|
|
38
|
+
const result = {};
|
|
39
|
+
for (const [role, modelName] of Object.entries(roleMapping)) {
|
|
40
|
+
const spec = allModels[modelName];
|
|
41
|
+
result[role] = await this.getOrCreate(aiService, modelName, spec);
|
|
42
|
+
}
|
|
43
|
+
return result;
|
|
44
|
+
}
|
|
29
45
|
/**
|
|
30
46
|
* Close and remove a specific driver from cache
|
|
31
47
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver-manager.js","sourceRoot":"","sources":["../../src/runner/driver-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"driver-manager.js","sourceRoot":"","sources":["../../src/runner/driver-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAEpD,MAAM,OAAO,aAAa;IAChB,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE5C;;;;;;;;;;OAUG;IACH,KAAK,CAAC,WAAW,CAAC,SAAoB,EAAE,SAAiB,EAAE,SAAoB;QAC7E,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,CAAC,OAAO,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;QACpC,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,2BAA2B,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;QAC/F,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,oBAAoB,CACxB,SAAoB,EACpB,WAAmC,EACnC,SAAoC;QAEpC,MAAM,MAAM,GAA6B,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,MAA8B,CAAC;IACxC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK,CAAC,SAAiB;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;gBACvC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM,CAAC,OAAO,CAAC,kBAAkB,SAAS,EAAE,CAAC,CAAC;YAChD,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,0BAA0B,SAAS,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAEjC,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;oBACjD,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;oBACrB,MAAM,CAAC,OAAO,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,0BAA0B,GAAG,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC1G,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IACrC,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAe,SAAS,
|
|
1
|
+
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAe,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAMhF,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAA4E,MAAM,aAAa,CAAC;AACpJ,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAiCnE,qBAAa,gBAAgB;IAEzB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,cAAc,CAAC;gBAPf,SAAS,EAAE,SAAS,EACpB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,gBAAgB,EAAE,EAC3B,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EACjC,WAAW,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,eAAe,EAAE,YAAA,EAC9B,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,SAAS,CAAA;KAAE,YAAA;IAG5D;;;;OAIG;IACG,GAAG,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;IAmBlC;;OAEG;IACH,OAAO,CAAC,aAAa;IA0DrB;;OAEG;YACW,WAAW;IAuGzB;;OAEG;YACW,aAAa;IA2E3B;;OAEG;YACW,kBAAkB;CA2BjC"}
|
|
@@ -3,10 +3,28 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { compile } from '@modular-prompt/core';
|
|
5
5
|
import { formatCompletionPrompt } from '@modular-prompt/driver';
|
|
6
|
-
import { defaultProcess } from '@modular-prompt/process';
|
|
6
|
+
import { defaultProcess, streamProcess, concatProcess, dialogueProcess, summarizeProcess, agenticProcess } from '@modular-prompt/process';
|
|
7
7
|
import { EvaluatorRunner } from './evaluator.js';
|
|
8
8
|
import { logger as baseLogger } from '../logger.js';
|
|
9
9
|
const logger = baseLogger.context('runner');
|
|
10
|
+
const builtinProcesses = {
|
|
11
|
+
defaultProcess,
|
|
12
|
+
streamProcess,
|
|
13
|
+
concatProcess,
|
|
14
|
+
dialogueProcess,
|
|
15
|
+
summarizeProcess,
|
|
16
|
+
agenticProcess,
|
|
17
|
+
};
|
|
18
|
+
function resolveProcess(process) {
|
|
19
|
+
if (!process)
|
|
20
|
+
return defaultProcess;
|
|
21
|
+
if (typeof process === 'function')
|
|
22
|
+
return process;
|
|
23
|
+
const fn = builtinProcesses[process];
|
|
24
|
+
if (!fn)
|
|
25
|
+
throw new Error(`Unknown builtin process: '${process}'`);
|
|
26
|
+
return fn;
|
|
27
|
+
}
|
|
10
28
|
export class ExperimentRunner {
|
|
11
29
|
aiService;
|
|
12
30
|
driverManager;
|
|
@@ -55,34 +73,47 @@ export class ExperimentRunner {
|
|
|
55
73
|
for (const testCase of this.testCases) {
|
|
56
74
|
// テストケースで使うモデルを決定
|
|
57
75
|
const modelsToTest = testCase.models
|
|
58
|
-
? testCase.models.map(
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
76
|
+
? testCase.models.map(entry => {
|
|
77
|
+
if (typeof entry === 'string') {
|
|
78
|
+
const spec = this.models[entry];
|
|
79
|
+
if (!spec) {
|
|
80
|
+
console.warn(`⚠️ Model '${entry}' not found in configuration, skipping`);
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
return { name: entry, modelSpec: spec };
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
// Inline DriverSet
|
|
87
|
+
const name = Object.entries(entry)
|
|
88
|
+
.map(([role, model]) => `${role}=${model}`)
|
|
89
|
+
.join(',');
|
|
90
|
+
return { name: `set(${name})`, driverSetMapping: entry };
|
|
63
91
|
}
|
|
64
|
-
return { name, spec };
|
|
65
92
|
}).filter(Boolean)
|
|
66
93
|
: Object.entries(this.models)
|
|
67
94
|
.filter(([_, spec]) => !spec.disabled)
|
|
68
|
-
.map(([name, spec]) => ({ name, spec }));
|
|
69
|
-
for (const
|
|
70
|
-
|
|
71
|
-
|
|
95
|
+
.map(([name, spec]) => ({ name, modelSpec: spec }));
|
|
96
|
+
for (const model of modelsToTest) {
|
|
97
|
+
// テストケースで使うモジュールを決定
|
|
98
|
+
const modulesToTest = testCase.modules
|
|
99
|
+
? this.modules.filter(m => testCase.modules.includes(m.name))
|
|
100
|
+
: this.modules;
|
|
101
|
+
for (const module of modulesToTest) {
|
|
72
102
|
const compiled = compile(module.module, testCase.input);
|
|
73
103
|
const prompt = formatCompletionPrompt(compiled);
|
|
74
104
|
plan.push({
|
|
75
105
|
order: order++,
|
|
76
106
|
testCase,
|
|
77
|
-
modelName,
|
|
78
|
-
modelSpec,
|
|
107
|
+
modelName: model.name,
|
|
108
|
+
modelSpec: model.modelSpec,
|
|
109
|
+
driverSetMapping: model.driverSetMapping,
|
|
79
110
|
module,
|
|
80
111
|
prompt,
|
|
81
112
|
});
|
|
82
113
|
}
|
|
83
114
|
}
|
|
84
115
|
}
|
|
85
|
-
logger.info(`Test plan: ${plan.length} items
|
|
116
|
+
logger.info(`Test plan: ${plan.length} items`);
|
|
86
117
|
return plan;
|
|
87
118
|
}
|
|
88
119
|
/**
|
|
@@ -104,18 +135,32 @@ export class ExperimentRunner {
|
|
|
104
135
|
}
|
|
105
136
|
// モデルごとに実行
|
|
106
137
|
for (const [modelName, items] of modelGroups) {
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
138
|
+
const firstItem = items[0];
|
|
139
|
+
let driverInput;
|
|
140
|
+
if (firstItem.driverSetMapping) {
|
|
141
|
+
// Inline DriverSet
|
|
142
|
+
console.log('='.repeat(80));
|
|
143
|
+
console.log(`🤖 DriverSet: ${modelName}`);
|
|
144
|
+
for (const [role, refName] of Object.entries(firstItem.driverSetMapping)) {
|
|
145
|
+
const refSpec = this.models[refName];
|
|
146
|
+
console.log(` ${role}: ${refName} (${refSpec.provider}:${refSpec.model})`);
|
|
147
|
+
}
|
|
148
|
+
console.log('='.repeat(80));
|
|
149
|
+
driverInput = await this.driverManager.getOrCreateDriverSet(this.aiService, firstItem.driverSetMapping, this.models);
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
const modelSpec = firstItem.modelSpec;
|
|
153
|
+
console.log('='.repeat(80));
|
|
154
|
+
console.log(`🤖 Model: ${modelName} (${modelSpec.provider}:${modelSpec.model})`);
|
|
155
|
+
console.log('='.repeat(80));
|
|
156
|
+
driverInput = await this.driverManager.getOrCreate(this.aiService, modelName, modelSpec);
|
|
157
|
+
}
|
|
113
158
|
for (const item of items) {
|
|
114
159
|
console.log(` ── ${item.testCase.name} ──`);
|
|
115
160
|
if (item.testCase.description) {
|
|
116
161
|
console.log(` ${item.testCase.description}`);
|
|
117
162
|
}
|
|
118
|
-
const runs = await this.runModuleTest(item.module.name, item.module.module,
|
|
163
|
+
const runs = await this.runModuleTest(item.module.name, item.module.module, driverInput, item.testCase);
|
|
119
164
|
allResults.push({
|
|
120
165
|
order: item.order,
|
|
121
166
|
result: {
|
|
@@ -145,9 +190,11 @@ export class ExperimentRunner {
|
|
|
145
190
|
});
|
|
146
191
|
}
|
|
147
192
|
}
|
|
148
|
-
//
|
|
149
|
-
|
|
150
|
-
|
|
193
|
+
// 単一モデルの場合のみclose（セットの場合はcleanupで処理）
|
|
194
|
+
if (!firstItem.driverSetMapping) {
|
|
195
|
+
logger.info(`Closing driver: ${modelName}`);
|
|
196
|
+
await this.driverManager.close(modelName);
|
|
197
|
+
}
|
|
151
198
|
console.log();
|
|
152
199
|
}
|
|
153
200
|
// Retire: 元の定義順にソートして返す
|
|
@@ -168,13 +215,17 @@ export class ExperimentRunner {
|
|
|
168
215
|
logger.verbose(`Run ${i + 1}/${this.repeatCount} for module: ${moduleName}`);
|
|
169
216
|
const startTime = Date.now();
|
|
170
217
|
try {
|
|
171
|
-
const
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
218
|
+
const processFn = resolveProcess(testCase.process);
|
|
219
|
+
const processOptions = testCase.process
|
|
220
|
+
? testCase.processOptions
|
|
221
|
+
: {
|
|
222
|
+
queryOptions: {
|
|
223
|
+
temperature: 0.7,
|
|
224
|
+
maxTokens: 2048,
|
|
225
|
+
...testCase.queryOptions,
|
|
226
|
+
},
|
|
227
|
+
};
|
|
228
|
+
const workflowResult = await processFn(driver, module, testCase.input, processOptions);
|
|
178
229
|
const elapsed = Date.now() - startTime;
|
|
179
230
|
// Convert workflow result to QueryResult-like structure
|
|
180
231
|
const result = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAEhE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAIzD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAW5C,MAAM,OAAO,gBAAgB;IAEjB;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IARV,YACU,SAAoB,EACpB,aAA4B,EAC5B,OAA2B,EAC3B,SAAqB,EACrB,MAAiC,EACjC,WAAmB,EACnB,UAA8B,EAC9B,cAAkD;QAPlD,cAAS,GAAT,SAAS,CAAW;QACpB,kBAAa,GAAb,aAAa,CAAe;QAC5B,YAAO,GAAP,OAAO,CAAoB;QAC3B,cAAS,GAAT,SAAS,CAAY;QACrB,WAAM,GAAN,MAAM,CAA2B;QACjC,gBAAW,GAAX,WAAW,CAAQ;QACnB,eAAU,GAAV,UAAU,CAAoB;QAC9B,mBAAc,GAAd,cAAc,CAAoC;IACzD,CAAC;IAEJ;;;;OAIG;IACH,KAAK,CAAC,GAAG;QACP,oBAAoB;QACpB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,2BAA2B;QAC3B,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAErE,kBAAkB;QAClB,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzE,MAAM,IAAI,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,MAAM,IAAI,GAAmB,EAAE,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,kBAAkB;YAClB,MAAM,YAAY,GAA6C,QAAQ,CAAC,MAAM;gBAC5E,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;oBACzB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;wBACV,OAAO,CAAC,IAAI,CAAC,cAAc,IAAI,wCAAwC,CAAC,CAAC;wBACzE,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACxB,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAA6C;gBAChE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;qBACxB,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;qBACrC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAE/C,KAAK,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,YAAY,EAAE,CAAC;gBAChE,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBAClC,0CAA0C;oBAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;oBACxD,MAAM,MAAM,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;oBAEhD,IAAI,CAAC,IAAI,CAAC;wBACR,KAAK,EAAE,KAAK,EAAE;wBACd,QAAQ;wBACR,SAAS;wBACT,SAAS;wBACT,MAAM;wBACN,MAAM;qBACP,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,MAAM,WAAW,IAAI,CAAC,SAAS,CAAC,MAAM,0BAA0B,IAAI,CAAC,OAAO,CAAC,MAAM,WAAW,CAAC,CAAC;QAC/H,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAC,IAAoB;QAI5C,MAAM,UAAU,GAAiD,EAAE,CAAC;QACpE,MAAM,eAAe,GAAyD,EAAE,CAAC;QAEjF,sBAAsB;QACtB,MAAM,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;QACtD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,WAAW;QACX,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,aAAa,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;YACjF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,MAAM,CAAC,IAAI,CAAC,uBAAuB,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;YAC3F,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YAE1F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;gBAC7C,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAC9B,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;gBACnD,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAEnG,UAAU,CAAC,IAAI,CAAC;oBACd,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE;wBACN,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;wBAC5B,KAAK,EAAE,SAAS;wBAChB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;wBACxB,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;4BACnB,OAAO,EAAE,CAAC,CAAC,OAAO;4BAClB,OAAO,EAAE,CAAC,CAAC,OAAO;4BAClB,OAAO,EAAE,CAAC,CAAC,WAAW,EAAE,OAAO,IAAI,EAAE;4BACrC,SAAS,EAAE,CAAC,CAAC,WAAW,EAAE,SAAS;4BACnC,YAAY,EAAE,CAAC,CAAC,WAAW,EAAE,YAAY;4BACzC,KAAK,EAAE,CAAC,CAAC,KAAK;yBACf,CAAC,CAAC;qBACJ;iBACF,CAAC,CAAC;gBAEH,yBAAyB;gBACzB,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBACnD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,eAAe,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,OAAO,EAAE;4BACP,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;4BAC5B,MAAM,EAAE,IAAI,CAAC,MAAM;4BACnB,IAAI,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAY,EAAE,CAAC,CAAC;yBACjE;qBACF,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,yBAAyB;YACzB,MAAM,CAAC,IAAI,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;YAC5C,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC1C,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,wBAAwB;QACxB,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC7C,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAElD,OAAO;YACL,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACtC,kBAAkB,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;SACxD,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,UAAkB,EAClB,MAAyB,EACzB,MAAgB,EAChB,QAAkB;QAElB,MAAM,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,WAAW,wBAAwB,UAAU,EAAE,CAAC,CAAC;QAEhF,MAAM,IAAI,GAA4F,EAAE,CAAC;QAEzG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,gBAAgB,UAAU,EAAE,CAAC,CAAC;YAE7E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,EAAE;oBAC1E,YAAY,EAAE;wBACZ,WAAW,EAAE,GAAG;wBAChB,SAAS,EAAE,IAAI;wBACf,GAAG,QAAQ,CAAC,YAAY;qBACzB;iBACF,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBAEvC,wDAAwD;gBACxD,MAAM,MAAM,GAAgB;oBAC1B,OAAO,EAAE,cAAc,CAAC,MAAM;oBAC9B,SAAS,EAAE,cAAc,CAAC,QAAQ,EAAE,SAAgB;oBACpD,YAAY,EAAE,cAAc,CAAC,QAAQ,EAAE,YAAmB;oBAC1D,KAAK,EAAE,cAAc,CAAC,QAAQ,EAAE,KAAY;iBAC7C,CAAC;gBAEF,MAAM,CAAC,OAAO,CAAC,UAAU,UAAU,QAAQ,CAAC,GAAG,CAAC,cAAc,OAAO,KAAK,CAAC,CAAC;gBAE5E,4CAA4C;gBAC5C,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO;qBAClC,OAAO,CAAC,8BAA8B,EAAE,EAAE,CAAC;qBAC3C,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;gBACzC,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,GAAG,GAAG;oBAChD,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;oBAC1C,CAAC,CAAC,cAAc,CAAC;gBACnB,OAAO,CAAC,GAAG,CAAC,SAAS,UAAU,SAAS,CAAC,GAAG,CAAC,KAAK,OAAO,oBAAoB,MAAM,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC,CAAC;gBACjH,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpD,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;wBAClC,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;oBAChF,CAAC;gBACH,CAAC;gBACD,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;oBAC1B,OAAO,CAAC,GAAG,CAAC,YAAY,cAAc,EAAE,CAAC,CAAC;gBAC5C,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC;oBACR,OAAO,EAAE,IAAI;oBACb,OAAO;oBACP,WAAW,EAAE,MAAM;iBACpB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBACvC,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5E,MAAM,CAAC,KAAK,CAAC,UAAU,UAAU,QAAQ,CAAC,GAAG,CAAC,YAAY,OAAO,QAAQ,YAAY,EAAE,CAAC,CAAC;gBACzF,IAAI,CAAC,IAAI,CAAC;oBACR,OAAO,EAAE,KAAK;oBACd,OAAO;oBACP,KAAK,EAAE,YAAY;iBACpB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAC9B,kBAAuC;QAEvC,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,cAAe,CAAC,IAAI,CAAC,CAAC;QACvF,MAAM,cAAc,GAAuB,EAAE,CAAC;QAE9C,2CAA2C;QAC3C,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,EAAE,CAAC;YAEd,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAW,EAAE,CAAC;gBACzC,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAClE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,eAAe,CAAC,cAAc,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAClE,CAAC;CAEF"}
|
|
1
|
+
{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAEhE,OAAO,EACL,cAAc,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAC7D,gBAAgB,EAAE,cAAc,EAEjC,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAE5C,MAAM,gBAAgB,GAAgD;IACpE,cAAc;IACd,aAAa;IACb,aAAa;IACb,eAAe;IACf,gBAAgB;IAChB,cAAc;CACf,CAAC;AAEF,SAAS,cAAc,CAAC,OAAyD;IAC/E,IAAI,CAAC,OAAO;QAAE,OAAO,cAAc,CAAC;IACpC,IAAI,OAAO,OAAO,KAAK,UAAU;QAAE,OAAO,OAAO,CAAC;IAClD,MAAM,EAAE,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,OAAO,GAAG,CAAC,CAAC;IAClE,OAAO,EAAE,CAAC;AACZ,CAAC;AAYD,MAAM,OAAO,gBAAgB;IAEjB;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IARV,YACU,SAAoB,EACpB,aAA4B,EAC5B,OAA2B,EAC3B,SAAqB,EACrB,MAAiC,EACjC,WAAmB,EACnB,UAA8B,EAC9B,cAAkD;QAPlD,cAAS,GAAT,SAAS,CAAW;QACpB,kBAAa,GAAb,aAAa,CAAe;QAC5B,YAAO,GAAP,OAAO,CAAoB;QAC3B,cAAS,GAAT,SAAS,CAAY;QACrB,WAAM,GAAN,MAAM,CAA2B;QACjC,gBAAW,GAAX,WAAW,CAAQ;QACnB,eAAU,GAAV,UAAU,CAAoB;QAC9B,mBAAc,GAAd,cAAc,CAAoC;IACzD,CAAC;IAEJ;;;;OAIG;IACH,KAAK,CAAC,GAAG;QACP,oBAAoB;QACpB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,2BAA2B;QAC3B,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAErE,kBAAkB;QAClB,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzE,MAAM,IAAI,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,MAAM,IAAI,GAAmB,EAAE,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,kBAAkB;YAClB,MAAM,YAAY,GAIb,QAAQ,CAAC,MAAM;gBAClB,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;oBAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;wBAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;wBAChC,IAAI,CAAC,IAAI,EAAE,CAAC;4BACV,OAAO,CAAC,IAAI,CAAC,cAAc,KAAK,wCAAwC,CAAC,CAAC;4BAC1E,OAAO,IAAI,CAAC;wBACd,CAAC;wBACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;oBAC1C,CAAC;yBAAM,CAAC;wBACN,mBAAmB;wBACnB,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;6BAC/B,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC;6BAC1C,IAAI,CAAC,GAAG,CAAC,CAAC;wBACb,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,GAAG,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC;oBAC3D,CAAC;gBACH,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAU;gBAC7B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;qBACxB,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;qBACrC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAE1D,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,oBAAoB;gBACpB,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO;oBACpC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBAC9D,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;gBAEjB,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;oBACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;oBACxD,MAAM,MAAM,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;oBAEhD,IAAI,CAAC,IAAI,CAAC;wBACR,KAAK,EAAE,KAAK,EAAE;wBACd,QAAQ;wBACR,SAAS,EAAE,KAAK,CAAC,IAAI;wBACrB,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;wBACxC,MAAM;wBACN,MAAM;qBACP,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAC,IAAoB;QAI5C,MAAM,UAAU,GAAiD,EAAE,CAAC;QACpE,MAAM,eAAe,GAAyD,EAAE,CAAC;QAEjF,sBAAsB;QACtB,MAAM,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;QACtD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,WAAW;QACX,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAE3B,IAAI,WAAwB,CAAC;YAE7B,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;gBAC/B,mBAAmB;gBACnB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,iBAAiB,SAAS,EAAE,CAAC,CAAC;gBAC1C,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,gBAAgB,CAAC,EAAE,CAAC;oBACzE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBACrC,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,KAAK,OAAO,KAAK,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;gBAC/E,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;gBAE5B,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,oBAAoB,CACzD,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,gBAAgB,EAAE,IAAI,CAAC,MAAM,CACxD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,SAAS,CAAC,SAAU,CAAC;gBACvC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,aAAa,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;gBACjF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;gBAE5B,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YAC3F,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;gBAC7C,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAC9B,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;gBACnD,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAExG,UAAU,CAAC,IAAI,CAAC;oBACd,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE;wBACN,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;wBAC5B,KAAK,EAAE,SAAS;wBAChB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;wBACxB,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;4BACnB,OAAO,EAAE,CAAC,CAAC,OAAO;4BAClB,OAAO,EAAE,CAAC,CAAC,OAAO;4BAClB,OAAO,EAAE,CAAC,CAAC,WAAW,EAAE,OAAO,IAAI,EAAE;4BACrC,SAAS,EAAE,CAAC,CAAC,WAAW,EAAE,SAAS;4BACnC,YAAY,EAAE,CAAC,CAAC,WAAW,EAAE,YAAY;4BACzC,KAAK,EAAE,CAAC,CAAC,KAAK;yBACf,CAAC,CAAC;qBACJ;iBACF,CAAC,CAAC;gBAEH,yBAAyB;gBACzB,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBACnD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,eAAe,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,OAAO,EAAE;4BACP,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;4BAC5B,MAAM,EAAE,IAAI,CAAC,MAAM;4BACnB,IAAI,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAY,EAAE,CAAC,CAAC;yBACjE;qBACF,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,qCAAqC;YACrC,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;gBAC5C,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC5C,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,wBAAwB;QACxB,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC7C,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAElD,OAAO;YACL,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACtC,kBAAkB,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;SACxD,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,UAAkB,EAClB,MAAyB,EACzB,MAAmB,EACnB,QAAkB;QAElB,MAAM,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,WAAW,wBAAwB,UAAU,EAAE,CAAC,CAAC;QAEhF,MAAM,IAAI,GAA4F,EAAE,CAAC;QAEzG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,gBAAgB,UAAU,EAAE,CAAC,CAAC;YAE7E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACnD,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO;oBACrC,CAAC,CAAC,QAAQ,CAAC,cAAc;oBACzB,CAAC,CAAC;wBACE,YAAY,EAAE;4BACZ,WAAW,EAAE,GAAG;4BAChB,SAAS,EAAE,IAAI;4BACf,GAAG,QAAQ,CAAC,YAAY;yBACzB;qBACF,CAAC;gBACN,MAAM,cAAc,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;gBACvF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBAEvC,wDAAwD;gBACxD,MAAM,MAAM,GAAgB;oBAC1B,OAAO,EAAE,cAAc,CAAC,MAAM;oBAC9B,SAAS,EAAE,cAAc,CAAC,QAAQ,EAAE,SAAgB;oBACpD,YAAY,EAAE,cAAc,CAAC,QAAQ,EAAE,YAAmB;oBAC1D,KAAK,EAAE,cAAc,CAAC,QAAQ,EAAE,KAAY;iBAC7C,CAAC;gBAEF,MAAM,CAAC,OAAO,CAAC,UAAU,UAAU,QAAQ,CAAC,GAAG,CAAC,cAAc,OAAO,KAAK,CAAC,CAAC;gBAE5E,4CAA4C;gBAC5C,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO;qBAClC,OAAO,CAAC,8BAA8B,EAAE,EAAE,CAAC;qBAC3C,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;gBACzC,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,GAAG,GAAG;oBAChD,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;oBAC1C,CAAC,CAAC,cAAc,CAAC;gBACnB,OAAO,CAAC,GAAG,CAAC,SAAS,UAAU,SAAS,CAAC,GAAG,CAAC,KAAK,OAAO,oBAAoB,MAAM,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC,CAAC;gBACjH,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpD,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;wBAClC,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;oBAChF,CAAC;gBACH,CAAC;gBACD,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;oBAC1B,OAAO,CAAC,GAAG,CAAC,YAAY,cAAc,EAAE,CAAC,CAAC;gBAC5C,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC;oBACR,OAAO,EAAE,IAAI;oBACb,OAAO;oBACP,WAAW,EAAE,MAAM;iBACpB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBACvC,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5E,MAAM,CAAC,KAAK,CAAC,UAAU,UAAU,QAAQ,CAAC,GAAG,CAAC,YAAY,OAAO,QAAQ,YAAY,EAAE,CAAC,CAAC;gBACzF,IAAI,CAAC,IAAI,CAAC;oBACR,OAAO,EAAE,KAAK;oBACd,OAAO;oBACP,KAAK,EAAE,YAAY;iBACpB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAC9B,kBAAuC;QAEvC,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,cAAe,CAAC,IAAI,CAAC,CAAC;QACvF,MAAM,cAAc,GAAuB,EAAE,CAAC;QAE9C,2CAA2C;QAC3C,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,EAAE,CAAC;YAEd,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAW,EAAE,CAAC;gBACzC,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAClE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,eAAe,CAAC,cAAc,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAClE,CAAC;CAEF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/trace/writer.ts"],"names":[],"mappings":"AAIA,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmEhE"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { mkdir, writeFile } from 'fs/promises';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { Logger } from '@modular-prompt/utils';
|
|
4
|
+
export async function writeTrace(traceDir) {
|
|
5
|
+
// Logger からすべての蓄積エントリを取得
|
|
6
|
+
const logger = new Logger();
|
|
7
|
+
const entries = logger.getLogEntries({ filterByContext: false });
|
|
8
|
+
if (entries.length === 0) {
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
// ディレクトリ作成
|
|
12
|
+
await mkdir(traceDir, { recursive: true });
|
|
13
|
+
// ファイル名を生成するヘルパー関数
|
|
14
|
+
const generateFilename = (prefix, context) => {
|
|
15
|
+
const p = prefix || '';
|
|
16
|
+
const c = context || '';
|
|
17
|
+
if (p && c) {
|
|
18
|
+
// prefix あり + context あり: {prefix}_{context}.log
|
|
19
|
+
return `${p}_${c.replace(/:/g, '_')}.log`;
|
|
20
|
+
}
|
|
21
|
+
else if (p) {
|
|
22
|
+
// prefix あり + context なし: {prefix}.log
|
|
23
|
+
return `${p}.log`;
|
|
24
|
+
}
|
|
25
|
+
else if (c) {
|
|
26
|
+
// prefix なし + context あり: {context}.log
|
|
27
|
+
return `${c.replace(/:/g, '_')}.log`;
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
// prefix なし + context なし: unknown.log
|
|
31
|
+
return 'unknown.log';
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
// summary.json を書き出す
|
|
35
|
+
const fileNames = [...new Set(entries.map(e => generateFilename(e.prefix, e.context)))];
|
|
36
|
+
const summary = {
|
|
37
|
+
timestamp: new Date().toISOString(),
|
|
38
|
+
totalEntries: entries.length,
|
|
39
|
+
fileNames,
|
|
40
|
+
};
|
|
41
|
+
await writeFile(join(traceDir, 'summary.json'), JSON.stringify(summary, null, 2));
|
|
42
|
+
// prefix + context ごとにグループ化してファイルに書き出す
|
|
43
|
+
const grouped = new Map();
|
|
44
|
+
for (const entry of entries) {
|
|
45
|
+
const filename = generateFilename(entry.prefix, entry.context);
|
|
46
|
+
if (!grouped.has(filename)) {
|
|
47
|
+
grouped.set(filename, []);
|
|
48
|
+
}
|
|
49
|
+
grouped.get(filename).push(entry);
|
|
50
|
+
}
|
|
51
|
+
for (const [filename, contextEntries] of grouped) {
|
|
52
|
+
// ファイル名はすでに生成済み
|
|
53
|
+
// 各エントリを読みやすい形式で書き出す
|
|
54
|
+
const lines = contextEntries.map(entry => {
|
|
55
|
+
const time = entry.timestamp;
|
|
56
|
+
const level = entry.level.toUpperCase().padEnd(7);
|
|
57
|
+
const message = entry.message;
|
|
58
|
+
const args = entry.args && entry.args.length > 0
|
|
59
|
+
? '\n' + entry.args.map(a => typeof a === 'object' ? JSON.stringify(a, null, 2) : String(a)).join('\n')
|
|
60
|
+
: '';
|
|
61
|
+
return `${time} ${level} ${message}${args}`;
|
|
62
|
+
});
|
|
63
|
+
await writeFile(join(traceDir, filename), lines.join('\n\n') + '\n');
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=writer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../src/trace/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB;IAC/C,yBAAyB;IACzB,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,aAAa,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;IAEjE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;IACT,CAAC;IAED,WAAW;IACX,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,mBAAmB;IACnB,MAAM,gBAAgB,GAAG,CAAC,MAAe,EAAE,OAAgB,EAAU,EAAE;QACrE,MAAM,CAAC,GAAG,MAAM,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,OAAO,IAAI,EAAE,CAAC;QAExB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACX,iDAAiD;YACjD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC;QAC5C,CAAC;aAAM,IAAI,CAAC,EAAE,CAAC;YACb,uCAAuC;YACvC,OAAO,GAAG,CAAC,MAAM,CAAC;QACpB,CAAC;aAAM,IAAI,CAAC,EAAE,CAAC;YACb,wCAAwC;YACxC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,sCAAsC;YACtC,OAAO,aAAa,CAAC;QACvB,CAAC;IACH,CAAC,CAAC;IAEF,qBAAqB;IACrB,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG;QACd,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,SAAS;KACV,CAAC;IACF,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAElF,uCAAuC;IACvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B,CAAC;IAClD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,IAAI,OAAO,EAAE,CAAC;QACjD,gBAAgB;QAEhB,qBAAqB;QACrB,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACvC,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC;YAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAClD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;YAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBAC9C,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBACvG,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,GAAG,IAAI,IAAI,KAAK,IAAI,OAAO,GAAG,IAAI,EAAE,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -3,6 +3,23 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import type { PromptModule } from '@modular-prompt/core';
|
|
5
5
|
import type { QueryResult, QueryOptions, ToolCall } from '@modular-prompt/driver';
|
|
6
|
+
import type { DriverInput, WorkflowResult } from '@modular-prompt/process';
|
|
7
|
+
/** テストケースのモデル指定: 文字列はモデル名、オブジェクトはロール→モデル名のマッピング */
|
|
8
|
+
export type TestModelEntry = string | {
|
|
9
|
+
default: string;
|
|
10
|
+
[role: string]: string;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* カスタムプロセス関数の型
|
|
14
|
+
*
|
|
15
|
+
* defaultProcess以外のワークフロー（agenticProcess等）を使用する場合に指定する。
|
|
16
|
+
* TestCase.processOptionsがoptionsとして渡される。
|
|
17
|
+
*/
|
|
18
|
+
export type ProcessFunction = (driver: DriverInput, module: PromptModule<any>, context: any, options?: any) => Promise<WorkflowResult<any>>;
|
|
19
|
+
/**
|
|
20
|
+
* 組み込みプロセス関数名
|
|
21
|
+
*/
|
|
22
|
+
export type BuiltinProcessName = 'defaultProcess' | 'streamProcess' | 'concatProcess' | 'dialogueProcess' | 'summarizeProcess' | 'agenticProcess';
|
|
6
23
|
/**
|
|
7
24
|
* Test case definition
|
|
8
25
|
*/
|
|
@@ -14,9 +31,15 @@ export interface TestCase {
|
|
|
14
31
|
/** Input context (passed to module.compile) */
|
|
15
32
|
input: any;
|
|
16
33
|
/** Model names to use for this test case (optional, uses all enabled models if not specified) */
|
|
17
|
-
models?:
|
|
18
|
-
/**
|
|
34
|
+
models?: TestModelEntry[];
|
|
35
|
+
/** Module names to use for this test case (optional, uses all modules if not specified) */
|
|
36
|
+
modules?: string[];
|
|
37
|
+
/** Query options for this test case (tools, temperature, etc.) — defaultProcess用 */
|
|
19
38
|
queryOptions?: Partial<QueryOptions>;
|
|
39
|
+
/** カスタムプロセス関数または組み込みプロセス名（未指定時はdefaultProcess） */
|
|
40
|
+
process?: ProcessFunction | BuiltinProcessName;
|
|
41
|
+
/** カスタムプロセス関数に渡すオプション（process指定時のみ有効） */
|
|
42
|
+
processOptions?: Record<string, unknown>;
|
|
20
43
|
}
|
|
21
44
|
/**
|
|
22
45
|
* Result of a single run
|
|
@@ -138,5 +161,6 @@ export interface ExtendedExperimentOptions extends ExperimentOptions {
|
|
|
138
161
|
dryRun?: boolean;
|
|
139
162
|
logFile?: string;
|
|
140
163
|
verbose?: boolean;
|
|
164
|
+
traceDir?: string;
|
|
141
165
|
}
|
|
142
166
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAElF;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,KAAK,EAAE,GAAG,CAAC;IACX,iGAAiG;IACjG,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAE3E,mDAAmD;AACnD,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,CAAC;AAElF;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,CAC5B,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,EACzB,OAAO,EAAE,GAAG,EACZ,OAAO,CAAC,EAAE,GAAG,KACV,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;AAElC;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B,gBAAgB,GAChB,eAAe,GACf,eAAe,GACf,iBAAiB,GACjB,kBAAkB,GAClB,gBAAgB,CAAC;AAErB;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,KAAK,EAAE,GAAG,CAAC;IACX,iGAAiG;IACjG,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;IAC1B,2FAA2F;IAC3F,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,oFAAoF;IACpF,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IACrC,kDAAkD;IAClD,OAAO,CAAC,EAAE,eAAe,GAAG,kBAAkB,CAAC;IAC/C,yCAAyC;IACzC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC1C;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,KAAK,CAAC;QACV,WAAW,EAAE,WAAW,CAAC;KAC1B,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;;OASG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACrE;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;;OASG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/E;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3C;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,iBAAiB;IAClE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB"}
|
|
@@ -13,7 +13,7 @@ models:
|
|
|
13
13
|
provider: "mlx"
|
|
14
14
|
capabilities: ["local", "tools"]
|
|
15
15
|
priority: 10
|
|
16
|
-
|
|
16
|
+
disabled: true
|
|
17
17
|
gemma3-12b:
|
|
18
18
|
model: "mlx-community/gemma-3-12b-it-qat-4bit"
|
|
19
19
|
provider: "mlx"
|
|
@@ -25,19 +25,19 @@ models:
|
|
|
25
25
|
provider: "mlx"
|
|
26
26
|
capabilities: ["local", "fast", "japanese"]
|
|
27
27
|
priority: 20
|
|
28
|
-
disabled: true
|
|
28
|
+
# disabled: true
|
|
29
29
|
lfm2.5-instruct:
|
|
30
30
|
model: LiquidAI/LFM2.5-1.2B-Instruct-MLX-8bit
|
|
31
31
|
provider: "mlx"
|
|
32
32
|
capabilities: ["local", "fast", "japanese"]
|
|
33
33
|
priority: 20
|
|
34
|
-
disabled: true
|
|
34
|
+
# disabled: true
|
|
35
35
|
lfm2.5-thinking:
|
|
36
36
|
model: LiquidAI/LFM2.5-1.2B-Thinking-MLX-8bit
|
|
37
37
|
provider: "mlx"
|
|
38
38
|
capabilities: ["local", "thinking", "japanese"]
|
|
39
39
|
priority: 20
|
|
40
|
-
disabled: true
|
|
40
|
+
# disabled: true
|
|
41
41
|
|
|
42
42
|
drivers:
|
|
43
43
|
mlx: {}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modular-prompt/experiment",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Experiment framework for comparing and evaluating prompt modules",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -25,10 +25,10 @@
|
|
|
25
25
|
"jiti": "^2.4.2",
|
|
26
26
|
"yaml": "^2.3.4",
|
|
27
27
|
"zod": "^3.22.4",
|
|
28
|
-
"@modular-prompt/core": "0.
|
|
29
|
-
"@modular-prompt/
|
|
30
|
-
"@modular-prompt/
|
|
31
|
-
"@modular-prompt/
|
|
28
|
+
"@modular-prompt/core": "0.2.0",
|
|
29
|
+
"@modular-prompt/driver": "0.10.0",
|
|
30
|
+
"@modular-prompt/utils": "0.3.0",
|
|
31
|
+
"@modular-prompt/process": "0.2.0"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"@eslint/js": "^9.34.0",
|
|
@@ -101,6 +101,15 @@ testCases:
|
|
|
101
101
|
queryOptions: # オプション
|
|
102
102
|
temperature: 0.5
|
|
103
103
|
|
|
104
|
+
- name: マルチモデルテスト
|
|
105
|
+
description: 役割別に異なるモデルを使用
|
|
106
|
+
input:
|
|
107
|
+
query: "複雑な問題を解決する"
|
|
108
|
+
models:
|
|
109
|
+
- default: gpt4o # インライン DriverSet 定義
|
|
110
|
+
thinking: gemini # thinking役割用のモデル
|
|
111
|
+
- gpt4o # 単一モデル指定（従来通り）
|
|
112
|
+
|
|
104
113
|
- name: ツール呼び出しテスト
|
|
105
114
|
input:
|
|
106
115
|
query: "東京の天気を調べて"
|
|
@@ -131,6 +140,25 @@ evaluation:
|
|
|
131
140
|
|
|
132
141
|
設定ファイル内のパス（modules, evaluators等）は設定ファイルのディレクトリからの相対パスで解決される。`~/` でホームディレクトリ、絶対パスも使用可能。
|
|
133
142
|
|
|
143
|
+
### テストケースのモデル指定
|
|
144
|
+
|
|
145
|
+
`testCases[].models` フィールドでは、以下の2種類の指定が可能です:
|
|
146
|
+
|
|
147
|
+
**1. 文字列（モデル名）**: 単一のドライバーを使用
|
|
148
|
+
```yaml
|
|
149
|
+
models: [gpt4o, gemini]
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
**2. オブジェクト（DriverSet）**: 役割別に異なるドライバーを指定
|
|
153
|
+
```yaml
|
|
154
|
+
models:
|
|
155
|
+
- default: gpt4o # 必須: デフォルトの役割
|
|
156
|
+
thinking: gemini # オプション: thinking役割用
|
|
157
|
+
chat: claude # オプション: chat役割用
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
役割は `ModelRole = 'default' | 'thinking' | 'instruct' | 'chat' | 'plan'` から選択できます。未指定の役割は自動的に `default` にフォールバックします。
|
|
161
|
+
|
|
134
162
|
## モジュール定義
|
|
135
163
|
|
|
136
164
|
テスト対象のモジュールファイルでは、PromptModule を直接 default export する:
|
|
@@ -151,6 +179,27 @@ export default module;
|
|
|
151
179
|
|
|
152
180
|
テストケースの `input` は実行時にコンテキストとして注入される。runner 内部で `defaultProcess` を使用してコンパイル・実行が行われる。
|
|
153
181
|
|
|
182
|
+
### 制限事項: ワークフロー関数の固定
|
|
183
|
+
|
|
184
|
+
現状、`ExperimentRunner` は **`defaultProcess` がハードコード**されており、`agenticProcess` や `streamProcess` などの別のワークフロー関数に差し替える機能は実装されていません。
|
|
185
|
+
|
|
186
|
+
**現在の実装** (`packages/experiment/src/runner/experiment.ts` L250):
|
|
187
|
+
- `defaultProcess` が直接呼び出される
|
|
188
|
+
- `TestCase`, `ModuleDefinition`, `ExperimentRunner` のいずれにもワークフロー指定フィールドなし
|
|
189
|
+
- YAML設定でもワークフロー関数を指定する手段なし
|
|
190
|
+
|
|
191
|
+
**今後の拡張方向**:
|
|
192
|
+
異なるワークフロー（agenticProcess等）を実験フレームワークで使用可能にするには、以下の機能追加が必要です:
|
|
193
|
+
|
|
194
|
+
1. **`TestCase` または `ModuleDefinition` に `process` フィールドを追加**
|
|
195
|
+
- YAML設定で `process: "./workflows/my-workflow.ts"` のようにパス指定
|
|
196
|
+
2. **`ExperimentRunner.runModuleTest()` でワークフロー関数を選択可能にする**
|
|
197
|
+
- デフォルトは `defaultProcess` を維持（後方互換）
|
|
198
|
+
3. **ワークフロー関数のロード機構**
|
|
199
|
+
- evaluator/moduleと同様に外部ファイルから関数をimport
|
|
200
|
+
|
|
201
|
+
この拡張により、実験フレームワーク上で異なる処理戦略（エージェント型、ストリーム処理等）を比較検証できるようになります。
|
|
202
|
+
|
|
154
203
|
## 評価器
|
|
155
204
|
|
|
156
205
|
### ビルトイン評価器
|