@modular-prompt/experiment 0.1.9 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/run-comparison.js +8 -2
- package/dist/run-comparison.js.map +1 -1
- package/dist/runner/experiment.d.ts.map +1 -1
- package/dist/runner/experiment.js +38 -13
- package/dist/runner/experiment.js.map +1 -1
- package/dist/types.d.ts +5 -1
- package/dist/types.d.ts.map +1 -1
- package/examples/experiment.yaml +1 -3
- package/examples/tools-experiment.yaml +73 -0
- package/examples/tools-test-module.mjs +29 -0
- package/examples/tools-test-module.ts +29 -0
- package/package.json +4 -4
package/dist/run-comparison.js
CHANGED
|
@@ -58,7 +58,7 @@ const { serverConfig, modules: configModules, testCases: configTestCases, evalua
|
|
|
58
58
|
// Keep models as object for experiment runner
|
|
59
59
|
const models = serverConfig.models;
|
|
60
60
|
// Display available models for logging
|
|
61
|
-
const modelEntries = Object.entries(models).filter(([_, spec]) => spec.
|
|
61
|
+
const modelEntries = Object.entries(models).filter(([_, spec]) => !spec.disabled && (!spec.role || spec.role === 'test'));
|
|
62
62
|
if (options.modelFilter) {
|
|
63
63
|
const filteredEntries = modelEntries.filter(([_, spec]) => spec.provider === options.modelFilter);
|
|
64
64
|
if (filteredEntries.length === 0) {
|
|
@@ -131,7 +131,7 @@ if (options.enableEvaluation) {
|
|
|
131
131
|
// Find the specified model by name
|
|
132
132
|
const modelName = evaluationConfig.model;
|
|
133
133
|
const modelSpec = serverConfig.models[modelName];
|
|
134
|
-
if (!modelSpec || modelSpec.
|
|
134
|
+
if (!modelSpec || modelSpec.disabled) {
|
|
135
135
|
console.error(`❌ Evaluator model not found or disabled: ${modelName}`);
|
|
136
136
|
console.error(' Please ensure the model is defined in the models section and enabled');
|
|
137
137
|
process.exit(1);
|
|
@@ -159,6 +159,12 @@ const results = await runner.run();
|
|
|
159
159
|
console.log('='.repeat(80));
|
|
160
160
|
console.log('✨ Experiment completed');
|
|
161
161
|
console.log('='.repeat(80));
|
|
162
|
+
// Flush log file if configured
|
|
163
|
+
if (options.logFile) {
|
|
164
|
+
const { logger } = await import('./logger.js');
|
|
165
|
+
await logger.flush();
|
|
166
|
+
console.log(`📄 Log file written: ${options.logFile}`);
|
|
167
|
+
}
|
|
162
168
|
// Cleanup drivers
|
|
163
169
|
await driverManager.cleanup();
|
|
164
170
|
// Display statistics if repeated
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-comparison.js","sourceRoot":"","sources":["../src/run-comparison.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,sBAAsB;AACtB,MAAM,OAAO,GAAG,SAAS,EAAE,CAAC;AAE5B,mBAAmB;AACnB,MAAM,CAAC,SAAS,CAAC;IACf,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;IACzC,eAAe,EAAE,OAAO;IACxB,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;IAC7B,UAAU,EAAE,KAAK;IACjB,OAAO,EAAE,OAAO,CAAC,OAAO;CACzB,CAAC,CAAC;AAEH,iBAAiB;AACjB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;AAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;AAC7C,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,CAAC,cAAc,IAAI,KAAK,EAAE,CAAC,CAAC;AACpE,OAAO,CAAC,GAAG,CAAC,iBAAiB,OAAO,CAAC,WAAW,IAAI,oBAAoB,EAAE,CAAC,CAAC;AAC5E,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;AACrE,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,WAAW,UAAU,CAAC,CAAC;AACtD,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;AAChF,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;AAC7E,CAAC;AACD,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;AAC/E,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,qBAAqB;AACrB,MAAM,EACJ,YAAY,EACZ,OAAO,EAAE,aAAa,EACtB,SAAS,EAAE,eAAe,EAC1B,UAAU,EAAE,gBAAgB,EAC5B,SAAS,EACT,SAAS,EACV,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAEnD,8CAA8C;AAC9C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAEnC,uCAAuC;AACvC,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAgB,EAAE,EAAE,CAC9E,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"run-comparison.js","sourceRoot":"","sources":["../src/run-comparison.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,sBAAsB;AACtB,MAAM,OAAO,GAAG,SAAS,EAAE,CAAC;AAE5B,mBAAmB;AACnB,MAAM,CAAC,SAAS,CAAC;IACf,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;IACzC,eAAe,EAAE,OAAO;IACxB,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;IAC7B,UAAU,EAAE,KAAK;IACjB,OAAO,EAAE,OAAO,CAAC,OAAO;CACzB,CAAC,CAAC;AAEH,iBAAiB;AACjB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;AAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;AAC7C,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,CAAC,cAAc,IAAI,KAAK,EAAE,CAAC,CAAC;AACpE,OAAO,CAAC,GAAG,CAAC,iBAAiB,OAAO,CAAC,WAAW,IAAI,oBAAoB,EAAE,CAAC,CAAC;AAC5E,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;AACrE,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,WAAW,UAAU,CAAC,CAAC;AACtD,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;AAChF,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;AAC7E,CAAC;AACD,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;AAC/E,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,qBAAqB;AACrB,MAAM,EACJ,YAAY,EACZ,OAAO,EAAE,aAAa,EACtB,SAAS,EAAE,eAAe,EAC1B,UAAU,EAAE,gBAAgB,EAC5B,SAAS,EACT,SAAS,EACV,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAEnD,8CAA8C;AAC9C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAEnC,uCAAuC;AACvC,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAgB,EAAE,EAAE,CAC9E,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CACvD,CAAC;AAEF,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IACxB,MAAM,eAAe,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAgB,EAAE,EAAE,CACvE,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,WAAW,CACtC,CAAC;IACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,gDAAgD,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,mBAAmB,eAAe,CAAC,MAAM,0BAA0B,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IACxG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAgB,EAAE,EAAE,CACtD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAC7D,CAAC;AACJ,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,GAAG,CAAC,mBAAmB,YAAY,CAAC,MAAM,YAAY,CAAC,CAAC;IAChE,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAgB,EAAE,EAAE,CACnD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAC7D,CAAC;AACJ,CAAC;AAED,gCAAgC;AAChC,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAgB,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC;AACxF,IAAI,MAAM,EAAE,CAAC;IACX,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,4GAA4G,CAAC,CAAC;AAC5H,CAAC;AACD,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,kBAAkB;AAClB,MAAM,YAAY,GAAG,eAAe,CAAC;AACrC,MAAM,SAAS,GAAG,OAAO,CAAC,cAAc;IACtC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,OAAO,CAAC,cAAc,CAAC;IACtE,CAAC,CAAC,YAAY,CAAC;AAEjB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IAC3B,OAAO,CAAC,KAAK,CAAC,wBAAwB,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9G,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,eAAe,CAAC,CAAC;AAC3D,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,wCAAwC;AACxC,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;AAC/D,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY;IAClC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,YAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAChE,CAAC,CAAC,UAAU,CAAC;AAEf,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IACzB,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACtC,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,MAAM,aAAa,CAAC,CAAC;AACvD,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACrE,OAAO,CAAC,GAAG,EAAE,CAAC;AAEd,8DAA8D;AAC9D,IAAI,UAAU,CAAC;AACf,IAAI,cAAc,CAAC;AACnB,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAC7B,8CAA8C;IAC9C,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC;IACxE,UAAU,GAAG,OAAO,CAAC,eAAe;QAClC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,eAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtE,CAAC,CAAC,aAAa,CAAC;IAElB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,8CAA8C;IAC9C,IAAI,CAAC,YAAY,CAAC,UAAU,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACjE,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAC/D,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;QACvE,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC1C,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,gBAAgB,GAAG,YAAY,CAAC,UAAU,CAAC;IAEjD,mCAAmC;IACnC,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC;IACzC,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAEjD,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QACrC,OAAO,CAAC,KAAK,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,KAAK,CAAC,yEAAyE,CAAC,CAAC;QACzF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,cAAc,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAEtD,OAAO,CAAC,GAAG,CAAC,8BAA8B,UAAU,CAAC,MAAM,gBAAgB,CAAC,CAAC;IAC7E,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACpF,OAAO,CAAC,GAAG,CAAC,uBAAuB,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC;AAED,kBAAkB;AAClB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,iBAAiB;AACjB,MAAM,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;AAC1C,MAAM,MAAM,GAAG,IAAI,gBAAgB,CACjC,SAAS,EACT,aAAa,EACb,OAAO,EACP,SAAS,EACT,MAAM,EACN,OAAO,CAAC,WAAW,EACnB,UAAU,EACV,cAAc,CACf,CAAC;AAEF,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;AAEnC,qBAAqB;AACrB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;AACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAE5B,+BAA+B;AAC/B,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;IAC/C,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;AACzD,CAAC;AAED,kBAAkB;AAClB,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;AAE9B,iCAAiC;AACjC,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACjD,QAAQ,CAAC,MAAM,EAAE,CAAC;AACpB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAe,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAChF,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAAuC,MAAM,aAAa,CAAC;AAC/G,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAMnE,qBAAa,gBAAgB;IAEzB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,cAAc,CAAC;gBAPf,SAAS,EAAE,SAAS,EACpB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,gBAAgB,EAAE,EAC3B,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EACjC,WAAW,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,eAAe,EAAE,YAAA,EAC9B,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,SAAS,CAAA;KAAE,YAAA;IAG5D;;;;OAIG;IACG,GAAG,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAe,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAChF,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAAuC,MAAM,aAAa,CAAC;AAC/G,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAMnE,qBAAa,gBAAgB;IAEzB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,cAAc,CAAC;gBAPf,SAAS,EAAE,SAAS,EACpB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,gBAAgB,EAAE,EAC3B,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EACjC,WAAW,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,eAAe,EAAE,YAAA,EAC9B,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,SAAS,CAAA;KAAE,YAAA;IAG5D;;;;OAIG;IACG,GAAG,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;IAkHlC;;OAEG;YACW,aAAa;IA8D3B;;OAEG;YACW,kBAAkB;IA2BhC;;OAEG;IACH,OAAO,CAAC,cAAc;CAiCvB"}
|
|
@@ -32,6 +32,8 @@ export class ExperimentRunner {
|
|
|
32
32
|
async run() {
|
|
33
33
|
const allResults = [];
|
|
34
34
|
const evaluationContexts = [];
|
|
35
|
+
// ドライバー切り替えをテストケースをまたいでトラッキング
|
|
36
|
+
let activeModelName = null;
|
|
35
37
|
for (const testCase of this.testCases) {
|
|
36
38
|
console.log('─'.repeat(80));
|
|
37
39
|
console.log(`Test Case: ${testCase.name}`);
|
|
@@ -67,7 +69,7 @@ export class ExperimentRunner {
|
|
|
67
69
|
return { name, spec };
|
|
68
70
|
}).filter(Boolean)
|
|
69
71
|
: Object.entries(this.models)
|
|
70
|
-
.filter(([_, spec]) => spec.
|
|
72
|
+
.filter(([_, spec]) => !spec.disabled)
|
|
71
73
|
.map(([name, spec]) => ({ name, spec }));
|
|
72
74
|
if (modelsToTest.length === 0) {
|
|
73
75
|
console.log('⚠️ No models to test for this test case, skipping');
|
|
@@ -75,23 +77,26 @@ export class ExperimentRunner {
|
|
|
75
77
|
continue;
|
|
76
78
|
}
|
|
77
79
|
// Test with each model
|
|
78
|
-
let previousDriver = null;
|
|
79
|
-
let previousModelName = null;
|
|
80
80
|
for (const { name: modelName, spec: modelSpec } of modelsToTest) {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
81
|
+
// モデル切り替え: 前のドライバーをクローズしてから新しいモデルを起動
|
|
82
|
+
if (activeModelName && activeModelName !== modelName) {
|
|
83
|
+
logger.info(`Closing driver: ${activeModelName} (switching to ${modelName})`);
|
|
84
|
+
await this.driverManager.close(activeModelName);
|
|
85
|
+
activeModelName = null;
|
|
86
|
+
}
|
|
87
|
+
if (activeModelName === modelName) {
|
|
88
|
+
logger.verbose(`Reusing driver for ${modelName}`);
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
logger.info(`Creating new driver for ${modelName} (${modelSpec.provider}:${modelSpec.model})`);
|
|
87
92
|
}
|
|
93
|
+
console.log(`🤖 Testing with ${modelName} (${modelSpec.provider}:${modelSpec.model})`);
|
|
88
94
|
// Get or create driver for this model
|
|
89
95
|
const driver = await this.driverManager.getOrCreate(this.aiService, modelName, modelSpec);
|
|
90
|
-
|
|
91
|
-
previousModelName = modelName;
|
|
96
|
+
activeModelName = modelName;
|
|
92
97
|
// Test each module
|
|
93
98
|
for (const { name, compiled, prompt } of compiledModules) {
|
|
94
|
-
const runs = await this.runModuleTest(name, compiled, driver);
|
|
99
|
+
const runs = await this.runModuleTest(name, compiled, driver, testCase);
|
|
95
100
|
allResults.push({
|
|
96
101
|
testCase: testCase.name,
|
|
97
102
|
model: modelName,
|
|
@@ -100,6 +105,8 @@ export class ExperimentRunner {
|
|
|
100
105
|
success: r.success,
|
|
101
106
|
elapsed: r.elapsed,
|
|
102
107
|
content: r.queryResult?.content || '',
|
|
108
|
+
toolCalls: r.queryResult?.toolCalls,
|
|
109
|
+
finishReason: r.queryResult?.finishReason,
|
|
103
110
|
error: r.error,
|
|
104
111
|
})),
|
|
105
112
|
});
|
|
@@ -124,7 +131,7 @@ export class ExperimentRunner {
|
|
|
124
131
|
/**
|
|
125
132
|
* Run module test with multiple repetitions
|
|
126
133
|
*/
|
|
127
|
-
async runModuleTest(moduleName, compiled, driver) {
|
|
134
|
+
async runModuleTest(moduleName, compiled, driver, testCase) {
|
|
128
135
|
logger.verbose(`Running ${this.repeatCount} time(s) for module: ${moduleName}`);
|
|
129
136
|
const runs = [];
|
|
130
137
|
for (let i = 0; i < this.repeatCount; i++) {
|
|
@@ -134,9 +141,27 @@ export class ExperimentRunner {
|
|
|
134
141
|
const result = await driver.query(compiled, {
|
|
135
142
|
temperature: 0.7,
|
|
136
143
|
maxTokens: 2048,
|
|
144
|
+
...testCase.queryOptions,
|
|
137
145
|
});
|
|
138
146
|
const elapsed = Date.now() - startTime;
|
|
139
147
|
logger.verbose(`Module ${moduleName} run ${i + 1}: Success (${elapsed}ms)`);
|
|
148
|
+
// Display result summary (思考ブロックはプレビューから除外)
|
|
149
|
+
// パターン: <think>...</think> または 先頭から</think>まで（テンプレートが<think>を付与する場合）
|
|
150
|
+
const displayContent = result.content
|
|
151
|
+
.replace(/<think>[\s\S]*?<\/think>\s*/g, '')
|
|
152
|
+
.replace(/^[\s\S]*?<\/think>\s*/g, '');
|
|
153
|
+
const contentPreview = displayContent.length > 200
|
|
154
|
+
? displayContent.substring(0, 200) + '...'
|
|
155
|
+
: displayContent;
|
|
156
|
+
console.log(` ✅ [${moduleName}] run ${i + 1} (${elapsed}ms) finishReason=${result.finishReason || 'unknown'}`);
|
|
157
|
+
if (result.toolCalls && result.toolCalls.length > 0) {
|
|
158
|
+
for (const tc of result.toolCalls) {
|
|
159
|
+
console.log(` 🔧 toolCall: ${tc.name}(${JSON.stringify(tc.arguments)})`);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
if (contentPreview.trim()) {
|
|
163
|
+
console.log(` 📝 ${contentPreview}`);
|
|
164
|
+
}
|
|
140
165
|
runs.push({
|
|
141
166
|
success: true,
|
|
142
167
|
elapsed,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAKhE,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAE5C,MAAM,OAAO,gBAAgB;IAEjB;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IARV,YACU,SAAoB,EACpB,aAA4B,EAC5B,OAA2B,EAC3B,SAAqB,EACrB,MAAiC,EACjC,WAAmB,EACnB,UAA8B,EAC9B,cAAkD;QAPlD,cAAS,GAAT,SAAS,CAAW;QACpB,kBAAa,GAAb,aAAa,CAAe;QAC5B,YAAO,GAAP,OAAO,CAAoB;QAC3B,cAAS,GAAT,SAAS,CAAY;QACrB,WAAM,GAAN,MAAM,CAA2B;QACjC,gBAAW,GAAX,WAAW,CAAQ;QACnB,eAAU,GAAV,UAAU,CAAoB;QAC9B,mBAAc,GAAd,cAAc,CAAoC;IACzD,CAAC;IAEJ;;;;OAIG;IACH,KAAK,CAAC,GAAG;QACP,MAAM,UAAU,GAAiB,EAAE,CAAC;QACpC,MAAM,kBAAkB,GAAwB,EAAE,CAAC;QAEnD,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3C,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,gBAAgB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;YAEd,qDAAqD;YACrD,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBAChD,MAAM,CAAC,OAAO,CAAC,gCAAgC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAChD,MAAM,MAAM,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;gBAChD,MAAM,CAAC,OAAO,CAAC,qBAAqB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAE3E,OAAO;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,QAAQ;oBACR,MAAM;iBACP,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,sCAAsC;YACtC,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;YACvC,CAAC;YAED,oDAAoD;YACpD,MAAM,YAAY,GAA6C,QAAQ,CAAC,MAAM;gBAC5E,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;oBACzB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;wBACV,OAAO,CAAC,IAAI,CAAC,cAAc,IAAI,wCAAwC,CAAC,CAAC;wBACzE,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACxB,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAA6C;gBAChE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;qBACxB,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/runner/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAKhE,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAE5C,MAAM,OAAO,gBAAgB;IAEjB;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IARV,YACU,SAAoB,EACpB,aAA4B,EAC5B,OAA2B,EAC3B,SAAqB,EACrB,MAAiC,EACjC,WAAmB,EACnB,UAA8B,EAC9B,cAAkD;QAPlD,cAAS,GAAT,SAAS,CAAW;QACpB,kBAAa,GAAb,aAAa,CAAe;QAC5B,YAAO,GAAP,OAAO,CAAoB;QAC3B,cAAS,GAAT,SAAS,CAAY;QACrB,WAAM,GAAN,MAAM,CAA2B;QACjC,gBAAW,GAAX,WAAW,CAAQ;QACnB,eAAU,GAAV,UAAU,CAAoB;QAC9B,mBAAc,GAAd,cAAc,CAAoC;IACzD,CAAC;IAEJ;;;;OAIG;IACH,KAAK,CAAC,GAAG;QACP,MAAM,UAAU,GAAiB,EAAE,CAAC;QACpC,MAAM,kBAAkB,GAAwB,EAAE,CAAC;QAEnD,8BAA8B;QAC9B,IAAI,eAAe,GAAkB,IAAI,CAAC;QAE1C,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3C,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,gBAAgB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;YAEd,qDAAqD;YACrD,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBAChD,MAAM,CAAC,OAAO,CAAC,gCAAgC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAChD,MAAM,MAAM,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;gBAChD,MAAM,CAAC,OAAO,CAAC,qBAAqB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAE3E,OAAO;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,QAAQ;oBACR,MAAM;iBACP,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,sCAAsC;YACtC,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;YACvC,CAAC;YAED,oDAAoD;YACpD,MAAM,YAAY,GAA6C,QAAQ,CAAC,MAAM;gBAC5E,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;oBACzB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;wBACV,OAAO,CAAC,IAAI,CAAC,cAAc,IAAI,wCAAwC,CAAC,CAAC;wBACzE,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACxB,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAA6C;gBAChE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;qBACxB,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;qBACrC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAE/C,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;gBAClE,OAAO,CAAC,GAAG,EAAE,CAAC;gBACd,SAAS;YACX,CAAC;YAED,uBAAuB;YACvB,KAAK,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,YAAY,EAAE,CAAC;gBAChE,qCAAqC;gBACrC,IAAI,eAAe,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;oBACrD,MAAM,CAAC,IAAI,CAAC,mBAAmB,eAAe,kBAAkB,SAAS,GAAG,CAAC,CAAC;oBAC9E,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;oBAChD,eAAe,GAAG,IAAI,CAAC;gBACzB,CAAC;gBAED,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;oBAClC,MAAM,CAAC,OAAO,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,2BAA2B,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;gBACjG,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,mBAAmB,SAAS,KAAK,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;gBAEvF,sCAAsC;gBACtC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;gBAC1F,eAAe,GAAG,SAAS,CAAC;gBAE5B,mBAAmB;gBACnB,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC;oBACzD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;oBAExE,UAAU,CAAC,IAAI,CAAC;wBACd,QAAQ,EAAE,QAAQ,CAAC,IAAI;wBACvB,KAAK,EAAE,SAAS;wBAChB,MAAM,EAAE,IAAI;wBACZ,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;4BACnB,OAAO,EAAE,CAAC,CAAC,OAAO;4BAClB,OAAO,EAAE,CAAC,CAAC,OAAO;4BAClB,OAAO,EAAE,CAAC,CAAC,WAAW,EAAE,OAAO,IAAI,EAAE;4BACrC,SAAS,EAAE,CAAC,CAAC,WAAW,EAAE,SAAS;4BACnC,YAAY,EAAE,CAAC,CAAC,WAAW,EAAE,YAAY;4BACzC,KAAK,EAAE,CAAC,CAAC,KAAK;yBACf,CAAC,CAAC;qBACJ,CAAC,CAAC;oBAEH,iDAAiD;oBACjD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;oBACnD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC9B,kBAAkB,CAAC,IAAI,CAAC;4BACtB,UAAU,EAAE,IAAI;4BAChB,MAAM;4BACN,IAAI,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAY,EAAE,CAAC,CAAC;yBACjE,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,kDAAkD;QAClD,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzE,MAAM,IAAI,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,UAAkB,EAClB,QAAa,EACb,MAAW,EACX,QAAkB;QAElB,MAAM,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,WAAW,wBAAwB,UAAU,EAAE,CAAC,CAAC;QAEhF,MAAM,IAAI,GAA4F,EAAE,CAAC;QAEzG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,gBAAgB,UAAU,EAAE,CAAC,CAAC;YAE7E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE;oBAC1C,WAAW,EAAE,GAAG;oBAChB,SAAS,EAAE,IAAI;oBACf,GAAG,QAAQ,CAAC,YAAY;iBACzB,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBAEvC,MAAM,CAAC,OAAO,CAAC,UAAU,UAAU,QAAQ,CAAC,GAAG,CAAC,cAAc,OAAO,KAAK,CAAC,CAAC;gBAE5E,4CAA4C;gBAC5C,qEAAqE;gBACrE,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO;qBAClC,OAAO,CAAC,8BAA8B,EAAE,EAAE,CAAC;qBAC3C,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;gBACzC,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,GAAG,GAAG;oBAChD,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;oBAC1C,CAAC,CAAC,cAAc,CAAC;gBACnB,OAAO,CAAC,GAAG,CAAC,SAAS,UAAU,SAAS,CAAC,GAAG,CAAC,KAAK,OAAO,oBAAoB,MAAM,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC,CAAC;gBACjH,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpD,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;wBAClC,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;oBAChF,CAAC;gBACH,CAAC;gBACD,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;oBAC1B,OAAO,CAAC,GAAG,CAAC,YAAY,cAAc,EAAE,CAAC,CAAC;gBAC5C,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC;oBACR,OAAO,EAAE,IAAI;oBACb,OAAO;oBACP,WAAW,EAAE,MAAM;iBACpB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBACvC,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5E,MAAM,CAAC,KAAK,CAAC,UAAU,UAAU,QAAQ,CAAC,GAAG,CAAC,YAAY,OAAO,QAAQ,YAAY,EAAE,CAAC,CAAC;gBACzF,IAAI,CAAC,IAAI,CAAC;oBACR,OAAO,EAAE,KAAK;oBACd,OAAO;oBACP,KAAK,EAAE,YAAY;iBACpB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAC9B,kBAAuC;QAEvC,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,cAAe,CAAC,IAAI,CAAC,CAAC;QACvF,MAAM,cAAc,GAAuB,EAAE,CAAC;QAE9C,2CAA2C;QAC3C,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,EAAE,CAAC;YAEd,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAW,EAAE,CAAC;gBACzC,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAClE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,eAAe,CAAC,cAAc,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,eAAwD;QAC7E,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,OAAO,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBAEnC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;oBACtC,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,IAAI,UAAU,OAAO,CAAC,IAAI,iBAAiB,CAAC,CAAC;gBAC5E,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,IAAI,UAAU,OAAO,CAAC,IAAI,UAAU,CAAC,CAAC;oBACrE,MAAM,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC;oBAC7C,MAAM,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;oBACpE,MAAM,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;oBACpE,MAAM,CAAC,OAAO,CAAC,WAAW,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;oBAEjF,uCAAuC;oBACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;wBAChF,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;4BAC5C,MAAM,CAAC,OAAO,CAAC,4BAA4B,CAAC,GAAG,CAAC,CAAC;4BACjD,MAAM,CAAC,OAAO,CAAC,OAAO,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;4BAC9F,MAAM,CAAC,OAAO,CAAC,OAAO,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;4BAC9F,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;CACF"}
|
package/dist/types.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Common type definitions for module comparison experiments
|
|
3
3
|
*/
|
|
4
4
|
import type { PromptModule } from '@modular-prompt/core';
|
|
5
|
-
import type { QueryResult } from '@modular-prompt/driver';
|
|
5
|
+
import type { QueryResult, QueryOptions, ToolCall } from '@modular-prompt/driver';
|
|
6
6
|
/**
|
|
7
7
|
* Test case definition
|
|
8
8
|
*/
|
|
@@ -15,6 +15,8 @@ export interface TestCase {
|
|
|
15
15
|
input: any;
|
|
16
16
|
/** Model names to use for this test case (optional, uses all enabled models if not specified) */
|
|
17
17
|
models?: string[];
|
|
18
|
+
/** Query options for this test case (tools, temperature, etc.) */
|
|
19
|
+
queryOptions?: Partial<QueryOptions>;
|
|
18
20
|
}
|
|
19
21
|
/**
|
|
20
22
|
* Result of a single run
|
|
@@ -23,6 +25,8 @@ export interface RunResult {
|
|
|
23
25
|
success: boolean;
|
|
24
26
|
elapsed: number;
|
|
25
27
|
content: string;
|
|
28
|
+
toolCalls?: ToolCall[];
|
|
29
|
+
finishReason?: string;
|
|
26
30
|
error?: string;
|
|
27
31
|
}
|
|
28
32
|
/**
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAElF;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,KAAK,EAAE,GAAG,CAAC;IACX,iGAAiG;IACjG,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,kEAAkE;IAClE,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,GAAG,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,KAAK,CAAC;QACV,WAAW,EAAE,WAAW,CAAC;KAC1B,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;;OASG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACrE;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;;OASG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/E;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3C;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,iBAAiB;IAClE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB"}
|
package/examples/experiment.yaml
CHANGED
|
@@ -7,7 +7,6 @@ models:
|
|
|
7
7
|
provider: "mlx"
|
|
8
8
|
capabilities: ["local", "fast", "tools"]
|
|
9
9
|
priority: 20
|
|
10
|
-
enabled: true
|
|
11
10
|
|
|
12
11
|
# Vertex AI Gemini - for testing with structuredOutput support
|
|
13
12
|
gemini-vertexai:
|
|
@@ -15,7 +14,6 @@ models:
|
|
|
15
14
|
provider: "vertexai"
|
|
16
15
|
capabilities: ["tools", "fast", "japanese"]
|
|
17
16
|
priority: 10
|
|
18
|
-
enabled: true
|
|
19
17
|
|
|
20
18
|
# GoogleGenAI Gemini - alternative for testing
|
|
21
19
|
gemini-googlegenai:
|
|
@@ -23,7 +21,7 @@ models:
|
|
|
23
21
|
provider: "googlegenai"
|
|
24
22
|
capabilities: ["tools", "fast", "japanese"]
|
|
25
23
|
priority: 15
|
|
26
|
-
|
|
24
|
+
disabled: true # Disabled by default
|
|
27
25
|
|
|
28
26
|
drivers:
|
|
29
27
|
mlx: {}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Tools実験設定
|
|
2
|
+
# MLXドライバーでのtool calling動作確認
|
|
3
|
+
|
|
4
|
+
models:
|
|
5
|
+
qwen3-4b:
|
|
6
|
+
model: "mlx-community/Qwen3-4B-Thinking-2507-heretic-8bit"
|
|
7
|
+
provider: "mlx"
|
|
8
|
+
capabilities: ["local", "tools"]
|
|
9
|
+
priority: 10
|
|
10
|
+
gemma3-12b:
|
|
11
|
+
model: "mlx-community/gemma-3-12b-it-qat-4bit"
|
|
12
|
+
provider: "mlx"
|
|
13
|
+
capabilities: ["local", "tools"]
|
|
14
|
+
priority: 20
|
|
15
|
+
|
|
16
|
+
drivers:
|
|
17
|
+
mlx: {}
|
|
18
|
+
|
|
19
|
+
modules:
|
|
20
|
+
- name: tools-test
|
|
21
|
+
path: ./tools-test-module.mjs
|
|
22
|
+
description: "ツール呼び出し実験用"
|
|
23
|
+
|
|
24
|
+
testCases:
|
|
25
|
+
# --- gemma3-12b ---
|
|
26
|
+
- name: "[gemma3] 天気ツール呼び出し"
|
|
27
|
+
description: "get_weatherツールを呼び出すことを期待"
|
|
28
|
+
models: ["gemma3-12b"]
|
|
29
|
+
input:
|
|
30
|
+
question: "東京の天気を教えてください。"
|
|
31
|
+
queryOptions: &tool_weather
|
|
32
|
+
temperature: 0.3
|
|
33
|
+
maxTokens: 512
|
|
34
|
+
tools: &tools_def
|
|
35
|
+
- name: get_weather
|
|
36
|
+
description: "指定された場所の現在の天気を取得する"
|
|
37
|
+
parameters:
|
|
38
|
+
type: object
|
|
39
|
+
properties:
|
|
40
|
+
location:
|
|
41
|
+
type: string
|
|
42
|
+
description: "天気を取得する場所（都市名）"
|
|
43
|
+
required:
|
|
44
|
+
- location
|
|
45
|
+
toolChoice: auto
|
|
46
|
+
|
|
47
|
+
- name: "[gemma3] ツール不要の質問"
|
|
48
|
+
description: "ツールを呼び出さずにテキストで回答することを期待"
|
|
49
|
+
models: ["gemma3-12b"]
|
|
50
|
+
input:
|
|
51
|
+
question: "1 + 1 は何ですか？"
|
|
52
|
+
queryOptions: &tool_math
|
|
53
|
+
temperature: 0.3
|
|
54
|
+
maxTokens: 1024
|
|
55
|
+
tools: *tools_def
|
|
56
|
+
toolChoice: auto
|
|
57
|
+
|
|
58
|
+
# --- qwen3-4b ---
|
|
59
|
+
- name: "[qwen3] 天気ツール呼び出し"
|
|
60
|
+
description: "get_weatherツールを呼び出すことを期待"
|
|
61
|
+
models: ["qwen3-4b"]
|
|
62
|
+
input:
|
|
63
|
+
question: "東京の天気を教えてください。"
|
|
64
|
+
queryOptions: *tool_weather
|
|
65
|
+
|
|
66
|
+
- name: "[qwen3] ツール不要の質問"
|
|
67
|
+
description: "ツールを呼び出さずにテキストで回答することを期待"
|
|
68
|
+
models: ["qwen3-4b"]
|
|
69
|
+
input:
|
|
70
|
+
question: "1 + 1 は何ですか？"
|
|
71
|
+
queryOptions: *tool_math
|
|
72
|
+
|
|
73
|
+
evaluators: []
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tools実験用モジュール
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { compile } from '@modular-prompt/core';
|
|
6
|
+
|
|
7
|
+
export default {
|
|
8
|
+
name: 'tools-test',
|
|
9
|
+
description: 'ツール呼び出し実験用モジュール',
|
|
10
|
+
compile: (context) => {
|
|
11
|
+
return compile({
|
|
12
|
+
objective: [
|
|
13
|
+
'- あなたは利用者からの質問に答えるアシスタントです。',
|
|
14
|
+
],
|
|
15
|
+
instructions: [
|
|
16
|
+
'- 質問の内容に応じて、適切なツールを使ってください。',
|
|
17
|
+
' - ツールの結果が返ってくるまで、推測で答えないでください。',
|
|
18
|
+
'- 必要がない場合は通常の応答を返します。',
|
|
19
|
+
],
|
|
20
|
+
messages: [
|
|
21
|
+
{
|
|
22
|
+
type: 'message',
|
|
23
|
+
role: 'user',
|
|
24
|
+
content: context.question || 'Hello',
|
|
25
|
+
},
|
|
26
|
+
],
|
|
27
|
+
});
|
|
28
|
+
},
|
|
29
|
+
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tools実験用モジュール
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { compile } from '@modular-prompt/core';
|
|
6
|
+
|
|
7
|
+
export default {
|
|
8
|
+
name: 'tools-test',
|
|
9
|
+
description: 'ツール呼び出し実験用モジュール',
|
|
10
|
+
compile: (context: any) => {
|
|
11
|
+
return compile({
|
|
12
|
+
objective: [
|
|
13
|
+
'あなたはツールを使って質問に答えるアシスタントです。',
|
|
14
|
+
'必要に応じてツールを呼び出してください。',
|
|
15
|
+
],
|
|
16
|
+
instructions: [
|
|
17
|
+
'質問の内容に応じて、適切なツールを使ってください。',
|
|
18
|
+
'ツールの結果が返ってくるまで、推測で答えないでください。',
|
|
19
|
+
],
|
|
20
|
+
messages: [
|
|
21
|
+
{
|
|
22
|
+
type: 'message' as const,
|
|
23
|
+
role: 'user' as const,
|
|
24
|
+
content: context.question || 'Hello',
|
|
25
|
+
},
|
|
26
|
+
],
|
|
27
|
+
});
|
|
28
|
+
},
|
|
29
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modular-prompt/experiment",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Experiment framework for comparing and evaluating prompt modules",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -24,9 +24,9 @@
|
|
|
24
24
|
"jiti": "^2.4.2",
|
|
25
25
|
"yaml": "^2.3.4",
|
|
26
26
|
"zod": "^3.22.4",
|
|
27
|
-
"@modular-prompt/
|
|
28
|
-
"@modular-prompt/
|
|
29
|
-
"@modular-prompt/utils": "0.2.
|
|
27
|
+
"@modular-prompt/driver": "0.8.0",
|
|
28
|
+
"@modular-prompt/core": "0.1.12",
|
|
29
|
+
"@modular-prompt/utils": "0.2.3"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@eslint/js": "^9.34.0",
|