@ikarmic/forge-eval 4.4.0 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/graders/types.d.ts +5 -0
- package/dist/graders/types.d.ts.map +1 -1
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +12 -2
- package/dist/runner.js.map +1 -1
- package/dist/tasks/index.d.ts.map +1 -1
- package/dist/tasks/index.js +3 -0
- package/dist/tasks/index.js.map +1 -1
- package/dist/tasks/memory.recall-relevant.d.ts +19 -0
- package/dist/tasks/memory.recall-relevant.d.ts.map +1 -0
- package/dist/tasks/memory.recall-relevant.js +36 -0
- package/dist/tasks/memory.recall-relevant.js.map +1 -0
- package/package.json +2 -2
package/dist/graders/types.d.ts
CHANGED
|
@@ -68,6 +68,11 @@ export interface TaskDefinition {
|
|
|
68
68
|
* starts. The cwd is a fresh temp directory per run.
|
|
69
69
|
*/
|
|
70
70
|
setup?: (cwd: string) => Promise<void> | void;
|
|
71
|
+
/**
|
|
72
|
+
* Run with project memory enabled. Default off: memory would add noise to the other tasks.
|
|
73
|
+
* The user's global memory is hidden for these runs so the result does not depend on the host.
|
|
74
|
+
*/
|
|
75
|
+
memory?: boolean;
|
|
71
76
|
/** Graders run after the agent finishes. All must pass for the task to pass. */
|
|
72
77
|
graders: Grader[];
|
|
73
78
|
/** Optional tags for filtering (e.g. "windows", "needs-network"). */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/graders/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,yEAAyE;IACzE,IAAI,EAAE,MAAM,CAAC;IACb,sCAAsC;IACtC,IAAI,EAAE,OAAO,CAAC;IACd,kFAAkF;IAClF,MAAM,EAAE,OAAO,CAAC;IAChB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,8DAA8D;IAC9D,GAAG,EAAE,MAAM,CAAC;IACZ,sEAAsE;IACtE,IAAI,EAAE,QAAQ,CAAC;IACf,kEAAkE;IAClE,MAAM,EAAE,cAAc,CAAC;IACvB,yEAAyE;IACzE,eAAe,EAAE,cAAc,EAAE,CAAC;CACnC;AAED;;;GAGG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,YAAY,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,YAAY,CAAC;AAE1D,uFAAuF;AACvF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,OAAO,CAAC;IAChB,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;IAChB,iFAAiF;IACjF,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,aAAa,KAAK,OAAO,CAAC,YAAY,CAAC,GAAG,YAAY,CAAC;AAElF;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,YAAY,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC9C,gFAAgF;IAChF,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,qEAAqE;IACrE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/graders/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,yEAAyE;IACzE,IAAI,EAAE,MAAM,CAAC;IACb,sCAAsC;IACtC,IAAI,EAAE,OAAO,CAAC;IACd,kFAAkF;IAClF,MAAM,EAAE,OAAO,CAAC;IAChB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,8DAA8D;IAC9D,GAAG,EAAE,MAAM,CAAC;IACZ,sEAAsE;IACtE,IAAI,EAAE,QAAQ,CAAC;IACf,kEAAkE;IAClE,MAAM,EAAE,cAAc,CAAC;IACvB,yEAAyE;IACzE,eAAe,EAAE,cAAc,EAAE,CAAC;CACnC;AAED;;;GAGG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,YAAY,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,YAAY,CAAC;AAE1D,uFAAuF;AACvF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,OAAO,CAAC;IAChB,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;IAChB,iFAAiF;IACjF,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,aAAa,KAAK,OAAO,CAAC,YAAY,CAAC,GAAG,YAAY,CAAC;AAElF;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,YAAY,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC9C;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gFAAgF;IAChF,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,qEAAqE;IACrE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB"}
|
package/dist/runner.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAIA,OAAO,EAAS,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EAEV,UAAU,EACV,cAAc,EACd,oBAAoB,EACpB,WAAW,EACZ,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EAEV,YAAY,EACZ,cAAc,EACd,QAAQ,EACR,cAAc,EACf,MAAM,oBAAoB,CAAC;AAE5B,uEAAuE;AACvE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,8DAA8D;IAC9D,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,6DAA6D;IAC7D,WAAW,EAAE,cAAc,CAAC;IAC5B,yEAAyE;IACzE,eAAe,EAAE,cAAc,EAAE,CAAC;IAClC,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IACZ,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,WAAW,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,kEAAkE;IAClE,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,mCAAmC;AACnC,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,YAAY,CAAC,EAAE,oBAAoB,CAAC;IACpC,yEAAyE;IACzE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;CACvC;AAED;;;;GAIG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,cAAc,EACpB,KAAK,EAAE,WAAW,EAClB,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC,
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAIA,OAAO,EAAS,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EAEV,UAAU,EACV,cAAc,EACd,oBAAoB,EACpB,WAAW,EACZ,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EAEV,YAAY,EACZ,cAAc,EACd,QAAQ,EACR,cAAc,EACf,MAAM,oBAAoB,CAAC;AAE5B,uEAAuE;AACvE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,8DAA8D;IAC9D,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,6DAA6D;IAC7D,WAAW,EAAE,cAAc,CAAC;IAC5B,yEAAyE;IACzE,eAAe,EAAE,cAAc,EAAE,CAAC;IAClC,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IACZ,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,WAAW,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,kEAAkE;IAClE,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,mCAAmC;AACnC,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,YAAY,CAAC,EAAE,oBAAoB,CAAC;IACpC,yEAAyE;IACzE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;CACvC;AAED;;;;GAIG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,cAAc,EACpB,KAAK,EAAE,WAAW,EAClB,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC,CA8GxB;AAED,gFAAgF;AAChF,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,SAAS,cAAc,EAAE,EAChC,KAAK,EAAE,WAAW,EAClB,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAgCzB;AAED,kEAAkE;AAClE,wBAAgB,YAAY,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,WAAW,CAW3D;AAED,6CAA6C;AAC7C,OAAO,EAAE,mBAAmB,EAAE,CAAC"}
|
package/dist/runner.js
CHANGED
|
@@ -17,6 +17,10 @@ export async function runTask(task, model, options = {}) {
|
|
|
17
17
|
let error;
|
|
18
18
|
let graderResults = [];
|
|
19
19
|
let passed = false;
|
|
20
|
+
// A memory task must not read the host's ~/.forge/MEMORY.md.
|
|
21
|
+
const prevGlobalMemory = process.env.FORGE_GLOBAL_MEMORY_DIR;
|
|
22
|
+
if (task.memory)
|
|
23
|
+
process.env.FORGE_GLOBAL_MEMORY_DIR = join(cwd, '.no-global-memory');
|
|
20
24
|
try {
|
|
21
25
|
// Setup the fixture files (if any).
|
|
22
26
|
if (task.setup) {
|
|
@@ -24,14 +28,14 @@ export async function runTask(task, model, options = {}) {
|
|
|
24
28
|
}
|
|
25
29
|
// Build an Agent with everything disabled that would add noise: cost
|
|
26
30
|
// tracking (we sum it ourselves), memory (per-task temp dir makes
|
|
27
|
-
// persistent state confusing), cache (each task is unique).
|
|
31
|
+
// persistent state confusing; a task can opt back in), cache (each task is unique).
|
|
28
32
|
const config = {
|
|
29
33
|
model,
|
|
30
34
|
cwd,
|
|
31
35
|
auto: true, // evals are autonomous: no interactive approval prompts
|
|
32
36
|
allowDestructive: true, // same reason
|
|
33
37
|
disableCost: true,
|
|
34
|
-
disableMemory: true,
|
|
38
|
+
disableMemory: task.memory !== true,
|
|
35
39
|
disableCache: true,
|
|
36
40
|
...(options.modelFactory ? { modelFactory: options.modelFactory } : {}),
|
|
37
41
|
};
|
|
@@ -81,6 +85,12 @@ export async function runTask(task, model, options = {}) {
|
|
|
81
85
|
error = err instanceof Error ? err.message : String(err);
|
|
82
86
|
}
|
|
83
87
|
finally {
|
|
88
|
+
if (task.memory) {
|
|
89
|
+
if (prevGlobalMemory === undefined)
|
|
90
|
+
delete process.env.FORGE_GLOBAL_MEMORY_DIR;
|
|
91
|
+
else
|
|
92
|
+
process.env.FORGE_GLOBAL_MEMORY_DIR = prevGlobalMemory;
|
|
93
|
+
}
|
|
84
94
|
if (ownsCwd) {
|
|
85
95
|
await rm(cwd, { recursive: true, force: true });
|
|
86
96
|
}
|
package/dist/runner.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AA+DjE;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAoB,EACpB,KAAkB,EAClB,UAA0B,EAAE;IAE5B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC;IAC1C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEvD,IAAI,WAA4B,CAAC;IACjC,IAAI,KAAyB,CAAC;IAC9B,IAAI,aAAa,GAAmB,EAAE,CAAC;IACvC,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,IAAI,CAAC;QACH,oCAAoC;QACpC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,qEAAqE;QACrE,kEAAkE;QAClE,
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AA+DjE;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAoB,EACpB,KAAkB,EAClB,UAA0B,EAAE;IAE5B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC;IAC1C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEvD,IAAI,WAA4B,CAAC;IACjC,IAAI,KAAyB,CAAC;IAC9B,IAAI,aAAa,GAAmB,EAAE,CAAC;IACvC,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,6DAA6D;IAC7D,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IAC7D,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,uBAAuB,GAAG,IAAI,CAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;IAEtF,IAAI,CAAC;QACH,oCAAoC;QACpC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,qEAAqE;QACrE,kEAAkE;QAClE,oFAAoF;QACpF,MAAM,MAAM,GAAgB;YAC1B,KAAK;YACL,GAAG;YACH,IAAI,EAAE,IAAI,EAAE,wDAAwD;YACpE,gBAAgB,EAAE,IAAI,EAAE,cAAc;YACtC,WAAW,EAAE,IAAI;YACjB,aAAa,EAAE,IAAI,CAAC,MAAM,KAAK,IAAI;YACnC,YAAY,EAAE,IAAI;YAClB,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;QAEhC,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC;YAC5B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,CAAC,KAAiB,EAAE,EAAE;gBAC7B,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBAC/B,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBACxG,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBACxC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC3C,IAAI,OAAO,EAAE,CAAC;wBACZ,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;wBAC9B,OAAO,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;wBAChC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC9B,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAChC,CAAC;gBACH,CAAC;gBACD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;SACF,CAAC,CAAC;QAEH,sEAAsE;QACtE,uCAAuC;QACvC,MAAM,GAAG,GAAkB;YACzB,GAAG;YACH,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;YACpB,MAAM,EAAE,WAAW;YACnB,eAAe;SAChB,CAAC;QAEF,MAAM,GAAG,IAAI,CAAC;QACd,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACtB,IAAI,CAAC,CAAC,CAAC,MAAM;oBAAE,MAAM,GAAG,KAAK,CAAC;YAChC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACjF,MAAM,GAAG,KAAK,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC;YAAS,CAAC;QACT,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,gBAAgB,KAAK,SAAS;gBAAE,OAAO,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;;gBAC1E,OAAO,CAAC,GAAG,CAAC,uBAAuB,GAAG,gBAAgB,CAAC;QAC9D,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,IAAI,KAAK,EAAE,CAAC;QACV,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;YACpB,MAAM,EAAE,KAAK;YACb,aAAa;YACb,WAAW,EAAE,WAAW,IAAI,gBAAgB,EAAE;YAC9C,eAAe;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC9B,GAAG;YACH,KAAK;SACN,CAAC;IACJ,CAAC;IAED,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;QACpB,MAAM;QACN,aAAa;QACb,WAAW;QACX,eAAe;QACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;QAC9B,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAgC,EAChC,KAAkB,EAClB,UAA0B,EAAE;IAE5B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACnD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,MAAM,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;YACjC,YAAY,IAAI,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;IACvC,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAE1E,OAAO;QACL,KAAK;QACL,SAAS;QACT,UAAU;QACV,OAAO;QACP,OAAO,EAAE;YACP,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM;YACN,MAAM;YACN,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;YAC5D,eAAe;YACf,YAAY;SACb;KACF,CAAC;AACJ,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,YAAY,CAAC,QAAiB;IAC5C,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9C,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,oBAAoB,QAAQ,4EAA4E,CACzG,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,QAAmC,EAAE,KAAK,EAAE,CAAC;IAClE,CAAC;IACD,OAAO,2BAA2B,EAAE,CAAC;AACvC,CAAC;AAED,6CAA6C;AAC7C,OAAO,EAAE,mBAAmB,EAAE,CAAC;AAE/B;;;;GAIG;AACH,SAAS,2BAA2B;IAClC,2DAA2D;IAC3D,4EAA4E;IAC5E,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACpD,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,UAAU,EAAE,CAAC;IACtF,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACjD,IAAI,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,mCAAmC,CAAC,CAAC;QAC/F,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,CAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC9D,CAAC;IACD,yEAAyE;IACzE,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,IAAI,0BAA0B,EAAE,CAAC;IACnH,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;QAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,aAAa,EAAE,CAAC;IAChG,IAAI,OAAO,CAAC,GAAG,CAAC,4BAA4B;QAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,sBAAsB,EAAE,CAAC;IACvH,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,IAAI,4BAA4B,EAAE,CAAC;IACvH,MAAM,IAAI,KAAK,CACb,uGAAuG;QACrG,8GAA8G,CACjH,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,IAAoB;IACpC,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB;IACvB,OAAO;QACL,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,UAAU,EAAE,CAAC;QACb,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC;QACV,SAAS,EAAE,KAAK;QAChB,QAAQ,EAAE,EAAE;KACb,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAc1D;;;GAGG;AACH,eAAO,MAAM,KAAK,EAAE,SAAS,cAAc,EAgB1C,CAAC;AAEF,kDAAkD;AAClD,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAE/D"}
|
package/dist/tasks/index.js
CHANGED
|
@@ -8,6 +8,7 @@ import editRenameSymbol from './edit.rename-symbol.js';
|
|
|
8
8
|
import multiRunAndReport from './multi.run-and-report.js';
|
|
9
9
|
import multiRefactorMove from './multi.refactor-move.js';
|
|
10
10
|
import multiSummarize from './multi.summarize.js';
|
|
11
|
+
import memoryRecallRelevant from './memory.recall-relevant.js';
|
|
11
12
|
/**
|
|
12
13
|
* The full task catalog. Order is the default run order. Keep grouped by
|
|
13
14
|
* category so the report reads naturally.
|
|
@@ -26,6 +27,8 @@ export const TASKS = [
|
|
|
26
27
|
multiRunAndReport,
|
|
27
28
|
multiRefactorMove,
|
|
28
29
|
multiSummarize,
|
|
30
|
+
// memory (appended so the indexes of the tasks above stay stable)
|
|
31
|
+
memoryRecallRelevant,
|
|
29
32
|
];
|
|
30
33
|
/** Lookup by id; returns undefined if missing. */
|
|
31
34
|
export function findTask(id) {
|
package/dist/tasks/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":"AAEA,OAAO,aAAa,MAAM,sBAAsB,CAAC;AACjD,OAAO,kBAAkB,MAAM,2BAA2B,CAAC;AAC3D,OAAO,eAAe,MAAM,wBAAwB,CAAC;AACrD,OAAO,cAAc,MAAM,uBAAuB,CAAC;AACnD,OAAO,mBAAmB,MAAM,4BAA4B,CAAC;AAC7D,OAAO,iBAAiB,MAAM,0BAA0B,CAAC;AACzD,OAAO,gBAAgB,MAAM,yBAAyB,CAAC;AACvD,OAAO,iBAAiB,MAAM,2BAA2B,CAAC;AAC1D,OAAO,iBAAiB,MAAM,0BAA0B,CAAC;AACzD,OAAO,cAAc,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":"AAEA,OAAO,aAAa,MAAM,sBAAsB,CAAC;AACjD,OAAO,kBAAkB,MAAM,2BAA2B,CAAC;AAC3D,OAAO,eAAe,MAAM,wBAAwB,CAAC;AACrD,OAAO,cAAc,MAAM,uBAAuB,CAAC;AACnD,OAAO,mBAAmB,MAAM,4BAA4B,CAAC;AAC7D,OAAO,iBAAiB,MAAM,0BAA0B,CAAC;AACzD,OAAO,gBAAgB,MAAM,yBAAyB,CAAC;AACvD,OAAO,iBAAiB,MAAM,2BAA2B,CAAC;AAC1D,OAAO,iBAAiB,MAAM,0BAA0B,CAAC;AACzD,OAAO,cAAc,MAAM,sBAAsB,CAAC;AAClD,OAAO,oBAAoB,MAAM,6BAA6B,CAAC;AAE/D;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAA8B;IAC9C,OAAO;IACP,aAAa;IACb,kBAAkB;IAClB,eAAe;IACf,OAAO;IACP,cAAc;IACd,mBAAmB;IACnB,iBAAiB;IACjB,gBAAgB;IAChB,aAAa;IACb,iBAAiB;IACjB,iBAAiB;IACjB,cAAc;IACd,kEAAkE;IAClE,oBAAoB;CACrB,CAAC;AAEF,kDAAkD;AAClD,MAAM,UAAU,QAAQ,CAAC,EAAU;IACjC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AACxC,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* memory / recall-relevant: the project's memory is larger than the prompt
|
|
3
|
+
* budget, so only an index plus the entries relevant to the question reach the
|
|
4
|
+
* model. The answer lives in one `correction` section ("use pnpm, never npm").
|
|
5
|
+
* Another section names a file that no longer exists, so it is injected with
|
|
6
|
+
* the "possibly stale" marker. Opts into memory (the only task that does).
|
|
7
|
+
*/
|
|
8
|
+
declare const _default: {
|
|
9
|
+
id: string;
|
|
10
|
+
name: string;
|
|
11
|
+
category: "read";
|
|
12
|
+
description: string;
|
|
13
|
+
task: string;
|
|
14
|
+
memory: true;
|
|
15
|
+
setup: (cwd: string) => Promise<void>;
|
|
16
|
+
graders: import("../index.js").Grader[];
|
|
17
|
+
};
|
|
18
|
+
export default _default;
|
|
19
|
+
//# sourceMappingURL=memory.recall-relevant.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.recall-relevant.d.ts","sourceRoot":"","sources":["../../src/tasks/memory.recall-relevant.ts"],"names":[],"mappings":"AAOA;;;;;;GAMG;;;;;;;;;;;AACH,wBAuBG"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { defineTask, regex, toolNotCalled, completed } from '../graders/index.js';
|
|
4
|
+
const FILLER_SENTENCE = 'Notes about this subsystem that have nothing to do with how anything is built or run. ';
|
|
5
|
+
/**
|
|
6
|
+
* memory / recall-relevant: the project's memory is larger than the prompt
|
|
7
|
+
* budget, so only an index plus the entries relevant to the question reach the
|
|
8
|
+
* model. The answer lives in one `correction` section ("use pnpm, never npm").
|
|
9
|
+
* Another section names a file that no longer exists, so it is injected with
|
|
10
|
+
* the "possibly stale" marker. Opts into memory (the only task that does).
|
|
11
|
+
*/
|
|
12
|
+
export default defineTask({
|
|
13
|
+
id: 'memory-recall-relevant',
|
|
14
|
+
name: 'Recall the right memory entry',
|
|
15
|
+
category: 'read',
|
|
16
|
+
description: 'Agent answers from project memory: pnpm correction wins; the stale entry is flagged, not trusted.',
|
|
17
|
+
task: 'Which command runs the tests in this project? Reply with just the command, nothing else.',
|
|
18
|
+
memory: true,
|
|
19
|
+
setup: async (cwd) => {
|
|
20
|
+
await mkdir(join(cwd, '.forge'), { recursive: true });
|
|
21
|
+
await writeFile(join(cwd, 'package.json'), JSON.stringify({ name: 'sample', version: '1.0.0' }, null, 2));
|
|
22
|
+
const filler = Array.from({ length: 10 }, (_, i) => {
|
|
23
|
+
const n = String(i + 1).padStart(2, '0');
|
|
24
|
+
return `## Subsystem ${n}\n\n<!-- forge: type=convention, updated=2026-09-01 -->\n${FILLER_SENTENCE.repeat(10)}`;
|
|
25
|
+
});
|
|
26
|
+
const memory = [
|
|
27
|
+
'# Project Memory',
|
|
28
|
+
...filler,
|
|
29
|
+
'## Package manager\n\n<!-- forge: type=correction, updated=2026-10-01, refs=package.json -->\nUse pnpm, never npm. Run `pnpm test` to run the tests in this workspace.',
|
|
30
|
+
'## Legacy API\n\n<!-- forge: type=decision, updated=2026-08-01, refs=src/legacy/api.ts, pin=true -->\nNever call the legacy API directly; go through the gateway.',
|
|
31
|
+
].join('\n\n');
|
|
32
|
+
await writeFile(join(cwd, '.forge', 'MEMORY.md'), `${memory}\n`);
|
|
33
|
+
},
|
|
34
|
+
graders: [completed(), regex(/pnpm test/), toolNotCalled('run_command')],
|
|
35
|
+
});
|
|
36
|
+
//# sourceMappingURL=memory.recall-relevant.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.recall-relevant.js","sourceRoot":"","sources":["../../src/tasks/memory.recall-relevant.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,MAAM,eAAe,GAAG,wFAAwF,CAAC;AAEjH;;;;;;GAMG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,wBAAwB;IAC5B,IAAI,EAAE,+BAA+B;IACrC,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,mGAAmG;IAChH,IAAI,EAAE,0FAA0F;IAChG,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1G,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACjD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACzC,OAAO,gBAAgB,CAAC,4DAA4D,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACnH,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG;YACb,kBAAkB;YAClB,GAAG,MAAM;YACT,wKAAwK;YACxK,mKAAmK;SACpK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACf,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,CAAC,WAAW,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,CAAC;CACzE,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ikarmic/forge-eval",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.5.0",
|
|
4
4
|
"description": "Eval harness for IKarmic AI Forge. Runs agent tasks against isolated workspaces and grades the results with deterministic checks.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
"dependencies": {
|
|
43
43
|
"ai": "^7.0.0",
|
|
44
44
|
"zod": "^3.23.8",
|
|
45
|
-
"@ikarmic/forge-core": "4.
|
|
45
|
+
"@ikarmic/forge-core": "4.5.0"
|
|
46
46
|
},
|
|
47
47
|
"devDependencies": {
|
|
48
48
|
"@types/node": "^22.7.0",
|