jiva-core 0.3.47 → 0.3.48-dev.0fd3c65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -1
- package/dist/code/agent.d.ts +13 -0
- package/dist/code/agent.d.ts.map +1 -1
- package/dist/code/agent.js +81 -7
- package/dist/code/agent.js.map +1 -1
- package/dist/code/benchmark/agent-factory.d.ts +18 -0
- package/dist/code/benchmark/agent-factory.d.ts.map +1 -0
- package/dist/code/benchmark/agent-factory.js +21 -0
- package/dist/code/benchmark/agent-factory.js.map +1 -0
- package/dist/code/benchmark/fixtures.d.ts +28 -0
- package/dist/code/benchmark/fixtures.d.ts.map +1 -0
- package/dist/code/benchmark/fixtures.js +177 -0
- package/dist/code/benchmark/fixtures.js.map +1 -0
- package/dist/code/benchmark/index.d.ts +16 -0
- package/dist/code/benchmark/index.d.ts.map +1 -0
- package/dist/code/benchmark/index.js +14 -0
- package/dist/code/benchmark/index.js.map +1 -0
- package/dist/code/benchmark/microcrm/assets/api.test.mjs +254 -0
- package/dist/code/benchmark/microcrm/assets/app.full.reference.mjs +267 -0
- package/dist/code/benchmark/microcrm/assets/app.reference.mjs +191 -0
- package/dist/code/benchmark/microcrm/assets/bulk.test.mjs +63 -0
- package/dist/code/benchmark/microcrm/assets/idempotency.test.mjs +64 -0
- package/dist/code/benchmark/microcrm/assets/query.test.mjs +96 -0
- package/dist/code/benchmark/microcrm/assets/stats.test.mjs +67 -0
- package/dist/code/benchmark/microcrm/index.d.ts +23 -0
- package/dist/code/benchmark/microcrm/index.d.ts.map +1 -0
- package/dist/code/benchmark/microcrm/index.js +72 -0
- package/dist/code/benchmark/microcrm/index.js.map +1 -0
- package/dist/code/benchmark/microcrm/spec.d.ts +13 -0
- package/dist/code/benchmark/microcrm/spec.d.ts.map +1 -0
- package/dist/code/benchmark/microcrm/spec.js +115 -0
- package/dist/code/benchmark/microcrm/spec.js.map +1 -0
- package/dist/code/benchmark/orchestrator-factory.d.ts +36 -0
- package/dist/code/benchmark/orchestrator-factory.d.ts.map +1 -0
- package/dist/code/benchmark/orchestrator-factory.js +57 -0
- package/dist/code/benchmark/orchestrator-factory.js.map +1 -0
- package/dist/code/benchmark/report.d.ts +8 -0
- package/dist/code/benchmark/report.d.ts.map +1 -0
- package/dist/code/benchmark/report.js +103 -0
- package/dist/code/benchmark/report.js.map +1 -0
- package/dist/code/benchmark/runner.d.ts +29 -0
- package/dist/code/benchmark/runner.d.ts.map +1 -0
- package/dist/code/benchmark/runner.js +219 -0
- package/dist/code/benchmark/runner.js.map +1 -0
- package/dist/code/benchmark/suites.d.ts +19 -0
- package/dist/code/benchmark/suites.d.ts.map +1 -0
- package/dist/code/benchmark/suites.js +48 -0
- package/dist/code/benchmark/suites.js.map +1 -0
- package/dist/code/benchmark/tasks.d.ts +16 -0
- package/dist/code/benchmark/tasks.d.ts.map +1 -0
- package/dist/code/benchmark/tasks.js +153 -0
- package/dist/code/benchmark/tasks.js.map +1 -0
- package/dist/code/benchmark/tests.d.ts +17 -0
- package/dist/code/benchmark/tests.d.ts.map +1 -0
- package/dist/code/benchmark/tests.js +234 -0
- package/dist/code/benchmark/tests.js.map +1 -0
- package/dist/code/benchmark/types.d.ts +161 -0
- package/dist/code/benchmark/types.d.ts.map +1 -0
- package/dist/code/benchmark/types.js +11 -0
- package/dist/code/benchmark/types.js.map +1 -0
- package/dist/code/benchmark/verify.d.ts +21 -0
- package/dist/code/benchmark/verify.d.ts.map +1 -0
- package/dist/code/benchmark/verify.js +108 -0
- package/dist/code/benchmark/verify.js.map +1 -0
- package/dist/code/tools/edit.d.ts.map +1 -1
- package/dist/code/tools/edit.js +12 -3
- package/dist/code/tools/edit.js.map +1 -1
- package/dist/code/tools/write.d.ts.map +1 -1
- package/dist/code/tools/write.js +13 -3
- package/dist/code/tools/write.js.map +1 -1
- package/dist/core/agent-interface.d.ts +5 -0
- package/dist/core/agent-interface.d.ts.map +1 -1
- package/dist/interfaces/cli/index.js +125 -0
- package/dist/interfaces/cli/index.js.map +1 -1
- package/dist/interfaces/cli/repl.d.ts.map +1 -1
- package/dist/interfaces/cli/repl.js +41 -15
- package/dist/interfaces/cli/repl.js.map +1 -1
- package/dist/interfaces/cli/setup-wizard.d.ts.map +1 -1
- package/dist/interfaces/cli/setup-wizard.js +2 -1
- package/dist/interfaces/cli/setup-wizard.js.map +1 -1
- package/dist/interfaces/http/index.d.ts.map +1 -1
- package/dist/interfaces/http/index.js +2 -0
- package/dist/interfaces/http/index.js.map +1 -1
- package/dist/interfaces/http/routes/benchmark.d.ts +14 -0
- package/dist/interfaces/http/routes/benchmark.d.ts.map +1 -0
- package/dist/interfaces/http/routes/benchmark.js +130 -0
- package/dist/interfaces/http/routes/benchmark.js.map +1 -0
- package/dist/models/model-client.d.ts +2 -1
- package/dist/models/model-client.d.ts.map +1 -1
- package/dist/models/model-client.js.map +1 -1
- package/package.json +2 -2
- package/scripts/bench-selftest.mjs +112 -0
- package/scripts/copy-benchmark-assets.mjs +17 -0
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical "taskstore" library fixtures.
|
|
3
|
+
*
|
|
4
|
+
* A single evolving Node library is the substrate for the whole benchmark. Each
|
|
5
|
+
* tier's scaffold is the canonical (golden) state of the prior tiers plus the
|
|
6
|
+
* new failing test(s). These helpers are the single source of truth for that
|
|
7
|
+
* golden state, so the suite stays DRY and the tiers genuinely build on one
|
|
8
|
+
* another.
|
|
9
|
+
*
|
|
10
|
+
* Layout in the workspace:
|
|
11
|
+
* package.json { "type": "module" }
|
|
12
|
+
* src/index.js public entry (single file for tiers 1-4, re-export after)
|
|
13
|
+
* src/model.js mutation functions (after the tier-5 refactor)
|
|
14
|
+
* src/query.js read functions (after the tier-5 refactor)
|
|
15
|
+
* src/storage.js serialize/deserialize (after tier 7)
|
|
16
|
+
* test/tNN.test.mjs cumulative node:test files
|
|
17
|
+
*/
|
|
18
|
+
export const PACKAGE_JSON = JSON.stringify({ name: 'taskstore', type: 'module', private: true }, null, 2) + '\n';
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
// Function source snippets (correct + buggy variants)
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
const createTaskFn = `export function createTask(title) {
|
|
23
|
+
if (typeof title !== 'string' || title.trim() === '') {
|
|
24
|
+
throw new Error('title is required');
|
|
25
|
+
}
|
|
26
|
+
return { title, done: false, priority: 'normal', due: null };
|
|
27
|
+
}`;
|
|
28
|
+
const addTaskFn = `export function addTask(list, title) {
|
|
29
|
+
const task = createTask(title);
|
|
30
|
+
const nextId = list.reduce((max, t) => Math.max(max, t.id || 0), 0) + 1;
|
|
31
|
+
return [...list, { ...task, id: nextId }];
|
|
32
|
+
}`;
|
|
33
|
+
/** Tier-8 planted bug: length-based id collides after a removal. */
|
|
34
|
+
const addTaskBuggyFn = `export function addTask(list, title) {
|
|
35
|
+
const task = createTask(title);
|
|
36
|
+
// BUG: uses list length for the id, which collides after a removal.
|
|
37
|
+
const nextId = list.length + 1;
|
|
38
|
+
return [...list, { ...task, id: nextId }];
|
|
39
|
+
}`;
|
|
40
|
+
const removeTaskFn = `export function removeTask(list, id) {
|
|
41
|
+
return list.filter((t) => t.id !== id);
|
|
42
|
+
}`;
|
|
43
|
+
const listTasksFn = `export function listTasks(list) {
|
|
44
|
+
return [...list];
|
|
45
|
+
}`;
|
|
46
|
+
const toggleTaskFn = `export function toggleTask(list, id) {
|
|
47
|
+
return list.map((t) => (t.id === id ? { ...t, done: !t.done } : t));
|
|
48
|
+
}`;
|
|
49
|
+
/** Tier-3 planted bug: treats id as an array index. */
|
|
50
|
+
const toggleTaskBuggyFn = `export function toggleTask(list, id) {
|
|
51
|
+
// BUG: treats id as an array index instead of matching task.id.
|
|
52
|
+
return list.map((t, index) => (index === id ? { ...t, done: !t.done } : t));
|
|
53
|
+
}`;
|
|
54
|
+
const setPriorityFn = `const PRIORITIES = ['low', 'normal', 'high'];
|
|
55
|
+
export function setPriority(list, id, priority) {
|
|
56
|
+
if (!PRIORITIES.includes(priority)) {
|
|
57
|
+
throw new Error('invalid priority: ' + priority);
|
|
58
|
+
}
|
|
59
|
+
return list.map((t) => (t.id === id ? { ...t, priority } : t));
|
|
60
|
+
}`;
|
|
61
|
+
const tasksByPriorityFn = `export function tasksByPriority(list, priority) {
|
|
62
|
+
return list.filter((t) => t.priority === priority);
|
|
63
|
+
}`;
|
|
64
|
+
const sortTasksFn = `export function sortTasks(list, options = {}) {
|
|
65
|
+
const by = options.by || 'due';
|
|
66
|
+
if (by !== 'due') throw new Error('unsupported sort key: ' + by);
|
|
67
|
+
// Array.prototype.sort is stable in Node, so equal keys keep their order.
|
|
68
|
+
return [...list].sort((a, b) => {
|
|
69
|
+
const av = a.due ?? null;
|
|
70
|
+
const bv = b.due ?? null;
|
|
71
|
+
if (av === null && bv === null) return 0;
|
|
72
|
+
if (av === null) return 1; // tasks without a due date sort last
|
|
73
|
+
if (bv === null) return -1;
|
|
74
|
+
if (av < bv) return -1;
|
|
75
|
+
if (av > bv) return 1;
|
|
76
|
+
return 0;
|
|
77
|
+
});
|
|
78
|
+
}`;
|
|
79
|
+
const storageFns = `export function serialize(list) {
|
|
80
|
+
return JSON.stringify(list);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function deserialize(str) {
|
|
84
|
+
const data = JSON.parse(str);
|
|
85
|
+
if (!Array.isArray(data)) throw new Error('expected an array');
|
|
86
|
+
return data;
|
|
87
|
+
}`;
|
|
88
|
+
// ---------------------------------------------------------------------------
|
|
89
|
+
// Golden src builders
|
|
90
|
+
// ---------------------------------------------------------------------------
|
|
91
|
+
/** Single-file `src/index.js` containing every function up to `tier` (<=4). */
|
|
92
|
+
function singleFileSrc(tier, opts = {}) {
|
|
93
|
+
const parts = [createTaskFn];
|
|
94
|
+
if (tier >= 2)
|
|
95
|
+
parts.push(addTaskFn, removeTaskFn, listTasksFn);
|
|
96
|
+
if (opts.buggyToggle)
|
|
97
|
+
parts.push(toggleTaskBuggyFn);
|
|
98
|
+
else if (tier >= 3)
|
|
99
|
+
parts.push(toggleTaskFn);
|
|
100
|
+
if (tier >= 4)
|
|
101
|
+
parts.push(setPriorityFn, tasksByPriorityFn);
|
|
102
|
+
return { 'src/index.js': parts.join('\n\n') + '\n' };
|
|
103
|
+
}
|
|
104
|
+
/** Multi-file layout (after the tier-5 refactor). */
|
|
105
|
+
function multiFileSrc(opts = {}) {
|
|
106
|
+
const model = [
|
|
107
|
+
createTaskFn,
|
|
108
|
+
opts.buggyAdd ? addTaskBuggyFn : addTaskFn,
|
|
109
|
+
removeTaskFn,
|
|
110
|
+
toggleTaskFn,
|
|
111
|
+
setPriorityFn,
|
|
112
|
+
].join('\n\n') + '\n';
|
|
113
|
+
const query = [listTasksFn, tasksByPriorityFn, ...(opts.sort ? [sortTasksFn] : [])].join('\n\n') + '\n';
|
|
114
|
+
const reexports = [`export * from './model.js';`, `export * from './query.js';`];
|
|
115
|
+
if (opts.storage)
|
|
116
|
+
reexports.push(`export * from './storage.js';`);
|
|
117
|
+
const files = {
|
|
118
|
+
'src/model.js': model,
|
|
119
|
+
'src/query.js': query,
|
|
120
|
+
'src/index.js': reexports.join('\n') + '\n',
|
|
121
|
+
};
|
|
122
|
+
if (opts.storage)
|
|
123
|
+
files['src/storage.js'] = storageFns + '\n';
|
|
124
|
+
return files;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Canonical golden source state *after* a tier is completed. Used both as the
|
|
128
|
+
* baseline scaffold for the next tier and (by the harness self-test) as a
|
|
129
|
+
* "known-good" set of files a mock agent can write to prove the verifier works.
|
|
130
|
+
*/
|
|
131
|
+
export function goldenSrcAfter(tier) {
|
|
132
|
+
switch (tier) {
|
|
133
|
+
case 0:
|
|
134
|
+
return {};
|
|
135
|
+
case 1:
|
|
136
|
+
return singleFileSrc(1);
|
|
137
|
+
case 2:
|
|
138
|
+
return singleFileSrc(2);
|
|
139
|
+
case 3:
|
|
140
|
+
return singleFileSrc(3);
|
|
141
|
+
case 4:
|
|
142
|
+
return singleFileSrc(4);
|
|
143
|
+
case 5:
|
|
144
|
+
return multiFileSrc();
|
|
145
|
+
case 6:
|
|
146
|
+
return multiFileSrc({ sort: true });
|
|
147
|
+
case 7:
|
|
148
|
+
case 8:
|
|
149
|
+
return multiFileSrc({ sort: true, storage: true });
|
|
150
|
+
default:
|
|
151
|
+
throw new Error(`no golden state for tier ${tier}`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
/** The scaffolded (pre-agent) src state for a given tier — golden(N-1) plus any planted bug. */
|
|
155
|
+
export function scaffoldSrc(tier) {
|
|
156
|
+
switch (tier) {
|
|
157
|
+
case 1:
|
|
158
|
+
return {}; // nothing yet — the agent creates src/index.js from scratch
|
|
159
|
+
case 2:
|
|
160
|
+
return singleFileSrc(1);
|
|
161
|
+
case 3:
|
|
162
|
+
return singleFileSrc(2, { buggyToggle: true }); // planted toggle bug
|
|
163
|
+
case 4:
|
|
164
|
+
return singleFileSrc(3);
|
|
165
|
+
case 5:
|
|
166
|
+
return singleFileSrc(4);
|
|
167
|
+
case 6:
|
|
168
|
+
return multiFileSrc();
|
|
169
|
+
case 7:
|
|
170
|
+
return multiFileSrc({ sort: true });
|
|
171
|
+
case 8:
|
|
172
|
+
return multiFileSrc({ sort: true, storage: true, buggyAdd: true }); // planted add bug
|
|
173
|
+
default:
|
|
174
|
+
throw new Error(`no scaffold for tier ${tier}`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
//# sourceMappingURL=fixtures.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../../../src/code/benchmark/fixtures.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAEjH,8EAA8E;AAC9E,sDAAsD;AACtD,8EAA8E;AAE9E,MAAM,YAAY,GAAG;;;;;EAKnB,CAAC;AAEH,MAAM,SAAS,GAAG;;;;EAIhB,CAAC;AAEH,oEAAoE;AACpE,MAAM,cAAc,GAAG;;;;;EAKrB,CAAC;AAEH,MAAM,YAAY,GAAG;;EAEnB,CAAC;AAEH,MAAM,WAAW,GAAG;;EAElB,CAAC;AAEH,MAAM,YAAY,GAAG;;EAEnB,CAAC;AAEH,uDAAuD;AACvD,MAAM,iBAAiB,GAAG;;;EAGxB,CAAC;AAEH,MAAM,aAAa,GAAG;;;;;;EAMpB,CAAC;AAEH,MAAM,iBAAiB,GAAG;;EAExB,CAAC;AAEH,MAAM,WAAW,GAAG;;;;;;;;;;;;;;EAclB,CAAC;AAEH,MAAM,UAAU,GAAG;;;;;;;;EAQjB,CAAC;AAEH,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,+EAA+E;AAC/E,SAAS,aAAa,CAAC,IAAY,EAAE,OAAkC,EAAE;IACvE,MAAM,KAAK,GAAG,CAAC,YAAY,CAAC,CAAC;IAC7B,IAAI,IAAI,IAAI,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IAChE,IAAI,IAAI,CAAC,WAAW;QAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;SAC/C,IAAI,IAAI,IAAI,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC7C,IAAI,IAAI,IAAI,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAC5D,OAAO,EAAE,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;AACvD,CAAC;AAED,qDAAqD;AACrD,SAAS,YAAY,CAAC,OAAkE,EAAE;IACxF,MAAM,KAAK,GAAG;QACZ,YAAY;QACZ,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;QAC1C,YAAY;QACZ,YAAY;QACZ,aAAa;KACd,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAEtB,MAAM,KAAK,GAAG,CAAC,WAAW,EAAE,iBAAiB,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAExG,MAAM,SAAS,GAAG,CAAC,6BAA6B,EAAE,6BAA6B,CAAC,CAAC;IACjF,IAAI,IAAI,CAAC,OAAO;QAAE,SAAS,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAElE,MAAM,KAAK,GAAa;QACtB,cAAc,EAAE,KAAK;QACrB,cAAc,EAAE,KAAK;QACrB,cAAc,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI;KAC5C,CAAC;IACF,IAAI,IAAI,CAAC,OAAO;QAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,UAAU,GAAG,IAAI,CAAC;IAC9D,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,CAAC;YACJ,OAAO,EAAE,CAAC;QACZ,KAAK,CAAC;YACJ,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,CAAC;YACJ,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,CAAC;YACJ,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,CAAC;YACJ,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,CAAC;YACJ,OAAO,YAAY,EAAE,CAAC;QACxB,KAAK,CAAC;YACJ,OAAO,YAAY,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,KAAK,CAAC,CAAC;QACP,KAAK,CAAC;YACJ,OAAO,YAAY,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD;YACE,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,gGAAgG;AAChG,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,CAAC;YACJ,OAAO,EAAE,CAAC,CAAC,4DAA4D;QACzE,KAAK,CAAC;YACJ,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,CAAC;YACJ,OAAO,aAAa,CAAC,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,qBAAqB;QACvE,KAAK,CAAC;YACJ,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,CAAC;YACJ,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,CAAC;YACJ,OAAO,YAAY,EAAE,CAAC;QACxB,KAAK,CAAC;YACJ,OAAO,YAAY,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,KAAK,CAAC;YACJ,OAAO,YAAY,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,kBAAkB;QACxF;YACE,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Code-mode benchmark suite — public entry point.
|
|
3
|
+
*
|
|
4
|
+
* Drives CodeAgent through a TDD-style suite of increasing complexity in
|
|
5
|
+
* isolated workspaces, scoring each task deterministically with Node's built-in
|
|
6
|
+
* test runner. Invokable from the CLI (`jiva benchmark`) and the HTTP API
|
|
7
|
+
* (`/api/benchmark/*`).
|
|
8
|
+
*/
|
|
9
|
+
export { BENCHMARK_TASKS, listTaskMetadata, selectTasks } from './tasks.js';
|
|
10
|
+
export { BENCHMARK_SUITES, DEFAULT_SUITE_ID, getSuite, listSuiteMetadata } from './suites.js';
|
|
11
|
+
export { runBenchmark, BenchmarkRunner } from './runner.js';
|
|
12
|
+
export { createOrchestrator } from './orchestrator-factory.js';
|
|
13
|
+
export type { OrchestratorInput, OrchestratorBundle, ModelConfigInput } from './orchestrator-factory.js';
|
|
14
|
+
export { formatReportForCLI, toReportJSON } from './report.js';
|
|
15
|
+
export type { BenchmarkTask, BenchmarkSuite, ScoringMode, TaskResult, SuiteResult, RunnerOptions, ProgressEvents, VerifyResult, } from './types.js';
|
|
16
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/code/benchmark/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC5E,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAC9F,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,YAAY,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AACzG,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC/D,YAAY,EACV,aAAa,EACb,cAAc,EACd,WAAW,EACX,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,YAAY,GACb,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Code-mode benchmark suite — public entry point.
|
|
3
|
+
*
|
|
4
|
+
* Drives CodeAgent through a TDD-style suite of increasing complexity in
|
|
5
|
+
* isolated workspaces, scoring each task deterministically with Node's built-in
|
|
6
|
+
* test runner. Invokable from the CLI (`jiva benchmark`) and the HTTP API
|
|
7
|
+
* (`/api/benchmark/*`).
|
|
8
|
+
*/
|
|
9
|
+
export { BENCHMARK_TASKS, listTaskMetadata, selectTasks } from './tasks.js';
|
|
10
|
+
export { BENCHMARK_SUITES, DEFAULT_SUITE_ID, getSuite, listSuiteMetadata } from './suites.js';
|
|
11
|
+
export { runBenchmark, BenchmarkRunner } from './runner.js';
|
|
12
|
+
export { createOrchestrator } from './orchestrator-factory.js';
|
|
13
|
+
export { formatReportForCLI, toReportJSON } from './report.js';
|
|
14
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/code/benchmark/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC5E,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAC9F,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { createApp } from '../src/app.js';
|
|
4
|
+
|
|
5
|
+
// Each test gets a fresh in-memory app on an ephemeral port, so tests are
|
|
6
|
+
// independent and a partial implementation still passes the subset it supports.
|
|
7
|
+
async function withServer(run) {
|
|
8
|
+
const server = createApp(':memory:');
|
|
9
|
+
await new Promise((res) => server.listen(0, res));
|
|
10
|
+
const { port } = server.address();
|
|
11
|
+
try {
|
|
12
|
+
await run(`http://127.0.0.1:${port}`);
|
|
13
|
+
} finally {
|
|
14
|
+
await new Promise((res) => server.close(res));
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
async function req(base, method, path, body, raw) {
|
|
19
|
+
const init = { method, headers: {} };
|
|
20
|
+
if (raw !== undefined) { init.headers['content-type'] = 'application/json'; init.body = raw; }
|
|
21
|
+
else if (body !== undefined) { init.headers['content-type'] = 'application/json'; init.body = JSON.stringify(body); }
|
|
22
|
+
const res = await fetch(base + path, init);
|
|
23
|
+
const text = await res.text();
|
|
24
|
+
let json = null;
|
|
25
|
+
try { json = text ? JSON.parse(text) : null; } catch { json = null; }
|
|
26
|
+
return { status: res.status, body: json };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const mkContact = (over = {}) => ({ name: 'Ada Lovelace', email: `ada${Math.random().toString(36).slice(2)}@example.com`, company: 'Analytical', ...over });
|
|
30
|
+
|
|
31
|
+
// ── Contacts: create & validation ─────────────────────────────────────────────
|
|
32
|
+
test('POST /contacts creates a contact (201) with id and created_at', () =>
|
|
33
|
+
withServer(async (base) => {
|
|
34
|
+
const c = mkContact();
|
|
35
|
+
const r = await req(base, 'POST', '/contacts', c);
|
|
36
|
+
assert.equal(r.status, 201);
|
|
37
|
+
assert.ok(Number.isInteger(r.body.id));
|
|
38
|
+
assert.equal(r.body.name, c.name);
|
|
39
|
+
assert.equal(r.body.email, c.email);
|
|
40
|
+
assert.ok(typeof r.body.created_at === 'string' && r.body.created_at.length > 0);
|
|
41
|
+
}));
|
|
42
|
+
|
|
43
|
+
test('POST /contacts without name → 400', () =>
|
|
44
|
+
withServer(async (base) => {
|
|
45
|
+
const r = await req(base, 'POST', '/contacts', { email: 'x@y.com' });
|
|
46
|
+
assert.equal(r.status, 400);
|
|
47
|
+
}));
|
|
48
|
+
|
|
49
|
+
test('POST /contacts without email → 400', () =>
|
|
50
|
+
withServer(async (base) => {
|
|
51
|
+
const r = await req(base, 'POST', '/contacts', { name: 'No Email' });
|
|
52
|
+
assert.equal(r.status, 400);
|
|
53
|
+
}));
|
|
54
|
+
|
|
55
|
+
test('POST /contacts with malformed email → 400', () =>
|
|
56
|
+
withServer(async (base) => {
|
|
57
|
+
const r = await req(base, 'POST', '/contacts', { name: 'Bad', email: 'not-an-email' });
|
|
58
|
+
assert.equal(r.status, 400);
|
|
59
|
+
}));
|
|
60
|
+
|
|
61
|
+
test('POST /contacts with duplicate email → 409', () =>
|
|
62
|
+
withServer(async (base) => {
|
|
63
|
+
const c = mkContact();
|
|
64
|
+
assert.equal((await req(base, 'POST', '/contacts', c)).status, 201);
|
|
65
|
+
const r = await req(base, 'POST', '/contacts', c);
|
|
66
|
+
assert.equal(r.status, 409);
|
|
67
|
+
}));
|
|
68
|
+
|
|
69
|
+
test('POST /contacts with malformed JSON body → 400', () =>
|
|
70
|
+
withServer(async (base) => {
|
|
71
|
+
const r = await req(base, 'POST', '/contacts', undefined, '{ not json');
|
|
72
|
+
assert.equal(r.status, 400);
|
|
73
|
+
}));
|
|
74
|
+
|
|
75
|
+
// ── Contacts: read ─────────────────────────────────────────────────────────────
|
|
76
|
+
test('GET /contacts/:id returns the contact (200)', () =>
|
|
77
|
+
withServer(async (base) => {
|
|
78
|
+
const created = await req(base, 'POST', '/contacts', mkContact());
|
|
79
|
+
const r = await req(base, 'GET', `/contacts/${created.body.id}`);
|
|
80
|
+
assert.equal(r.status, 200);
|
|
81
|
+
assert.equal(r.body.id, created.body.id);
|
|
82
|
+
}));
|
|
83
|
+
|
|
84
|
+
test('GET /contacts/:id unknown → 404', () =>
|
|
85
|
+
withServer(async (base) => {
|
|
86
|
+
const r = await req(base, 'GET', '/contacts/99999');
|
|
87
|
+
assert.equal(r.status, 404);
|
|
88
|
+
}));
|
|
89
|
+
|
|
90
|
+
test('GET /contacts returns { data, total }', () =>
|
|
91
|
+
withServer(async (base) => {
|
|
92
|
+
await req(base, 'POST', '/contacts', mkContact());
|
|
93
|
+
await req(base, 'POST', '/contacts', mkContact());
|
|
94
|
+
const r = await req(base, 'GET', '/contacts');
|
|
95
|
+
assert.equal(r.status, 200);
|
|
96
|
+
assert.ok(Array.isArray(r.body.data));
|
|
97
|
+
assert.equal(r.body.total, 2);
|
|
98
|
+
assert.equal(r.body.data.length, 2);
|
|
99
|
+
}));
|
|
100
|
+
|
|
101
|
+
test('GET /contacts?q= filters by name or email substring', () =>
|
|
102
|
+
withServer(async (base) => {
|
|
103
|
+
await req(base, 'POST', '/contacts', mkContact({ name: 'Grace Hopper', email: 'grace@navy.mil' }));
|
|
104
|
+
await req(base, 'POST', '/contacts', mkContact({ name: 'Alan Turing', email: 'alan@bletchley.uk' }));
|
|
105
|
+
const r = await req(base, 'GET', '/contacts?q=Hopper');
|
|
106
|
+
assert.equal(r.status, 200);
|
|
107
|
+
assert.equal(r.body.total, 1);
|
|
108
|
+
assert.equal(r.body.data[0].name, 'Grace Hopper');
|
|
109
|
+
}));
|
|
110
|
+
|
|
111
|
+
test('GET /contacts?limit=&offset= paginates while total reflects the full count', () =>
|
|
112
|
+
withServer(async (base) => {
|
|
113
|
+
for (let i = 0; i < 5; i++) await req(base, 'POST', '/contacts', mkContact());
|
|
114
|
+
const r = await req(base, 'GET', '/contacts?limit=2&offset=0');
|
|
115
|
+
assert.equal(r.status, 200);
|
|
116
|
+
assert.equal(r.body.data.length, 2);
|
|
117
|
+
assert.equal(r.body.total, 5);
|
|
118
|
+
}));
|
|
119
|
+
|
|
120
|
+
// ── Contacts: update & delete ──────────────────────────────────────────────────
|
|
121
|
+
test('PATCH /contacts/:id updates a field (200)', () =>
|
|
122
|
+
withServer(async (base) => {
|
|
123
|
+
const created = await req(base, 'POST', '/contacts', mkContact({ company: 'Old' }));
|
|
124
|
+
const r = await req(base, 'PATCH', `/contacts/${created.body.id}`, { company: 'New Co' });
|
|
125
|
+
assert.equal(r.status, 200);
|
|
126
|
+
assert.equal(r.body.company, 'New Co');
|
|
127
|
+
}));
|
|
128
|
+
|
|
129
|
+
test('PATCH /contacts/:id to an existing email → 409', () =>
|
|
130
|
+
withServer(async (base) => {
|
|
131
|
+
const a = await req(base, 'POST', '/contacts', mkContact());
|
|
132
|
+
const b = await req(base, 'POST', '/contacts', mkContact());
|
|
133
|
+
const r = await req(base, 'PATCH', `/contacts/${b.body.id}`, { email: a.body.email });
|
|
134
|
+
assert.equal(r.status, 409);
|
|
135
|
+
}));
|
|
136
|
+
|
|
137
|
+
test('PATCH /contacts/:id unknown → 404', () =>
|
|
138
|
+
withServer(async (base) => {
|
|
139
|
+
const r = await req(base, 'PATCH', '/contacts/99999', { company: 'x' });
|
|
140
|
+
assert.equal(r.status, 404);
|
|
141
|
+
}));
|
|
142
|
+
|
|
143
|
+
test('DELETE /contacts/:id → 204 then GET → 404', () =>
|
|
144
|
+
withServer(async (base) => {
|
|
145
|
+
const created = await req(base, 'POST', '/contacts', mkContact());
|
|
146
|
+
const del = await req(base, 'DELETE', `/contacts/${created.body.id}`);
|
|
147
|
+
assert.equal(del.status, 204);
|
|
148
|
+
const r = await req(base, 'GET', `/contacts/${created.body.id}`);
|
|
149
|
+
assert.equal(r.status, 404);
|
|
150
|
+
}));
|
|
151
|
+
|
|
152
|
+
test('DELETE /contacts/:id unknown → 404', () =>
|
|
153
|
+
withServer(async (base) => {
|
|
154
|
+
const r = await req(base, 'DELETE', '/contacts/99999');
|
|
155
|
+
assert.equal(r.status, 404);
|
|
156
|
+
}));
|
|
157
|
+
|
|
158
|
+
// ── Deals ──────────────────────────────────────────────────────────────────────
|
|
159
|
+
test('POST /contacts/:id/deals creates a deal (201) defaulting stage to "lead"', () =>
|
|
160
|
+
withServer(async (base) => {
|
|
161
|
+
const c = await req(base, 'POST', '/contacts', mkContact());
|
|
162
|
+
const r = await req(base, 'POST', `/contacts/${c.body.id}/deals`, { title: 'Big deal', amount: 1000 });
|
|
163
|
+
assert.equal(r.status, 201);
|
|
164
|
+
assert.equal(r.body.stage, 'lead');
|
|
165
|
+
assert.equal(r.body.amount, 1000);
|
|
166
|
+
assert.equal(r.body.contact_id, c.body.id);
|
|
167
|
+
}));
|
|
168
|
+
|
|
169
|
+
test('POST deal for unknown contact → 404', () =>
|
|
170
|
+
withServer(async (base) => {
|
|
171
|
+
const r = await req(base, 'POST', '/contacts/99999/deals', { title: 'x' });
|
|
172
|
+
assert.equal(r.status, 404);
|
|
173
|
+
}));
|
|
174
|
+
|
|
175
|
+
test('POST deal without title → 400', () =>
|
|
176
|
+
withServer(async (base) => {
|
|
177
|
+
const c = await req(base, 'POST', '/contacts', mkContact());
|
|
178
|
+
const r = await req(base, 'POST', `/contacts/${c.body.id}/deals`, { amount: 10 });
|
|
179
|
+
assert.equal(r.status, 400);
|
|
180
|
+
}));
|
|
181
|
+
|
|
182
|
+
test('POST deal with invalid stage → 400', () =>
|
|
183
|
+
withServer(async (base) => {
|
|
184
|
+
const c = await req(base, 'POST', '/contacts', mkContact());
|
|
185
|
+
const r = await req(base, 'POST', `/contacts/${c.body.id}/deals`, { title: 'x', stage: 'banana' });
|
|
186
|
+
assert.equal(r.status, 400);
|
|
187
|
+
}));
|
|
188
|
+
|
|
189
|
+
test('POST deal with negative amount → 400', () =>
|
|
190
|
+
withServer(async (base) => {
|
|
191
|
+
const c = await req(base, 'POST', '/contacts', mkContact());
|
|
192
|
+
const r = await req(base, 'POST', `/contacts/${c.body.id}/deals`, { title: 'x', amount: -5 });
|
|
193
|
+
assert.equal(r.status, 400);
|
|
194
|
+
}));
|
|
195
|
+
|
|
196
|
+
test('GET /contacts/:id/deals lists only that contact’s deals', () =>
|
|
197
|
+
withServer(async (base) => {
|
|
198
|
+
const a = await req(base, 'POST', '/contacts', mkContact());
|
|
199
|
+
const b = await req(base, 'POST', '/contacts', mkContact());
|
|
200
|
+
await req(base, 'POST', `/contacts/${a.body.id}/deals`, { title: 'A1' });
|
|
201
|
+
await req(base, 'POST', `/contacts/${a.body.id}/deals`, { title: 'A2' });
|
|
202
|
+
await req(base, 'POST', `/contacts/${b.body.id}/deals`, { title: 'B1' });
|
|
203
|
+
const r = await req(base, 'GET', `/contacts/${a.body.id}/deals`);
|
|
204
|
+
assert.equal(r.status, 200);
|
|
205
|
+
assert.equal(r.body.data.length, 2);
|
|
206
|
+
}));
|
|
207
|
+
|
|
208
|
+
test('GET /deals?stage= filters by stage', () =>
|
|
209
|
+
withServer(async (base) => {
|
|
210
|
+
const c = await req(base, 'POST', '/contacts', mkContact());
|
|
211
|
+
await req(base, 'POST', `/contacts/${c.body.id}/deals`, { title: 'won1', stage: 'won', amount: 100 });
|
|
212
|
+
await req(base, 'POST', `/contacts/${c.body.id}/deals`, { title: 'lead1', stage: 'lead' });
|
|
213
|
+
const r = await req(base, 'GET', '/deals?stage=won');
|
|
214
|
+
assert.equal(r.status, 200);
|
|
215
|
+
assert.equal(r.body.data.length, 1);
|
|
216
|
+
assert.equal(r.body.data[0].stage, 'won');
|
|
217
|
+
}));
|
|
218
|
+
|
|
219
|
+
test('PATCH /deals/:id updates the stage (200)', () =>
|
|
220
|
+
withServer(async (base) => {
|
|
221
|
+
const c = await req(base, 'POST', '/contacts', mkContact());
|
|
222
|
+
const d = await req(base, 'POST', `/contacts/${c.body.id}/deals`, { title: 'x', stage: 'lead' });
|
|
223
|
+
const r = await req(base, 'PATCH', `/deals/${d.body.id}`, { stage: 'won' });
|
|
224
|
+
assert.equal(r.status, 200);
|
|
225
|
+
assert.equal(r.body.stage, 'won');
|
|
226
|
+
}));
|
|
227
|
+
|
|
228
|
+
test('GET /deals/summary aggregates count and totalAmount per stage', () =>
|
|
229
|
+
withServer(async (base) => {
|
|
230
|
+
const c = await req(base, 'POST', '/contacts', mkContact());
|
|
231
|
+
await req(base, 'POST', `/contacts/${c.body.id}/deals`, { title: 'w1', stage: 'won', amount: 100 });
|
|
232
|
+
await req(base, 'POST', `/contacts/${c.body.id}/deals`, { title: 'w2', stage: 'won', amount: 250 });
|
|
233
|
+
const r = await req(base, 'GET', '/deals/summary');
|
|
234
|
+
assert.equal(r.status, 200);
|
|
235
|
+
assert.equal(r.body.summary.won.count, 2);
|
|
236
|
+
assert.equal(r.body.summary.won.totalAmount, 350);
|
|
237
|
+
assert.equal(r.body.summary.lead.count, 0);
|
|
238
|
+
}));
|
|
239
|
+
|
|
240
|
+
test('DELETE contact cascades to its deals', () =>
|
|
241
|
+
withServer(async (base) => {
|
|
242
|
+
const c = await req(base, 'POST', '/contacts', mkContact());
|
|
243
|
+
await req(base, 'POST', `/contacts/${c.body.id}/deals`, { title: 'd1', stage: 'won', amount: 10 });
|
|
244
|
+
await req(base, 'DELETE', `/contacts/${c.body.id}`);
|
|
245
|
+
const r = await req(base, 'GET', '/deals');
|
|
246
|
+
assert.equal(r.status, 200);
|
|
247
|
+
assert.equal(r.body.data.length, 0);
|
|
248
|
+
}));
|
|
249
|
+
|
|
250
|
+
test('unknown route → 404', () =>
|
|
251
|
+
withServer(async (base) => {
|
|
252
|
+
const r = await req(base, 'GET', '/nope');
|
|
253
|
+
assert.equal(r.status, 404);
|
|
254
|
+
}));
|