@forwardimpact/libharness 0.1.20 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -201
- package/README.md +196 -80
- package/bin/fit-benchmark.js +44 -0
- package/bin/fit-harness.js +358 -0
- package/bin/fit-selfedit.js +165 -0
- package/bin/fit-trace.js +510 -0
- package/package.json +42 -12
- package/src/agent-runner.js +256 -0
- package/src/benchmark/apm-installer.js +207 -0
- package/src/benchmark/env-loader.js +158 -0
- package/src/benchmark/hook-env.js +40 -0
- package/src/benchmark/invariants.js +141 -0
- package/src/benchmark/judge.js +187 -0
- package/src/benchmark/npm-installer.js +87 -0
- package/src/benchmark/report.js +522 -0
- package/src/benchmark/result.js +127 -0
- package/src/benchmark/runner.js +583 -0
- package/src/benchmark/task-family.js +260 -0
- package/src/benchmark/workdir.js +298 -0
- package/src/commands/assert.js +153 -0
- package/src/commands/benchmark-definition.js +165 -0
- package/src/commands/benchmark-invariants.js +73 -0
- package/src/commands/benchmark-report.js +51 -0
- package/src/commands/benchmark-run.js +111 -0
- package/src/commands/by-discussion.js +94 -0
- package/src/commands/callback.js +119 -0
- package/src/commands/discuss.js +132 -0
- package/src/commands/facilitate.js +123 -0
- package/src/commands/output.js +36 -0
- package/src/commands/run.js +152 -0
- package/src/commands/supervise.js +136 -0
- package/src/commands/task-input.js +54 -0
- package/src/commands/tee.js +53 -0
- package/src/commands/trace.js +630 -0
- package/src/commands/work-tracker.js +35 -0
- package/src/cost.js +79 -0
- package/src/discuss-tools.js +173 -0
- package/src/discusser.js +394 -0
- package/src/events/github.js +161 -0
- package/src/facilitator.js +205 -0
- package/src/inbox-poller.js +81 -0
- package/src/index.js +72 -2
- package/src/judge.js +210 -0
- package/src/message-bus.js +118 -0
- package/src/orchestration-loop.js +330 -0
- package/src/orchestration-toolkit.js +441 -0
- package/src/orchestrator-helpers.js +23 -0
- package/src/profile-prompt.js +266 -0
- package/src/redaction.js +253 -0
- package/src/render/line-renderer.js +54 -0
- package/src/render/orchestrator-filter.js +19 -0
- package/src/render/palette.js +63 -0
- package/src/render/tool-hints.js +154 -0
- package/src/render/turn-renderer.js +96 -0
- package/src/reply-emitter.js +47 -0
- package/src/sequence-counter.js +21 -0
- package/src/signature-filter.js +27 -0
- package/src/supervisor.js +236 -0
- package/src/tee-writer.js +150 -0
- package/src/trace-collector.js +444 -0
- package/src/trace-github.js +473 -0
- package/src/trace-multi.js +101 -0
- package/src/trace-query.js +748 -0
- package/src/trace-render.js +211 -0
- package/src/trace-usage.js +249 -0
- package/src/fixture/assertions.js +0 -42
- package/src/fixture/cache.js +0 -50
- package/src/fixture/eval.js +0 -146
- package/src/fixture/index.js +0 -9
- package/src/fixture/pathway.js +0 -451
- package/src/fixture/services.js +0 -56
- package/src/mock/clients.js +0 -135
- package/src/mock/config.js +0 -45
- package/src/mock/data.js +0 -46
- package/src/mock/fs.js +0 -111
- package/src/mock/grpc.js +0 -94
- package/src/mock/http.js +0 -60
- package/src/mock/index.js +0 -36
- package/src/mock/infra.js +0 -219
- package/src/mock/logger.js +0 -42
- package/src/mock/observer.js +0 -74
- package/src/mock/resource-index.js +0 -95
- package/src/mock/service-callbacks.js +0 -39
- package/src/mock/services.js +0 -79
- package/src/mock/spy.js +0 -44
- package/src/mock/storage.js +0 -118
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
import { common, tool } from "@forwardimpact/libtype";
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Creates a mock resource index with common test data
|
|
5
|
-
* @param {object} options - Setup options
|
|
6
|
-
* @returns {object} Mock resource index
|
|
7
|
-
*/
|
|
8
|
-
export function createMockResourceIndex(options = {}) {
|
|
9
|
-
const resources = new Map();
|
|
10
|
-
|
|
11
|
-
const index = {
|
|
12
|
-
resources,
|
|
13
|
-
|
|
14
|
-
async get(identifiers, _actor) {
|
|
15
|
-
if (!identifiers || identifiers.length === 0) return [];
|
|
16
|
-
return identifiers
|
|
17
|
-
.map((id) => {
|
|
18
|
-
const key = typeof id === "string" ? id : id.toString?.() || id.name;
|
|
19
|
-
return resources.get(key);
|
|
20
|
-
})
|
|
21
|
-
.filter(Boolean);
|
|
22
|
-
},
|
|
23
|
-
|
|
24
|
-
put(resource) {
|
|
25
|
-
const key = resource.id?.toString?.() || resource.id?.name;
|
|
26
|
-
if (key) resources.set(key, resource);
|
|
27
|
-
},
|
|
28
|
-
|
|
29
|
-
async has(id) {
|
|
30
|
-
const key = typeof id === "string" ? id : id.toString?.();
|
|
31
|
-
return resources.has(key);
|
|
32
|
-
},
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* Sets up default test resources
|
|
36
|
-
* @param {object} setupOptions - Setup options
|
|
37
|
-
* @param {string[]} [setupOptions.tools] - Tool names to seed
|
|
38
|
-
* @param {string} [setupOptions.conversationId] - Conversation ID
|
|
39
|
-
*/
|
|
40
|
-
setupDefaults(setupOptions = {}) {
|
|
41
|
-
const { tools = [], conversationId = "test-conversation" } = setupOptions;
|
|
42
|
-
|
|
43
|
-
resources.set(
|
|
44
|
-
conversationId,
|
|
45
|
-
common.Conversation.fromObject({
|
|
46
|
-
id: { name: conversationId },
|
|
47
|
-
}),
|
|
48
|
-
);
|
|
49
|
-
|
|
50
|
-
for (const name of tools) {
|
|
51
|
-
resources.set(
|
|
52
|
-
`tool.ToolFunction.${name}`,
|
|
53
|
-
tool.ToolFunction.fromObject({
|
|
54
|
-
id: { name, tokens: 20 },
|
|
55
|
-
name,
|
|
56
|
-
description: `${name} tool`,
|
|
57
|
-
}),
|
|
58
|
-
);
|
|
59
|
-
}
|
|
60
|
-
},
|
|
61
|
-
|
|
62
|
-
/**
|
|
63
|
-
* Adds a message resource
|
|
64
|
-
* @param {object} msg - Message to add
|
|
65
|
-
*/
|
|
66
|
-
addMessage(msg) {
|
|
67
|
-
const id =
|
|
68
|
-
msg.id?.type && msg.id?.toString
|
|
69
|
-
? msg.id.toString()
|
|
70
|
-
: msg.id?.name || String(msg.id);
|
|
71
|
-
resources.set(id, msg);
|
|
72
|
-
},
|
|
73
|
-
|
|
74
|
-
/**
|
|
75
|
-
* Find resources by prefix
|
|
76
|
-
* @param {string} prefix - Prefix to search for
|
|
77
|
-
* @returns {Promise<string[]>} List of matching keys
|
|
78
|
-
*/
|
|
79
|
-
async findByPrefix(prefix) {
|
|
80
|
-
const keys = [];
|
|
81
|
-
for (const key of resources.keys()) {
|
|
82
|
-
if (key.startsWith(prefix)) {
|
|
83
|
-
keys.push(key);
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
return keys;
|
|
87
|
-
},
|
|
88
|
-
};
|
|
89
|
-
|
|
90
|
-
if (options.tools || options.conversationId) {
|
|
91
|
-
index.setupDefaults(options);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
return index;
|
|
95
|
-
}
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Creates a mock service callbacks object for agent testing
|
|
3
|
-
* @param {object} overrides - Method overrides per service
|
|
4
|
-
* @returns {object} Service callbacks object
|
|
5
|
-
*/
|
|
6
|
-
export function createMockServiceCallbacks(overrides = {}) {
|
|
7
|
-
return {
|
|
8
|
-
memory: {
|
|
9
|
-
append: async () => ({}),
|
|
10
|
-
get: async () => ({
|
|
11
|
-
messages: [{ role: "system", content: "You are an assistant" }],
|
|
12
|
-
tools: [],
|
|
13
|
-
}),
|
|
14
|
-
...overrides.memory,
|
|
15
|
-
},
|
|
16
|
-
llm: {
|
|
17
|
-
createCompletions: async () => ({
|
|
18
|
-
choices: [
|
|
19
|
-
{
|
|
20
|
-
message: {
|
|
21
|
-
role: "assistant",
|
|
22
|
-
content: "Test response",
|
|
23
|
-
tool_calls: [],
|
|
24
|
-
},
|
|
25
|
-
},
|
|
26
|
-
],
|
|
27
|
-
}),
|
|
28
|
-
...overrides.llm,
|
|
29
|
-
},
|
|
30
|
-
tool: {
|
|
31
|
-
call: async () => ({
|
|
32
|
-
role: "tool",
|
|
33
|
-
content: "Tool result",
|
|
34
|
-
}),
|
|
35
|
-
...overrides.tool,
|
|
36
|
-
},
|
|
37
|
-
...overrides,
|
|
38
|
-
};
|
|
39
|
-
}
|
package/src/mock/services.js
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Mock service utilities for testing
|
|
3
|
-
*/
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Creates a mock vector service
|
|
7
|
-
* @param {object} overrides - Properties to override in the mock
|
|
8
|
-
* @returns {object} Mock vector service
|
|
9
|
-
*/
|
|
10
|
-
export function mockVectorService(overrides = {}) {
|
|
11
|
-
return {
|
|
12
|
-
QueryItems: async (request, callback) => {
|
|
13
|
-
callback(null, { results: [], total: 0 });
|
|
14
|
-
},
|
|
15
|
-
close: () => {},
|
|
16
|
-
...overrides,
|
|
17
|
-
};
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* Creates a mock history service
|
|
22
|
-
* @param {object} overrides - Properties to override in the mock
|
|
23
|
-
* @returns {object} Mock history service
|
|
24
|
-
*/
|
|
25
|
-
export function mockHistoryService(overrides = {}) {
|
|
26
|
-
return {
|
|
27
|
-
GetHistory: async (request, callback) => {
|
|
28
|
-
callback(null, { messages: [] });
|
|
29
|
-
},
|
|
30
|
-
UpdateHistory: async (request, callback) => {
|
|
31
|
-
callback(null, {});
|
|
32
|
-
},
|
|
33
|
-
close: () => {},
|
|
34
|
-
...overrides,
|
|
35
|
-
};
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Creates a mock LLM service
|
|
40
|
-
* @param {object} overrides - Properties to override in the mock
|
|
41
|
-
* @returns {object} Mock LLM service
|
|
42
|
-
*/
|
|
43
|
-
export function mockLlmService(overrides = {}) {
|
|
44
|
-
return {
|
|
45
|
-
CreateCompletions: async (request, callback) => {
|
|
46
|
-
callback(null, {
|
|
47
|
-
choices: [
|
|
48
|
-
{
|
|
49
|
-
index: 0,
|
|
50
|
-
message: { role: "assistant", content: "Test response" },
|
|
51
|
-
finish_reason: "stop",
|
|
52
|
-
},
|
|
53
|
-
],
|
|
54
|
-
});
|
|
55
|
-
},
|
|
56
|
-
CreateEmbeddings: async (request, callback) => {
|
|
57
|
-
callback(null, {
|
|
58
|
-
data: [{ index: 0, embedding: [0.1, 0.2, 0.3] }],
|
|
59
|
-
});
|
|
60
|
-
},
|
|
61
|
-
close: () => {},
|
|
62
|
-
...overrides,
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
* Creates a mock text service
|
|
68
|
-
* @param {object} overrides - Properties to override in the mock
|
|
69
|
-
* @returns {object} Mock text service
|
|
70
|
-
*/
|
|
71
|
-
export function mockTextService(overrides = {}) {
|
|
72
|
-
return {
|
|
73
|
-
GetChunks: async (request, callback) => {
|
|
74
|
-
callback(null, { chunks: {} });
|
|
75
|
-
},
|
|
76
|
-
close: () => {},
|
|
77
|
-
...overrides,
|
|
78
|
-
};
|
|
79
|
-
}
|
package/src/mock/spy.js
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Portable mock function helper. Replaces `mock.fn` from `node:test` so the
|
|
3
|
-
* test suite can run under either node:test or bun:test.
|
|
4
|
-
*
|
|
5
|
-
* Shape matches node:test's `mock.fn` to keep call-inspection sites
|
|
6
|
-
* (`fn.mock.calls[0].arguments`, `fn.mock.callCount()`, `fn.mock.resetCalls()`)
|
|
7
|
-
* unchanged across the codebase.
|
|
8
|
-
*
|
|
9
|
-
* @template T
|
|
10
|
-
* @param {(...args: any[]) => T} [impl] - Initial implementation.
|
|
11
|
-
* @returns {((...args: any[]) => T) & { mock: { calls: Array<{arguments: any[], result?: T, error?: unknown, this: unknown}>, callCount: () => number, resetCalls: () => void, mockImplementation: (newImpl: (...args: any[]) => T) => void } }}
|
|
12
|
-
*/
|
|
13
|
-
export function spy(impl) {
|
|
14
|
-
let _impl = impl;
|
|
15
|
-
const calls = [];
|
|
16
|
-
const fn = function (...args) {
|
|
17
|
-
const rec = { arguments: args, this: this };
|
|
18
|
-
if (!_impl) {
|
|
19
|
-
calls.push(rec);
|
|
20
|
-
return undefined;
|
|
21
|
-
}
|
|
22
|
-
try {
|
|
23
|
-
const result = _impl.apply(this, args);
|
|
24
|
-
rec.result = result;
|
|
25
|
-
calls.push(rec);
|
|
26
|
-
return result;
|
|
27
|
-
} catch (err) {
|
|
28
|
-
rec.error = err;
|
|
29
|
-
calls.push(rec);
|
|
30
|
-
throw err;
|
|
31
|
-
}
|
|
32
|
-
};
|
|
33
|
-
fn.mock = {
|
|
34
|
-
calls,
|
|
35
|
-
callCount: () => calls.length,
|
|
36
|
-
resetCalls: () => {
|
|
37
|
-
calls.length = 0;
|
|
38
|
-
},
|
|
39
|
-
mockImplementation: (newImpl) => {
|
|
40
|
-
_impl = newImpl;
|
|
41
|
-
},
|
|
42
|
-
};
|
|
43
|
-
return fn;
|
|
44
|
-
}
|
package/src/mock/storage.js
DELETED
|
@@ -1,118 +0,0 @@
|
|
|
1
|
-
import { spy } from "./spy.js";
|
|
2
|
-
/**
|
|
3
|
-
* Creates a mock storage instance with tracking
|
|
4
|
-
* @param {object} overrides - Method overrides
|
|
5
|
-
* @returns {object} Mock storage with data tracking
|
|
6
|
-
*/
|
|
7
|
-
export function createMockStorage(overrides = {}) {
|
|
8
|
-
const data = new Map();
|
|
9
|
-
|
|
10
|
-
return {
|
|
11
|
-
data,
|
|
12
|
-
exists: spy((key) => Promise.resolve(data.has(key))),
|
|
13
|
-
get: spy((key) => {
|
|
14
|
-
const value = data.get(key);
|
|
15
|
-
if (!value) return Promise.reject(new Error("Not found"));
|
|
16
|
-
|
|
17
|
-
if (key.endsWith(".json")) {
|
|
18
|
-
return Promise.resolve(value ? JSON.parse(value) : {});
|
|
19
|
-
}
|
|
20
|
-
if (key.endsWith(".jsonl")) {
|
|
21
|
-
const lines = value.split("\n").filter(Boolean);
|
|
22
|
-
return Promise.resolve(lines.map((line) => JSON.parse(line)));
|
|
23
|
-
}
|
|
24
|
-
return Promise.resolve(value);
|
|
25
|
-
}),
|
|
26
|
-
put: spy((key, value) => {
|
|
27
|
-
data.set(key, value);
|
|
28
|
-
return Promise.resolve();
|
|
29
|
-
}),
|
|
30
|
-
append: spy((key, value) => {
|
|
31
|
-
const existing = data.get(key) || "";
|
|
32
|
-
data.set(key, existing ? `${existing}\n${value}` : value);
|
|
33
|
-
return Promise.resolve();
|
|
34
|
-
}),
|
|
35
|
-
delete: spy((key) => {
|
|
36
|
-
data.delete(key);
|
|
37
|
-
return Promise.resolve();
|
|
38
|
-
}),
|
|
39
|
-
findByPrefix: spy(() => Promise.resolve([])),
|
|
40
|
-
...overrides,
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
* MockStorage class for OOP-style usage
|
|
46
|
-
*/
|
|
47
|
-
export class MockStorage {
|
|
48
|
-
/**
|
|
49
|
-
* Creates a new MockStorage instance
|
|
50
|
-
*/
|
|
51
|
-
constructor() {
|
|
52
|
-
this.data = new Map();
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
/**
|
|
56
|
-
* Gets a value by key
|
|
57
|
-
* @param {string} key - The storage key
|
|
58
|
-
* @returns {Promise<*>} The stored value
|
|
59
|
-
*/
|
|
60
|
-
async get(key) {
|
|
61
|
-
const value = this.data.get(key);
|
|
62
|
-
if (!value) throw new Error("Not found");
|
|
63
|
-
|
|
64
|
-
if (key.endsWith(".jsonl")) {
|
|
65
|
-
return value.split("\n").map((line) => JSON.parse(line));
|
|
66
|
-
}
|
|
67
|
-
return value;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
/**
|
|
71
|
-
* Stores a value by key
|
|
72
|
-
* @param {string} key - The storage key
|
|
73
|
-
* @param {*} value - The value to store
|
|
74
|
-
* @returns {Promise<void>}
|
|
75
|
-
*/
|
|
76
|
-
async put(key, value) {
|
|
77
|
-
this.data.set(key, value);
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Alias for put - stores a value by key
|
|
82
|
-
* @param {string} key - The storage key
|
|
83
|
-
* @param {*} value - The value to store
|
|
84
|
-
* @returns {Promise<void>}
|
|
85
|
-
*/
|
|
86
|
-
async set(key, value) {
|
|
87
|
-
this.data.set(key, value);
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* Appends a value to an existing key
|
|
92
|
-
* @param {string} key - The storage key
|
|
93
|
-
* @param {string} value - The value to append
|
|
94
|
-
* @returns {Promise<void>}
|
|
95
|
-
*/
|
|
96
|
-
async append(key, value) {
|
|
97
|
-
const existing = this.data.get(key) || "";
|
|
98
|
-
this.data.set(key, existing ? `${existing}\n${value}` : value);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
/**
|
|
102
|
-
* Checks if a key exists
|
|
103
|
-
* @param {string} key - The storage key
|
|
104
|
-
* @returns {Promise<boolean>} True if key exists
|
|
105
|
-
*/
|
|
106
|
-
async exists(key) {
|
|
107
|
-
return this.data.has(key);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
/**
|
|
111
|
-
* Deletes a value by key
|
|
112
|
-
* @param {string} key - The storage key
|
|
113
|
-
* @returns {Promise<void>}
|
|
114
|
-
*/
|
|
115
|
-
async delete(key) {
|
|
116
|
-
this.data.delete(key);
|
|
117
|
-
}
|
|
118
|
-
}
|