@forwardimpact/libharness 0.1.22 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/LICENSE +21 -201
  2. package/README.md +196 -80
  3. package/bin/fit-benchmark.js +44 -0
  4. package/bin/fit-harness.js +358 -0
  5. package/bin/fit-selfedit.js +165 -0
  6. package/bin/fit-trace.js +510 -0
  7. package/package.json +41 -11
  8. package/src/agent-runner.js +256 -0
  9. package/src/benchmark/apm-installer.js +207 -0
  10. package/src/benchmark/env-loader.js +158 -0
  11. package/src/benchmark/hook-env.js +40 -0
  12. package/src/benchmark/invariants.js +141 -0
  13. package/src/benchmark/judge.js +187 -0
  14. package/src/benchmark/npm-installer.js +87 -0
  15. package/src/benchmark/report.js +522 -0
  16. package/src/benchmark/result.js +127 -0
  17. package/src/benchmark/runner.js +583 -0
  18. package/src/benchmark/task-family.js +260 -0
  19. package/src/benchmark/workdir.js +298 -0
  20. package/src/commands/assert.js +153 -0
  21. package/src/commands/benchmark-definition.js +165 -0
  22. package/src/commands/benchmark-invariants.js +73 -0
  23. package/src/commands/benchmark-report.js +51 -0
  24. package/src/commands/benchmark-run.js +111 -0
  25. package/src/commands/by-discussion.js +94 -0
  26. package/src/commands/callback.js +119 -0
  27. package/src/commands/discuss.js +132 -0
  28. package/src/commands/facilitate.js +123 -0
  29. package/src/commands/output.js +36 -0
  30. package/src/commands/run.js +152 -0
  31. package/src/commands/supervise.js +136 -0
  32. package/src/commands/task-input.js +54 -0
  33. package/src/commands/tee.js +53 -0
  34. package/src/commands/trace.js +630 -0
  35. package/src/commands/work-tracker.js +35 -0
  36. package/src/cost.js +79 -0
  37. package/src/discuss-tools.js +173 -0
  38. package/src/discusser.js +394 -0
  39. package/src/events/github.js +161 -0
  40. package/src/facilitator.js +205 -0
  41. package/src/inbox-poller.js +81 -0
  42. package/src/index.js +72 -2
  43. package/src/judge.js +210 -0
  44. package/src/message-bus.js +118 -0
  45. package/src/orchestration-loop.js +330 -0
  46. package/src/orchestration-toolkit.js +441 -0
  47. package/src/orchestrator-helpers.js +23 -0
  48. package/src/profile-prompt.js +266 -0
  49. package/src/redaction.js +253 -0
  50. package/src/render/line-renderer.js +54 -0
  51. package/src/render/orchestrator-filter.js +19 -0
  52. package/src/render/palette.js +63 -0
  53. package/src/render/tool-hints.js +154 -0
  54. package/src/render/turn-renderer.js +96 -0
  55. package/src/reply-emitter.js +47 -0
  56. package/src/sequence-counter.js +21 -0
  57. package/src/signature-filter.js +27 -0
  58. package/src/supervisor.js +236 -0
  59. package/src/tee-writer.js +150 -0
  60. package/src/trace-collector.js +444 -0
  61. package/src/trace-github.js +473 -0
  62. package/src/trace-multi.js +101 -0
  63. package/src/trace-query.js +748 -0
  64. package/src/trace-render.js +211 -0
  65. package/src/trace-usage.js +249 -0
  66. package/src/fixture/assertions.js +0 -42
  67. package/src/fixture/cache.js +0 -50
  68. package/src/fixture/eval.js +0 -146
  69. package/src/fixture/index.js +0 -9
  70. package/src/fixture/pathway.js +0 -451
  71. package/src/fixture/services.js +0 -56
  72. package/src/mock/clients.js +0 -135
  73. package/src/mock/config.js +0 -45
  74. package/src/mock/data.js +0 -46
  75. package/src/mock/fs.js +0 -111
  76. package/src/mock/grpc.js +0 -94
  77. package/src/mock/http.js +0 -60
  78. package/src/mock/index.js +0 -36
  79. package/src/mock/infra.js +0 -219
  80. package/src/mock/logger.js +0 -42
  81. package/src/mock/observer.js +0 -74
  82. package/src/mock/resource-index.js +0 -95
  83. package/src/mock/service-callbacks.js +0 -39
  84. package/src/mock/services.js +0 -79
  85. package/src/mock/spy.js +0 -44
  86. package/src/mock/storage.js +0 -118
@@ -1,95 +0,0 @@
1
- import { common, tool } from "@forwardimpact/libtype";
2
-
3
- /**
4
- * Creates a mock resource index with common test data
5
- * @param {object} options - Setup options
6
- * @returns {object} Mock resource index
7
- */
8
- export function createMockResourceIndex(options = {}) {
9
- const resources = new Map();
10
-
11
- const index = {
12
- resources,
13
-
14
- async get(identifiers, _actor) {
15
- if (!identifiers || identifiers.length === 0) return [];
16
- return identifiers
17
- .map((id) => {
18
- const key = typeof id === "string" ? id : id.toString?.() || id.name;
19
- return resources.get(key);
20
- })
21
- .filter(Boolean);
22
- },
23
-
24
- put(resource) {
25
- const key = resource.id?.toString?.() || resource.id?.name;
26
- if (key) resources.set(key, resource);
27
- },
28
-
29
- async has(id) {
30
- const key = typeof id === "string" ? id : id.toString?.();
31
- return resources.has(key);
32
- },
33
-
34
- /**
35
- * Sets up default test resources
36
- * @param {object} setupOptions - Setup options
37
- * @param {string[]} [setupOptions.tools] - Tool names to seed
38
- * @param {string} [setupOptions.conversationId] - Conversation ID
39
- */
40
- setupDefaults(setupOptions = {}) {
41
- const { tools = [], conversationId = "test-conversation" } = setupOptions;
42
-
43
- resources.set(
44
- conversationId,
45
- common.Conversation.fromObject({
46
- id: { name: conversationId },
47
- }),
48
- );
49
-
50
- for (const name of tools) {
51
- resources.set(
52
- `tool.ToolFunction.${name}`,
53
- tool.ToolFunction.fromObject({
54
- id: { name, tokens: 20 },
55
- name,
56
- description: `${name} tool`,
57
- }),
58
- );
59
- }
60
- },
61
-
62
- /**
63
- * Adds a message resource
64
- * @param {object} msg - Message to add
65
- */
66
- addMessage(msg) {
67
- const id =
68
- msg.id?.type && msg.id?.toString
69
- ? msg.id.toString()
70
- : msg.id?.name || String(msg.id);
71
- resources.set(id, msg);
72
- },
73
-
74
- /**
75
- * Find resources by prefix
76
- * @param {string} prefix - Prefix to search for
77
- * @returns {Promise<string[]>} List of matching keys
78
- */
79
- async findByPrefix(prefix) {
80
- const keys = [];
81
- for (const key of resources.keys()) {
82
- if (key.startsWith(prefix)) {
83
- keys.push(key);
84
- }
85
- }
86
- return keys;
87
- },
88
- };
89
-
90
- if (options.tools || options.conversationId) {
91
- index.setupDefaults(options);
92
- }
93
-
94
- return index;
95
- }
@@ -1,39 +0,0 @@
1
- /**
2
- * Creates a mock service callbacks object for agent testing
3
- * @param {object} overrides - Method overrides per service
4
- * @returns {object} Service callbacks object
5
- */
6
- export function createMockServiceCallbacks(overrides = {}) {
7
- return {
8
- memory: {
9
- append: async () => ({}),
10
- get: async () => ({
11
- messages: [{ role: "system", content: "You are an assistant" }],
12
- tools: [],
13
- }),
14
- ...overrides.memory,
15
- },
16
- llm: {
17
- createCompletions: async () => ({
18
- choices: [
19
- {
20
- message: {
21
- role: "assistant",
22
- content: "Test response",
23
- tool_calls: [],
24
- },
25
- },
26
- ],
27
- }),
28
- ...overrides.llm,
29
- },
30
- tool: {
31
- call: async () => ({
32
- role: "tool",
33
- content: "Tool result",
34
- }),
35
- ...overrides.tool,
36
- },
37
- ...overrides,
38
- };
39
- }
@@ -1,79 +0,0 @@
1
- /**
2
- * Mock service utilities for testing
3
- */
4
-
5
- /**
6
- * Creates a mock vector service
7
- * @param {object} overrides - Properties to override in the mock
8
- * @returns {object} Mock vector service
9
- */
10
- export function mockVectorService(overrides = {}) {
11
- return {
12
- QueryItems: async (request, callback) => {
13
- callback(null, { results: [], total: 0 });
14
- },
15
- close: () => {},
16
- ...overrides,
17
- };
18
- }
19
-
20
- /**
21
- * Creates a mock history service
22
- * @param {object} overrides - Properties to override in the mock
23
- * @returns {object} Mock history service
24
- */
25
- export function mockHistoryService(overrides = {}) {
26
- return {
27
- GetHistory: async (request, callback) => {
28
- callback(null, { messages: [] });
29
- },
30
- UpdateHistory: async (request, callback) => {
31
- callback(null, {});
32
- },
33
- close: () => {},
34
- ...overrides,
35
- };
36
- }
37
-
38
- /**
39
- * Creates a mock LLM service
40
- * @param {object} overrides - Properties to override in the mock
41
- * @returns {object} Mock LLM service
42
- */
43
- export function mockLlmService(overrides = {}) {
44
- return {
45
- CreateCompletions: async (request, callback) => {
46
- callback(null, {
47
- choices: [
48
- {
49
- index: 0,
50
- message: { role: "assistant", content: "Test response" },
51
- finish_reason: "stop",
52
- },
53
- ],
54
- });
55
- },
56
- CreateEmbeddings: async (request, callback) => {
57
- callback(null, {
58
- data: [{ index: 0, embedding: [0.1, 0.2, 0.3] }],
59
- });
60
- },
61
- close: () => {},
62
- ...overrides,
63
- };
64
- }
65
-
66
- /**
67
- * Creates a mock text service
68
- * @param {object} overrides - Properties to override in the mock
69
- * @returns {object} Mock text service
70
- */
71
- export function mockTextService(overrides = {}) {
72
- return {
73
- GetChunks: async (request, callback) => {
74
- callback(null, { chunks: {} });
75
- },
76
- close: () => {},
77
- ...overrides,
78
- };
79
- }
package/src/mock/spy.js DELETED
@@ -1,44 +0,0 @@
1
- /**
2
- * Portable mock function helper. Replaces `mock.fn` from `node:test` so the
3
- * test suite can run under either node:test or bun:test.
4
- *
5
- * Shape matches node:test's `mock.fn` to keep call-inspection sites
6
- * (`fn.mock.calls[0].arguments`, `fn.mock.callCount()`, `fn.mock.resetCalls()`)
7
- * unchanged across the codebase.
8
- *
9
- * @template T
10
- * @param {(...args: any[]) => T} [impl] - Initial implementation.
11
- * @returns {((...args: any[]) => T) & { mock: { calls: Array<{arguments: any[], result?: T, error?: unknown, this: unknown}>, callCount: () => number, resetCalls: () => void, mockImplementation: (newImpl: (...args: any[]) => T) => void } }}
12
- */
13
- export function spy(impl) {
14
- let _impl = impl;
15
- const calls = [];
16
- const fn = function (...args) {
17
- const rec = { arguments: args, this: this };
18
- if (!_impl) {
19
- calls.push(rec);
20
- return undefined;
21
- }
22
- try {
23
- const result = _impl.apply(this, args);
24
- rec.result = result;
25
- calls.push(rec);
26
- return result;
27
- } catch (err) {
28
- rec.error = err;
29
- calls.push(rec);
30
- throw err;
31
- }
32
- };
33
- fn.mock = {
34
- calls,
35
- callCount: () => calls.length,
36
- resetCalls: () => {
37
- calls.length = 0;
38
- },
39
- mockImplementation: (newImpl) => {
40
- _impl = newImpl;
41
- },
42
- };
43
- return fn;
44
- }
@@ -1,118 +0,0 @@
1
- import { spy } from "./spy.js";
2
- /**
3
- * Creates a mock storage instance with tracking
4
- * @param {object} overrides - Method overrides
5
- * @returns {object} Mock storage with data tracking
6
- */
7
- export function createMockStorage(overrides = {}) {
8
- const data = new Map();
9
-
10
- return {
11
- data,
12
- exists: spy((key) => Promise.resolve(data.has(key))),
13
- get: spy((key) => {
14
- const value = data.get(key);
15
- if (!value) return Promise.reject(new Error("Not found"));
16
-
17
- if (key.endsWith(".json")) {
18
- return Promise.resolve(value ? JSON.parse(value) : {});
19
- }
20
- if (key.endsWith(".jsonl")) {
21
- const lines = value.split("\n").filter(Boolean);
22
- return Promise.resolve(lines.map((line) => JSON.parse(line)));
23
- }
24
- return Promise.resolve(value);
25
- }),
26
- put: spy((key, value) => {
27
- data.set(key, value);
28
- return Promise.resolve();
29
- }),
30
- append: spy((key, value) => {
31
- const existing = data.get(key) || "";
32
- data.set(key, existing ? `${existing}\n${value}` : value);
33
- return Promise.resolve();
34
- }),
35
- delete: spy((key) => {
36
- data.delete(key);
37
- return Promise.resolve();
38
- }),
39
- findByPrefix: spy(() => Promise.resolve([])),
40
- ...overrides,
41
- };
42
- }
43
-
44
- /**
45
- * MockStorage class for OOP-style usage
46
- */
47
- export class MockStorage {
48
- /**
49
- * Creates a new MockStorage instance
50
- */
51
- constructor() {
52
- this.data = new Map();
53
- }
54
-
55
- /**
56
- * Gets a value by key
57
- * @param {string} key - The storage key
58
- * @returns {Promise<*>} The stored value
59
- */
60
- async get(key) {
61
- const value = this.data.get(key);
62
- if (!value) throw new Error("Not found");
63
-
64
- if (key.endsWith(".jsonl")) {
65
- return value.split("\n").map((line) => JSON.parse(line));
66
- }
67
- return value;
68
- }
69
-
70
- /**
71
- * Stores a value by key
72
- * @param {string} key - The storage key
73
- * @param {*} value - The value to store
74
- * @returns {Promise<void>}
75
- */
76
- async put(key, value) {
77
- this.data.set(key, value);
78
- }
79
-
80
- /**
81
- * Alias for put - stores a value by key
82
- * @param {string} key - The storage key
83
- * @param {*} value - The value to store
84
- * @returns {Promise<void>}
85
- */
86
- async set(key, value) {
87
- this.data.set(key, value);
88
- }
89
-
90
- /**
91
- * Appends a value to an existing key
92
- * @param {string} key - The storage key
93
- * @param {string} value - The value to append
94
- * @returns {Promise<void>}
95
- */
96
- async append(key, value) {
97
- const existing = this.data.get(key) || "";
98
- this.data.set(key, existing ? `${existing}\n${value}` : value);
99
- }
100
-
101
- /**
102
- * Checks if a key exists
103
- * @param {string} key - The storage key
104
- * @returns {Promise<boolean>} True if key exists
105
- */
106
- async exists(key) {
107
- return this.data.has(key);
108
- }
109
-
110
- /**
111
- * Deletes a value by key
112
- * @param {string} key - The storage key
113
- * @returns {Promise<void>}
114
- */
115
- async delete(key) {
116
- this.data.delete(key);
117
- }
118
- }