@crewhaus/runtime-core 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-markdown.d.ts +56 -0
- package/dist/cli-markdown.js +164 -0
- package/dist/index.d.ts +368 -0
- package/dist/index.js +1976 -0
- package/dist/observability.d.ts +38 -0
- package/dist/observability.js +69 -0
- package/dist/project-memory.d.ts +28 -0
- package/dist/project-memory.js +104 -0
- package/package.json +46 -43
- package/src/cli-markdown.test.ts +0 -207
- package/src/cli-markdown.ts +0 -190
- package/src/cost-inline.test.ts +0 -58
- package/src/cost-persist.test.ts +0 -113
- package/src/cost-wire-model.test.ts +0 -138
- package/src/index-coverage.test.ts +0 -1671
- package/src/index.test.ts +0 -2471
- package/src/index.ts +0 -2365
- package/src/observability-coverage.test.ts +0 -303
- package/src/observability.test.ts +0 -138
- package/src/observability.ts +0 -116
- package/src/project-memory-coverage.test.ts +0 -119
- package/src/project-memory.test.ts +0 -145
- package/src/project-memory.ts +0 -134
- package/src/tenant-isolation.test.ts +0 -80
|
@@ -1,303 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Coverage tests for `attachDefaultSubscribers` and `logFlushError`
|
|
3
|
-
* (observability.ts). Drives the cost-tracker / cost-inline / flushAll /
|
|
4
|
-
* shutdownAll branches deterministically by:
|
|
5
|
-
* - mocking the metrics-collector + otel-exporter attach helpers so their
|
|
6
|
-
* `flush`/`shutdown` resolve or reject under test control (NO real
|
|
7
|
-
* network sinks, NO leaked handles), and
|
|
8
|
-
* - publishing real `cost_accrual` events through a real TraceEventBus to
|
|
9
|
-
* exercise the inline-cost stdout handler.
|
|
10
|
-
*
|
|
11
|
-
* The mock.module replacements are reinstalled to the REAL modules in
|
|
12
|
-
* afterAll — `mock.restore()` (run in afterEach for the spies) does not undo
|
|
13
|
-
* `mock.module`, and Bun shares one module registry across all test files in
|
|
14
|
-
* nondeterministic order, so without that re-mock the fakes would leak into
|
|
15
|
-
* the rest of the runtime-core suite.
|
|
16
|
-
*/
|
|
17
|
-
import { afterAll, afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test";
|
|
18
|
-
// Real modules captured for the afterAll restore — as plain-object SNAPSHOTS:
|
|
19
|
-
// an `import * as` namespace is a live view that resolves to the fakes once
|
|
20
|
-
// mock.module patches the module, so restoring from it would be a no-op.
|
|
21
|
-
import * as realMetricsCollectorNS from "@crewhaus/metrics-collector";
|
|
22
|
-
import * as realOtelExporterNS from "@crewhaus/otel-exporter";
|
|
23
|
-
import { createRunContext } from "@crewhaus/run-context";
|
|
24
|
-
import { TraceEventBus } from "@crewhaus/trace-event-bus";
|
|
25
|
-
import type { CostAccrualEvent } from "@crewhaus/trace-event-bus";
|
|
26
|
-
|
|
27
|
-
const realMetricsCollectorSnapshot = { ...realMetricsCollectorNS };
|
|
28
|
-
const realOtelExporterSnapshot = { ...realOtelExporterNS };
|
|
29
|
-
|
|
30
|
-
// Controllable fakes for the two heavyweight subscribers. The booleans let
|
|
31
|
-
// each test decide whether flush()/shutdown() reject so the catch → logFlushError
|
|
32
|
-
// branches (102, 105, 113-114) are reachable without real I/O.
|
|
33
|
-
const metricsState = {
|
|
34
|
-
flushRejects: false,
|
|
35
|
-
shutdownRejects: false,
|
|
36
|
-
flushCalls: 0,
|
|
37
|
-
shutdownCalls: 0,
|
|
38
|
-
unsubscribeCalls: 0,
|
|
39
|
-
attached: true,
|
|
40
|
-
};
|
|
41
|
-
const otelState = {
|
|
42
|
-
flushRejects: false,
|
|
43
|
-
shutdownRejects: false,
|
|
44
|
-
flushCalls: 0,
|
|
45
|
-
shutdownCalls: 0,
|
|
46
|
-
unsubscribeCalls: 0,
|
|
47
|
-
attached: true,
|
|
48
|
-
};
|
|
49
|
-
|
|
50
|
-
function resetState(): void {
|
|
51
|
-
for (const s of [metricsState, otelState]) {
|
|
52
|
-
s.flushRejects = false;
|
|
53
|
-
s.shutdownRejects = false;
|
|
54
|
-
s.flushCalls = 0;
|
|
55
|
-
s.shutdownCalls = 0;
|
|
56
|
-
s.unsubscribeCalls = 0;
|
|
57
|
-
s.attached = true;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
beforeEach(() => {
|
|
62
|
-
resetState();
|
|
63
|
-
mock.module("@crewhaus/metrics-collector", () => ({
|
|
64
|
-
attachIfEnvSet: async () =>
|
|
65
|
-
metricsState.attached
|
|
66
|
-
? {
|
|
67
|
-
registry: {},
|
|
68
|
-
unsubscribe: () => {
|
|
69
|
-
metricsState.unsubscribeCalls += 1;
|
|
70
|
-
},
|
|
71
|
-
flush: async () => {
|
|
72
|
-
metricsState.flushCalls += 1;
|
|
73
|
-
if (metricsState.flushRejects) throw new Error("metrics flush boom");
|
|
74
|
-
},
|
|
75
|
-
shutdown: async () => {
|
|
76
|
-
metricsState.shutdownCalls += 1;
|
|
77
|
-
if (metricsState.shutdownRejects) throw new Error("metrics shutdown boom");
|
|
78
|
-
},
|
|
79
|
-
}
|
|
80
|
-
: undefined,
|
|
81
|
-
}));
|
|
82
|
-
mock.module("@crewhaus/otel-exporter", () => ({
|
|
83
|
-
attachIfEnvSet: () =>
|
|
84
|
-
otelState.attached
|
|
85
|
-
? {
|
|
86
|
-
unsubscribe: () => {
|
|
87
|
-
otelState.unsubscribeCalls += 1;
|
|
88
|
-
},
|
|
89
|
-
flush: async () => {
|
|
90
|
-
otelState.flushCalls += 1;
|
|
91
|
-
if (otelState.flushRejects) throw new Error("otel flush boom");
|
|
92
|
-
},
|
|
93
|
-
shutdown: async () => {
|
|
94
|
-
otelState.shutdownCalls += 1;
|
|
95
|
-
if (otelState.shutdownRejects) throw new Error("otel shutdown boom");
|
|
96
|
-
},
|
|
97
|
-
}
|
|
98
|
-
: undefined,
|
|
99
|
-
}));
|
|
100
|
-
});
|
|
101
|
-
|
|
102
|
-
afterEach(() => {
|
|
103
|
-
mock.restore();
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
afterAll(() => {
|
|
107
|
-
// Reinstall the real modules — mock.restore() only undoes spies, not
|
|
108
|
-
// mock.module, so without these the fakes leak into sibling test files.
|
|
109
|
-
mock.module("@crewhaus/metrics-collector", () => realMetricsCollectorSnapshot);
|
|
110
|
-
mock.module("@crewhaus/otel-exporter", () => realOtelExporterSnapshot);
|
|
111
|
-
});
|
|
112
|
-
|
|
113
|
-
function makeAccrual(overrides: Partial<CostAccrualEvent> = {}): CostAccrualEvent {
|
|
114
|
-
return {
|
|
115
|
-
kind: "cost_accrual",
|
|
116
|
-
traceId: "trace_a",
|
|
117
|
-
runId: "run_a",
|
|
118
|
-
sessionId: "sess_0000000000000001",
|
|
119
|
-
timestamp: 0,
|
|
120
|
-
provider: "anthropic",
|
|
121
|
-
modelId: "claude-sonnet-4-6",
|
|
122
|
-
inputTokens: 2_000,
|
|
123
|
-
outputTokens: 500,
|
|
124
|
-
cachedReadTokens: 0,
|
|
125
|
-
costUsdMicros: 4_200,
|
|
126
|
-
...overrides,
|
|
127
|
-
} as CostAccrualEvent;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
describe("attachDefaultSubscribers — cost tracking + inline cost line", () => {
|
|
131
|
-
test("cost tracker is created with tenantId when CREWHAUS_TENANT_ID is set", async () => {
|
|
132
|
-
const { attachDefaultSubscribers } = await import("./observability");
|
|
133
|
-
const bus = new TraceEventBus({ runId: "run_a", sessionId: "sess_0000000000000001" });
|
|
134
|
-
const ctx = createRunContext();
|
|
135
|
-
const subs = await attachDefaultSubscribers(bus, ctx, {
|
|
136
|
-
CREWHAUS_COST_TRACKING: "1",
|
|
137
|
-
CREWHAUS_TENANT_ID: "tenant-xyz",
|
|
138
|
-
});
|
|
139
|
-
// The cost tracker exists (lines 67-74 — including the tenantId spread).
|
|
140
|
-
expect(subs.costTracker).toBeDefined();
|
|
141
|
-
await subs.flushAll();
|
|
142
|
-
await subs.shutdownAll();
|
|
143
|
-
});
|
|
144
|
-
|
|
145
|
-
test("cost tracker is created without tenantId when the env var is absent", async () => {
|
|
146
|
-
const { attachDefaultSubscribers } = await import("./observability");
|
|
147
|
-
const bus = new TraceEventBus({ runId: "run_a", sessionId: "sess_0000000000000001" });
|
|
148
|
-
const ctx = createRunContext();
|
|
149
|
-
const subs = await attachDefaultSubscribers(bus, ctx, {
|
|
150
|
-
CREWHAUS_COST_TRACKING: "true",
|
|
151
|
-
});
|
|
152
|
-
expect(subs.costTracker).toBeDefined();
|
|
153
|
-
await subs.shutdownAll();
|
|
154
|
-
});
|
|
155
|
-
|
|
156
|
-
test("inline cost handler prints one line per cost_accrual and ignores other events", async () => {
|
|
157
|
-
const { attachDefaultSubscribers } = await import("./observability");
|
|
158
|
-
const bus = new TraceEventBus({ runId: "run_a", sessionId: "sess_0000000000000001" });
|
|
159
|
-
const ctx = createRunContext();
|
|
160
|
-
const writes: string[] = [];
|
|
161
|
-
const writeSpy = spyOn(process.stdout, "write").mockImplementation(((chunk: unknown) => {
|
|
162
|
-
writes.push(String(chunk));
|
|
163
|
-
return true;
|
|
164
|
-
}) as typeof process.stdout.write);
|
|
165
|
-
try {
|
|
166
|
-
const subs = await attachDefaultSubscribers(bus, ctx, {
|
|
167
|
-
CREWHAUS_COST_TRACKING: "1",
|
|
168
|
-
CREWHAUS_COST_INLINE: "1",
|
|
169
|
-
});
|
|
170
|
-
expect(subs.costInlineUnsubscribe).toBeDefined();
|
|
171
|
-
|
|
172
|
-
// A non-cost event must hit the early `return` (line 85) → no write.
|
|
173
|
-
bus.publish({ ...bus.envelope(), kind: "turn_start", turn: 1, messageCount: 0 });
|
|
174
|
-
const beforeCost = writes.length;
|
|
175
|
-
// A cost_accrual event triggers the inline line (lines 86-88).
|
|
176
|
-
bus.publish(makeAccrual({ costUsdMicros: 4_200 }));
|
|
177
|
-
const costLines = writes.filter((w) => w.includes("💸"));
|
|
178
|
-
expect(costLines.length).toBe(1);
|
|
179
|
-
expect(costLines[0]).toContain("$0.0042");
|
|
180
|
-
expect(beforeCost).toBe(0); // turn_start wrote nothing
|
|
181
|
-
|
|
182
|
-
// Unsubscribe stops further inline lines.
|
|
183
|
-
subs.costInlineUnsubscribe?.();
|
|
184
|
-
bus.publish(makeAccrual({ costUsdMicros: 9_999 }));
|
|
185
|
-
expect(writes.filter((w) => w.includes("💸")).length).toBe(1);
|
|
186
|
-
await subs.shutdownAll();
|
|
187
|
-
} finally {
|
|
188
|
-
writeSpy.mockRestore();
|
|
189
|
-
}
|
|
190
|
-
});
|
|
191
|
-
|
|
192
|
-
test("inline cost is NOT enabled when CREWHAUS_COST_TRACKING is missing", async () => {
|
|
193
|
-
const { attachDefaultSubscribers } = await import("./observability");
|
|
194
|
-
const bus = new TraceEventBus({ runId: "run_a", sessionId: "sess_0000000000000001" });
|
|
195
|
-
const ctx = createRunContext();
|
|
196
|
-
// COST_INLINE set but no COST_TRACKING → costTracker undefined → inline off.
|
|
197
|
-
const subs = await attachDefaultSubscribers(bus, ctx, { CREWHAUS_COST_INLINE: "1" });
|
|
198
|
-
expect(subs.costTracker).toBeUndefined();
|
|
199
|
-
expect(subs.costInlineUnsubscribe).toBeUndefined();
|
|
200
|
-
await subs.shutdownAll();
|
|
201
|
-
});
|
|
202
|
-
});
|
|
203
|
-
|
|
204
|
-
describe("attachDefaultSubscribers — flushAll / shutdownAll error handling", () => {
|
|
205
|
-
test("flushAll flushes metrics + otel + bus and tolerates rejections (logFlushError)", async () => {
|
|
206
|
-
const { attachDefaultSubscribers } = await import("./observability");
|
|
207
|
-
const bus = new TraceEventBus({ runId: "run_a", sessionId: "sess_0000000000000001" });
|
|
208
|
-
const ctx = createRunContext();
|
|
209
|
-
const errSpy = spyOn(ctx.logger, "error");
|
|
210
|
-
|
|
211
|
-
metricsState.flushRejects = true;
|
|
212
|
-
otelState.flushRejects = true;
|
|
213
|
-
// Both subscribers attached (their attach helpers are mocked to return
|
|
214
|
-
// truthy objects). bus.flush is also exercised; force it to reject too so
|
|
215
|
-
// the bus-flush catch arm (logFlushError "bus") runs.
|
|
216
|
-
const busFlushSpy = spyOn(bus, "flush").mockImplementation(async () => {
|
|
217
|
-
throw new Error("bus flush boom");
|
|
218
|
-
});
|
|
219
|
-
|
|
220
|
-
const subs = await attachDefaultSubscribers(bus, ctx, {
|
|
221
|
-
CREWHAUS_METRICS: "stdout",
|
|
222
|
-
OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318",
|
|
223
|
-
});
|
|
224
|
-
// Should resolve despite all three flush paths rejecting.
|
|
225
|
-
await subs.flushAll();
|
|
226
|
-
|
|
227
|
-
expect(metricsState.flushCalls).toBe(1);
|
|
228
|
-
expect(otelState.flushCalls).toBe(1);
|
|
229
|
-
expect(busFlushSpy).toHaveBeenCalled();
|
|
230
|
-
// logFlushError logged once per failing surface (metrics, otel, bus).
|
|
231
|
-
const flushFailedLogs = errSpy.mock.calls.filter((c) => c[0] === "observability.flush_failed");
|
|
232
|
-
expect(flushFailedLogs.length).toBe(3);
|
|
233
|
-
busFlushSpy.mockRestore();
|
|
234
|
-
});
|
|
235
|
-
|
|
236
|
-
test("shutdownAll awaits metrics + otel shutdown and tolerates rejections", async () => {
|
|
237
|
-
const { attachDefaultSubscribers } = await import("./observability");
|
|
238
|
-
const bus = new TraceEventBus({ runId: "run_a", sessionId: "sess_0000000000000001" });
|
|
239
|
-
const ctx = createRunContext();
|
|
240
|
-
const errSpy = spyOn(ctx.logger, "error");
|
|
241
|
-
|
|
242
|
-
metricsState.shutdownRejects = true;
|
|
243
|
-
otelState.shutdownRejects = true;
|
|
244
|
-
|
|
245
|
-
const subs = await attachDefaultSubscribers(bus, ctx, {
|
|
246
|
-
CREWHAUS_METRICS: "stdout",
|
|
247
|
-
OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318",
|
|
248
|
-
CREWHAUS_COST_TRACKING: "1",
|
|
249
|
-
CREWHAUS_COST_INLINE: "1",
|
|
250
|
-
});
|
|
251
|
-
await subs.shutdownAll();
|
|
252
|
-
|
|
253
|
-
// Both shutdowns ran (lines 101-106), both failures logged (113-114),
|
|
254
|
-
// and the cost tracker + inline subscription were torn down (107-108).
|
|
255
|
-
expect(metricsState.shutdownCalls).toBe(1);
|
|
256
|
-
expect(otelState.shutdownCalls).toBe(1);
|
|
257
|
-
const shutdownFailedLogs = errSpy.mock.calls.filter(
|
|
258
|
-
(c) => c[0] === "observability.flush_failed",
|
|
259
|
-
);
|
|
260
|
-
expect(shutdownFailedLogs.length).toBe(2);
|
|
261
|
-
});
|
|
262
|
-
|
|
263
|
-
test("logFlushError stringifies a non-Error rejection value", async () => {
|
|
264
|
-
const { attachDefaultSubscribers } = await import("./observability");
|
|
265
|
-
const bus = new TraceEventBus({ runId: "run_a", sessionId: "sess_0000000000000001" });
|
|
266
|
-
const ctx = createRunContext();
|
|
267
|
-
const errSpy = spyOn(ctx.logger, "error");
|
|
268
|
-
|
|
269
|
-
// Reject with a plain string (not an Error) so the `String(err)` branch
|
|
270
|
-
// of logFlushError (line 114) is exercised. Use Promise.reject rather than
|
|
271
|
-
// `throw "literal"` to avoid the throw-literal lint while still rejecting
|
|
272
|
-
// with a non-Error value.
|
|
273
|
-
const busFlushSpy = spyOn(bus, "flush").mockImplementation(() =>
|
|
274
|
-
Promise.reject("plain-string-failure"),
|
|
275
|
-
);
|
|
276
|
-
|
|
277
|
-
const subs = await attachDefaultSubscribers(bus, ctx, {});
|
|
278
|
-
await subs.flushAll();
|
|
279
|
-
|
|
280
|
-
const log = errSpy.mock.calls.find((c) => c[0] === "observability.flush_failed");
|
|
281
|
-
expect(log).toBeDefined();
|
|
282
|
-
expect((log?.[1] as { message: string }).message).toBe("plain-string-failure");
|
|
283
|
-
busFlushSpy.mockRestore();
|
|
284
|
-
});
|
|
285
|
-
|
|
286
|
-
test("flushAll with no subscribers attached only flushes the bus", async () => {
|
|
287
|
-
const { attachDefaultSubscribers } = await import("./observability");
|
|
288
|
-
const bus = new TraceEventBus({ runId: "run_a", sessionId: "sess_0000000000000001" });
|
|
289
|
-
const ctx = createRunContext();
|
|
290
|
-
// No env → metrics/otel attach helpers (mocked) still return objects, so
|
|
291
|
-
// flip `attached` off to model the genuine no-subscriber case (the `if`
|
|
292
|
-
// guards on lines 93-95 / 101-106 skip their bodies).
|
|
293
|
-
metricsState.attached = false;
|
|
294
|
-
otelState.attached = false;
|
|
295
|
-
const subs = await attachDefaultSubscribers(bus, ctx, {});
|
|
296
|
-
expect(subs.metrics).toBeUndefined();
|
|
297
|
-
expect(subs.otel).toBeUndefined();
|
|
298
|
-
await subs.flushAll();
|
|
299
|
-
await subs.shutdownAll();
|
|
300
|
-
expect(metricsState.flushCalls).toBe(0);
|
|
301
|
-
expect(otelState.flushCalls).toBe(0);
|
|
302
|
-
});
|
|
303
|
-
});
|
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* T3 — single-turn `runChatLoop` round-trip emits the expected ordered
|
|
3
|
-
* sequence of TraceEvents under one traceId.
|
|
4
|
-
*/
|
|
5
|
-
import { describe, expect, test } from "bun:test";
|
|
6
|
-
import type Anthropic from "@anthropic-ai/sdk";
|
|
7
|
-
import { createRunContext } from "@crewhaus/run-context";
|
|
8
|
-
import { buildTool } from "@crewhaus/tool-builder";
|
|
9
|
-
import type { TraceEvent, TraceEventKind } from "@crewhaus/trace-event-bus";
|
|
10
|
-
import { z } from "zod";
|
|
11
|
-
import { runChatLoop } from "./index";
|
|
12
|
-
|
|
13
|
-
function makeOneTurnAdapter(
|
|
14
|
-
toolUseId: string,
|
|
15
|
-
): import("@crewhaus/adapter-anthropic").ProviderAdapter {
|
|
16
|
-
let i = 0;
|
|
17
|
-
return {
|
|
18
|
-
providerId: "anthropic",
|
|
19
|
-
features: {
|
|
20
|
-
caching: "explicit",
|
|
21
|
-
tool_use: true,
|
|
22
|
-
vision: true,
|
|
23
|
-
thinking: true,
|
|
24
|
-
web_search: true,
|
|
25
|
-
},
|
|
26
|
-
estimateTokens: () => 0,
|
|
27
|
-
stream: () => {
|
|
28
|
-
const isFirst = i === 0;
|
|
29
|
-
i += 1;
|
|
30
|
-
return (async function* () {
|
|
31
|
-
yield {
|
|
32
|
-
kind: "message_start",
|
|
33
|
-
usage: { input: isFirst ? 100 : 150, output: 0 },
|
|
34
|
-
} as const;
|
|
35
|
-
if (isFirst) {
|
|
36
|
-
yield {
|
|
37
|
-
kind: "content_block_start",
|
|
38
|
-
index: 0,
|
|
39
|
-
block: { type: "tool_use", id: toolUseId, name: "noop", input: {} },
|
|
40
|
-
} as const;
|
|
41
|
-
yield {
|
|
42
|
-
kind: "content_block_delta",
|
|
43
|
-
index: 0,
|
|
44
|
-
delta: { type: "input_json_delta", partial_json: JSON.stringify({ x: 1 }) },
|
|
45
|
-
} as const;
|
|
46
|
-
yield { kind: "content_block_stop", index: 0 } as const;
|
|
47
|
-
yield {
|
|
48
|
-
kind: "message_delta",
|
|
49
|
-
stopReason: "tool_use",
|
|
50
|
-
usage: { input: 100, output: 30 },
|
|
51
|
-
} as const;
|
|
52
|
-
} else {
|
|
53
|
-
yield {
|
|
54
|
-
kind: "content_block_start",
|
|
55
|
-
index: 0,
|
|
56
|
-
block: { type: "text", text: "" },
|
|
57
|
-
} as const;
|
|
58
|
-
yield {
|
|
59
|
-
kind: "content_block_delta",
|
|
60
|
-
index: 0,
|
|
61
|
-
delta: { type: "text_delta", text: "done" },
|
|
62
|
-
} as const;
|
|
63
|
-
yield { kind: "content_block_stop", index: 0 } as const;
|
|
64
|
-
yield {
|
|
65
|
-
kind: "message_delta",
|
|
66
|
-
stopReason: "end_turn",
|
|
67
|
-
usage: { input: 150, output: 10 },
|
|
68
|
-
} as const;
|
|
69
|
-
}
|
|
70
|
-
yield { kind: "message_stop" } as const;
|
|
71
|
-
})();
|
|
72
|
-
},
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
describe("runChatLoop observability", () => {
|
|
77
|
-
test("single turn with one tool call emits the expected event sequence", async () => {
|
|
78
|
-
const noop = buildTool({
|
|
79
|
-
name: "noop",
|
|
80
|
-
description: "does nothing",
|
|
81
|
-
inputSchema: z.object({ x: z.number() }),
|
|
82
|
-
execute: async () => "ok",
|
|
83
|
-
});
|
|
84
|
-
const runContext = createRunContext();
|
|
85
|
-
const seen: TraceEvent[] = [];
|
|
86
|
-
runContext.eventBus.subscribe((e) => {
|
|
87
|
-
seen.push(e);
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
await runChatLoop({
|
|
91
|
-
model: "test-model",
|
|
92
|
-
instructions: "test",
|
|
93
|
-
_adapter: makeOneTurnAdapter("toolu_a"),
|
|
94
|
-
runContext,
|
|
95
|
-
singleTurn: true,
|
|
96
|
-
seedMessages: [{ role: "user", content: "hello" }],
|
|
97
|
-
tools: [noop],
|
|
98
|
-
permissionMode: "default",
|
|
99
|
-
});
|
|
100
|
-
|
|
101
|
-
const kinds = seen.map((e) => e.kind);
|
|
102
|
-
// Permissive ordering check: the spine of the run should appear in this order.
|
|
103
|
-
const wantOrder: TraceEventKind[] = [
|
|
104
|
-
"turn_start",
|
|
105
|
-
"model_request",
|
|
106
|
-
"model_response",
|
|
107
|
-
"tool_call_start",
|
|
108
|
-
"permission_decision",
|
|
109
|
-
"tool_call_end",
|
|
110
|
-
"model_request",
|
|
111
|
-
"model_response",
|
|
112
|
-
"turn_end",
|
|
113
|
-
];
|
|
114
|
-
let cursor = 0;
|
|
115
|
-
for (const target of wantOrder) {
|
|
116
|
-
const idx = kinds.indexOf(target, cursor);
|
|
117
|
-
expect(idx).toBeGreaterThanOrEqual(cursor);
|
|
118
|
-
cursor = idx + 1;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
// All events share one traceId.
|
|
122
|
-
const traceIds = new Set(seen.map((e) => e.traceId));
|
|
123
|
-
expect(traceIds.size).toBe(1);
|
|
124
|
-
|
|
125
|
-
// Tool span ids match for the start/end pair.
|
|
126
|
-
const toolStart = seen.find((e) => e.kind === "tool_call_start" && e.toolUseId === "toolu_a");
|
|
127
|
-
const toolEnd = seen.find((e) => e.kind === "tool_call_end" && e.toolUseId === "toolu_a");
|
|
128
|
-
expect(toolStart?.spanId).toBeDefined();
|
|
129
|
-
expect(toolEnd?.spanId).toBe(toolStart?.spanId);
|
|
130
|
-
|
|
131
|
-
// The non-streaming path carries usage onto model_response.
|
|
132
|
-
const response = seen.find(
|
|
133
|
-
(e): e is Extract<TraceEvent, { kind: "model_response" }> => e.kind === "model_response",
|
|
134
|
-
);
|
|
135
|
-
expect(response?.usage.input).toBe(100);
|
|
136
|
-
expect(response?.usage.output).toBe(30);
|
|
137
|
-
});
|
|
138
|
-
});
|
package/src/observability.ts
DELETED
|
@@ -1,116 +0,0 @@
|
|
|
1
|
-
import { type CostTracker, createCostTracker, formatUsdMicros } from "@crewhaus/cost-tracker";
|
|
2
|
-
import {
|
|
3
|
-
type AttachedMetrics,
|
|
4
|
-
attachIfEnvSet as attachMetricsIfEnvSet,
|
|
5
|
-
} from "@crewhaus/metrics-collector";
|
|
6
|
-
import {
|
|
7
|
-
type AttachedOtelExporter,
|
|
8
|
-
attachIfEnvSet as attachOtelIfEnvSet,
|
|
9
|
-
} from "@crewhaus/otel-exporter";
|
|
10
|
-
/**
|
|
11
|
-
* Section 15 — attach the default observability subscribers to a
|
|
12
|
-
* `TraceEventBus` based on environment variables. Returns a single `flushAll`
|
|
13
|
-
* the orchestrator awaits in its `finally` block.
|
|
14
|
-
*
|
|
15
|
-
* CREWHAUS_TRACE=pretty|json → structured-event-printer
|
|
16
|
-
* CREWHAUS_METRICS=stdout|... → metrics-collector (buffered stdout JSON / textfile / http)
|
|
17
|
-
* OTEL_EXPORTER_OTLP_ENDPOINT → otel-exporter (OTLP/HTTP, gen_ai/* attributes)
|
|
18
|
-
* CREWHAUS_COST_TRACKING=1 → cost-tracker (Section 27 — emits cost_accrual events)
|
|
19
|
-
* CREWHAUS_COST_INLINE=1 → CLI inline cost line per turn (requires CREWHAUS_COST_TRACKING)
|
|
20
|
-
*
|
|
21
|
-
* All are opt-in. With no env vars set this function returns a no-op
|
|
22
|
-
* `flushAll` so observability adds zero output and zero overhead.
|
|
23
|
-
*/
|
|
24
|
-
import type { RunContext } from "@crewhaus/run-context";
|
|
25
|
-
import {
|
|
26
|
-
type AttachedPrinter,
|
|
27
|
-
attachIfEnvSet as attachPrinterIfEnvSet,
|
|
28
|
-
} from "@crewhaus/structured-event-printer";
|
|
29
|
-
import type {
|
|
30
|
-
CostAccrualEvent,
|
|
31
|
-
TraceEvent,
|
|
32
|
-
TraceEventBus,
|
|
33
|
-
Unsubscribe,
|
|
34
|
-
} from "@crewhaus/trace-event-bus";
|
|
35
|
-
|
|
36
|
-
export type AttachedSubscribers = {
|
|
37
|
-
printer: AttachedPrinter | undefined;
|
|
38
|
-
metrics: AttachedMetrics | undefined;
|
|
39
|
-
otel: AttachedOtelExporter | undefined;
|
|
40
|
-
costTracker: CostTracker | undefined;
|
|
41
|
-
costInlineUnsubscribe: Unsubscribe | undefined;
|
|
42
|
-
flushAll(): Promise<void>;
|
|
43
|
-
shutdownAll(): Promise<void>;
|
|
44
|
-
};
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Format a single `cost_accrual` event as a one-line CLI status:
|
|
48
|
-
* `[💸 $0.0042 · 12.3k in / 1.1k out · model=claude-sonnet-4-6]`
|
|
49
|
-
*
|
|
50
|
-
* Exported so target-cli's emitted bundle can render the same format
|
|
51
|
-
* without re-implementing it.
|
|
52
|
-
*/
|
|
53
|
-
export function formatCostInlineLine(ev: CostAccrualEvent): string {
|
|
54
|
-
const cost = formatUsdMicros(ev.costUsdMicros);
|
|
55
|
-
const fmtTokens = (n: number): string => (n >= 1000 ? `${(n / 1000).toFixed(1)}k` : `${n}`);
|
|
56
|
-
return `[💸 ${cost} · ${fmtTokens(ev.inputTokens)} in / ${fmtTokens(ev.outputTokens)} out · model=${ev.modelId}]`;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
export async function attachDefaultSubscribers(
|
|
60
|
-
bus: TraceEventBus,
|
|
61
|
-
runContext: RunContext,
|
|
62
|
-
env: NodeJS.ProcessEnv = process.env,
|
|
63
|
-
): Promise<AttachedSubscribers> {
|
|
64
|
-
const printer = attachPrinterIfEnvSet(bus, env);
|
|
65
|
-
const metrics = await attachMetricsIfEnvSet(bus, env);
|
|
66
|
-
const otel = attachOtelIfEnvSet(bus, env);
|
|
67
|
-
const costTracker =
|
|
68
|
-
env["CREWHAUS_COST_TRACKING"] === "1" || env["CREWHAUS_COST_TRACKING"] === "true"
|
|
69
|
-
? createCostTracker(bus, {
|
|
70
|
-
...(env["CREWHAUS_TENANT_ID"] !== undefined
|
|
71
|
-
? { tenantId: env["CREWHAUS_TENANT_ID"] }
|
|
72
|
-
: {}),
|
|
73
|
-
})
|
|
74
|
-
: undefined;
|
|
75
|
-
// CLI inline cost line — opt-in via CREWHAUS_COST_INLINE=1. Subscribes to
|
|
76
|
-
// the `cost_accrual` events that cost-tracker emits, prints one tidy line
|
|
77
|
-
// per accrual, and stays out of the trace bus's main printer surface so
|
|
78
|
-
// it doesn't double-print when CREWHAUS_TRACE=pretty is also set.
|
|
79
|
-
const costInlineEnabled =
|
|
80
|
-
(env["CREWHAUS_COST_INLINE"] === "1" || env["CREWHAUS_COST_INLINE"] === "true") &&
|
|
81
|
-
costTracker !== undefined;
|
|
82
|
-
let costInlineUnsubscribe: Unsubscribe | undefined;
|
|
83
|
-
if (costInlineEnabled) {
|
|
84
|
-
const handler = (event: TraceEvent): void => {
|
|
85
|
-
if (event.kind !== "cost_accrual") return;
|
|
86
|
-
process.stdout.write(`${formatCostInlineLine(event as CostAccrualEvent)}\n`);
|
|
87
|
-
};
|
|
88
|
-
costInlineUnsubscribe = bus.subscribe(handler);
|
|
89
|
-
}
|
|
90
|
-
const flushAll = async (): Promise<void> => {
|
|
91
|
-
printer?.finalize();
|
|
92
|
-
const tasks: Promise<void>[] = [];
|
|
93
|
-
if (metrics)
|
|
94
|
-
tasks.push(metrics.flush().catch((err) => logFlushError(runContext, "metrics", err)));
|
|
95
|
-
if (otel) tasks.push(otel.flush().catch((err) => logFlushError(runContext, "otel", err)));
|
|
96
|
-
tasks.push(bus.flush().catch((err) => logFlushError(runContext, "bus", err)));
|
|
97
|
-
await Promise.all(tasks);
|
|
98
|
-
};
|
|
99
|
-
const shutdownAll = async (): Promise<void> => {
|
|
100
|
-
printer?.unsubscribe();
|
|
101
|
-
if (metrics) {
|
|
102
|
-
await metrics.shutdown().catch((err) => logFlushError(runContext, "metrics", err));
|
|
103
|
-
}
|
|
104
|
-
if (otel) {
|
|
105
|
-
await otel.shutdown().catch((err) => logFlushError(runContext, "otel", err));
|
|
106
|
-
}
|
|
107
|
-
costTracker?.unsubscribe();
|
|
108
|
-
costInlineUnsubscribe?.();
|
|
109
|
-
};
|
|
110
|
-
return { printer, metrics, otel, costTracker, costInlineUnsubscribe, flushAll, shutdownAll };
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
function logFlushError(runContext: RunContext, name: string, err: unknown): void {
|
|
114
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
115
|
-
runContext.logger.error("observability.flush_failed", { name, message });
|
|
116
|
-
}
|
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Coverage tests for the defensive / error branches of loadProjectMemory
|
|
3
|
-
* that the happy-path suite (project-memory.test.ts) does not reach:
|
|
4
|
-
* - a matched path that is a directory, not a file (`!st.isFile()`)
|
|
5
|
-
* - `readFile()` throwing (the `catch { continue }` during read)
|
|
6
|
-
* - `classifyBoundary()` rejecting (the `.catch(() => undefined)` arrow)
|
|
7
|
-
*
|
|
8
|
-
* Module mocks are installed ONCE at file scope (before loadProjectMemory is
|
|
9
|
-
* first imported) and gated behind mutable flags so each test toggles only
|
|
10
|
-
* the failure it needs — this sidesteps ESM live-binding caching, where a
|
|
11
|
-
* mock.module call made after the importer is already loaded would not be
|
|
12
|
-
* observed. Flags are reset in afterEach; afterAll reinstalls the REAL
|
|
13
|
-
* modules — `mock.restore()` does not undo `mock.module`, and Bun shares one
|
|
14
|
-
* module registry across all test files (nondeterministic order), so only
|
|
15
|
-
* the re-mock keeps the stubs from leaking into sibling test files.
|
|
16
|
-
*/
|
|
17
|
-
import { afterAll, afterEach, describe, expect, mock, test } from "bun:test";
|
|
18
|
-
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
19
|
-
import * as realFsPromises from "node:fs/promises";
|
|
20
|
-
import { tmpdir } from "node:os";
|
|
21
|
-
import { join } from "node:path";
|
|
22
|
-
// Real module captured for the afterAll restore.
|
|
23
|
-
import * as realBoundaryClassifierNS from "@crewhaus/boundary-classifier";
|
|
24
|
-
|
|
25
|
-
const realStat = realFsPromises.stat;
|
|
26
|
-
const realReadFile = realFsPromises.readFile;
|
|
27
|
-
// Plain-object SNAPSHOTS for the afterAll restore: an `import * as` namespace
|
|
28
|
-
// is a live view that resolves to the stubs once mock.module patches the
|
|
29
|
-
// module, so restoring from it would be a no-op.
|
|
30
|
-
const realFsPromisesSnapshot = { ...realFsPromises };
|
|
31
|
-
const realBoundaryClassifierSnapshot = { ...realBoundaryClassifierNS };
|
|
32
|
-
|
|
33
|
-
// Per-test toggles. When a `*Fails` is set, the corresponding mocked fn
|
|
34
|
-
// rejects for the named file; otherwise it delegates to the real impl.
|
|
35
|
-
const flags = {
|
|
36
|
-
readFileFailsFor: undefined as string | undefined,
|
|
37
|
-
classifyFails: false,
|
|
38
|
-
classifyCalls: 0,
|
|
39
|
-
};
|
|
40
|
-
|
|
41
|
-
mock.module("node:fs/promises", () => ({
|
|
42
|
-
...realFsPromises,
|
|
43
|
-
stat: realStat,
|
|
44
|
-
readFile: (p: string, enc: BufferEncoding) => {
|
|
45
|
-
if (flags.readFileFailsFor !== undefined && String(p).endsWith(flags.readFileFailsFor)) {
|
|
46
|
-
return Promise.reject(new Error("readFile EIO"));
|
|
47
|
-
}
|
|
48
|
-
return realReadFile(p, enc);
|
|
49
|
-
},
|
|
50
|
-
}));
|
|
51
|
-
|
|
52
|
-
mock.module("@crewhaus/boundary-classifier", () => ({
|
|
53
|
-
classifyBoundary: (..._args: unknown[]) => {
|
|
54
|
-
flags.classifyCalls += 1;
|
|
55
|
-
if (flags.classifyFails) return Promise.reject(new Error("classifier offline"));
|
|
56
|
-
return Promise.resolve({ classification: "clean", hits: [] });
|
|
57
|
-
},
|
|
58
|
-
}));
|
|
59
|
-
|
|
60
|
-
afterEach(() => {
|
|
61
|
-
flags.readFileFailsFor = undefined;
|
|
62
|
-
flags.classifyFails = false;
|
|
63
|
-
flags.classifyCalls = 0;
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
afterAll(() => {
|
|
67
|
-
// Reinstall the real modules — mock.restore() would only undo spies, not
|
|
68
|
-
// mock.module, so re-mocking is the only way to prevent cross-file leaks.
|
|
69
|
-
mock.module("node:fs/promises", () => realFsPromisesSnapshot);
|
|
70
|
-
mock.module("@crewhaus/boundary-classifier", () => realBoundaryClassifierSnapshot);
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
describe("loadProjectMemory — defensive branches", () => {
|
|
74
|
-
test("skips a canonical name that resolves to a directory (not a file)", async () => {
|
|
75
|
-
const { loadProjectMemory } = await import("./project-memory");
|
|
76
|
-
const dir = mkdtempSync(join(tmpdir(), "crewhaus-pm-dir-"));
|
|
77
|
-
try {
|
|
78
|
-
// A *directory* named CLAUDE.md: existsSync true, stat ok, isFile false.
|
|
79
|
-
mkdirSync(join(dir, "CLAUDE.md"));
|
|
80
|
-
writeFileSync(join(dir, "AGENTS.md"), "# real file");
|
|
81
|
-
const result = await loadProjectMemory({ cwd: dir });
|
|
82
|
-
expect(result.files.map((f) => f.filename)).toEqual(["AGENTS.md"]);
|
|
83
|
-
} finally {
|
|
84
|
-
rmSync(dir, { recursive: true, force: true });
|
|
85
|
-
}
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
test("continues past a readFile() failure during read (catch { continue })", async () => {
|
|
89
|
-
const dir = mkdtempSync(join(tmpdir(), "crewhaus-pm-read-"));
|
|
90
|
-
try {
|
|
91
|
-
writeFileSync(join(dir, "AGENTS.md"), "# unreadable");
|
|
92
|
-
writeFileSync(join(dir, "CLAUDE.md"), "# readable");
|
|
93
|
-
flags.readFileFailsFor = "AGENTS.md";
|
|
94
|
-
const { loadProjectMemory } = await import("./project-memory");
|
|
95
|
-
const result = await loadProjectMemory({ cwd: dir });
|
|
96
|
-
// AGENTS.md's read threw → `continue`; only CLAUDE.md is loaded.
|
|
97
|
-
expect(result.files.map((f) => f.filename)).toEqual(["CLAUDE.md"]);
|
|
98
|
-
expect(result.files[0]?.content).toContain("# readable");
|
|
99
|
-
} finally {
|
|
100
|
-
rmSync(dir, { recursive: true, force: true });
|
|
101
|
-
}
|
|
102
|
-
});
|
|
103
|
-
|
|
104
|
-
test("tolerates classifyBoundary rejecting (.catch arrow returns undefined)", async () => {
|
|
105
|
-
const dir = mkdtempSync(join(tmpdir(), "crewhaus-pm-classify-"));
|
|
106
|
-
try {
|
|
107
|
-
writeFileSync(join(dir, "AGENTS.md"), "# content to classify");
|
|
108
|
-
flags.classifyFails = true;
|
|
109
|
-
const { loadProjectMemory } = await import("./project-memory");
|
|
110
|
-
const result = await loadProjectMemory({ cwd: dir });
|
|
111
|
-
// The classifier threw but the load still succeeded (best-effort).
|
|
112
|
-
expect(flags.classifyCalls).toBe(1);
|
|
113
|
-
expect(result.files.length).toBe(1);
|
|
114
|
-
expect(result.files[0]?.content).toContain("# content to classify");
|
|
115
|
-
} finally {
|
|
116
|
-
rmSync(dir, { recursive: true, force: true });
|
|
117
|
-
}
|
|
118
|
-
});
|
|
119
|
-
});
|