@crewhaus/runtime-core 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,138 +0,0 @@
1
- /**
2
- * Regression — Section 27 cost tracking over the live `runChatLoop` path
3
- * with provider-prefixed spec model strings.
4
- *
5
- * `model_response` used to publish the raw spec string
6
- * (`"bedrock/us.anthropic.claude-…"`, `"openai/gpt-4o"`); cost-tracker
7
- * pricing keys are bare wire model ids, so every prefixed run was a
8
- * silent pricing miss charged $0 — only unprefixed `claude-*` strings
9
- * priced. These tests run the real loop (stub adapter) against a
10
- * cost-tracker on the run bus and assert non-zero cost, zero misses, and
11
- * the wire-id/spec-string split on the published events.
12
- */
13
- import { afterAll, beforeAll, describe, expect, test } from "bun:test";
14
- import { mkdtempSync, rmSync } from "node:fs";
15
- import { tmpdir } from "node:os";
16
- import { join } from "node:path";
17
- import type { ProviderAdapter } from "@crewhaus/adapter-anthropic";
18
- import { createCostTracker } from "@crewhaus/cost-tracker";
19
- import { createRunContext } from "@crewhaus/run-context";
20
- import type {
21
- CostAccrualEvent,
22
- ModelResponseEvent,
23
- ProviderId,
24
- TraceEvent,
25
- } from "@crewhaus/trace-event-bus";
26
- import { runChatLoop } from "./index";
27
-
28
- const SHARED_SESSION_ROOT = mkdtempSync(join(tmpdir(), "crewhaus-cost-wire-model-tests-"));
29
- beforeAll(() => {
30
- process.env["CREWHAUS_SESSION_DIR"] = SHARED_SESSION_ROOT;
31
- });
32
- afterAll(() => {
33
- process.env["CREWHAUS_SESSION_DIR"] = undefined;
34
- rmSync(SHARED_SESSION_ROOT, { recursive: true, force: true });
35
- });
36
-
37
- /** Single text-only turn: 100 input tokens, 10 output tokens. */
38
- function makeTextAdapter(providerId: ProviderId): ProviderAdapter {
39
- return {
40
- providerId,
41
- features: {
42
- caching: "explicit",
43
- tool_use: true,
44
- vision: true,
45
- thinking: false,
46
- web_search: false,
47
- },
48
- estimateTokens: () => 0,
49
- stream: () =>
50
- (async function* () {
51
- yield { kind: "message_start", usage: { input: 100, output: 0 } } as const;
52
- yield {
53
- kind: "content_block_start",
54
- index: 0,
55
- block: { type: "text", text: "" },
56
- } as const;
57
- yield {
58
- kind: "content_block_delta",
59
- index: 0,
60
- delta: { type: "text_delta", text: "ok" },
61
- } as const;
62
- yield { kind: "content_block_stop", index: 0 } as const;
63
- yield {
64
- kind: "message_delta",
65
- stopReason: "end_turn",
66
- usage: { input: 100, output: 10 },
67
- } as const;
68
- yield { kind: "message_stop" } as const;
69
- })(),
70
- };
71
- }
72
-
73
- async function runOnce(model: string, providerId: ProviderId) {
74
- const runContext = createRunContext();
75
- const seen: TraceEvent[] = [];
76
- runContext.eventBus.subscribe((e) => {
77
- seen.push(e);
78
- });
79
- const tracker = createCostTracker(runContext.eventBus);
80
- await runChatLoop({
81
- model,
82
- instructions: "test",
83
- _adapter: makeTextAdapter(providerId),
84
- runContext,
85
- singleTurn: true,
86
- seedMessages: [{ role: "user", content: "hello" }],
87
- });
88
- tracker.unsubscribe();
89
- const responses = seen.filter((e): e is ModelResponseEvent => e.kind === "model_response");
90
- const accruals = seen.filter((e): e is CostAccrualEvent => e.kind === "cost_accrual");
91
- return { tracker, responses, accruals, runId: runContext.eventBus.runId };
92
- }
93
-
94
- describe("cost tracking with provider-prefixed spec model strings", () => {
95
- test("bedrock inference-profile spec string prices — no $0 pricing miss", async () => {
96
- const spec = "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0";
97
- const wire = "us.anthropic.claude-sonnet-4-5-20250929-v1:0";
98
- const { tracker, responses, accruals, runId } = await runOnce(spec, "bedrock");
99
-
100
- expect(responses.length).toBe(1);
101
- expect(responses[0]?.model).toBe(wire);
102
- expect(responses[0]?.specModel).toBe(spec);
103
-
104
- expect(tracker.pricingMisses()).toBe(0);
105
- expect(tracker.observed()).toBe(1);
106
- // anthropic.claude-sonnet-4 row: 100 in × $3/M + 10 out × $15/M = 450 micros
107
- expect(tracker.getRunCost(runId).totalUsdMicros).toBe(450);
108
- expect(tracker.getRunCost(runId).byProvider.bedrock).toBe(450);
109
-
110
- expect(accruals.length).toBe(1);
111
- expect(accruals[0]?.modelId).toBe(wire);
112
- expect(accruals[0]?.specModel).toBe(spec);
113
- expect(accruals[0]?.costUsdMicros).toBe(450);
114
- });
115
-
116
- test("openai/* spec string prices — no $0 pricing miss", async () => {
117
- const { tracker, responses, accruals, runId } = await runOnce("openai/gpt-4o", "openai");
118
-
119
- expect(responses[0]?.model).toBe("gpt-4o");
120
- expect(responses[0]?.specModel).toBe("openai/gpt-4o");
121
-
122
- expect(tracker.pricingMisses()).toBe(0);
123
- // gpt-4o row: 100 in × $2.5/M + 10 out × $10/M = 350 micros
124
- expect(tracker.getRunCost(runId).byProvider.openai).toBe(350);
125
- expect(accruals[0]?.modelId).toBe("gpt-4o");
126
- });
127
-
128
- test("unprefixed claude-* spec string still prices; no specModel published", async () => {
129
- const { tracker, responses, accruals, runId } = await runOnce("claude-sonnet-4-6", "anthropic");
130
-
131
- expect(responses[0]?.model).toBe("claude-sonnet-4-6");
132
- expect(responses[0]?.specModel).toBeUndefined();
133
-
134
- expect(tracker.pricingMisses()).toBe(0);
135
- expect(tracker.getRunCost(runId).byProvider.anthropic).toBe(450);
136
- expect(accruals[0]?.specModel).toBeUndefined();
137
- });
138
- });