@johpaz/hive-sdk 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +167 -0
- package/README.md +11 -1
- package/package.json +26 -10
- package/packages/core/src/agent/acceptance-checks.ts +16 -10
- package/packages/core/src/agent/agent-catalog.ts +3 -3
- package/packages/core/src/agent/agent-loop.ts +115 -26
- package/packages/core/src/agent/capability-search.ts +2 -2
- package/packages/core/src/agent/catalog-selector.ts +4 -4
- package/packages/core/src/agent/compaction.ts +10 -9
- package/packages/core/src/agent/context-compiler.ts +58 -34
- package/packages/core/src/agent/conversation-store.ts +31 -7
- package/packages/core/src/agent/curator.ts +4 -4
- package/packages/core/src/agent/delegation-runtime.ts +5 -5
- package/packages/core/src/agent/goal-runner.ts +9 -9
- package/packages/core/src/agent/index.ts +1 -0
- package/packages/core/src/agent/llm-client.ts +98 -37
- package/packages/core/src/agent/llm-providers/anthropic.ts +4 -4
- package/packages/core/src/agent/llm-providers/deepseek.ts +1 -1
- package/packages/core/src/agent/llm-providers/gemini.ts +4 -4
- package/packages/core/src/agent/llm-providers/groq.ts +1 -1
- package/packages/core/src/agent/llm-providers/hiveagents.ts +3 -3
- package/packages/core/src/agent/llm-providers/interface.ts +2 -2
- package/packages/core/src/agent/llm-providers/kimi.ts +1 -1
- package/packages/core/src/agent/llm-providers/minimax.ts +1 -1
- package/packages/core/src/agent/llm-providers/mistral.ts +1 -1
- package/packages/core/src/agent/llm-providers/modelscope.ts +1 -1
- package/packages/core/src/agent/llm-providers/nvidia.ts +1 -1
- package/packages/core/src/agent/llm-providers/ollama.ts +4 -4
- package/packages/core/src/agent/llm-providers/openai-compat-base.ts +9 -5
- package/packages/core/src/agent/llm-providers/openai.ts +1 -1
- package/packages/core/src/agent/llm-providers/opencode-go.ts +1 -1
- package/packages/core/src/agent/llm-providers/openrouter.ts +1 -1
- package/packages/core/src/agent/llm-providers/qwen.ts +1 -1
- package/packages/core/src/agent/llm-providers/z-ai.ts +1 -1
- package/packages/core/src/agent/mcp-result-normalizer.ts +192 -0
- package/packages/core/src/agent/playbook-selector.ts +4 -4
- package/packages/core/src/agent/prompt-builder.ts +5 -5
- package/packages/core/src/agent/proof-packet.ts +5 -5
- package/packages/core/src/agent/providers/index.ts +4 -4
- package/packages/core/src/agent/realtime-providers/gemini-live.ts +238 -0
- package/packages/core/src/agent/realtime-providers/index.ts +29 -0
- package/packages/core/src/agent/realtime-providers/interface.ts +108 -0
- package/packages/core/src/agent/reflector.ts +6 -6
- package/packages/core/src/agent/run-store.ts +8 -8
- package/packages/core/src/agent/service.ts +10 -10
- package/packages/core/src/agent/skill-selector.ts +6 -6
- package/packages/core/src/agent/thread-id.ts +71 -0
- package/packages/core/src/agent/thread-store.ts +250 -0
- package/packages/core/src/agent/tool-selector.ts +7 -5
- package/packages/core/src/agent/tracer.ts +5 -5
- package/packages/core/src/api/createAgent.ts +1 -1
- package/packages/core/src/artifacts/store.ts +84 -3
- package/packages/core/src/canvas/emitter.ts +2 -2
- package/packages/core/src/channels/telegram.ts +1 -1
- package/packages/core/src/channels/webchat.ts +1 -1
- package/packages/core/src/config/loader.ts +15 -1
- package/packages/core/src/events/agent-bus.ts +3 -3
- package/packages/core/src/events/channel-narration.ts +3 -3
- package/packages/core/src/events/event-bus.ts +1 -1
- package/packages/core/src/events/narration.ts +3 -3
- package/packages/core/src/gateway/delegation-groups.ts +4 -4
- package/packages/core/src/gateway/durable-queue.ts +5 -5
- package/packages/core/src/gateway/job-store.ts +5 -5
- package/packages/core/src/gateway/notification-inbox.ts +2 -2
- package/packages/core/src/gateway/server.ts +2 -2
- package/packages/core/src/mcp/MCPClient.ts +3 -3
- package/packages/core/src/mcp/hot-reload.ts +5 -5
- package/packages/core/src/mcp/tool-sync.ts +5 -5
- package/packages/core/src/mcp/transports/index.ts +2 -2
- package/packages/core/src/mcp/transports/sse.ts +1 -1
- package/packages/core/src/models/index.ts +36 -0
- package/packages/core/src/multimodal/index.ts +2 -2
- package/packages/core/src/multimodal/vision-service.ts +6 -6
- package/packages/core/src/plugins/loader.ts +4 -1
- package/packages/core/src/resilience/circuit-breaker.ts +16 -5
- package/packages/core/src/resilience/retry.ts +1 -1
- package/packages/core/src/scheduler/CronScheduler.ts +6 -6
- package/packages/core/src/scheduler/integration.ts +9 -9
- package/packages/core/src/sessions/index.ts +266 -0
- package/packages/core/src/storage/bootstrap.ts +34 -8
- package/packages/core/src/storage/causal-events.ts +1 -1
- package/packages/core/src/storage/collections.ts +32 -1
- package/packages/core/src/storage/crypto.ts +16 -2
- package/packages/core/src/storage/hive.ts +1 -1
- package/packages/core/src/storage/hivedb.ts +10 -1
- package/packages/core/src/storage/onboarding.ts +6 -6
- package/packages/core/src/storage/reconcile.ts +5 -5
- package/packages/core/src/storage/seed.ts +103 -13
- package/packages/core/src/storage/usage.ts +3 -3
- package/packages/core/src/swarm/AgentExecutor.ts +2 -2
- package/packages/core/src/swarm/Coordinator.ts +8 -8
- package/packages/core/src/swarm/EventBridge.ts +2 -2
- package/packages/core/src/swarm/RoleSwarm.ts +234 -0
- package/packages/core/src/swarm/TaskGraph.ts +2 -2
- package/packages/core/src/swarm/index.ts +7 -0
- package/packages/core/src/swarm/presets/HiveLearnPreset.ts +2 -2
- package/packages/core/src/swarm/presets/ResearchPreset.ts +2 -2
- package/packages/core/src/swarm/strategies/ParallelStrategy.ts +1 -1
- package/packages/core/src/swarm/strategies/PriorityStrategy.ts +3 -3
- package/packages/core/src/tools/ToolExecutor.ts +7 -3
- package/packages/core/src/tools/core/index.ts +2 -2
- package/packages/core/src/tools/cron/index.ts +4 -4
- package/packages/core/src/tools/web/artifact-inspect.ts +2 -2
- package/packages/core/src/tools/web/artifact-read.ts +162 -0
- package/packages/core/src/tools/web/browser-backend.ts +226 -0
- package/packages/core/src/tools/web/browser-click.ts +2 -2
- package/packages/core/src/tools/web/browser-extract.ts +2 -2
- package/packages/core/src/tools/web/browser-navigate.ts +2 -2
- package/packages/core/src/tools/web/browser-screenshot.ts +12 -5
- package/packages/core/src/tools/web/browser-script.ts +2 -2
- package/packages/core/src/tools/web/browser-service.ts +85 -366
- package/packages/core/src/tools/web/browser-session.ts +125 -0
- package/packages/core/src/tools/web/browser-type.ts +2 -2
- package/packages/core/src/tools/web/browser-wait.ts +2 -2
- package/packages/core/src/tools/web/computer-use.ts +553 -0
- package/packages/core/src/tools/web/index.ts +8 -1
- package/packages/core/src/tools/web/webview-backend.ts +851 -0
- package/packages/core/src/utils/index.ts +1 -0
- package/packages/core/src/utils/logger.ts +12 -4
- package/packages/core/src/utils/redact-binary.ts +17 -0
- package/packages/core/src/utils/toon.ts +1 -1
- package/packages/core/src/voice/index.ts +6 -6
- package/bun.lock +0 -833
- package/bunfig.toml +0 -9
- package/docs/API-AGENTS.md +0 -367
- package/docs/API-CONTEXT-COMPILER.md +0 -249
- package/docs/API-DAG-SCHEDULER.md +0 -273
- package/docs/API-TOOLS-SKILLS-CHANNELS.md +0 -446
- package/docs/API-WORKERS-EVENTS.md +0 -299
- package/docs/HIVE-HARNESS.md +0 -113
- package/docs/INDEX.md +0 -190
- package/docs/TEMPLATE-HIVE-APP.md +0 -360
- package/packages/cli/package.json +0 -17
- package/packages/cli/src/commands/create-app.test.ts +0 -180
- package/packages/core/package.json +0 -70
- package/packages/core/src/api/createAgent.test.ts +0 -160
- package/packages/core/src/canvas/canvas.test.ts +0 -36
- package/packages/core/src/channels/channels.test.ts +0 -18
- package/packages/core/src/ethics/EthicsGuard.test.ts +0 -108
- package/packages/core/src/gateway/gateway.test.ts +0 -38
- package/packages/core/src/memory/Scratchpad.test.ts +0 -68
- package/packages/core/src/scheduler/scheduler.test.ts +0 -15
- package/packages/core/src/skills/skills.test.ts +0 -62
- package/packages/core/src/swarm/swarm.test.ts +0 -24
- package/packages/core/src/tool-runtime/tool-runtime.test.ts +0 -99
- package/packages/core/src/tools/ToolRegistry.test.ts +0 -98
- package/packages/core/src/tools/api/api-request.test.ts +0 -164
- package/packages/core/src/tools/web/browser-service.test.ts +0 -83
- package/packages/core/src/workers/workers.test.ts +0 -41
- package/scripts/bump-version.ts +0 -248
- package/scripts/generate-skill-bundle.ts +0 -108
- package/test/agent-loop-terminal-synthesis.test.ts +0 -32
- package/test/catalog-agents-stay-enabled.test.ts +0 -117
- package/test/causal-events.test.ts +0 -117
- package/test/compaction.test.ts +0 -105
- package/test/context-compiler.test.ts +0 -269
- package/test/curator.test.ts +0 -130
- package/test/durable-queue.test.ts +0 -114
- package/test/harness-barrel.test.ts +0 -64
- package/test/hive-helpers.test.ts +0 -130
- package/test/hivedb-search.test.ts +0 -189
- package/test/internal-turns.test.ts +0 -166
- package/test/job-idempotency.test.ts +0 -68
- package/test/job-retry-backoff.test.ts +0 -184
- package/test/job-store.test.ts +0 -381
- package/test/llm-retry.test.ts +0 -97
- package/test/memory-perf.test.ts +0 -774
- package/test/minimal-loadout.test.ts +0 -78
- package/test/model-catalog.test.ts +0 -105
- package/test/preload.ts +0 -12
- package/test/reflector.test.ts +0 -320
- package/test/retention-cap.test.ts +0 -91
- package/test/retired-capabilities-pruned.test.ts +0 -192
- package/test/run-store.test.ts +0 -355
- package/test/scratchpad.test.ts +0 -74
- package/test/secrets-durability.test.ts +0 -119
- package/test/seed-model-reseed.test.ts +0 -155
- package/test/setup-agent-seed.test.ts +0 -264
- package/test/tool-inventory.test.ts +0 -65
- package/test/tool-runtime.test.ts +0 -258
- package/test/toon.test.ts +0 -429
- package/tsconfig.json +0 -42
|
@@ -1,184 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Tests for gateway/job-store.ts — failJobOrRetry: logical-failure retries
|
|
3
|
-
* with exponential backoff + jitter, separate from `attempts` (crash/lease
|
|
4
|
-
* only). Plus DurableLaneQueue wiring: chat_turn never retries, other types
|
|
5
|
-
* retry via the queue when the executor returns retryable:false-or-absent.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
process.env.HIVE_DB_PATH = ":memory:";
|
|
9
|
-
|
|
10
|
-
import { describe, test, expect, beforeEach, afterEach } from "bun:test";
|
|
11
|
-
import { closeHiveDb } from "../packages/core/src/storage/hivedb";
|
|
12
|
-
import { ensureHiveDb } from "../packages/core/src/storage/bootstrap";
|
|
13
|
-
import { resetBootId, getBootId } from "../packages/core/src/storage/boot-id";
|
|
14
|
-
import {
|
|
15
|
-
createJob,
|
|
16
|
-
claimJob,
|
|
17
|
-
failJobOrRetry,
|
|
18
|
-
getJob,
|
|
19
|
-
computeBackoffDelay,
|
|
20
|
-
type JobRetryPolicy,
|
|
21
|
-
} from "../packages/core/src/gateway/job-store";
|
|
22
|
-
import { DurableLaneQueue, registerExecutor } from "../packages/core/src/gateway/durable-queue";
|
|
23
|
-
import { col } from "../packages/core/src/storage/hive";
|
|
24
|
-
import type { JobDoc } from "../packages/core/src/storage/collections";
|
|
25
|
-
|
|
26
|
-
beforeEach(async () => {
|
|
27
|
-
closeHiveDb();
|
|
28
|
-
resetBootId();
|
|
29
|
-
await ensureHiveDb();
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
afterEach(() => {
|
|
33
|
-
closeHiveDb();
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
const FAST_POLICY: JobRetryPolicy = {
|
|
37
|
-
maxRetries: 2,
|
|
38
|
-
initialDelayMs: 10,
|
|
39
|
-
backoffMultiplier: 2,
|
|
40
|
-
maxDelayMs: 1000,
|
|
41
|
-
jitter: 0,
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
describe("job-store: computeBackoffDelay", () => {
|
|
45
|
-
test("grows exponentially and caps at maxDelayMs", () => {
|
|
46
|
-
const policy: JobRetryPolicy = { maxRetries: 10, initialDelayMs: 100, backoffMultiplier: 2, maxDelayMs: 500, jitter: 0 };
|
|
47
|
-
expect(computeBackoffDelay(0, policy)).toBe(100);
|
|
48
|
-
expect(computeBackoffDelay(1, policy)).toBe(200);
|
|
49
|
-
expect(computeBackoffDelay(2, policy)).toBe(400);
|
|
50
|
-
expect(computeBackoffDelay(3, policy)).toBe(500); // capped
|
|
51
|
-
expect(computeBackoffDelay(10, policy)).toBe(500); // still capped
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
test("jitter adds a bounded random amount on top of the base delay", () => {
|
|
55
|
-
const policy: JobRetryPolicy = { maxRetries: 10, initialDelayMs: 100, backoffMultiplier: 2, maxDelayMs: 10_000, jitter: 0.5 };
|
|
56
|
-
for (let i = 0; i < 20; i++) {
|
|
57
|
-
const delay = computeBackoffDelay(0, policy);
|
|
58
|
-
expect(delay).toBeGreaterThanOrEqual(100);
|
|
59
|
-
expect(delay).toBeLessThanOrEqual(150);
|
|
60
|
-
}
|
|
61
|
-
});
|
|
62
|
-
});
|
|
63
|
-
|
|
64
|
-
describe("job-store: failJobOrRetry", () => {
|
|
65
|
-
test("schedules a retry: pending with a future not_before, retry_count incremented", async () => {
|
|
66
|
-
const job = await createJob({ lane: "s1", type: "worker_task", payload: {}, run_id: "r1" });
|
|
67
|
-
await claimJob(job.id);
|
|
68
|
-
|
|
69
|
-
const before = Date.now();
|
|
70
|
-
const result = await failJobOrRetry(job.id, "transient error", getBootId(), FAST_POLICY);
|
|
71
|
-
|
|
72
|
-
expect(result).not.toBeNull();
|
|
73
|
-
expect(result!.status).toBe("pending");
|
|
74
|
-
expect(result!.retry_count).toBe(1);
|
|
75
|
-
expect(result!.last_error).toBe("transient error");
|
|
76
|
-
expect(result!.not_before).toBeGreaterThan(before);
|
|
77
|
-
expect(result!.boot_id).toBeNull();
|
|
78
|
-
expect(result!.lease_expires_at).toBeNull();
|
|
79
|
-
|
|
80
|
-
const fromDb = await getJob(job.id);
|
|
81
|
-
expect(fromDb!.status).toBe("pending");
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
test("does not touch `attempts` (crash counter) — only `retry_count`", async () => {
|
|
85
|
-
const job = await createJob({ lane: "s1", type: "worker_task", payload: {}, run_id: "r1" });
|
|
86
|
-
const claimed = await claimJob(job.id);
|
|
87
|
-
expect(claimed!.attempts).toBe(1);
|
|
88
|
-
|
|
89
|
-
const result = await failJobOrRetry(job.id, "boom", getBootId(), FAST_POLICY);
|
|
90
|
-
expect(result!.attempts).toBe(1); // unchanged
|
|
91
|
-
expect(result!.retry_count).toBe(1);
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
test("fails terminally once retry_count reaches policy.maxRetries", async () => {
|
|
95
|
-
const job = await createJob({ lane: "s1", type: "worker_task", payload: {}, run_id: "r1" });
|
|
96
|
-
const c = await col<JobDoc>("jobQueue");
|
|
97
|
-
|
|
98
|
-
// Exhaust retries one at a time (each retry re-pends the job; re-claim before the next failure).
|
|
99
|
-
// Force not_before into the past after each retry so claimJob doesn't block on the backoff delay.
|
|
100
|
-
for (let i = 0; i < FAST_POLICY.maxRetries; i++) {
|
|
101
|
-
await claimJob(job.id);
|
|
102
|
-
const r = await failJobOrRetry(job.id, `attempt ${i}`, getBootId(), FAST_POLICY);
|
|
103
|
-
expect(r!.status).toBe("pending");
|
|
104
|
-
const entry = await c.get(job.id);
|
|
105
|
-
await c.put(job.id, { ...entry!.doc, not_before: Date.now() - 1 }, { expectedVersion: entry!.version });
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
await claimJob(job.id);
|
|
109
|
-
const final = await failJobOrRetry(job.id, "final failure", getBootId(), FAST_POLICY);
|
|
110
|
-
expect(final!.status).toBe("failed");
|
|
111
|
-
expect(final!.error).toBe("final failure");
|
|
112
|
-
expect(final!.last_error).toBe("final failure");
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
test("ignores a job owned by a different boot_id", async () => {
|
|
116
|
-
const job = await createJob({ lane: "s1", type: "worker_task", payload: {}, run_id: "r1" });
|
|
117
|
-
await claimJob(job.id, "boot-A");
|
|
118
|
-
|
|
119
|
-
const result = await failJobOrRetry(job.id, "err", "boot-B", FAST_POLICY);
|
|
120
|
-
expect(result).toBeNull();
|
|
121
|
-
|
|
122
|
-
const fromDb = await getJob(job.id);
|
|
123
|
-
expect(fromDb!.status).toBe("running");
|
|
124
|
-
});
|
|
125
|
-
});
|
|
126
|
-
|
|
127
|
-
async function waitFor(predicate: () => Promise<boolean>, timeoutMs = 3000): Promise<void> {
|
|
128
|
-
const start = Date.now();
|
|
129
|
-
while (Date.now() - start < timeoutMs) {
|
|
130
|
-
if (await predicate()) return;
|
|
131
|
-
await new Promise((r) => setTimeout(r, 10));
|
|
132
|
-
}
|
|
133
|
-
throw new Error("waitFor timed out");
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
describe("durable-queue: retry-on-logical-failure wiring", () => {
|
|
137
|
-
let queue: DurableLaneQueue | null = null;
|
|
138
|
-
|
|
139
|
-
afterEach(() => {
|
|
140
|
-
queue?.stop();
|
|
141
|
-
queue = null;
|
|
142
|
-
});
|
|
143
|
-
|
|
144
|
-
test("a retryable logical failure eventually completes after retrying", async () => {
|
|
145
|
-
let calls = 0;
|
|
146
|
-
registerExecutor("worker_task", async () => {
|
|
147
|
-
calls++;
|
|
148
|
-
if (calls < 2) return { ok: false, error: "flaky", retryable: true };
|
|
149
|
-
return { ok: true, result: "recovered" };
|
|
150
|
-
});
|
|
151
|
-
|
|
152
|
-
queue = new DurableLaneQueue({ maxGlobalConcurrency: 2, jobRetryPolicy: FAST_POLICY });
|
|
153
|
-
const job = await queue.enqueue({ lane: "task:retry-1", type: "worker_task", run_id: "r1", payload: {} });
|
|
154
|
-
|
|
155
|
-
// Wait past the (10ms) backoff delay, then use the public start() — it
|
|
156
|
-
// re-dispatches any due-pending lane — instead of the real 10s timer.
|
|
157
|
-
// The extra buffer avoids racing the exact not_before boundary.
|
|
158
|
-
await waitFor(async () => (await getJob(job.id))?.status === "pending" && (await getJob(job.id))!.retry_count === 1);
|
|
159
|
-
await new Promise((r) => setTimeout(r, 30));
|
|
160
|
-
queue.start();
|
|
161
|
-
|
|
162
|
-
await waitFor(async () => (await getJob(job.id))?.status === "completed");
|
|
163
|
-
const final = await getJob(job.id);
|
|
164
|
-
expect(calls).toBe(2);
|
|
165
|
-
expect(final!.status).toBe("completed");
|
|
166
|
-
expect(JSON.parse(final!.result_json!)).toBe("recovered");
|
|
167
|
-
});
|
|
168
|
-
|
|
169
|
-
test("chat_turn jobs never auto-retry a logical failure", async () => {
|
|
170
|
-
let calls = 0;
|
|
171
|
-
registerExecutor("chat_turn", async () => {
|
|
172
|
-
calls++;
|
|
173
|
-
return { ok: false, error: "user-facing failure" };
|
|
174
|
-
});
|
|
175
|
-
|
|
176
|
-
queue = new DurableLaneQueue({ maxGlobalConcurrency: 2, jobRetryPolicy: FAST_POLICY });
|
|
177
|
-
const job = await queue.enqueue({ lane: "session-1", type: "chat_turn", run_id: "r1", payload: {} });
|
|
178
|
-
|
|
179
|
-
await waitFor(async () => (await getJob(job.id))?.status === "failed");
|
|
180
|
-
expect(calls).toBe(1);
|
|
181
|
-
const final = await getJob(job.id);
|
|
182
|
-
expect(final!.retry_count).toBe(0);
|
|
183
|
-
});
|
|
184
|
-
});
|
package/test/job-store.test.ts
DELETED
|
@@ -1,381 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Tests for gateway/job-store.ts — JobDoc lifecycle, claim OCC, lease renewal,
|
|
3
|
-
* reclaim after expiry, cancel.
|
|
4
|
-
* Uses HIVE_DB_PATH=":memory:" so no state persists between runs.
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
process.env.HIVE_DB_PATH = ":memory:";
|
|
8
|
-
|
|
9
|
-
import { describe, test, expect, beforeEach, afterEach } from "bun:test";
|
|
10
|
-
import { closeHiveDb } from "../packages/core/src/storage/hivedb";
|
|
11
|
-
import { ensureHiveDb } from "../packages/core/src/storage/bootstrap";
|
|
12
|
-
import { col } from "../packages/core/src/storage/hive";
|
|
13
|
-
import type { JobDoc } from "../packages/core/src/storage/collections";
|
|
14
|
-
import { getBootId, resetBootId } from "../packages/core/src/storage/boot-id";
|
|
15
|
-
import {
|
|
16
|
-
createJob,
|
|
17
|
-
claimJob,
|
|
18
|
-
renewLease,
|
|
19
|
-
completeJob,
|
|
20
|
-
failJob,
|
|
21
|
-
reclaimOrInterrupt,
|
|
22
|
-
cancelJob,
|
|
23
|
-
findPendingJobsByLane,
|
|
24
|
-
findExpiredLeases,
|
|
25
|
-
getJob,
|
|
26
|
-
} from "../packages/core/src/gateway/job-store";
|
|
27
|
-
|
|
28
|
-
beforeEach(async () => {
|
|
29
|
-
closeHiveDb();
|
|
30
|
-
resetBootId();
|
|
31
|
-
await ensureHiveDb();
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
afterEach(() => {
|
|
35
|
-
closeHiveDb();
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
describe("job-store: createJob", () => {
|
|
39
|
-
test("creates a pending job with correct defaults", async () => {
|
|
40
|
-
const job = await createJob({
|
|
41
|
-
lane: "session-1",
|
|
42
|
-
type: "chat_turn",
|
|
43
|
-
payload: { threadId: "t1", message: "hello" },
|
|
44
|
-
run_id: "run1",
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
expect(job.status).toBe("pending");
|
|
48
|
-
expect(job.lane).toBe("session-1");
|
|
49
|
-
expect(job.type).toBe("chat_turn");
|
|
50
|
-
expect(job.attempts).toBe(0);
|
|
51
|
-
expect(job.max_attempts).toBe(2);
|
|
52
|
-
expect(job.priority).toBe(0);
|
|
53
|
-
expect(job.boot_id).toBeNull();
|
|
54
|
-
expect(job.lease_expires_at).toBeNull();
|
|
55
|
-
expect(job.result_json).toBeNull();
|
|
56
|
-
expect(job.error).toBeNull();
|
|
57
|
-
|
|
58
|
-
const payload = JSON.parse(job.payload_json);
|
|
59
|
-
expect(payload.threadId).toBe("t1");
|
|
60
|
-
expect(payload.message).toBe("hello");
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
test("creates with priority and max_attempts overrides", async () => {
|
|
64
|
-
const job = await createJob({
|
|
65
|
-
lane: "task:1",
|
|
66
|
-
type: "worker_task",
|
|
67
|
-
payload: { task: "build" },
|
|
68
|
-
run_id: "run2",
|
|
69
|
-
priority: 10,
|
|
70
|
-
max_attempts: 5,
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
expect(job.priority).toBe(10);
|
|
74
|
-
expect(job.max_attempts).toBe(5);
|
|
75
|
-
});
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
describe("job-store: claimJob OCC", () => {
|
|
79
|
-
test("claims a pending job → running with boot_id + lease", async () => {
|
|
80
|
-
const job = await createJob({
|
|
81
|
-
lane: "session-1",
|
|
82
|
-
type: "chat_turn",
|
|
83
|
-
payload: {},
|
|
84
|
-
run_id: "run1",
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
const claimed = await claimJob(job.id);
|
|
88
|
-
expect(claimed).not.toBeNull();
|
|
89
|
-
expect(claimed!.status).toBe("running");
|
|
90
|
-
expect(claimed!.attempts).toBe(1);
|
|
91
|
-
expect(claimed!.boot_id).toBe(getBootId());
|
|
92
|
-
expect(claimed!.lease_expires_at).toBeGreaterThan(Date.now());
|
|
93
|
-
|
|
94
|
-
const fromDb = await getJob(job.id);
|
|
95
|
-
expect(fromDb!.status).toBe("running");
|
|
96
|
-
});
|
|
97
|
-
|
|
98
|
-
test("claiming a non-pending job returns null", async () => {
|
|
99
|
-
const job = await createJob({
|
|
100
|
-
lane: "session-1",
|
|
101
|
-
type: "chat_turn",
|
|
102
|
-
payload: {},
|
|
103
|
-
run_id: "run1",
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
await claimJob(job.id);
|
|
107
|
-
const secondAttempt = await claimJob(job.id);
|
|
108
|
-
expect(secondAttempt).toBeNull();
|
|
109
|
-
});
|
|
110
|
-
|
|
111
|
-
test("claiming with not_before in future returns null", async () => {
|
|
112
|
-
const job = await createJob({
|
|
113
|
-
lane: "session-1",
|
|
114
|
-
type: "chat_turn",
|
|
115
|
-
payload: {},
|
|
116
|
-
run_id: "run1",
|
|
117
|
-
not_before: Date.now() + 60_000,
|
|
118
|
-
});
|
|
119
|
-
|
|
120
|
-
const claimed = await claimJob(job.id);
|
|
121
|
-
expect(claimed).toBeNull();
|
|
122
|
-
});
|
|
123
|
-
|
|
124
|
-
test("two bootIds race — only one wins (OCC guarantee)", async () => {
|
|
125
|
-
const job = await createJob({
|
|
126
|
-
lane: "session-1",
|
|
127
|
-
type: "chat_turn",
|
|
128
|
-
payload: {},
|
|
129
|
-
run_id: "run1",
|
|
130
|
-
});
|
|
131
|
-
|
|
132
|
-
// Simulate two different boots attempting claim simultaneously
|
|
133
|
-
const [claim1, claim2] = await Promise.all([
|
|
134
|
-
claimJob(job.id, "boot-A"),
|
|
135
|
-
claimJob(job.id, "boot-B"),
|
|
136
|
-
]);
|
|
137
|
-
|
|
138
|
-
// At least one should win, at most one should win
|
|
139
|
-
const winners = [claim1, claim2].filter((c) => c !== null);
|
|
140
|
-
expect(winners.length).toBe(1);
|
|
141
|
-
const winningBoot = winners[0]!.boot_id;
|
|
142
|
-
expect(["boot-A", "boot-B"]).toContain(winningBoot);
|
|
143
|
-
});
|
|
144
|
-
});
|
|
145
|
-
|
|
146
|
-
describe("job-store: lease renewal", () => {
|
|
147
|
-
test("renewLease extends lease_expires_at", async () => {
|
|
148
|
-
const job = await createJob({
|
|
149
|
-
lane: "s1",
|
|
150
|
-
type: "chat_turn",
|
|
151
|
-
payload: {},
|
|
152
|
-
run_id: "r1",
|
|
153
|
-
});
|
|
154
|
-
await claimJob(job.id);
|
|
155
|
-
const before = await getJob(job.id);
|
|
156
|
-
const oldLease = before!.lease_expires_at!;
|
|
157
|
-
|
|
158
|
-
// Wait a bit so the new lease is strictly greater
|
|
159
|
-
await new Promise((r) => setTimeout(r, 10));
|
|
160
|
-
const ok = await renewLease(job.id);
|
|
161
|
-
expect(ok).toBe(true);
|
|
162
|
-
|
|
163
|
-
const after = await getJob(job.id);
|
|
164
|
-
expect(after!.lease_expires_at!).toBeGreaterThan(oldLease);
|
|
165
|
-
});
|
|
166
|
-
|
|
167
|
-
test("renewLease fails for wrong boot_id", async () => {
|
|
168
|
-
const job = await createJob({
|
|
169
|
-
lane: "s1",
|
|
170
|
-
type: "chat_turn",
|
|
171
|
-
payload: {},
|
|
172
|
-
run_id: "r1",
|
|
173
|
-
});
|
|
174
|
-
await claimJob(job.id, "boot-A");
|
|
175
|
-
|
|
176
|
-
const ok = await renewLease(job.id, "boot-B");
|
|
177
|
-
expect(ok).toBe(false);
|
|
178
|
-
});
|
|
179
|
-
});
|
|
180
|
-
|
|
181
|
-
describe("job-store: completeJob / failJob", () => {
|
|
182
|
-
test("completeJob saves result_json and clears lease", async () => {
|
|
183
|
-
const job = await createJob({
|
|
184
|
-
lane: "s1",
|
|
185
|
-
type: "chat_turn",
|
|
186
|
-
payload: {},
|
|
187
|
-
run_id: "r1",
|
|
188
|
-
});
|
|
189
|
-
await claimJob(job.id);
|
|
190
|
-
|
|
191
|
-
await completeJob(job.id, { response: "done" });
|
|
192
|
-
|
|
193
|
-
const saved = await getJob(job.id);
|
|
194
|
-
expect(saved!.status).toBe("completed");
|
|
195
|
-
expect(saved!.result_json).not.toBeNull();
|
|
196
|
-
const result = JSON.parse(saved!.result_json!);
|
|
197
|
-
expect(result.response).toBe("done");
|
|
198
|
-
expect(saved!.boot_id).toBeNull();
|
|
199
|
-
expect(saved!.lease_expires_at).toBeNull();
|
|
200
|
-
expect(saved!.finished_at).not.toBeNull();
|
|
201
|
-
});
|
|
202
|
-
|
|
203
|
-
test("failJob saves error and clears lease", async () => {
|
|
204
|
-
const job = await createJob({
|
|
205
|
-
lane: "s1",
|
|
206
|
-
type: "worker_task",
|
|
207
|
-
payload: {},
|
|
208
|
-
run_id: "r1",
|
|
209
|
-
});
|
|
210
|
-
await claimJob(job.id);
|
|
211
|
-
|
|
212
|
-
await failJob(job.id, "Worker crashed");
|
|
213
|
-
|
|
214
|
-
const saved = await getJob(job.id);
|
|
215
|
-
expect(saved!.status).toBe("failed");
|
|
216
|
-
expect(saved!.error).toBe("Worker crashed");
|
|
217
|
-
expect(saved!.boot_id).toBeNull();
|
|
218
|
-
});
|
|
219
|
-
|
|
220
|
-
test("completeJob refuses to overwrite a job owned by different boot", async () => {
|
|
221
|
-
const job = await createJob({
|
|
222
|
-
lane: "s1",
|
|
223
|
-
type: "chat_turn",
|
|
224
|
-
payload: {},
|
|
225
|
-
run_id: "r1",
|
|
226
|
-
});
|
|
227
|
-
await claimJob(job.id, "boot-A");
|
|
228
|
-
|
|
229
|
-
// boot-B tries to complete — should be ignored
|
|
230
|
-
await completeJob(job.id, { response: "bad" }, "boot-B");
|
|
231
|
-
|
|
232
|
-
const saved = await getJob(job.id);
|
|
233
|
-
expect(saved!.status).toBe("running");
|
|
234
|
-
expect(saved!.result_json).toBeNull();
|
|
235
|
-
});
|
|
236
|
-
});
|
|
237
|
-
|
|
238
|
-
describe("job-store: reclaimOrInterrupt", () => {
|
|
239
|
-
test("reclaims job back to pending when lease expired and attempts < max", async () => {
|
|
240
|
-
const job = await createJob({
|
|
241
|
-
lane: "s1",
|
|
242
|
-
type: "chat_turn",
|
|
243
|
-
payload: {},
|
|
244
|
-
run_id: "r1",
|
|
245
|
-
max_attempts: 3,
|
|
246
|
-
});
|
|
247
|
-
await claimJob(job.id, "boot-A");
|
|
248
|
-
|
|
249
|
-
// Expire the lease
|
|
250
|
-
const c = await col<JobDoc>("jobQueue");
|
|
251
|
-
const entry = await c.get(job.id);
|
|
252
|
-
await c.put(job.id, { ...entry!.doc, lease_expires_at: Date.now() - 1000 }, { expectedVersion: entry!.version });
|
|
253
|
-
|
|
254
|
-
const result = await reclaimOrInterrupt(job.id);
|
|
255
|
-
expect(result).not.toBeNull();
|
|
256
|
-
expect(result!.status).toBe("pending");
|
|
257
|
-
expect(result!.attempts).toBe(1);
|
|
258
|
-
expect(result!.boot_id).toBeNull();
|
|
259
|
-
});
|
|
260
|
-
|
|
261
|
-
test("interrupts job when attempts exhausted", async () => {
|
|
262
|
-
const job = await createJob({
|
|
263
|
-
lane: "s1",
|
|
264
|
-
type: "worker_task",
|
|
265
|
-
payload: {},
|
|
266
|
-
run_id: "r1",
|
|
267
|
-
max_attempts: 2,
|
|
268
|
-
});
|
|
269
|
-
// Claim twice to exhaust attempts
|
|
270
|
-
await claimJob(job.id, "boot-A");
|
|
271
|
-
const c = await col<JobDoc>("jobQueue");
|
|
272
|
-
let entry = await c.get(job.id);
|
|
273
|
-
await c.put(job.id, { ...entry!.doc, lease_expires_at: Date.now() - 1000 }, { expectedVersion: entry!.version });
|
|
274
|
-
await reclaimOrInterrupt(job.id); // attempt 1 → pending
|
|
275
|
-
|
|
276
|
-
entry = await c.get(job.id);
|
|
277
|
-
expect(entry!.doc.status).toBe("pending");
|
|
278
|
-
|
|
279
|
-
await claimJob(job.id, "boot-B"); // attempt 2 → running
|
|
280
|
-
entry = await c.get(job.id);
|
|
281
|
-
await c.put(job.id, { ...entry!.doc, lease_expires_at: Date.now() - 1000 }, { expectedVersion: entry!.version });
|
|
282
|
-
const result = await reclaimOrInterrupt(job.id); // attempt 2 = max → interrupted
|
|
283
|
-
|
|
284
|
-
expect(result!.status).toBe("interrupted");
|
|
285
|
-
expect(result!.error).toContain("Max attempts");
|
|
286
|
-
});
|
|
287
|
-
});
|
|
288
|
-
|
|
289
|
-
describe("job-store: cancelJob", () => {
|
|
290
|
-
test("cancels a pending job", async () => {
|
|
291
|
-
const job = await createJob({
|
|
292
|
-
lane: "s1",
|
|
293
|
-
type: "chat_turn",
|
|
294
|
-
payload: {},
|
|
295
|
-
run_id: "r1",
|
|
296
|
-
});
|
|
297
|
-
|
|
298
|
-
const ok = await cancelJob(job.id);
|
|
299
|
-
expect(ok).toBe(true);
|
|
300
|
-
|
|
301
|
-
const saved = await getJob(job.id);
|
|
302
|
-
expect(saved!.status).toBe("cancelled");
|
|
303
|
-
});
|
|
304
|
-
|
|
305
|
-
test("cancels a running job", async () => {
|
|
306
|
-
const job = await createJob({
|
|
307
|
-
lane: "s1",
|
|
308
|
-
type: "chat_turn",
|
|
309
|
-
payload: {},
|
|
310
|
-
run_id: "r1",
|
|
311
|
-
});
|
|
312
|
-
await claimJob(job.id);
|
|
313
|
-
|
|
314
|
-
const ok = await cancelJob(job.id);
|
|
315
|
-
expect(ok).toBe(true);
|
|
316
|
-
|
|
317
|
-
const saved = await getJob(job.id);
|
|
318
|
-
expect(saved!.status).toBe("cancelled");
|
|
319
|
-
});
|
|
320
|
-
|
|
321
|
-
test("refuses to cancel an already completed job", async () => {
|
|
322
|
-
const job = await createJob({
|
|
323
|
-
lane: "s1",
|
|
324
|
-
type: "chat_turn",
|
|
325
|
-
payload: {},
|
|
326
|
-
run_id: "r1",
|
|
327
|
-
});
|
|
328
|
-
await claimJob(job.id);
|
|
329
|
-
await completeJob(job.id, { response: "done" });
|
|
330
|
-
|
|
331
|
-
const ok = await cancelJob(job.id);
|
|
332
|
-
expect(ok).toBe(false);
|
|
333
|
-
});
|
|
334
|
-
});
|
|
335
|
-
|
|
336
|
-
describe("job-store: findPendingJobsByLane", () => {
|
|
337
|
-
test("returns pending jobs ordered by priority desc then FIFO by id", async () => {
|
|
338
|
-
const jobLow = await createJob({ lane: "sports", type: "chat_turn", payload: {}, run_id: "r1", priority: 0 });
|
|
339
|
-
const jobHigh = await createJob({ lane: "sports", type: "chat_turn", payload: {}, run_id: "r2", priority: 10 });
|
|
340
|
-
const jobMid = await createJob({ lane: "sports", type: "chat_turn", payload: {}, run_id: "r3", priority: 5 });
|
|
341
|
-
const other = await createJob({ lane: "news", type: "chat_turn", payload: {}, run_id: "r4", priority: 100 });
|
|
342
|
-
|
|
343
|
-
const pending = await findPendingJobsByLane("sports");
|
|
344
|
-
expect(pending.length).toBe(3);
|
|
345
|
-
expect(pending[0].id).toBe(jobHigh.id); // priority 10
|
|
346
|
-
expect(pending[1].id).toBe(jobMid.id); // priority 5
|
|
347
|
-
expect(pending[2].id).toBe(jobLow.id); // priority 0
|
|
348
|
-
});
|
|
349
|
-
|
|
350
|
-
test("excludes pending jobs with not_before in future", async () => {
|
|
351
|
-
await createJob({ lane: "s1", type: "chat_turn", payload: {}, run_id: "r1" });
|
|
352
|
-
await createJob({ lane: "s1", type: "chat_turn", payload: {}, run_id: "r2", not_before: Date.now() + 60_000 });
|
|
353
|
-
|
|
354
|
-
const pending = await findPendingJobsByLane("s1");
|
|
355
|
-
expect(pending.length).toBe(1);
|
|
356
|
-
});
|
|
357
|
-
});
|
|
358
|
-
|
|
359
|
-
describe("job-store: findExpiredLeases", () => {
|
|
360
|
-
test("finds running jobs with expired leases", async () => {
|
|
361
|
-
const job = await createJob({ lane: "s1", type: "chat_turn", payload: {}, run_id: "r1" });
|
|
362
|
-
await claimJob(job.id, "boot-A");
|
|
363
|
-
|
|
364
|
-
// Expire the lease
|
|
365
|
-
const c = await col<JobDoc>("jobQueue");
|
|
366
|
-
const entry = await c.get(job.id);
|
|
367
|
-
await c.put(job.id, { ...entry!.doc, lease_expires_at: Date.now() - 1000 }, { expectedVersion: entry!.version });
|
|
368
|
-
|
|
369
|
-
const expired = await findExpiredLeases();
|
|
370
|
-
expect(expired.length).toBe(1);
|
|
371
|
-
expect(expired[0].id).toBe(job.id);
|
|
372
|
-
});
|
|
373
|
-
|
|
374
|
-
test("excludes jobs with active leases", async () => {
|
|
375
|
-
const job = await createJob({ lane: "s1", type: "chat_turn", payload: {}, run_id: "r1" });
|
|
376
|
-
await claimJob(job.id);
|
|
377
|
-
|
|
378
|
-
const expired = await findExpiredLeases();
|
|
379
|
-
expect(expired.length).toBe(0);
|
|
380
|
-
});
|
|
381
|
-
});
|
package/test/llm-retry.test.ts
DELETED
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Tests for resilience/retry.ts — isRetryableError classification and
|
|
3
|
-
* withRetry's backoff/retry-after behavior, plus llm-client.ts's callLLM
|
|
4
|
-
* wiring (retries a 429 then succeeds; never retries after abort).
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
import { describe, test, expect } from "bun:test";
|
|
8
|
-
import { isRetryableError, withRetry, computeRetryDelay, type RetryPolicy } from "../packages/core/src/resilience/retry";
|
|
9
|
-
|
|
10
|
-
describe("resilience/retry: isRetryableError", () => {
|
|
11
|
-
test("classifies HTTP status codes", () => {
|
|
12
|
-
expect(isRetryableError({ status: 429 })).toBe(true);
|
|
13
|
-
expect(isRetryableError({ status: 408 })).toBe(true);
|
|
14
|
-
expect(isRetryableError({ status: 500 })).toBe(true);
|
|
15
|
-
expect(isRetryableError({ status: 503 })).toBe(true);
|
|
16
|
-
expect(isRetryableError({ status: 400 })).toBe(false);
|
|
17
|
-
expect(isRetryableError({ status: 401 })).toBe(false);
|
|
18
|
-
expect(isRetryableError({ status: 404 })).toBe(false);
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
test("reads status from a nested response object", () => {
|
|
22
|
-
expect(isRetryableError({ response: { status: 429 } })).toBe(true);
|
|
23
|
-
expect(isRetryableError({ response: { status: 403 } })).toBe(false);
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
test("classifies network/timeout errors by name/message when no status is present", () => {
|
|
27
|
-
expect(isRetryableError(new Error("fetch failed"))).toBe(true);
|
|
28
|
-
expect(isRetryableError({ name: "TimeoutError", message: "Request timed out" })).toBe(true);
|
|
29
|
-
expect(isRetryableError({ message: "ECONNRESET" })).toBe(true);
|
|
30
|
-
expect(isRetryableError({ name: "AbortError", message: "The operation was aborted" })).toBe(false);
|
|
31
|
-
});
|
|
32
|
-
|
|
33
|
-
test("unclassifiable errors default to non-retryable", () => {
|
|
34
|
-
expect(isRetryableError(new Error("invalid API key"))).toBe(false);
|
|
35
|
-
});
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
describe("resilience/retry: computeRetryDelay", () => {
|
|
39
|
-
test("honors an explicit Retry-After over the computed backoff", () => {
|
|
40
|
-
const policy: RetryPolicy = { maxAttempts: 3, initialDelayMs: 1000, backoffMultiplier: 2, maxDelayMs: 30_000 };
|
|
41
|
-
expect(computeRetryDelay(0, policy, 5000)).toBe(5000);
|
|
42
|
-
expect(computeRetryDelay(0, policy, 60_000)).toBe(30_000); // still capped
|
|
43
|
-
});
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
describe("resilience/retry: withRetry", () => {
|
|
47
|
-
const FAST_POLICY: RetryPolicy = { maxAttempts: 3, initialDelayMs: 1, backoffMultiplier: 1, maxDelayMs: 5 };
|
|
48
|
-
|
|
49
|
-
test("retries a transient failure then returns the eventual success", async () => {
|
|
50
|
-
let calls = 0;
|
|
51
|
-
const result = await withRetry(async () => {
|
|
52
|
-
calls++;
|
|
53
|
-
if (calls < 2) throw { status: 429, message: "rate limited" };
|
|
54
|
-
return "ok";
|
|
55
|
-
}, FAST_POLICY);
|
|
56
|
-
|
|
57
|
-
expect(result).toBe("ok");
|
|
58
|
-
expect(calls).toBe(2);
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
test("stops retrying once maxAttempts is reached and throws the last error", async () => {
|
|
62
|
-
let calls = 0;
|
|
63
|
-
await expect(
|
|
64
|
-
withRetry(async () => {
|
|
65
|
-
calls++;
|
|
66
|
-
throw { status: 500, message: "always fails" };
|
|
67
|
-
}, FAST_POLICY)
|
|
68
|
-
).rejects.toThrow("always fails");
|
|
69
|
-
expect(calls).toBe(FAST_POLICY.maxAttempts);
|
|
70
|
-
});
|
|
71
|
-
|
|
72
|
-
test("does not retry a non-retryable (terminal) error", async () => {
|
|
73
|
-
let calls = 0;
|
|
74
|
-
await expect(
|
|
75
|
-
withRetry(async () => {
|
|
76
|
-
calls++;
|
|
77
|
-
throw { status: 401, message: "unauthorized" };
|
|
78
|
-
}, FAST_POLICY)
|
|
79
|
-
).rejects.toThrow("unauthorized");
|
|
80
|
-
expect(calls).toBe(1);
|
|
81
|
-
});
|
|
82
|
-
|
|
83
|
-
test("a custom isRetryable classifier overrides the default", async () => {
|
|
84
|
-
let calls = 0;
|
|
85
|
-
const result = await withRetry(
|
|
86
|
-
async () => {
|
|
87
|
-
calls++;
|
|
88
|
-
if (calls < 2) throw new Error("domain-specific transient");
|
|
89
|
-
return "recovered";
|
|
90
|
-
},
|
|
91
|
-
FAST_POLICY,
|
|
92
|
-
(err) => (err as Error).message === "domain-specific transient"
|
|
93
|
-
);
|
|
94
|
-
expect(result).toBe("recovered");
|
|
95
|
-
expect(calls).toBe(2);
|
|
96
|
-
});
|
|
97
|
-
});
|