@humwick/runner 0.0.0-stage → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +167 -0
- package/agent-bin/agent-relay-browser +10 -0
- package/agent-bin/humwick +10 -0
- package/hooks/fff-search-redirect/fff-search-redirect.cjs +161 -0
- package/hooks/fff-search-redirect/hooks.json +13 -0
- package/hooks/rg-r-guard/hooks.json +18 -0
- package/hooks/rg-r-guard/rg-r-guard.cjs +144 -0
- package/hooks/web-tools-redirect/hooks.json +13 -0
- package/hooks/web-tools-redirect/web-tools-redirect.cjs +84 -0
- package/node_modules/@humwick/offload/LICENSE +167 -0
- package/node_modules/@humwick/offload/README.md +57 -0
- package/node_modules/@humwick/offload/dist/async-runner.d.ts +18 -0
- package/node_modules/@humwick/offload/dist/async-runner.d.ts.map +1 -0
- package/node_modules/@humwick/offload/dist/async-runner.js +158 -0
- package/node_modules/@humwick/offload/dist/async-runner.js.map +1 -0
- package/node_modules/@humwick/offload/dist/envelope.d.ts +25 -0
- package/node_modules/@humwick/offload/dist/envelope.d.ts.map +1 -0
- package/node_modules/@humwick/offload/dist/envelope.js +80 -0
- package/node_modules/@humwick/offload/dist/envelope.js.map +1 -0
- package/node_modules/@humwick/offload/dist/fixtures/worker-task.d.ts +10 -0
- package/node_modules/@humwick/offload/dist/fixtures/worker-task.d.ts.map +1 -0
- package/node_modules/@humwick/offload/dist/fixtures/worker-task.js +59 -0
- package/node_modules/@humwick/offload/dist/fixtures/worker-task.js.map +1 -0
- package/node_modules/@humwick/offload/dist/index.d.ts +5 -0
- package/node_modules/@humwick/offload/dist/index.d.ts.map +1 -0
- package/node_modules/@humwick/offload/dist/index.js +5 -0
- package/node_modules/@humwick/offload/dist/index.js.map +1 -0
- package/node_modules/@humwick/offload/dist/subprocess-runner.d.ts +62 -0
- package/node_modules/@humwick/offload/dist/subprocess-runner.d.ts.map +1 -0
- package/node_modules/@humwick/offload/dist/subprocess-runner.js +295 -0
- package/node_modules/@humwick/offload/dist/subprocess-runner.js.map +1 -0
- package/node_modules/@humwick/offload/dist/types.d.ts +111 -0
- package/node_modules/@humwick/offload/dist/types.d.ts.map +1 -0
- package/node_modules/@humwick/offload/dist/types.js +176 -0
- package/node_modules/@humwick/offload/dist/types.js.map +1 -0
- package/node_modules/@humwick/offload/dist/worker-pool.d.ts +23 -0
- package/node_modules/@humwick/offload/dist/worker-pool.d.ts.map +1 -0
- package/node_modules/@humwick/offload/dist/worker-pool.js +435 -0
- package/node_modules/@humwick/offload/dist/worker-pool.js.map +1 -0
- package/node_modules/@humwick/offload/dist/worker-runtime.d.ts +2 -0
- package/node_modules/@humwick/offload/dist/worker-runtime.d.ts.map +1 -0
- package/node_modules/@humwick/offload/dist/worker-runtime.js +27 -0
- package/node_modules/@humwick/offload/dist/worker-runtime.js.map +1 -0
- package/node_modules/@humwick/offload/dist/worker.d.ts +5 -0
- package/node_modules/@humwick/offload/dist/worker.d.ts.map +1 -0
- package/node_modules/@humwick/offload/dist/worker.js +1 -0
- package/node_modules/@humwick/offload/dist/worker.js.map +1 -0
- package/node_modules/@humwick/offload/package.json +46 -0
- package/node_modules/@humwick/offload/src/async-runner.ts +170 -0
- package/node_modules/@humwick/offload/src/envelope.ts +92 -0
- package/node_modules/@humwick/offload/src/fixtures/worker-task.ts +49 -0
- package/node_modules/@humwick/offload/src/index.ts +4 -0
- package/node_modules/@humwick/offload/src/subprocess-runner.ts +334 -0
- package/node_modules/@humwick/offload/src/types.ts +235 -0
- package/node_modules/@humwick/offload/src/worker-pool.ts +463 -0
- package/node_modules/@humwick/offload/src/worker-runtime.ts +32 -0
- package/node_modules/@humwick/offload/src/worker.ts +5 -0
- package/package.json +47 -4
- package/plugins/browser/skills/playwright-cli/SKILL.md +53 -0
- package/plugins/claude/.claude-plugin/plugin.json +8 -0
- package/plugins/claude/claude-statusline-probe.cjs +144 -0
- package/plugins/claude/hooks/gate-run-guard.provisioned.mjs +914 -0
- package/plugins/claude/hooks/gate-run-guard.sh +9 -0
- package/plugins/claude/hooks/gate-run-guard.ts +237 -0
- package/plugins/claude/hooks/hooks.json +158 -0
- package/plugins/claude/hooks/kill-guard.provisioned.mjs +7027 -0
- package/plugins/claude/hooks/kill-guard.sh +341 -0
- package/plugins/claude/hooks/kill-guard.ts +337 -0
- package/plugins/claude/hooks/permission-request.sh +22 -0
- package/plugins/claude/hooks/post-compact.sh +10 -0
- package/plugins/claude/hooks/post-tool-use.sh +8 -0
- package/plugins/claude/hooks/pre-compact.sh +12 -0
- package/plugins/claude/hooks/read-only-bash-guard.sh +587 -0
- package/plugins/claude/hooks/relay-status.sh +373 -0
- package/plugins/claude/hooks/session-end.sh +26 -0
- package/plugins/claude/hooks/session-start.sh +31 -0
- package/plugins/claude/hooks/shell-trust.ts +28 -0
- package/plugins/claude/hooks/stop-failure.sh +47 -0
- package/plugins/claude/hooks/stop.sh +48 -0
- package/plugins/claude/hooks/subagent-guard.cjs +26 -0
- package/plugins/claude/hooks/subagent-skill-policy.cjs +160 -0
- package/plugins/claude/hooks/subagent-start.sh +13 -0
- package/plugins/claude/hooks/subagent-stop.sh +19 -0
- package/plugins/claude/hooks/user-prompt-submit.sh +18 -0
- package/plugins/claude/hooks/wait-loop-guard.ts +120 -0
- package/plugins/claude/monitors/monitors.json +7 -0
- package/plugins/claude/monitors/relay-monitor.provisioned.mjs +786 -0
- package/plugins/claude/monitors/relay-monitor.ts +58 -0
- package/plugins/claude/skills/disconnect/SKILL.md +16 -0
- package/plugins/claude/skills/label/SKILL.md +23 -0
- package/plugins/claude/skills/message/SKILL.md +40 -0
- package/plugins/claude/skills/pair/SKILL.md +30 -0
- package/plugins/claude/skills/react/SKILL.md +18 -0
- package/plugins/claude/skills/read-message/SKILL.md +24 -0
- package/plugins/claude/skills/reply/SKILL.md +44 -0
- package/plugins/claude/skills/send-claimable/SKILL.md +39 -0
- package/plugins/claude/skills/slice-first/SKILL.md +32 -0
- package/plugins/claude/skills/slice-first/references/plan-recipe.md +46 -0
- package/plugins/claude/skills/status/SKILL.md +16 -0
- package/plugins/claude/skills/tags/SKILL.md +25 -0
- package/plugins/claude/skills/task/SKILL.md +28 -0
- package/plugins/claude-output-style/.claude-plugin/plugin.json +5 -0
- package/plugins/claude-output-style/output-styles/agent-relay.md +5 -0
- package/plugins/codex/skills/disconnect/SKILL.md +15 -0
- package/plugins/codex/skills/guide/SKILL.md +15 -0
- package/plugins/codex/skills/label/SKILL.md +22 -0
- package/plugins/codex/skills/message/SKILL.md +40 -0
- package/plugins/codex/skills/pair/SKILL.md +29 -0
- package/plugins/codex/skills/react/SKILL.md +17 -0
- package/plugins/codex/skills/read-message/SKILL.md +23 -0
- package/plugins/codex/skills/reply/SKILL.md +44 -0
- package/plugins/codex/skills/send-claimable/SKILL.md +39 -0
- package/plugins/codex/skills/slice-first/SKILL.md +32 -0
- package/plugins/codex/skills/slice-first/references/plan-recipe.md +46 -0
- package/plugins/codex/skills/status/SKILL.md +15 -0
- package/plugins/codex/skills/tags/SKILL.md +24 -0
- package/plugins/codex/skills/task/SKILL.md +28 -0
- package/src/adapter.ts +1061 -0
- package/src/adapters/__fixtures__/codex-app-server.ts +47 -0
- package/src/adapters/claude-api.ts +243 -0
- package/src/adapters/claude-context-command.ts +16 -0
- package/src/adapters/claude-credentials.ts +22 -0
- package/src/adapters/claude-delivery-lanes.ts +28 -0
- package/src/adapters/claude-delivery.ts +166 -0
- package/src/adapters/claude-effort.ts +94 -0
- package/src/adapters/claude-exit.ts +290 -0
- package/src/adapters/claude-headless-mirror.ts +88 -0
- package/src/adapters/claude-headless-refusal.ts +51 -0
- package/src/adapters/claude-headless.ts +617 -0
- package/src/adapters/claude-monitor-delivery-plan.ts +317 -0
- package/src/adapters/claude-permission.ts +243 -0
- package/src/adapters/claude-prompt-gates.ts +269 -0
- package/src/adapters/claude-prompt-hook.ts +145 -0
- package/src/adapters/claude-session-capture.ts +293 -0
- package/src/adapters/claude-session-probe.ts +414 -0
- package/src/adapters/claude-shutdown.ts +190 -0
- package/src/adapters/claude-status-detectors.ts +426 -0
- package/src/adapters/claude-subagents.ts +93 -0
- package/src/adapters/claude-submit-confirmation.ts +287 -0
- package/src/adapters/claude-tmux.ts +548 -0
- package/src/adapters/claude-token-usage-capture.ts +253 -0
- package/src/adapters/claude-tool-policy.ts +165 -0
- package/src/adapters/claude-transcript-tail.ts +193 -0
- package/src/adapters/claude-transcript.ts +812 -0
- package/src/adapters/claude.ts +1097 -0
- package/src/adapters/codex-agent-message-capture.ts +189 -0
- package/src/adapters/codex-app-server.ts +70 -0
- package/src/adapters/codex-client.ts +362 -0
- package/src/adapters/codex-effective-config.ts +87 -0
- package/src/adapters/codex-launch-config.ts +153 -0
- package/src/adapters/codex-rollout-context.ts +272 -0
- package/src/adapters/codex-skill-discovery.ts +18 -0
- package/src/adapters/codex-token-usage-capture.ts +163 -0
- package/src/adapters/codex-tool-output.ts +53 -0
- package/src/adapters/codex-turn-outcome.ts +27 -0
- package/src/adapters/codex-version.ts +64 -0
- package/src/adapters/codex.ts +1442 -0
- package/src/adapters/incremental-file.ts +110 -0
- package/src/adapters/opencode-api.ts +71 -0
- package/src/adapters/opencode-context.ts +277 -0
- package/src/adapters/opencode-model-registry.ts +126 -0
- package/src/adapters/opencode-provider.ts +169 -0
- package/src/adapters/opencode-server-retry.ts +50 -0
- package/src/adapters/opencode-soft-steer.ts +302 -0
- package/src/adapters/opencode-spawn.ts +145 -0
- package/src/adapters/opencode-subscribe.ts +62 -0
- package/src/adapters/opencode-token-usage-capture.ts +187 -0
- package/src/adapters/opencode-tool-events.ts +302 -0
- package/src/adapters/opencode-tools-config.ts +17 -0
- package/src/adapters/opencode.ts +2264 -0
- package/src/adapters/project-shell.ts +80 -0
- package/src/adapters/provider-launch-config.ts +21 -0
- package/src/adapters/provider-quota-harvest.ts +127 -0
- package/src/adapters/service-access.ts +153 -0
- package/src/adapters/tmux-offload.ts +207 -0
- package/src/adapters/tool-summary.ts +214 -0
- package/src/agent-redact.ts +75 -0
- package/src/agent-runner/commands/dispatch.ts +185 -0
- package/src/agent-runner/commands/lifecycle.ts +84 -0
- package/src/agent-runner/commands/prompt.ts +284 -0
- package/src/agent-runner/credentials.ts +208 -0
- package/src/agent-runner/http-liveness.ts +85 -0
- package/src/agent-runner/lifecycle-milestones.ts +207 -0
- package/src/agent-runner/lifecycle-phase.ts +80 -0
- package/src/agent-runner/registration.ts +163 -0
- package/src/agent-runner/runner-context.ts +158 -0
- package/src/agent-runner/session-mirror.ts +278 -0
- package/src/agent-runner/task-claims.ts +116 -0
- package/src/agent-runner/token-usage-coverage.ts +54 -0
- package/src/agent-runner/turn-report.ts +269 -0
- package/src/api-budget-accounting.ts +91 -0
- package/src/area-memory.ts +255 -0
- package/src/artifact-path-materialize.ts +52 -0
- package/src/async-guard.ts +35 -0
- package/src/attachment-cache-sweep.ts +31 -0
- package/src/attachment-cache.ts +195 -0
- package/src/background-script-reconcile.ts +106 -0
- package/src/bounded-log.ts +59 -0
- package/src/bounded-spawn-sync.ts +18 -0
- package/src/busy-reconciler.ts +282 -0
- package/src/claim-tracker.ts +140 -0
- package/src/claude-settings.ts +167 -0
- package/src/claude-workspace-trust.ts +289 -0
- package/src/config.ts +547 -0
- package/src/context-boundary.ts +101 -0
- package/src/context-command.ts +151 -0
- package/src/context-probe-state.ts +6 -0
- package/src/continuation-archive.ts +179 -0
- package/src/control-server.ts +730 -0
- package/src/delivery-policy.ts +9 -0
- package/src/delivery-result.ts +24 -0
- package/src/effort-command.ts +125 -0
- package/src/empty-turn.ts +816 -0
- package/src/free-port.ts +25 -0
- package/src/fs-offload.ts +26 -0
- package/src/gate-run-config.ts +35 -0
- package/src/gate-run-lease-probe.ts +38 -0
- package/src/host-asset-plan.ts +450 -0
- package/src/host-claude-config-worker.ts +34 -0
- package/src/host-sources.ts +40 -0
- package/src/inbox-carrier.ts +7 -0
- package/src/index.ts +386 -0
- package/src/launch-assembly.ts +955 -0
- package/src/liveness.ts +105 -0
- package/src/logger.ts +108 -0
- package/src/mcp-outbox.ts +204 -0
- package/src/mcp-preflight.ts +86 -0
- package/src/mcp-source-selection.ts +54 -0
- package/src/mcp-tool-deferral.ts +35 -0
- package/src/mcp-vault-env.ts +25 -0
- package/src/message-body-config/config.ts +16 -0
- package/src/message-body-config.ts +1 -0
- package/src/mid-turn-delivery.ts +43 -0
- package/src/native-memory-lever.ts +329 -0
- package/src/native-self-resume.ts +244 -0
- package/src/outbox.ts +453 -0
- package/src/park-post.ts +17 -0
- package/src/pending-permission-store.ts +148 -0
- package/src/process-meta.ts +9 -0
- package/src/profile-home.ts +419 -0
- package/src/profile-projection.ts +411 -0
- package/src/project-hooks.ts +65 -0
- package/src/project-instructions.ts +119 -0
- package/src/provider-double.ts +59 -0
- package/src/provider-hold.ts +228 -0
- package/src/provider-home-marker.ts +3 -0
- package/src/provider-home-tombstone.ts +4 -0
- package/src/provider-input-queue.ts +134 -0
- package/src/provider-integration.ts +93 -0
- package/src/provider-processes.ts +17 -0
- package/src/provider-submit-failure.ts +15 -0
- package/src/providers.ts +35 -0
- package/src/provisioning.ts +703 -0
- package/src/question-decision.ts +18 -0
- package/src/quota.ts +22 -0
- package/src/rate-limit.ts +351 -0
- package/src/refused-prompt-store.ts +36 -0
- package/src/relay-injection-events.ts +161 -0
- package/src/relay-instructions.ts +107 -0
- package/src/relay-mcp-proxy.long-turn-e2e.ts +183 -0
- package/src/relay-mcp-proxy.ts +704 -0
- package/src/relay-mcp.ts +583 -0
- package/src/reply-obligation-cache.ts +211 -0
- package/src/response-capture-report.ts +111 -0
- package/src/response-routing.ts +6 -0
- package/src/runner-core.ts +2955 -0
- package/src/runner-exit.ts +12 -0
- package/src/runner-helpers.ts +582 -0
- package/src/runner-insights.ts +105 -0
- package/src/runner-mcp-proxy.ts +46 -0
- package/src/runner-spawn-config.ts +68 -0
- package/src/runner-stale-scratch-sweep.ts +21 -0
- package/src/runner.ts +9 -0
- package/src/runtime-token-file.ts +45 -0
- package/src/runtime-token-remint.ts +56 -0
- package/src/service-access.ts +88 -0
- package/src/session-destroy.ts +27 -0
- package/src/session-insights.ts +118 -0
- package/src/session-scratch.ts +296 -0
- package/src/spawn-input.ts +147 -0
- package/src/startup-config-fetch.ts +48 -0
- package/src/subreaper.ts +113 -0
- package/src/template-resolver.ts +29 -0
- package/src/test-run-dedup.ts +435 -0
- package/src/transcript-context.ts +99 -0
- package/src/transport-errors.ts +48 -0
- package/src/turn-opener.ts +55 -0
- package/src/turn-reconcile-event.ts +88 -0
- package/src/vault-proxy-hop.ts +99 -0
- package/src/version.ts +44 -0
- package/src/web-tool-call-proxy.ts +128 -0
- package/src/web-tools.ts +59 -0
- package/README.md +0 -3
package/src/outbox.ts
ADDED
|
@@ -0,0 +1,453 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite";
|
|
2
|
+
import { mkdirSync } from "node:fs";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { sanitizeFsName } from "@humwick/sdk/fs-name";
|
|
6
|
+
import { errMessage } from "@humwick/sdk";
|
|
7
|
+
import { logger } from "./logger";
|
|
8
|
+
import { runnerOutboxDirFromEnv } from "./config";
|
|
9
|
+
import { isDeterministicClientError, isTransientDeliveryError } from "./runner-helpers";
|
|
10
|
+
|
|
11
|
+
// Phase 2 (#196) — the "nothing is ever lost" half. Runner→server events that used to be
|
|
12
|
+
// fire-and-forget over HTTP (session turns, reasoning/tool traces, prompt echoes, insights,
|
|
13
|
+
// hook-fatal reports) were silently dropped whenever the server was momentarily down. This
|
|
14
|
+
// is a durable, FIFO, disk-backed queue that:
|
|
15
|
+
// - survives Runner/server restart (bun:sqlite file in the runtime dir),
|
|
16
|
+
// - stamps true event time (`occurredAt`) once at enqueue and preserves it through retries,
|
|
17
|
+
// - retries with capped exponential backoff, strictly in order (an append log must not
|
|
18
|
+
// reorder turns),
|
|
19
|
+
// - poisons a DETERMINISTICALLY-failing head immediately and an unknown-failure head after
|
|
20
|
+
// maxAttempts so it can't block the queue — but NEVER poisons on attempt count for
|
|
21
|
+
// TRANSIENT (connection-class / 5xx / 429) failures (#2433): report-up chat rides this
|
|
22
|
+
// queue, so a poisoned row is real message loss. A transient head instead parks onto a
|
|
23
|
+
// slow periodic re-drive after a long CONTINUOUS-failure wall-clock budget, so an ordinary
|
|
24
|
+
// relay-restart window can't drop anything no matter how hard flush() hammers,
|
|
25
|
+
// - is bounded with a logged drop policy (never silently truncates).
|
|
26
|
+
//
|
|
27
|
+
// Status deliberately does NOT go through here: it rides the WebSocket bus, which is
|
|
28
|
+
// last-wins and self-heals on reconnect (so it already satisfies "coalesce, don't replay
|
|
29
|
+
// stale busyes"). The coalesce mode below exists so a future state event could migrate here.
|
|
30
|
+
|
|
31
|
+
type OutboxMode = "append" | "coalesce";
|
|
32
|
+
|
|
33
|
+
interface OutboxEventInput {
|
|
34
|
+
kind: string;
|
|
35
|
+
payload: unknown;
|
|
36
|
+
mode?: OutboxMode;
|
|
37
|
+
// Required for coalesce mode: prior un-poisoned rows with the same dedupeKey are replaced.
|
|
38
|
+
dedupeKey?: string;
|
|
39
|
+
// Defaults to now. Set explicitly only to backdate (e.g. replaying a captured timestamp).
|
|
40
|
+
occurredAt?: number;
|
|
41
|
+
// Defaults to a stable derived key so server-side dedup makes retries exactly-once.
|
|
42
|
+
idempotencyKey?: string;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface OutboxRecord {
|
|
46
|
+
seq: number;
|
|
47
|
+
kind: string;
|
|
48
|
+
mode: OutboxMode;
|
|
49
|
+
occurredAt: number;
|
|
50
|
+
idempotencyKey: string;
|
|
51
|
+
payload: unknown;
|
|
52
|
+
attempts: number;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// The transport. Resolve = delivered (row deleted). Reject = failed (retried with backoff).
|
|
56
|
+
type OutboxSend = (record: OutboxRecord) => Promise<void>;
|
|
57
|
+
|
|
58
|
+
interface OutboxOptions {
|
|
59
|
+
agentId: string;
|
|
60
|
+
send: OutboxSend;
|
|
61
|
+
// Storage directory. Defaults to AGENT_RELAY_RUNNER_OUTBOX_DIR, else a per-host temp dir.
|
|
62
|
+
dir?: string;
|
|
63
|
+
maxRows?: number;
|
|
64
|
+
maxAttempts?: number;
|
|
65
|
+
baseBackoffMs?: number;
|
|
66
|
+
maxBackoffMs?: number;
|
|
67
|
+
pollMs?: number;
|
|
68
|
+
// #2433 — transient (connection-class / 5xx / 429) failures NEVER poison on attempt count.
|
|
69
|
+
// After this much CONTINUOUS wall-clock failure on a row, park it onto a slow periodic
|
|
70
|
+
// re-drive (every transientRedriveMs) instead of hot-looping, so a long relay outage still
|
|
71
|
+
// recovers every frame once the relay returns. Sized to outlast any ordinary restart/deploy
|
|
72
|
+
// window (~11s) many times over.
|
|
73
|
+
transientParkAfterMs?: number;
|
|
74
|
+
transientRedriveMs?: number;
|
|
75
|
+
// Observability seam: called when a row is actually POISONED (permanently dropped). Only
|
|
76
|
+
// deterministic client rejections and unknown-class failures past maxAttempts get here —
|
|
77
|
+
// never transient ones. The session outbox uses it to surface mirror/report-up loss
|
|
78
|
+
// in-session (#2433) rather than only in a FATAL host log.
|
|
79
|
+
onPoison?: (record: OutboxRecord, reason: string) => void | Promise<void>;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const DEFAULTS = {
|
|
83
|
+
maxRows: 5000,
|
|
84
|
+
maxAttempts: 12,
|
|
85
|
+
baseBackoffMs: 1_000,
|
|
86
|
+
maxBackoffMs: 60_000,
|
|
87
|
+
pollMs: 5_000,
|
|
88
|
+
// ~5 minutes of CONTINUOUS transient failure before parking (a CD relay restart is ~11s;
|
|
89
|
+
// even a bad deploy clears well inside this), then one re-drive attempt per minute.
|
|
90
|
+
transientParkAfterMs: 300_000,
|
|
91
|
+
transientRedriveMs: 60_000,
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
// Pause between forced delivery attempts during flush() — keeps a brief server outage
|
|
95
|
+
// from hot-looping while still retrying fast enough to land within the flush deadline.
|
|
96
|
+
const FLUSH_RETRY_PAUSE_MS = 150;
|
|
97
|
+
|
|
98
|
+
interface Row {
|
|
99
|
+
seq: number;
|
|
100
|
+
kind: string;
|
|
101
|
+
mode: string;
|
|
102
|
+
occurred_at: number;
|
|
103
|
+
idempotency_key: string;
|
|
104
|
+
payload: string;
|
|
105
|
+
attempts: number;
|
|
106
|
+
next_attempt_at: number;
|
|
107
|
+
first_failed_at: number;
|
|
108
|
+
poisoned: number;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export class Outbox {
|
|
112
|
+
private readonly db: Database;
|
|
113
|
+
private readonly agentId: string;
|
|
114
|
+
private readonly send: OutboxSend;
|
|
115
|
+
private readonly maxRows: number;
|
|
116
|
+
private readonly maxAttempts: number;
|
|
117
|
+
private readonly baseBackoffMs: number;
|
|
118
|
+
private readonly maxBackoffMs: number;
|
|
119
|
+
private readonly pollMs: number;
|
|
120
|
+
private readonly transientParkAfterMs: number;
|
|
121
|
+
private readonly transientRedriveMs: number;
|
|
122
|
+
private readonly onPoison?: (record: OutboxRecord, reason: string) => void | Promise<void>;
|
|
123
|
+
readonly path: string;
|
|
124
|
+
|
|
125
|
+
private draining = false;
|
|
126
|
+
private rerun = false;
|
|
127
|
+
// Set by flush() to make the next drain pass ignore per-row backoff (deliver NOW).
|
|
128
|
+
private forceDue = false;
|
|
129
|
+
private pollTimer?: ReturnType<typeof setInterval>;
|
|
130
|
+
private dueTimer?: ReturnType<typeof setTimeout>;
|
|
131
|
+
private stopped = false;
|
|
132
|
+
|
|
133
|
+
constructor(options: OutboxOptions) {
|
|
134
|
+
this.agentId = options.agentId;
|
|
135
|
+
this.send = options.send;
|
|
136
|
+
this.maxRows = options.maxRows ?? DEFAULTS.maxRows;
|
|
137
|
+
this.maxAttempts = options.maxAttempts ?? DEFAULTS.maxAttempts;
|
|
138
|
+
this.baseBackoffMs = options.baseBackoffMs ?? DEFAULTS.baseBackoffMs;
|
|
139
|
+
this.maxBackoffMs = options.maxBackoffMs ?? DEFAULTS.maxBackoffMs;
|
|
140
|
+
this.pollMs = options.pollMs ?? DEFAULTS.pollMs;
|
|
141
|
+
this.transientParkAfterMs = options.transientParkAfterMs ?? DEFAULTS.transientParkAfterMs;
|
|
142
|
+
this.transientRedriveMs = options.transientRedriveMs ?? DEFAULTS.transientRedriveMs;
|
|
143
|
+
this.onPoison = options.onPoison;
|
|
144
|
+
|
|
145
|
+
const dir = options.dir ?? runnerOutboxDirFromEnv() ?? join(tmpdir(), "agent-relay-outbox");
|
|
146
|
+
this.path = options.dir === ":memory:" ? ":memory:" : join(dir, `outbox-${safeName(this.agentId)}.sqlite`);
|
|
147
|
+
if (this.path !== ":memory:") mkdirSync(dirname(this.path), { recursive: true });
|
|
148
|
+
|
|
149
|
+
this.db = new Database(this.path, { create: true });
|
|
150
|
+
this.db.exec("PRAGMA journal_mode = WAL");
|
|
151
|
+
this.db.exec("PRAGMA busy_timeout = 2000");
|
|
152
|
+
this.db.exec(`
|
|
153
|
+
CREATE TABLE IF NOT EXISTS outbox (
|
|
154
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
155
|
+
kind TEXT NOT NULL,
|
|
156
|
+
mode TEXT NOT NULL DEFAULT 'append',
|
|
157
|
+
dedupe_key TEXT,
|
|
158
|
+
occurred_at INTEGER NOT NULL,
|
|
159
|
+
idempotency_key TEXT NOT NULL,
|
|
160
|
+
payload TEXT NOT NULL,
|
|
161
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
162
|
+
next_attempt_at INTEGER NOT NULL DEFAULT 0,
|
|
163
|
+
first_failed_at INTEGER NOT NULL DEFAULT 0,
|
|
164
|
+
poisoned INTEGER NOT NULL DEFAULT 0,
|
|
165
|
+
created_at INTEGER NOT NULL
|
|
166
|
+
)
|
|
167
|
+
`);
|
|
168
|
+
// Pre-#2433 databases lack the transient-park bookkeeping column (fresh CREATEs include it).
|
|
169
|
+
const columns = this.db.query("PRAGMA table_info(outbox)").all() as Array<{ name: string }>;
|
|
170
|
+
if (!columns.some((column) => column.name === "first_failed_at")) {
|
|
171
|
+
this.db.exec("ALTER TABLE outbox ADD COLUMN first_failed_at INTEGER NOT NULL DEFAULT 0");
|
|
172
|
+
}
|
|
173
|
+
// A restart is a fresh start: clear any backoff/park timers left by the prior process so
|
|
174
|
+
// pending events get an immediate retry (the down server may now be back). `attempts`
|
|
175
|
+
// is kept so the deterministic/unknown poison thresholds still count cumulative failures;
|
|
176
|
+
// `first_failed_at` restarts the CONTINUOUS-failure clock, since whatever the prior
|
|
177
|
+
// generation was failing against may be fixed by the restart itself.
|
|
178
|
+
this.db.exec("UPDATE outbox SET next_attempt_at = 0, first_failed_at = 0 WHERE next_attempt_at > 0 OR first_failed_at > 0");
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Persist an event. Returns the assigned seq. Triggers a drain.
|
|
182
|
+
enqueue(input: OutboxEventInput): number {
|
|
183
|
+
if (this.stopped) throw new Error("outbox is stopped");
|
|
184
|
+
const mode: OutboxMode = input.mode ?? "append";
|
|
185
|
+
const occurredAt = input.occurredAt ?? Date.now();
|
|
186
|
+
const payloadJson = JSON.stringify(input.payload ?? null);
|
|
187
|
+
const idempotencyKey = input.idempotencyKey ?? `${this.agentId}:${input.kind}:${occurredAt}:${shortHash(payloadJson)}`;
|
|
188
|
+
|
|
189
|
+
if (mode === "coalesce") {
|
|
190
|
+
if (!input.dedupeKey) throw new Error("coalesce mode requires a dedupeKey");
|
|
191
|
+
this.db.query("DELETE FROM outbox WHERE dedupe_key = ? AND poisoned = 0").run(input.dedupeKey);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const info = this.db
|
|
195
|
+
.query(`INSERT INTO outbox (kind, mode, dedupe_key, occurred_at, idempotency_key, payload, created_at)
|
|
196
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
|
197
|
+
.run(input.kind, mode, input.dedupeKey ?? null, occurredAt, idempotencyKey, payloadJson, Date.now());
|
|
198
|
+
const seq = Number(info.lastInsertRowid);
|
|
199
|
+
|
|
200
|
+
this.enforceBound();
|
|
201
|
+
// Defer the drain to a microtask so a synchronous burst of enqueues (e.g. several
|
|
202
|
+
// coalesce updates) all land — and coalesce — before the pump pulls the head.
|
|
203
|
+
queueMicrotask(() => this.pump());
|
|
204
|
+
return seq;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// Bounded ring buffer: if over capacity, drop the oldest rows (defined overflow policy).
|
|
208
|
+
// Logged, never silent. Prefers dropping already-poisoned rows first, then oldest by seq.
|
|
209
|
+
private enforceBound(): void {
|
|
210
|
+
const { n } = this.db.query("SELECT count(*) AS n FROM outbox").get() as { n: number };
|
|
211
|
+
if (n <= this.maxRows) return;
|
|
212
|
+
const overflow = n - this.maxRows;
|
|
213
|
+
// Oldest poisoned first, then oldest live — both by seq.
|
|
214
|
+
const victims = this.db
|
|
215
|
+
.query("SELECT seq FROM outbox ORDER BY poisoned DESC, seq ASC LIMIT ?")
|
|
216
|
+
.all(overflow) as Array<{ seq: number }>;
|
|
217
|
+
const ids = victims.map((v) => v.seq);
|
|
218
|
+
if (ids.length === 0) return;
|
|
219
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
220
|
+
this.db.query(`DELETE FROM outbox WHERE seq IN (${placeholders})`).run(...ids);
|
|
221
|
+
logger.warn("outbox", `bound exceeded (${n}/${this.maxRows}) — dropped ${ids.length} oldest event(s)`);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// Begin the background pump: an initial drain plus a poll timer as a backstop.
|
|
225
|
+
start(): void {
|
|
226
|
+
if (this.pollTimer || this.stopped) return;
|
|
227
|
+
this.pump();
|
|
228
|
+
this.pollTimer = setInterval(() => this.pump(), this.pollMs);
|
|
229
|
+
this.pollTimer.unref?.();
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// (#1617) — the ONE way a background caller starts a drain. `drain()` is a bare try/finally with
|
|
233
|
+
// no catch, and drainOnce() touches SQLite outside the send() try/catch (the head SELECT, and the
|
|
234
|
+
// attempt/poison UPDATEs in the failure path), so a closed or erroring database rejects it. Every
|
|
235
|
+
// pump site is fire-and-forget — an enqueue microtask, start()'s initial pump and poll timer, the
|
|
236
|
+
// scheduleDue wake-up — with nobody to await the promise, and Bun exits(1) on an unhandled
|
|
237
|
+
// rejection with no handler installed anywhere in the runner (#1614). That crashed the whole
|
|
238
|
+
// agent over a queue error, on a runner nothing respawns (#1618).
|
|
239
|
+
//
|
|
240
|
+
// Reported through this module's own existing channel (logger, same as the bound-exceeded warn
|
|
241
|
+
// and the poison fatal) rather than swallowed: a pump that stops pumping is its own bug, and
|
|
242
|
+
// it's the only reason the durable queue exists. `drain()` itself keeps its rejecting contract
|
|
243
|
+
// for flush(), which awaits it directly and is bounded by its own deadline.
|
|
244
|
+
private pump(): void {
|
|
245
|
+
void this.drain().catch((error) => {
|
|
246
|
+
logger.error("outbox", `drain failed: ${errMessage(error)}`);
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Drain everything we can before the process exits, ignoring per-row backoff. Used on
|
|
251
|
+
// shutdown/kill/crash (#183): the capture seam durably enqueues the end-of-session
|
|
252
|
+
// Insights datapoint, but the per-agent outbox is never reopened once the agent is gone,
|
|
253
|
+
// so a row that hasn't been POSTed yet is stranded forever. flush() blocks (bounded) on
|
|
254
|
+
// delivery — the opposite intent of the polite, backoff-respecting steady-state drain.
|
|
255
|
+
// Returns true if the queue fully drained within the deadline.
|
|
256
|
+
//
|
|
257
|
+
// #2433: the forced 150ms retry loop here used to consume the per-row attempt/poison
|
|
258
|
+
// budget, so <2s of relay downtime at a flush boundary poisoned frames. Force-drain
|
|
259
|
+
// failures no longer burn the TRANSIENT poison budget (see drainOnce) — flush() can
|
|
260
|
+
// hammer as hard as it likes during an outage without ever pushing a row toward poison.
|
|
261
|
+
async flush(timeoutMs: number): Promise<boolean> {
|
|
262
|
+
const deadline = Date.now() + Math.max(0, timeoutMs);
|
|
263
|
+
for (;;) {
|
|
264
|
+
if (this.stopped) return this.pendingCount() === 0;
|
|
265
|
+
if (this.pendingCount() === 0) return true;
|
|
266
|
+
await this.drain(true);
|
|
267
|
+
if (this.pendingCount() === 0) return true;
|
|
268
|
+
if (Date.now() >= deadline) return false;
|
|
269
|
+
await new Promise<void>((resolve) => setTimeout(resolve, FLUSH_RETRY_PAUSE_MS));
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Process the queue strictly oldest-first. Coalesces concurrent calls; if a drain is
|
|
274
|
+
// requested while one is running, it re-runs once at the end (so an enqueue during a
|
|
275
|
+
// send isn't missed). `force` (from flush) makes this pass ignore per-row backoff.
|
|
276
|
+
async drain(force = false): Promise<void> {
|
|
277
|
+
if (this.stopped) return;
|
|
278
|
+
if (force) this.forceDue = true;
|
|
279
|
+
if (this.draining) { this.rerun = true; return; }
|
|
280
|
+
this.draining = true;
|
|
281
|
+
try {
|
|
282
|
+
do {
|
|
283
|
+
this.rerun = false;
|
|
284
|
+
const due = this.forceDue;
|
|
285
|
+
this.forceDue = false;
|
|
286
|
+
await this.drainOnce(due);
|
|
287
|
+
} while ((this.rerun || this.forceDue) && !this.stopped);
|
|
288
|
+
} finally {
|
|
289
|
+
this.draining = false;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
private async drainOnce(force = false): Promise<void> {
|
|
294
|
+
for (;;) {
|
|
295
|
+
if (this.stopped) return;
|
|
296
|
+
const row = this.db
|
|
297
|
+
.query("SELECT * FROM outbox WHERE poisoned = 0 ORDER BY seq ASC LIMIT 1")
|
|
298
|
+
.get() as Row | null;
|
|
299
|
+
if (!row) return;
|
|
300
|
+
|
|
301
|
+
const now = Date.now();
|
|
302
|
+
if (!force && row.next_attempt_at > now) {
|
|
303
|
+
// Head isn't due yet. Don't reorder past it (FIFO) — schedule a wake-up and stop.
|
|
304
|
+
this.scheduleDue(row.next_attempt_at - now);
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
const record = this.toRecord(row);
|
|
309
|
+
|
|
310
|
+
try {
|
|
311
|
+
await this.send(record);
|
|
312
|
+
this.db.query("DELETE FROM outbox WHERE seq = ?").run(row.seq);
|
|
313
|
+
} catch (error) {
|
|
314
|
+
const reason = errMessage(error);
|
|
315
|
+
const transient = isTransientDeliveryError(error);
|
|
316
|
+
const deterministic = !transient && isDeterministicClientError(error);
|
|
317
|
+
// #2433: flush()'s forced loop hammers every 150ms while ignoring backoff; letting
|
|
318
|
+
// those passes consume the TRANSIENT attempt budget is what let a <2s relay restart
|
|
319
|
+
// window poison frames at a flush boundary. Forced drains of transient failures
|
|
320
|
+
// therefore burn NO attempt budget — classification still applies, counting does
|
|
321
|
+
// not (deterministic/unknown classes keep counting, preserving their poison paths).
|
|
322
|
+
const attempts = transient && force ? row.attempts : row.attempts + 1;
|
|
323
|
+
|
|
324
|
+
if (deterministic || (!transient && attempts >= this.maxAttempts)) {
|
|
325
|
+
await this.poisonRow(row, attempts, reason, deterministic);
|
|
326
|
+
// Move on — the next iteration picks the new head (poison no longer blocks).
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
if (transient) {
|
|
331
|
+
// Never poisoned on attempt count (#2433): retry under backoff until either the
|
|
332
|
+
// relay returns (the row delivers) or the CONTINUOUS-failure wall-clock budget
|
|
333
|
+
// runs out and the row parks onto a slow periodic re-drive cadence.
|
|
334
|
+
const firstFailedAt = row.first_failed_at > 0 ? row.first_failed_at : now;
|
|
335
|
+
if (now - firstFailedAt >= this.transientParkAfterMs) {
|
|
336
|
+
this.db.query("UPDATE outbox SET attempts = ?, next_attempt_at = ?, first_failed_at = ? WHERE seq = ?")
|
|
337
|
+
.run(attempts, now + this.transientRedriveMs, firstFailedAt, row.seq);
|
|
338
|
+
logger.error("outbox", `event seq=${row.seq} kind=${row.kind} parked after ${Math.round((now - firstFailedAt) / 1000)}s of continuous transient failures — re-driving every ${Math.round(this.transientRedriveMs / 1000)}s until the relay returns (last error: ${reason})`);
|
|
339
|
+
this.scheduleDue(this.transientRedriveMs);
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
const delay = this.backoff(attempts);
|
|
343
|
+
this.db.query("UPDATE outbox SET attempts = ?, next_attempt_at = ?, first_failed_at = ? WHERE seq = ?")
|
|
344
|
+
.run(attempts, now + delay, firstFailedAt, row.seq);
|
|
345
|
+
logger.debug("outbox", `event seq=${row.seq} kind=${row.kind} transient retry ${attempts} (no poison cap) in ${delay}ms: ${reason}`);
|
|
346
|
+
this.scheduleDue(delay);
|
|
347
|
+
return; // head is now scheduled; stop until it's due (preserve order)
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
const delay = this.backoff(attempts);
|
|
351
|
+
this.db.query("UPDATE outbox SET attempts = ?, next_attempt_at = ? WHERE seq = ?").run(attempts, now + delay, row.seq);
|
|
352
|
+
logger.debug("outbox", `event seq=${row.seq} kind=${row.kind} retry ${attempts}/${this.maxAttempts} in ${delay}ms: ${reason}`);
|
|
353
|
+
this.scheduleDue(delay);
|
|
354
|
+
return; // head is now scheduled; stop until it's due (preserve order)
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
private toRecord(row: Row): OutboxRecord {
|
|
360
|
+
return {
|
|
361
|
+
seq: row.seq,
|
|
362
|
+
kind: row.kind,
|
|
363
|
+
mode: row.mode as OutboxMode,
|
|
364
|
+
occurredAt: row.occurred_at,
|
|
365
|
+
idempotencyKey: row.idempotency_key,
|
|
366
|
+
payload: safeParse(row.payload),
|
|
367
|
+
attempts: row.attempts,
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// Permanently drop a row. Only reachable for deterministic client rejections (immediately)
|
|
372
|
+
// or unknown-class failures past maxAttempts — NEVER for transient ones (#2433). The
|
|
373
|
+
// onPoison seam lets the session outbox surface the loss in-session instead of only in
|
|
374
|
+
// this FATAL host log.
|
|
375
|
+
private async poisonRow(row: Row, attempts: number, reason: string, immediate: boolean): Promise<void> {
|
|
376
|
+
this.db.query("UPDATE outbox SET attempts = ?, poisoned = 1 WHERE seq = ?").run(attempts, row.seq);
|
|
377
|
+
logger.fatal("outbox", immediate
|
|
378
|
+
? `event seq=${row.seq} kind=${row.kind} poisoned immediately on a deterministic client rejection (${attempts} attempt(s)): ${reason}`
|
|
379
|
+
: `event seq=${row.seq} kind=${row.kind} poisoned after ${attempts} attempts: ${reason}`);
|
|
380
|
+
if (!this.onPoison) return;
|
|
381
|
+
try {
|
|
382
|
+
const record = this.toRecord(row);
|
|
383
|
+
record.attempts = attempts;
|
|
384
|
+
await this.onPoison(record, reason);
|
|
385
|
+
} catch (error) {
|
|
386
|
+
logger.error("outbox", `onPoison handler failed for seq=${row.seq}: ${errMessage(error)}`);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
// Pure per-row delay calc: the attempt count lives in SQLite (one outbox row
|
|
391
|
+
// at a time may be due), so this stays a stateless function rather than the
|
|
392
|
+
// stateful SDK ReconnectionManager (single in-memory counter) used for the
|
|
393
|
+
// orchestrator/voice reconnect loops — different model, deliberately not shared.
|
|
394
|
+
private backoff(attempts: number): number {
|
|
395
|
+
const exp = Math.min(this.maxBackoffMs, this.baseBackoffMs * 2 ** (attempts - 1));
|
|
396
|
+
return Math.round(exp / 2 + Math.random() * (exp / 2)); // full-ish jitter, never below half
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
private scheduleDue(delayMs: number): void {
|
|
400
|
+
if (this.stopped || this.dueTimer) return;
|
|
401
|
+
this.dueTimer = setTimeout(() => {
|
|
402
|
+
this.dueTimer = undefined;
|
|
403
|
+
this.pump();
|
|
404
|
+
}, Math.max(0, delayMs));
|
|
405
|
+
this.dueTimer.unref?.();
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// Observability / tests.
|
|
409
|
+
pendingCount(): number {
|
|
410
|
+
return (this.db.query("SELECT count(*) AS n FROM outbox WHERE poisoned = 0").get() as { n: number }).n;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
updatePayload(seq: number, payload: unknown): void {
|
|
414
|
+
const payloadJson = JSON.stringify(payload ?? null);
|
|
415
|
+
this.db.query("UPDATE outbox SET payload = ? WHERE seq = ? AND poisoned = 0").run(payloadJson, seq);
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
poisonedCount(): number {
|
|
419
|
+
return (this.db.query("SELECT count(*) AS n FROM outbox WHERE poisoned = 1").get() as { n: number }).n;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
stop(): void {
|
|
423
|
+
this.stopped = true;
|
|
424
|
+
if (this.pollTimer) clearInterval(this.pollTimer);
|
|
425
|
+
this.pollTimer = undefined;
|
|
426
|
+
if (this.dueTimer) clearTimeout(this.dueTimer);
|
|
427
|
+
this.dueTimer = undefined;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
close(): void {
|
|
431
|
+
this.stop();
|
|
432
|
+
try { this.db.close(); } catch { /* already closed */ }
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
function safeName(value: string): string {
|
|
437
|
+
return sanitizeFsName(value, { replacement: "_", maxLen: 180, fallback: "agent" });
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
function safeParse(json: string): unknown {
|
|
441
|
+
try { return JSON.parse(json); } catch { return null; }
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// Small, fast, stable string hash (FNV-1a, 32-bit) — enough to disambiguate identical
|
|
445
|
+
// kind+timestamp payloads in the idempotency key. Not security-sensitive.
|
|
446
|
+
function shortHash(value: string): string {
|
|
447
|
+
let h = 0x811c9dc5;
|
|
448
|
+
for (let i = 0; i < value.length; i++) {
|
|
449
|
+
h ^= value.charCodeAt(i);
|
|
450
|
+
h = Math.imul(h, 0x01000193);
|
|
451
|
+
}
|
|
452
|
+
return (h >>> 0).toString(36);
|
|
453
|
+
}
|
package/src/park-post.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { errMessage } from "@humwick/sdk";
|
|
2
|
+
import { logger } from "./logger";
|
|
3
|
+
|
|
4
|
+
const PARK_PERSIST_ATTEMPTS = 3;
|
|
5
|
+
|
|
6
|
+
/** A parked prompt's self-post, retried with backoff; undefined once every attempt failed. */
|
|
7
|
+
export async function retryParkPost<T>(what: string, post: () => Promise<T>): Promise<T | undefined> {
|
|
8
|
+
for (let attempt = 0; attempt < PARK_PERSIST_ATTEMPTS; attempt++) {
|
|
9
|
+
if (attempt > 0) await new Promise((resolve) => setTimeout(resolve, 500 * 2 ** (attempt - 1)));
|
|
10
|
+
try {
|
|
11
|
+
return await post();
|
|
12
|
+
} catch (error) {
|
|
13
|
+
logger.warn("runner", `${what} failed (attempt ${attempt + 1}/${PARK_PERSIST_ATTEMPTS}): ${errMessage(error)}`);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
return undefined;
|
|
17
|
+
}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite";
|
|
2
|
+
import type { InteractivePrompt } from "@humwick/sdk";
|
|
3
|
+
import { mkdirSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { dirname, join } from "node:path";
|
|
6
|
+
import { sanitizeFsName } from "@humwick/sdk/fs-name";
|
|
7
|
+
import { runnerOutboxDirFromEnv } from "./config";
|
|
8
|
+
import { logger } from "./logger";
|
|
9
|
+
import { errMessage } from "@humwick/sdk";
|
|
10
|
+
|
|
11
|
+
export interface PendingPermissionRequestRecord {
|
|
12
|
+
approvalId: string;
|
|
13
|
+
view: InteractivePrompt;
|
|
14
|
+
hookDeadlineAt: number;
|
|
15
|
+
ownerInstanceId: string;
|
|
16
|
+
occurredAt: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
interface PendingPermissionRow {
|
|
20
|
+
approval_id: string;
|
|
21
|
+
view: string;
|
|
22
|
+
hook_deadline_at: number;
|
|
23
|
+
owner_instance_id: string;
|
|
24
|
+
occurred_at: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export class PendingPermissionStore {
|
|
28
|
+
private db?: Database;
|
|
29
|
+
readonly path: string;
|
|
30
|
+
|
|
31
|
+
constructor(options: { agentId: string; dir?: string }) {
|
|
32
|
+
const dir = options.dir ?? runnerOutboxDirFromEnv() ?? join(tmpdir(), "agent-relay-outbox");
|
|
33
|
+
this.path = options.dir === ":memory:" ? ":memory:" : join(dir, `pending-permissions-${safeName(options.agentId)}.sqlite`);
|
|
34
|
+
try {
|
|
35
|
+
if (this.path !== ":memory:") mkdirSync(dirname(this.path), { recursive: true });
|
|
36
|
+
const db = new Database(this.path, { create: true });
|
|
37
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
38
|
+
db.exec("PRAGMA busy_timeout = 2000");
|
|
39
|
+
db.exec(`
|
|
40
|
+
CREATE TABLE IF NOT EXISTS pending_permission_requests (
|
|
41
|
+
approval_id TEXT PRIMARY KEY,
|
|
42
|
+
view TEXT NOT NULL,
|
|
43
|
+
hook_deadline_at INTEGER NOT NULL,
|
|
44
|
+
owner_instance_id TEXT NOT NULL,
|
|
45
|
+
occurred_at INTEGER NOT NULL,
|
|
46
|
+
created_at INTEGER NOT NULL
|
|
47
|
+
)
|
|
48
|
+
`);
|
|
49
|
+
this.db = db;
|
|
50
|
+
} catch (error) {
|
|
51
|
+
logger.error("pending-permission-store", `disabled after open failed path=${this.path}: ${errMessage(error)}`);
|
|
52
|
+
this.db = undefined;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
upsert(record: PendingPermissionRequestRecord): void {
|
|
57
|
+
if (!this.db) return;
|
|
58
|
+
try {
|
|
59
|
+
this.db.query(`
|
|
60
|
+
INSERT INTO pending_permission_requests (approval_id, view, hook_deadline_at, owner_instance_id, occurred_at, created_at)
|
|
61
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
62
|
+
ON CONFLICT(approval_id) DO UPDATE SET
|
|
63
|
+
view = excluded.view,
|
|
64
|
+
hook_deadline_at = excluded.hook_deadline_at,
|
|
65
|
+
owner_instance_id = excluded.owner_instance_id,
|
|
66
|
+
occurred_at = excluded.occurred_at
|
|
67
|
+
`).run(record.approvalId, JSON.stringify(record.view), record.hookDeadlineAt, record.ownerInstanceId, record.occurredAt, Date.now());
|
|
68
|
+
} catch (error) {
|
|
69
|
+
logger.error("pending-permission-store", `upsert failed approvalId=${record.approvalId}; continuing in-memory only: ${errMessage(error)}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
delete(approvalId: string): void {
|
|
74
|
+
if (!this.db) return;
|
|
75
|
+
try {
|
|
76
|
+
this.db.query("DELETE FROM pending_permission_requests WHERE approval_id = ?").run(approvalId);
|
|
77
|
+
} catch (error) {
|
|
78
|
+
logger.error("pending-permission-store", `delete failed approvalId=${approvalId}; continuing in-memory only: ${errMessage(error)}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
list(): PendingPermissionRequestRecord[] {
|
|
83
|
+
if (!this.db) return [];
|
|
84
|
+
try {
|
|
85
|
+
const rows = this.db.query("SELECT * FROM pending_permission_requests ORDER BY occurred_at ASC").all() as PendingPermissionRow[];
|
|
86
|
+
return rows.map((row) => ({
|
|
87
|
+
approvalId: row.approval_id,
|
|
88
|
+
view: safeParseView(row.view),
|
|
89
|
+
hookDeadlineAt: row.hook_deadline_at,
|
|
90
|
+
ownerInstanceId: row.owner_instance_id,
|
|
91
|
+
occurredAt: row.occurred_at,
|
|
92
|
+
}));
|
|
93
|
+
} catch (error) {
|
|
94
|
+
logger.error("pending-permission-store", `list failed; continuing without persisted approvals: ${errMessage(error)}`);
|
|
95
|
+
return [];
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
close(): void {
|
|
100
|
+
try { this.db?.close(); } catch { /* already closed */ }
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export function bestEffortListPendingPermissionRequests(store: PendingPermissionStore | undefined): PendingPermissionRequestRecord[] {
|
|
105
|
+
try {
|
|
106
|
+
return store?.list() ?? [];
|
|
107
|
+
} catch (error) {
|
|
108
|
+
logger.error("pending-permission-store", `list failed; continuing without persisted approvals: ${errMessage(error)}`);
|
|
109
|
+
return [];
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function bestEffortUpsertPendingPermissionRequest(store: PendingPermissionStore | undefined, record: PendingPermissionRequestRecord): void {
|
|
114
|
+
try {
|
|
115
|
+
store?.upsert(record);
|
|
116
|
+
} catch (error) {
|
|
117
|
+
logger.error("pending-permission-store", `upsert failed approvalId=${record.approvalId}; continuing in-memory only: ${errMessage(error)}`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function bestEffortDeletePendingPermissionRequest(store: PendingPermissionStore | undefined, approvalId: string): void {
|
|
122
|
+
try {
|
|
123
|
+
store?.delete(approvalId);
|
|
124
|
+
} catch (error) {
|
|
125
|
+
logger.error("pending-permission-store", `delete failed approvalId=${approvalId}; continuing in-memory only: ${errMessage(error)}`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function safeName(value: string): string {
|
|
130
|
+
return sanitizeFsName(value, { replacement: "_", maxLen: 180, fallback: "agent" });
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function safeParseView(json: string): InteractivePrompt {
|
|
134
|
+
try {
|
|
135
|
+
const parsed = JSON.parse(json);
|
|
136
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed as InteractivePrompt;
|
|
137
|
+
} catch {
|
|
138
|
+
// Fall through to a valid placeholder so recovery can still emit a notice and clear the row.
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
id: "unknown",
|
|
142
|
+
kind: "tool",
|
|
143
|
+
title: "Recovered permission request",
|
|
144
|
+
body: "The original permission request could not be decoded.",
|
|
145
|
+
choices: [],
|
|
146
|
+
reasoningSettled: false,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ManagedProcess } from "./adapter";
|
|
2
|
+
|
|
3
|
+
export function providerTerminalSession(process?: ManagedProcess): string | undefined {
|
|
4
|
+
return typeof process?.meta?.tmuxSession === "string" ? process.meta.tmuxSession : undefined;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export function providerTerminalSocket(process?: ManagedProcess): string | undefined {
|
|
8
|
+
return typeof process?.meta?.tmuxSocket === "string" ? process.meta.tmuxSocket : undefined;
|
|
9
|
+
}
|