@edgehero/pi-dispatch 4.3.1 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +84 -3
- package/README.md +20 -7
- package/deploy/console.service +52 -0
- package/deploy/docker-compose.yml +3 -1
- package/deploy/pi-dispatch-valkey.container +4 -2
- package/deploy/sandbox-gateway.service +23 -0
- package/deploy/worker-env-wrapper.cmd +6 -0
- package/deploy/worker-env-wrapper.sh +10 -1
- package/package.json +117 -19
- package/src/agent-snapshot.mjs +49 -0
- package/src/answerer-runtime.mjs +194 -0
- package/src/answerer.mjs +178 -0
- package/src/approval-policy.mjs +76 -0
- package/src/approval-runtime.mjs +188 -0
- package/src/approval-session.mjs +137 -0
- package/src/approval-state.mjs +170 -0
- package/src/auth-cli.mjs +86 -0
- package/src/auth-connections.mjs +145 -0
- package/src/auth-policy.mjs +32 -0
- package/src/auth-store.mjs +97 -0
- package/src/azure-host.mjs +66 -4
- package/src/azure-prompt.mjs +5 -4
- package/src/backend-conformance.mjs +34 -2
- package/src/backend-local.mjs +103 -34
- package/src/backend-openshell.mjs +1540 -0
- package/src/backend-podman.mjs +63 -2
- package/src/backend-registry.mjs +61 -4
- package/src/backends.mjs +395 -6
- package/src/bullmq-internals.mjs +208 -0
- package/src/canary-cli.mjs +28 -0
- package/src/canary-lock.mjs +41 -0
- package/src/canary-runtime.mjs +237 -0
- package/src/canary.mjs +53 -0
- package/src/cancel-cli.mjs +47 -34
- package/src/cancel-state.mjs +213 -15
- package/src/capabilities.mjs +36 -1
- package/src/capacity-cli.mjs +28 -8
- package/src/capacity-records.mjs +75 -16
- package/src/capacity.mjs +65 -0
- package/src/chat-cli.mjs +288 -0
- package/src/chat-fork.mjs +547 -0
- package/src/chat-mode.mjs +156 -0
- package/src/chat-view.mjs +720 -0
- package/src/checkpoint-paths.mjs +113 -0
- package/src/cli.mjs +128 -41
- package/src/comment-tables.mjs +322 -0
- package/src/community-auth-adapters.mjs +87 -0
- package/src/community-auth-helper.mjs +76 -0
- package/src/community-auth-metadata.mjs +6 -0
- package/src/community-auth-output-pins.mjs +38 -0
- package/src/community-auth-patches.mjs +168 -0
- package/src/community-auth-pins.mjs +38 -0
- package/src/config.mjs +190 -29
- package/src/connection.mjs +83 -7
- package/src/console-answer.mjs +113 -0
- package/src/console-audit.mjs +604 -0
- package/src/console-bootstrap.mjs +159 -0
- package/src/console-client.mjs +848 -0
- package/src/console-probe.mjs +106 -0
- package/src/console-tokens.mjs +459 -0
- package/src/container-names.mjs +51 -0
- package/src/container-spec.mjs +20 -1
- package/src/cpu-reserve.mjs +4 -1
- package/src/defer-reasons.mjs +200 -0
- package/src/defer-write.mjs +208 -0
- package/src/deployment-venue.mjs +23 -6
- package/src/docker-run.mjs +11 -0
- package/src/doctor-bundles.mjs +130 -0
- package/src/doctor-model-proposals.mjs +109 -0
- package/src/doctor-openshell.mjs +238 -0
- package/src/doctor-results.mjs +602 -0
- package/src/doctor-spawner.mjs +215 -0
- package/src/doctor.mjs +1639 -196
- package/src/dollar-budget.mjs +212 -3
- package/src/endpoint-status.mjs +100 -0
- package/src/env-allowlist.mjs +96 -6
- package/src/env-file.mjs +6 -4
- package/src/file-expect.mjs +47 -0
- package/src/forgejo-host.mjs +42 -5
- package/src/forgejo-prompt.mjs +5 -4
- package/src/forges.mjs +33 -3
- package/src/fork-job.mjs +58 -0
- package/src/fork-policy.mjs +23 -0
- package/src/fork-store.mjs +338 -0
- package/src/get-token.mjs +1 -1
- package/src/git-dirty.mjs +119 -14
- package/src/git-hardening.mjs +85 -0
- package/src/github-host.mjs +31 -6
- package/src/github-prompt.mjs +7 -16
- package/src/gitlab-host.mjs +42 -3
- package/src/gitlab-prompt.mjs +5 -4
- package/src/held-document.mjs +23 -0
- package/src/held-replies.mjs +325 -0
- package/src/held-settle.mjs +266 -0
- package/src/hook-runs.mjs +188 -0
- package/src/host-budget.mjs +8 -5
- package/src/host-clone.mjs +72 -0
- package/src/host-loads.mjs +557 -0
- package/src/host-registry.mjs +66 -6
- package/src/host-telemetry.mjs +92 -0
- package/src/http-body.mjs +78 -0
- package/src/image-preflight.mjs +156 -13
- package/src/import-pi.mjs +60 -172
- package/src/inbox-relay.mjs +482 -0
- package/src/inbox-settle.mjs +196 -0
- package/src/index.mjs +734 -129
- package/src/init.mjs +35 -5
- package/src/job-data-version.mjs +75 -0
- package/src/job-id.mjs +30 -15
- package/src/job-layout.mjs +22 -0
- package/src/job-size.mjs +16 -0
- package/src/live-jobs.mjs +14 -6
- package/src/live-probes.mjs +6 -2
- package/src/live-transcript.mjs +872 -0
- package/src/loaded-record.mjs +609 -0
- package/src/mcp-config.mjs +1 -1
- package/src/mcp-preflight.mjs +36 -5
- package/src/model-attempts.mjs +84 -0
- package/src/model-catalog.mjs +83 -0
- package/src/model-cooldowns.mjs +22 -0
- package/src/model-failover-protocol.mjs +11 -0
- package/src/model-failover.mjs +134 -0
- package/src/model-observation.mjs +50 -0
- package/src/model-proposal-protocol.mjs +21 -0
- package/src/model-proposals.mjs +75 -0
- package/src/model-ref.mjs +67 -10
- package/src/model-roles-file.mjs +24 -0
- package/src/model-roles.mjs +73 -0
- package/src/model-status.mjs +34 -0
- package/src/models-json.mjs +4 -1
- package/src/money.mjs +1 -1
- package/src/oauth-openai.mjs +121 -0
- package/src/older-state.mjs +249 -0
- package/src/on-failure.mjs +17 -13
- package/src/open-browser.mjs +31 -2
- package/src/opener-cli.mjs +135 -0
- package/src/opener-protocol.mjs +27 -0
- package/src/opener-terminal.mjs +128 -0
- package/src/openshell-cli.mjs +155 -0
- package/src/openshell-policy.mjs +163 -0
- package/src/outbox.mjs +14 -0
- package/src/output-cap.mjs +1 -0
- package/src/package-canary-record.mjs +26 -0
- package/src/package-canary.mjs +135 -0
- package/src/package-names.mjs +9 -0
- package/src/package-stage.mjs +418 -0
- package/src/packages.mjs +24 -16
- package/src/parked.mjs +2 -1
- package/src/pause-windows.mjs +136 -17
- package/src/podman-stack.mjs +40 -14
- package/src/portfolio-snapshot.mjs +26 -7
- package/src/prepare-github.mjs +41 -77
- package/src/prepare-local.mjs +32 -2
- package/src/prepare.mjs +45 -11
- package/src/private-store-boundary.mjs +70 -0
- package/src/processor.mjs +1550 -173
- package/src/projects.mjs +23 -0
- package/src/prompt-fence.mjs +18 -0
- package/src/provider-key.mjs +7 -0
- package/src/provider-steering.mjs +1 -0
- package/src/question-context.mjs +35 -0
- package/src/question-group-runtime.mjs +91 -0
- package/src/question-groups.mjs +171 -0
- package/src/queue.mjs +264 -55
- package/src/reason-catalog.mjs +448 -0
- package/src/receiver-status.mjs +230 -0
- package/src/record-reason.mjs +37 -0
- package/src/recovery-queue.mjs +566 -0
- package/src/redact-secrets.mjs +219 -0
- package/src/reserve-estimate.mjs +101 -0
- package/src/reserved-env.mjs +36 -1
- package/src/retention.mjs +67 -0
- package/src/run-budget-ledger.mjs +283 -0
- package/src/run-checkpoint.mjs +627 -0
- package/src/run-container.mjs +749 -33
- package/src/run-history.mjs +791 -45
- package/src/run-mirror.mjs +75 -7
- package/src/run-progress.mjs +751 -0
- package/src/runtime-observations.mjs +3 -1
- package/src/runtime-settings.mjs +272 -37
- package/src/sandbox-cli.mjs +78 -2
- package/src/sandbox-deployment.mjs +51 -0
- package/src/sandbox-gateway-protocol.mjs +378 -0
- package/src/sandbox-hosts.mjs +100 -0
- package/src/sandbox-inventory.mjs +83 -0
- package/src/sandbox-policy.mjs +225 -0
- package/src/sandbox-prepare-requests.mjs +145 -0
- package/src/sandbox-prepare.mjs +285 -0
- package/src/sandbox-refs.mjs +70 -0
- package/src/sandbox-session-monitor.mjs +4 -0
- package/src/sandbox-session-watch.mjs +124 -0
- package/src/sandbox-sessions.mjs +161 -0
- package/src/sandbox-store.mjs +63 -6
- package/src/sandbox-workspace-guard.mjs +39 -0
- package/src/sandbox.mjs +27 -200
- package/src/scheduler-stall-guard.mjs +14 -8
- package/src/schedules.mjs +19 -6
- package/src/secret-bundles.mjs +245 -0
- package/src/secret-profiles.mjs +103 -3
- package/src/secrets.mjs +19 -9
- package/src/service-env.mjs +38 -5
- package/src/service.mjs +638 -106
- package/src/session-key.mjs +60 -0
- package/src/session-store.mjs +168 -10
- package/src/setup-contract.mjs +40 -0
- package/src/setup-egress.mjs +113 -0
- package/src/setup-host-entry.mjs +33 -0
- package/src/setup-host.mjs +242 -0
- package/src/setup-lock.mjs +41 -0
- package/src/size-records.mjs +15 -3
- package/src/start.mjs +1353 -135
- package/src/status-comment.mjs +415 -0
- package/src/stderr-transcript.mjs +318 -0
- package/src/steer-ack.mjs +122 -0
- package/src/steer-cli.mjs +213 -0
- package/src/steer-framing.mjs +185 -0
- package/src/steer-inbox.mjs +693 -0
- package/src/store-version.mjs +139 -0
- package/src/subscription-config.mjs +56 -0
- package/src/subscription-limits.mjs +3 -0
- package/src/subscription-quota.mjs +259 -0
- package/src/subscription-runtime.mjs +123 -0
- package/src/subscription-status.mjs +42 -0
- package/src/subscription-wait.mjs +11 -0
- package/src/subscriptions.mjs +7 -1
- package/src/thinking-level.mjs +19 -0
- package/src/touch-outbox.mjs +61 -0
- package/src/touch-state.mjs +133 -0
- package/src/triggers-file.mjs +299 -39
- package/src/triggers.mjs +780 -38
- package/src/trust-paths.mjs +171 -0
- package/src/up.mjs +398 -51
- package/src/updates-cli.mjs +61 -0
- package/src/updates-look.mjs +127 -0
- package/src/updates-reconcile.mjs +182 -0
- package/src/updates-subscription-adapters.mjs +8 -0
- package/src/updates.mjs +252 -0
- package/src/valkey-auth.mjs +32 -9
- package/src/valkey-endpoint.mjs +14 -4
- package/src/wait-for.mjs +1 -1
- package/src/wait-state.mjs +62 -3
- package/src/watch-board.mjs +38 -0
- package/src/watch-closer.mjs +1 -1
- package/src/watch-contract.mjs +102 -0
- package/src/watch-decision.mjs +78 -0
- package/src/watch-drainer.mjs +14 -0
- package/src/watch-issue-adapter.mjs +44 -0
- package/src/watch-issue.mjs +111 -0
- package/src/watch-record.mjs +10 -0
- package/src/watch-runtime.mjs +167 -0
- package/src/watch-store.mjs +183 -0
- package/src/watch-trace.mjs +34 -0
- package/src/workspace-checkpoint.mjs +1750 -0
- package/src/zero-touch-fixes.mjs +99 -0
- package/src/zero-touch.mjs +225 -0
- package/src/repeat-slot.mjs +0 -13
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Held replies (issue #638, DES-AWAIT-INPUT-ENDS-THE-JOB, INT-STEER-INBOX-CONTRACT): an answer to a run that is parking
|
|
3
|
+
* right now is never swallowed.
|
|
4
|
+
*
|
|
5
|
+
* THE GAP IT CLOSES. A run armed with `run.awaitInput` posts its question as a forge comment itself, and only then calls
|
|
6
|
+
* the tool and ends. The processor parks it (promotes the session with the `parked` mark, releases the run's semantic
|
|
7
|
+
* window, posts the fixed comment, sets the parked marker), and only from then on does a reply start a resumed job. A
|
|
8
|
+
* person can answer in the seconds before that, and such a reply met one of two walls: deduplicated against the run
|
|
9
|
+
* that still holds the window (the receiver answered `deduplicated` and nothing ran), or, on an issue without the
|
|
10
|
+
* phrase, dropped by the filter because the marker was not on the issue yet (`no-trigger-phrase`). Both are held here
|
|
11
|
+
* instead, and the run's park takes the first one as its resumed job (held-settle.mjs).
|
|
12
|
+
*
|
|
13
|
+
* ONE module for both sides, exported as `@edgehero/pi-dispatch/held-replies` the way steer-inbox.mjs is: the worker
|
|
14
|
+
* registers its awaitInput runs live and settles them (held-settle.mjs), the receiver finds a target's live runs and
|
|
15
|
+
* holds a reply under one. Light on purpose (no queue library, no processor), because the receiver imports it.
|
|
16
|
+
*
|
|
17
|
+
* KEYS OF THEIR OWN, NOT THE STEER INBOX. A held reply is not a steer: the run it waits on has stopped taking input (it
|
|
18
|
+
* asked and is ending), and the job it becomes is a new run, never a message written into this one. Sharing
|
|
19
|
+
* `run:steer:target:*` would put every awaitInput run in front of `trySteer`, which would push a phrase comment into an
|
|
20
|
+
* inbox nothing reads (an awaitInput run need not be steer-armed); writing a held reply into `run:inbox:*` would let
|
|
21
|
+
* the relay frame it as a steer for a run that is already gone. So the keys are spelled apart, and nothing that reads
|
|
22
|
+
* one set reads the other (runId is `runIdOf(job)`, spelled through `sanitizeJobId`):
|
|
23
|
+
* run:await:target:<the same 32-hex hash steerTargetKey makes> ZSET runId -> expiresAtMs, PEXPIRE 60 s
|
|
24
|
+
* run:await:open:<runId> STRING host name, PX 30 s: the one authority on whether a run takes a held reply
|
|
25
|
+
* run:held:<runId> LIST of entry JSON { id, at, route, fallback }, append only, PEXPIRE 1 h
|
|
26
|
+
* run:held:closed:<runId> STRING, set by the settle's close, PX 1 h
|
|
27
|
+
* run:await:parked:<the same 32-hex hash> STRING the run id, PX 60 s: set by the park's close, so a reply delivered
|
|
28
|
+
* after the run left its target is still routed as a parked reply
|
|
29
|
+
* The target hash is the steer one's, derived from it (`awaitTargetKeyOf`), never spelled again: a key computed two
|
|
30
|
+
* ways is a key that one day differs, and then a reply is held under a target no run registered.
|
|
31
|
+
*
|
|
32
|
+
* THE HOLD AND THE CLOSE ARE EACH ONE LUA SCRIPT, the steer inbox's push and close pair for its reason: `holdReply`
|
|
33
|
+
* refuses `closed` once the close ran and `gone` once the run stopped refreshing, and `closeHeld` sets `closed`, drops
|
|
34
|
+
* the liveness keys and returns the list in one step. So a reply is either in the close's reply (and the settle answers
|
|
35
|
+
* it) or refused at the hold (and the receiver answers exactly as it did before this feature). There is no interleaving
|
|
36
|
+
* in which a reply is accepted and then never seen.
|
|
37
|
+
*
|
|
38
|
+
* BOUNDED. At most HELD_MAX_ENTRIES per run, enforced in the script so concurrent receivers cannot race past it (the
|
|
39
|
+
* oldest entry is evicted to make room), and HELD_ENTRY_MAX_BYTES per entry, checked before Valkey is touched: an entry
|
|
40
|
+
* over it is refused, never cut, and the receiver falls back to today's answer.
|
|
41
|
+
*
|
|
42
|
+
* NEVER THROWS. Every export returns a value or `{ error }`: a validation refusal is `{ error: "refused", reason }`, a
|
|
43
|
+
* Valkey fault `{ error: <code> }`, naming no key and no value.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
import { steerTargetOf } from "./inbox-relay.mjs";
|
|
47
|
+
import { sanitizeJobId } from "./run-history.mjs";
|
|
48
|
+
import { STEER_ID_RE } from "./steer-framing.mjs";
|
|
49
|
+
|
|
50
|
+
const STEER_TARGET_PREFIX = "run:steer:target:";
|
|
51
|
+
export const AWAIT_TARGET_PREFIX = "run:await:target:";
|
|
52
|
+
/** The open key's life: the keeper refreshes it every 2 s, so 30 s of silence means the run is gone. */
|
|
53
|
+
export const AWAIT_OPEN_TTL_MS = 30 * 1000;
|
|
54
|
+
/** The target pointer's own expiry, twice a member's score horizon (steer-inbox.mjs's STEER_TARGET_TTL_MS rule). */
|
|
55
|
+
export const AWAIT_TARGET_TTL_MS = 60 * 1000;
|
|
56
|
+
/**
|
|
57
|
+
* The held list and the closed key live an hour, above the 10-minute semantic window (queue.mjs SEMANTIC_WINDOW_MS):
|
|
58
|
+
* a reply can be held only while that window could swallow it, so an hour outlives every case, and a crashed worker
|
|
59
|
+
* leaves no residue past it.
|
|
60
|
+
*/
|
|
61
|
+
export const HELD_TTL_MS = 60 * 60 * 1000;
|
|
62
|
+
/** The most replies one run holds: the first is started, the rest are answered once, so more buys nothing. */
|
|
63
|
+
export const HELD_MAX_ENTRIES = 20;
|
|
64
|
+
/**
|
|
65
|
+
* The most one entry may weigh as JSON. The fallback job carries the comment and the issue's title and body (a forge
|
|
66
|
+
* caps each near 64 K characters, so up to four times that in UTF-8, and JSON escaping can double it), so 1 MiB fits
|
|
67
|
+
* any real delivery and still bounds a run at HELD_MAX_ENTRIES MiB.
|
|
68
|
+
*/
|
|
69
|
+
export const HELD_ENTRY_MAX_BYTES = 1024 * 1024;
|
|
70
|
+
export const HELD_ROUTES = Object.freeze(["dedup", "pre-label"]);
|
|
71
|
+
export const HELD_OUTCOMES = Object.freeze(["ok", "dup", "closed", "gone"]);
|
|
72
|
+
/**
|
|
73
|
+
* How long a target says "a run just parked here" after the park's take closed its list (review round 2). A phrase-free
|
|
74
|
+
* reply posted just before the label went on can be DELIVERED after the take, when the run is already off its target, so
|
|
75
|
+
* the receiver finds no live run and the filter dropped it for the missing label: lost. While this marker stands the
|
|
76
|
+
* receiver enqueues such a reply as an ordinary parked reply instead (it then coalesces against the taken job, or
|
|
77
|
+
* resumes the parked session itself). A minute covers webhook delivery delay; after it the label is on, and the reply
|
|
78
|
+
* routes normally.
|
|
79
|
+
*/
|
|
80
|
+
export const AWAIT_PARKED_TTL_MS = 60 * 1000;
|
|
81
|
+
export const AWAIT_PARKED_PREFIX = "run:await:parked:";
|
|
82
|
+
|
|
83
|
+
const HOST_MAX = 255;
|
|
84
|
+
const nowOf = (now) => (typeof now === "function" ? now() : now);
|
|
85
|
+
const isMs = (n) => Number.isSafeInteger(n) && n >= 0;
|
|
86
|
+
const validRunId = (runId) => (typeof runId === "string" && runId !== "") || (Number.isSafeInteger(runId) && runId >= 0);
|
|
87
|
+
const codeOf = (error) => (typeof error?.code === "string" && error.code !== "" ? error.code : "valkey");
|
|
88
|
+
const refused = (reason) => ({ error: "refused", reason });
|
|
89
|
+
const isAwaitTarget = (k) => typeof k === "string" && k.startsWith(AWAIT_TARGET_PREFIX) && /^[0-9a-f]{32}$/.test(k.slice(AWAIT_TARGET_PREFIX.length));
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* The await target pointer for a job (job data as queued, `kind` included), or null for a local or cron job, a job with
|
|
93
|
+
* no target number, or anything not plain. The steer target's hash with this module's prefix: derived, never spelled.
|
|
94
|
+
*/
|
|
95
|
+
export function awaitTargetKeyOf(job, { targetOf = steerTargetOf } = {}) {
|
|
96
|
+
try {
|
|
97
|
+
const { targetKey } = targetOf(job);
|
|
98
|
+
if (typeof targetKey !== "string" || !targetKey.startsWith(STEER_TARGET_PREFIX)) return null;
|
|
99
|
+
return `${AWAIT_TARGET_PREFIX}${targetKey.slice(STEER_TARGET_PREFIX.length)}`;
|
|
100
|
+
} catch {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function awaitOpenKey(runId) {
|
|
106
|
+
return `run:await:open:${sanitizeJobId(runId)}`;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function heldKeys(runId) {
|
|
110
|
+
const id = sanitizeJobId(runId);
|
|
111
|
+
return { list: `run:held:${id}`, closed: `run:held:closed:${id}` };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The entry as the list stores it, or `{ refusal }`. Only the contract's fields, in a FIXED order with `id` first, so
|
|
116
|
+
* the hold script can recognise a listed id by the JSON's prefix without parsing it in Lua. `fallback` is the job the
|
|
117
|
+
* receiver would have enqueued (receiver/src/steer.mjs `fallbackJobOf`): it holds the comment and the issue text, which
|
|
118
|
+
* is why this list is read by the settle alone and never shown on any surface.
|
|
119
|
+
*/
|
|
120
|
+
export function normalizeHeldEntry(entry, now = Date.now) {
|
|
121
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) return { refusal: "the held reply is not an object" };
|
|
122
|
+
if (typeof entry.id !== "string" || !STEER_ID_RE.test(entry.id)) return { refusal: "the held reply id is not 32 lowercase hex characters" };
|
|
123
|
+
if (!HELD_ROUTES.includes(entry.route)) return { refusal: `the route is not one of ${HELD_ROUTES.join(", ")}` };
|
|
124
|
+
const at = entry.at === undefined ? nowOf(now) : entry.at;
|
|
125
|
+
if (!isMs(at)) return { refusal: "the held reply time is not a millisecond timestamp" };
|
|
126
|
+
const fb = entry.fallback;
|
|
127
|
+
if (!fb || typeof fb !== "object" || Array.isArray(fb)) return { refusal: "the fallback job is not an object" };
|
|
128
|
+
const out = { id: entry.id, at, route: entry.route, fallback: fb };
|
|
129
|
+
let json;
|
|
130
|
+
try {
|
|
131
|
+
json = JSON.stringify(out);
|
|
132
|
+
} catch {
|
|
133
|
+
return { refusal: "the fallback job does not serialise" };
|
|
134
|
+
}
|
|
135
|
+
if (typeof json !== "string") return { refusal: "the fallback job does not serialise" };
|
|
136
|
+
const bytes = Buffer.byteLength(json, "utf8");
|
|
137
|
+
if (bytes > HELD_ENTRY_MAX_BYTES) return { refusal: `the held reply is ${bytes} bytes, above the ${HELD_ENTRY_MAX_BYTES}-byte limit` };
|
|
138
|
+
return { entry: out, json };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// closed FIRST: the close also deletes the open key, and "the run has settled" is the truer of the two answers. Then
|
|
142
|
+
// gone, then dup (an entry whose JSON starts with this id's prefix: a redelivered webhook GUID is the same reply).
|
|
143
|
+
// AT THE CAP THE OLDEST ENTRY GOES, never the new one (review round 2): the keeper registers a run for its whole pickup,
|
|
144
|
+
// so on a busy issue a long run's list fills with comments made long before the agent asked anything, and refusing the
|
|
145
|
+
// newest (`full`, the first version) refused exactly the one reply likely to be the answer. The oldest is the one the
|
|
146
|
+
// take's lookback would drop anyway (held-settle.mjs HELD_LOOKBACK_MS).
|
|
147
|
+
// KEYS: open, closed, list. ARGV: id prefix, entry JSON, max entries, ttl ms.
|
|
148
|
+
const HOLD_SCRIPT = `if redis.call('EXISTS', KEYS[2]) == 1 then return 'closed' end
|
|
149
|
+
if redis.call('EXISTS', KEYS[1]) == 0 then return 'gone' end
|
|
150
|
+
local items = redis.call('LRANGE', KEYS[3], 0, -1)
|
|
151
|
+
for i = 1, #items do
|
|
152
|
+
if string.sub(items[i], 1, #ARGV[1]) == ARGV[1] then return 'dup' end
|
|
153
|
+
end
|
|
154
|
+
redis.call('RPUSH', KEYS[3], ARGV[2])
|
|
155
|
+
redis.call('LTRIM', KEYS[3], -tonumber(ARGV[3]), -1)
|
|
156
|
+
redis.call('PEXPIRE', KEYS[3], ARGV[4])
|
|
157
|
+
return 'ok'`;
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Hold one reply for a live run: "ok", "dup", "closed" or "gone", else `{ error }` (at the cap the oldest entry is evicted). Validated before Valkey is
|
|
161
|
+
* touched. `ok` and `dup` mean the run's settle will see it; every other answer means it will not, and the caller
|
|
162
|
+
* answers exactly as it did before held replies existed.
|
|
163
|
+
*/
|
|
164
|
+
export async function holdReply({ redis, runId, entry, now = Date.now } = {}) {
|
|
165
|
+
if (!validRunId(runId)) return refused("no run id");
|
|
166
|
+
const norm = normalizeHeldEntry(entry, now);
|
|
167
|
+
if (norm.refusal) return refused(norm.refusal);
|
|
168
|
+
const k = heldKeys(runId);
|
|
169
|
+
try {
|
|
170
|
+
const r = await redis.eval(HOLD_SCRIPT, 3, awaitOpenKey(runId), k.closed, k.list, `{"id":"${norm.entry.id}"`, norm.json, String(HELD_MAX_ENTRIES), String(HELD_TTL_MS));
|
|
171
|
+
return HELD_OUTCOMES.includes(r) ? r : { error: "bad-reply" };
|
|
172
|
+
} catch (error) {
|
|
173
|
+
return { error: codeOf(error) };
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// The settle's close. KEYS: closed, open, list[, target]. ARGV: target member (the raw runId), ttl ms. `first` says
|
|
178
|
+
// whether THIS call made the closing transition: a run settled twice (the processor, then BullMQ's failed listener for
|
|
179
|
+
// the same thrown failure) gets the list back both times, and only the first may act on it (steer-inbox.mjs's rule).
|
|
180
|
+
// KEYS[5], when present, is the target's parked marker, set in the same script as the close (the park's take only), so
|
|
181
|
+
// there is no instant in which the run is off its target and the marker is not yet up. ARGV[3]: its ttl ms.
|
|
182
|
+
const CLOSE_SCRIPT = `local first = 1 - redis.call('EXISTS', KEYS[1])
|
|
183
|
+
redis.call('SET', KEYS[1], '1', 'PX', ARGV[2])
|
|
184
|
+
redis.call('DEL', KEYS[2])
|
|
185
|
+
if #KEYS >= 4 then redis.call('ZREM', KEYS[4], ARGV[1]) end
|
|
186
|
+
if #KEYS >= 5 then redis.call('SET', KEYS[5], ARGV[1], 'PX', ARGV[3]) end
|
|
187
|
+
redis.call('PEXPIRE', KEYS[3], ARGV[2])
|
|
188
|
+
return {redis.call('LRANGE', KEYS[3], 0, -1), first}`;
|
|
189
|
+
|
|
190
|
+
/** The parked marker of an await target (`run:await:parked:<the same hex32>`), or null for anything else. */
|
|
191
|
+
export function awaitParkedKeyOf(targetKey) {
|
|
192
|
+
return isAwaitTarget(targetKey) ? `${AWAIT_PARKED_PREFIX}${targetKey.slice(AWAIT_TARGET_PREFIX.length)}` : null;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** Whether a run parked on this await target within AWAIT_PARKED_TTL_MS: true, false, or `{ error }`. */
|
|
196
|
+
export async function recentlyParked({ redis, targetKey } = {}) {
|
|
197
|
+
const key = awaitParkedKeyOf(targetKey);
|
|
198
|
+
if (key === null) return refused("the target key is not an await target key");
|
|
199
|
+
try {
|
|
200
|
+
return Number(await redis.exists(key)) === 1;
|
|
201
|
+
} catch (error) {
|
|
202
|
+
return { error: codeOf(error) };
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** LRANGE reply to entries; a line that does not parse as an entry is dropped, never surfaced as text. */
|
|
207
|
+
function entriesOf(list) {
|
|
208
|
+
const out = [];
|
|
209
|
+
for (const line of Array.isArray(list) ? list : []) {
|
|
210
|
+
try {
|
|
211
|
+
const e = JSON.parse(line);
|
|
212
|
+
if (e && typeof e === "object" && typeof e.id === "string" && STEER_ID_RE.test(e.id) && HELD_ROUTES.includes(e.route) && e.fallback && typeof e.fallback === "object") out.push(e);
|
|
213
|
+
} catch {}
|
|
214
|
+
}
|
|
215
|
+
return out;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* The close (the park's take, else the settle at a FINAL outcome): `{ entries, first }`, every reply the run ever held, in the order held.
|
|
220
|
+
* After it every hold is refused. `targetKey` is the run's await pointer, null for a run without one.
|
|
221
|
+
*/
|
|
222
|
+
export async function closeHeld({ redis, runId, targetKey = null, parked = false } = {}) {
|
|
223
|
+
if (!validRunId(runId)) return refused("no run id");
|
|
224
|
+
if (targetKey !== null && !isAwaitTarget(targetKey)) return refused("the target key is not an await target key");
|
|
225
|
+
const k = heldKeys(runId);
|
|
226
|
+
// `parked` (the park's take): also raise the target's parked marker, in the same script.
|
|
227
|
+
const keys = [k.closed, awaitOpenKey(runId), k.list, ...(targetKey ? [targetKey, ...(parked === true ? [awaitParkedKeyOf(targetKey)] : [])] : [])];
|
|
228
|
+
try {
|
|
229
|
+
const r = await redis.eval(CLOSE_SCRIPT, keys.length, ...keys, String(runId), String(HELD_TTL_MS), String(AWAIT_PARKED_TTL_MS));
|
|
230
|
+
if (!Array.isArray(r) || r.length !== 2) return { error: "bad-reply" };
|
|
231
|
+
return { entries: entriesOf(r[0]), first: Number(r[1]) === 1 };
|
|
232
|
+
} catch (error) {
|
|
233
|
+
return { error: codeOf(error) };
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// KEYS: list, closed. Acts only while `closed` exists, so no reply held for the new run can be in what it deletes.
|
|
238
|
+
const RESET_SCRIPT = `if redis.call('EXISTS', KEYS[2]) == 0 then return 0 end
|
|
239
|
+
redis.call('DEL', KEYS[1], KEYS[2])
|
|
240
|
+
return 1`;
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Reset a CLOSED held list for a new run under the same run id (a console retry keeps the job id): the previous run is
|
|
244
|
+
* settled and its replies answered, and while its `closed` stood every hold for the new run would be refused for an
|
|
245
|
+
* hour. Called by the keeper before it registers, the steer relay's `resetClosedInbox` rule. `{ reset }` or `{ error }`.
|
|
246
|
+
*/
|
|
247
|
+
export async function resetClosedHeld({ redis, runId } = {}) {
|
|
248
|
+
if (!validRunId(runId)) return refused("no run id");
|
|
249
|
+
const k = heldKeys(runId);
|
|
250
|
+
try {
|
|
251
|
+
return { reset: Number(await redis.eval(RESET_SCRIPT, 2, k.list, k.closed)) === 1 };
|
|
252
|
+
} catch (error) {
|
|
253
|
+
return { error: codeOf(error) };
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Register a run as live on its await target, or refresh it (the keeper calls this every 2 s): ONE MULTI that sets the
|
|
259
|
+
* open key (PX 30 s), scores the run in the target's ZSET at `now + 30 s` (PEXPIRE 60 s, members already expired
|
|
260
|
+
* dropped) and keeps the held list's hour. `targetKey` null for a run no forge reply can reach: nothing to register,
|
|
261
|
+
* so `{ ok: true }` without a call. `{ ok: true }` or `{ error }`.
|
|
262
|
+
*/
|
|
263
|
+
export async function registerAwaitLive({ redis, runId, targetKey, host, now = Date.now } = {}) {
|
|
264
|
+
if (!validRunId(runId)) return refused("no run id");
|
|
265
|
+
if (targetKey === null || targetKey === undefined) return { ok: true };
|
|
266
|
+
if (!isAwaitTarget(targetKey)) return refused("the target key is not an await target key");
|
|
267
|
+
if (typeof host !== "string" || host === "" || host.length > HOST_MAX) return refused("the host name is missing or too long");
|
|
268
|
+
const t = nowOf(now);
|
|
269
|
+
if (!isMs(t)) return refused("now is not a millisecond timestamp");
|
|
270
|
+
try {
|
|
271
|
+
const res = await redis
|
|
272
|
+
.multi()
|
|
273
|
+
.set(awaitOpenKey(runId), host, "PX", AWAIT_OPEN_TTL_MS)
|
|
274
|
+
.zremrangebyscore(targetKey, "-inf", t)
|
|
275
|
+
.zadd(targetKey, t + AWAIT_OPEN_TTL_MS, String(runId))
|
|
276
|
+
.pexpire(targetKey, AWAIT_TARGET_TTL_MS)
|
|
277
|
+
.pexpire(heldKeys(runId).list, HELD_TTL_MS)
|
|
278
|
+
.exec();
|
|
279
|
+
if (!Array.isArray(res)) return { error: "bad-reply" };
|
|
280
|
+
const failed = res.find(([err]) => err);
|
|
281
|
+
if (failed) return { error: codeOf(failed[0]) };
|
|
282
|
+
return { ok: true };
|
|
283
|
+
} catch (error) {
|
|
284
|
+
return { error: codeOf(error) };
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/** The keeper's 2 s refresh: the same writes as `registerAwaitLive`, named apart so a log line says which. */
|
|
289
|
+
export function refreshAwaitLive(args) {
|
|
290
|
+
return registerAwaitLive(args);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** Drop the run from its target and delete the open key, now. Not a close: the held list stays for the settle. */
|
|
294
|
+
export async function unregisterAwaitLive({ redis, runId, targetKey = null } = {}) {
|
|
295
|
+
if (!validRunId(runId)) return refused("no run id");
|
|
296
|
+
if (targetKey !== null && !isAwaitTarget(targetKey)) return refused("the target key is not an await target key");
|
|
297
|
+
try {
|
|
298
|
+
const m = redis.multi().del(awaitOpenKey(runId));
|
|
299
|
+
if (targetKey) m.zrem(targetKey, String(runId));
|
|
300
|
+
const res = await m.exec();
|
|
301
|
+
if (!Array.isArray(res)) return { error: "bad-reply" };
|
|
302
|
+
const failed = res.find(([err]) => err);
|
|
303
|
+
if (failed) return { error: codeOf(failed[0]) };
|
|
304
|
+
return { ok: true };
|
|
305
|
+
} catch (error) {
|
|
306
|
+
return { error: codeOf(error) };
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* The awaitInput runs live on a target: at most `limit` runIds whose score is after `now`, else `{ error }`. A member
|
|
312
|
+
* whose keeper stopped refreshing is ignored the moment its score passes, whatever the ZSET's own TTL says.
|
|
313
|
+
*/
|
|
314
|
+
export async function liveAwaitRunsForTarget({ redis, targetKey, now = Date.now, limit = 4 } = {}) {
|
|
315
|
+
if (!isAwaitTarget(targetKey)) return refused("the target key is not an await target key");
|
|
316
|
+
const t = nowOf(now);
|
|
317
|
+
if (!isMs(t)) return refused("now is not a millisecond timestamp");
|
|
318
|
+
const n = Number.isSafeInteger(limit) && limit > 0 ? Math.min(limit, 64) : 4;
|
|
319
|
+
try {
|
|
320
|
+
const r = await redis.zrangebyscore(targetKey, `(${t}`, "+inf", "LIMIT", 0, n);
|
|
321
|
+
return Array.isArray(r) ? r.filter((m) => typeof m === "string" && m !== "") : { error: "bad-reply" };
|
|
322
|
+
} catch (error) {
|
|
323
|
+
return { error: codeOf(error) };
|
|
324
|
+
}
|
|
325
|
+
}
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { isFinalOutcome } from "./inbox-settle.mjs";
|
|
2
|
+
import { enqueueForgeJob, QUEUE } from "./queue.mjs";
|
|
3
|
+
import { runIdOf } from "./recovery-queue.mjs";
|
|
4
|
+
import { sanitizeJobId } from "./run-history.mjs";
|
|
5
|
+
import * as heldModule from "./held-replies.mjs";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The worker's side of held replies (issue #638, DES-AWAIT-INPUT-ENDS-THE-JOB; the keys and the scripts are
|
|
9
|
+
* held-replies.mjs's): a keeper that keeps an awaitInput run reachable while it runs, the take inside the park that
|
|
10
|
+
* turns what it held into at most one resumed job, and the final settle that drops what a run that did not park held.
|
|
11
|
+
*
|
|
12
|
+
* WHY THE WORKER AND NOT THE RECEIVER DECIDES. The receiver only knows that a reply met a run that is still going; it
|
|
13
|
+
* cannot know whether that run will park (the agent may finish, fail, or be cancelled after asking). The worker learns
|
|
14
|
+
* it in the park sequence, or at the run's final outcome, so that is where a held reply becomes a job or is dropped. Dropping is today's
|
|
15
|
+
* coalescing exactly: a reply that met a run which then did NOT park was always deduplicated against it, and still is.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** The keeper's refresh period, the steer relay's: 2 s against a 30 s open key. */
|
|
19
|
+
export const AWAIT_KEEPER_POLL_MS = 2_000;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* How far back before the park a held reply may have been written and still be taken as the answer (issue #638 review,
|
|
23
|
+
* M1). The keeper registers the run for its whole pickup, so without a bound a phrase-free comment made ten minutes
|
|
24
|
+
* into a long run, before the agent had asked anything, would be taken as "the answer" to a question posted later.
|
|
25
|
+
* Five minutes covers the seconds between the agent's question and the park with a wide margin; an older entry is
|
|
26
|
+
* dropped silently, which is the coalescing it always got. Residual, named: a comment written in the last five minutes
|
|
27
|
+
* before the question and not meant as an answer is taken as one (the resumed agent reads it as a reply and may ask
|
|
28
|
+
* again).
|
|
29
|
+
*/
|
|
30
|
+
export const HELD_LOOKBACK_MS = 5 * 60 * 1000;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The one comment the replies after the first get, however many there are (fixed words, no mention, no quote of any
|
|
34
|
+
* reply). The first reply is the answer the parked run resumes with; a run asks for one answer, and starting a second
|
|
35
|
+
* resumed job from the same parked session would race the first for it.
|
|
36
|
+
*/
|
|
37
|
+
export const HELD_EXTRA_REPLY = "This run was waiting for one answer and took the first reply it was given. If this comment adds to it, comment again once the run has replied.";
|
|
38
|
+
|
|
39
|
+
/** The comment when the held answer could not be started (an enqueue that failed): the reply is answered, never lost silently. */
|
|
40
|
+
export const HELD_RESUME_FAILED_REPLY = "This run was waiting for an answer and could not start from the reply it was given. Comment again to answer it.";
|
|
41
|
+
|
|
42
|
+
const codeOf = (e) => (typeof e?.code === "string" && e.code !== "" ? e.code : typeof e?.error === "string" ? e.error : "error");
|
|
43
|
+
const targetNumber = (t) => (typeof t?.number === "number" ? t.number : typeof t?.number === "string" && /^[1-9][0-9]{0,15}$/.test(t.number) ? Number(t.number) : null);
|
|
44
|
+
|
|
45
|
+
/** Whether a processor result is a run that PARKED: the completed branch's `reason: "awaiting-input"` (processor.mjs). */
|
|
46
|
+
export function isParkedOutcome(result) {
|
|
47
|
+
return result !== null && typeof result === "object" && result.outcome === "completed" && result.reason === "awaiting-input";
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* `makeAwaitKeeper({ redis, job, host, now, log, pollMs, held })` -> `{ start(), stop() }`: keeps an awaitInput run
|
|
52
|
+
* live on its await target for the whole pickup (registered at start, refreshed every `pollMs`), so a reply in the
|
|
53
|
+
* seconds between the agent's question and the park can be held. Started by the processor only for a job whose
|
|
54
|
+
* `data.awaitInput === true` and whose target a forge reply can reach; stopped by the park's take, else at the settle
|
|
55
|
+
* (index.mjs).
|
|
56
|
+
*
|
|
57
|
+
* STOP LEAVES THE KEYS TO EXPIRE rather than deleting them: the settle's close removes them at a final outcome, and an
|
|
58
|
+
* attempt that will be retried (an InfraRetry with attempts left) keeps taking a held reply for the open key's 30 s,
|
|
59
|
+
* after which the receiver answers as it always did. Never throws; faults are logged once per step, by code.
|
|
60
|
+
*/
|
|
61
|
+
export function makeAwaitKeeper({ redis, job, host, now = Date.now, log = () => {}, pollMs = AWAIT_KEEPER_POLL_MS, held = heldModule } = {}) {
|
|
62
|
+
const runId = runIdOf(job);
|
|
63
|
+
const targetKey = held.awaitTargetKeyOf(job?.data ?? null);
|
|
64
|
+
const jobId = sanitizeJobId(job?.id);
|
|
65
|
+
let timer = null;
|
|
66
|
+
let running = null;
|
|
67
|
+
let stopped = false;
|
|
68
|
+
const said = new Set();
|
|
69
|
+
const fault = (step, code) => {
|
|
70
|
+
if (said.has(step)) return;
|
|
71
|
+
said.add(step);
|
|
72
|
+
log("held_keeper_fault", { jobId, step, code });
|
|
73
|
+
};
|
|
74
|
+
const refresh = async (step) => {
|
|
75
|
+
try {
|
|
76
|
+
const r = await held.refreshAwaitLive({ redis, runId, targetKey, host, now });
|
|
77
|
+
if (r?.error) fault(step, codeOf(r));
|
|
78
|
+
} catch (e) {
|
|
79
|
+
fault(step, codeOf(e));
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
const tick = () => {
|
|
83
|
+
if (running || stopped) return running ?? Promise.resolve();
|
|
84
|
+
running = refresh("refresh").finally(() => {
|
|
85
|
+
running = null;
|
|
86
|
+
});
|
|
87
|
+
return running;
|
|
88
|
+
};
|
|
89
|
+
return {
|
|
90
|
+
targetKey,
|
|
91
|
+
async start() {
|
|
92
|
+
if (timer !== null || stopped || targetKey === null) return;
|
|
93
|
+
// A closed list under this run id belongs to a run that already settled (a console retry keeps the job id).
|
|
94
|
+
const reset = await held.resetClosedHeld({ redis, runId });
|
|
95
|
+
if (reset?.error) fault("reset", codeOf(reset));
|
|
96
|
+
await refresh("register");
|
|
97
|
+
if (stopped) return;
|
|
98
|
+
timer = setInterval(() => void tick(), pollMs);
|
|
99
|
+
timer.unref?.();
|
|
100
|
+
},
|
|
101
|
+
async stop() {
|
|
102
|
+
stopped = true;
|
|
103
|
+
if (timer !== null) clearInterval(timer);
|
|
104
|
+
timer = null;
|
|
105
|
+
try {
|
|
106
|
+
await running;
|
|
107
|
+
} catch {}
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* `makeHeldSettle({ redis, queues, comment, log, now, enqueue, held })` -> `settleHeld({ job, runId, result, error,
|
|
114
|
+
* queueName })`, with `settleHeld.take({ job, runId, queueName })`: what an awaitInput run's held replies become.
|
|
115
|
+
*
|
|
116
|
+
* THE TAKE, INSIDE THE PARK (issue #638 review, H1). The processor's parked branch calls `take` after it has set the
|
|
117
|
+
* marker and released the window and before its comment (processor.mjs, the park sequence). `closeHeld` first (one
|
|
118
|
+
* script: no reply can be held after it, and every reply held before it is in its reply); only the FIRST close acts:
|
|
119
|
+
* - entries older than HELD_LOOKBACK_MS before now are dropped silently (`held_replies_dropped`, `stale`);
|
|
120
|
+
* - the first recent entry whose fallback names this run's forge, repo, target type and number (it was held under
|
|
121
|
+
* them; one that does not is skipped with `held_reply_mismatch` and the next is tried) is enqueued as the resumed job
|
|
122
|
+
* WITH the semantic window (`enqueueForgeJob`), on the queue that ran the run (a host queue stays on its host). The
|
|
123
|
+
* window is the point: the park released this run's window a moment before, so the taken reply now holds it, and
|
|
124
|
+
* every later answer (a phrase-free reply the marker routes, a phrase reply) is coalesced against that one job
|
|
125
|
+
* instead of starting a second resumed run from the same parked transcript. An add the window swallowed against
|
|
126
|
+
* ANOTHER job (an answer that came in between the release and the take, which is then the run) is not taken;
|
|
127
|
+
* against THIS run's own job (the release failed) it is not taken either, and is answered as not started;
|
|
128
|
+
* - every other recent entry gets ONE fixed comment, HELD_EXTRA_REPLY; when nothing could be started at all, one
|
|
129
|
+
* HELD_RESUME_FAILED_REPLY instead (also when the close itself failed: an answer may have been held and lost).
|
|
130
|
+
* The take's close also raises the target's parked marker. Returns `{ resumed, answered, dropped, taken }`; the
|
|
131
|
+
* processor posts HELD_TAKEN_COMMENT instead of the parked comment when `taken` is set ("here" or "other").
|
|
132
|
+
*
|
|
133
|
+
* THE SETTLE, AT THE FINAL OUTCOME (the inbox settle's three sites and `isFinalOutcome`), is the backstop. After a take
|
|
134
|
+
* the list is closed and it acts on nothing (`first: false`), and when the take's own close failed it only drops
|
|
135
|
+
* (`afterTake`). Otherwise: a parked outcome whose take never ran (an unwired seam) gets the take's logic; ANY other outcome drops every held reply, which is exactly the coalescing a reply
|
|
136
|
+
* into a still-running run always got. That includes an operator cancel acknowledged before the result: the record says
|
|
137
|
+
* the run was cancelled, not parked, so its held answers are dropped (review, L2).
|
|
138
|
+
*
|
|
139
|
+
* Logs, ids and counts only: `held_reply_resumed { jobId, runId, n, queue }`, `held_replies_dropped { runId, n,
|
|
140
|
+
* stale? }`, `held_reply_mismatch { runId }`, `held_reply_not_taken { runId, survivingJobId }`, `held_reply_resume_failed
|
|
141
|
+
* { runId, queue, code }`, `held_settle_failed { jobId, code }`. Never throws; the counts or null.
|
|
142
|
+
*/
|
|
143
|
+
export function makeHeldSettle({ redis, queues, comment = async () => {}, log = () => {}, now = Date.now, enqueue = enqueueForgeJob, held = heldModule } = {}) {
|
|
144
|
+
const queueFor = (name) => {
|
|
145
|
+
if (typeof name === "string" && name.startsWith(`${QUEUE}@`) && typeof queues?.host === "function") return { queue: queues.host(name), name };
|
|
146
|
+
return { queue: queues?.shared?.(), name: QUEUE };
|
|
147
|
+
};
|
|
148
|
+
const answer = async (job, text) => {
|
|
149
|
+
try {
|
|
150
|
+
await comment({ ...job.data, id: job.id }, text);
|
|
151
|
+
} catch {}
|
|
152
|
+
};
|
|
153
|
+
// The same forge, repo, target type and number, and (review round 2) the same flow and command: the held fallback was
|
|
154
|
+
// held under this run's target, and one that would run different work is no answer to this run's question.
|
|
155
|
+
const sameTarget = (fields, d) => !((fields?.kind !== undefined && fields.kind !== d.kind) || fields?.repo !== d.repo || typeof fields?.target?.type !== "string" || fields.target.type !== d.target?.type || targetNumber(fields.target) === null || targetNumber(fields.target) !== targetNumber(d.target) || fields.flow !== d.flow || fields.command !== d.command);
|
|
156
|
+
const close = async (job, runId, { parked = false } = {}) => {
|
|
157
|
+
const closed = await held.closeHeld({ redis, runId, targetKey: held.awaitTargetKeyOf(job.data), parked });
|
|
158
|
+
if (!closed || closed.error) {
|
|
159
|
+
log("held_settle_failed", { jobId: sanitizeJobId(job.id), code: codeOf(closed) });
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
return { entries: Array.isArray(closed.entries) ? closed.entries : [], first: closed.first === true };
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
// The take's body, on entries a first close returned.
|
|
166
|
+
const start = async ({ job, runId, entries, queueName }) => {
|
|
167
|
+
// `taken`: "here" when this take started the resumed job, "other" when the window showed another job already is
|
|
168
|
+
// the answer (one that got in between the release and the take), null otherwise. Either word means the run has
|
|
169
|
+
// its resumed job, so the processor says so and a late steer message must not start a second one (review round 2).
|
|
170
|
+
const counts = { resumed: 0, answered: 0, dropped: 0, taken: null };
|
|
171
|
+
if (entries.length === 0) return counts;
|
|
172
|
+
const rid = sanitizeJobId(runId);
|
|
173
|
+
const t = typeof now === "function" ? now() : now;
|
|
174
|
+
const recent = entries.filter((e) => Number.isSafeInteger(e?.at) && e.at >= t - HELD_LOOKBACK_MS);
|
|
175
|
+
const stale = entries.length - recent.length;
|
|
176
|
+
if (stale > 0) {
|
|
177
|
+
counts.dropped = stale;
|
|
178
|
+
log("held_replies_dropped", { runId: rid, n: stale, stale });
|
|
179
|
+
}
|
|
180
|
+
if (recent.length === 0) return counts;
|
|
181
|
+
const route = queueFor(queueName ?? job.queueName);
|
|
182
|
+
let takenAt = -1;
|
|
183
|
+
let failed = false;
|
|
184
|
+
let tried = false;
|
|
185
|
+
for (let i = 0; i < recent.length; i++) {
|
|
186
|
+
const fields = recent[i].fallback;
|
|
187
|
+
if (!sameTarget(fields, job.data)) {
|
|
188
|
+
log("held_reply_mismatch", { runId: rid });
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
tried = true;
|
|
192
|
+
try {
|
|
193
|
+
if (!route.queue) throw Object.assign(new Error("no queue"), { code: "no-queue" });
|
|
194
|
+
// The queue's enqueue timestamp is later than the receiver's trusted receipt. Keep that receipt
|
|
195
|
+
// outside the payload so a held answer does not acquire artificial human waiting time.
|
|
196
|
+
const r = await enqueue(route.queue, job.data.kind, { ...fields, replyReceivedAt: recent[i].at });
|
|
197
|
+
if (r?.deduplicated === true) {
|
|
198
|
+
// Swallowed by the window: another job holds it (an answer between the release and the take, which is
|
|
199
|
+
// then the resumed run), or this run's own (its release failed, so nothing was started).
|
|
200
|
+
log("held_reply_not_taken", { runId: rid, survivingJobId: sanitizeJobId(r.survivingJobId) });
|
|
201
|
+
// The run's own job id, or its run id for a #645 recovery whose original holds the window.
|
|
202
|
+
failed = r.survivingJobId === job.id || r.survivingJobId === runId;
|
|
203
|
+
if (!failed) counts.taken = "other";
|
|
204
|
+
} else {
|
|
205
|
+
takenAt = i;
|
|
206
|
+
counts.resumed = 1;
|
|
207
|
+
counts.taken = "here";
|
|
208
|
+
log("held_reply_resumed", { jobId: sanitizeJobId(r?.jobId), runId: rid, n: recent.length, queue: route.name });
|
|
209
|
+
}
|
|
210
|
+
} catch (e) {
|
|
211
|
+
log("held_reply_resume_failed", { runId: rid, queue: route.name, code: codeOf(e) });
|
|
212
|
+
failed = true;
|
|
213
|
+
}
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
const rest = recent.length - (takenAt >= 0 ? 1 : 0);
|
|
217
|
+
if (rest > 0) {
|
|
218
|
+
// "Took the first reply" is true when one was taken here or by the job that holds the window; otherwise the
|
|
219
|
+
// replies were not started at all and say so.
|
|
220
|
+
await answer(job, takenAt >= 0 || (tried && !failed) ? HELD_EXTRA_REPLY : HELD_RESUME_FAILED_REPLY);
|
|
221
|
+
counts.answered = rest;
|
|
222
|
+
}
|
|
223
|
+
return counts;
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
// `afterTake`: the park's take ran for this outcome (the processor says so). A take whose close FAILED already told the
|
|
227
|
+
// person nothing was started (HELD_RESUME_FAILED_REPLY), so a backstop close that succeeds afterwards must not start a
|
|
228
|
+
// job anyway and contradict it (review round 2): after a take, the backstop only ever drops.
|
|
229
|
+
async function settleHeld({ job, runId = runIdOf(job), result = null, error = null, queueName = null, afterTake = false } = {}) {
|
|
230
|
+
try {
|
|
231
|
+
if (job?.data?.awaitInput !== true) return null;
|
|
232
|
+
if (!isFinalOutcome({ job, result, error })) return null;
|
|
233
|
+
if (runId === null || runId === undefined || runId === "") return null;
|
|
234
|
+
const closed = await close(job, runId);
|
|
235
|
+
if (closed === null) return null;
|
|
236
|
+
if (!closed.first || closed.entries.length === 0) return { resumed: 0, answered: 0, dropped: 0, taken: null };
|
|
237
|
+
if (isParkedOutcome(result) && afterTake !== true) return await start({ job, runId, entries: closed.entries, queueName });
|
|
238
|
+
log("held_replies_dropped", { runId: sanitizeJobId(runId), n: closed.entries.length });
|
|
239
|
+
return { resumed: 0, answered: 0, dropped: closed.entries.length, taken: null };
|
|
240
|
+
} catch (e) {
|
|
241
|
+
log("held_settle_failed", { jobId: sanitizeJobId(job?.id ?? ""), code: codeOf(e) });
|
|
242
|
+
return null;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
settleHeld.take = async function takeHeld({ job, runId = runIdOf(job), queueName = null } = {}) {
|
|
247
|
+
try {
|
|
248
|
+
if (job?.data?.awaitInput !== true) return null;
|
|
249
|
+
if (runId === null || runId === undefined || runId === "") return null;
|
|
250
|
+
// `parked`: the close also raises the target's parked marker (held-replies.mjs), so a reply delivered after the run
|
|
251
|
+
// left its target is routed as a parked reply rather than dropped for the label it raced.
|
|
252
|
+
const closed = await close(job, runId, { parked: true });
|
|
253
|
+
if (closed === null) {
|
|
254
|
+
// L1: the close failed, so a reply may have been held and will never be seen: say so once.
|
|
255
|
+
await answer(job, HELD_RESUME_FAILED_REPLY);
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
if (!closed.first) return { resumed: 0, answered: 0, dropped: 0, taken: null };
|
|
259
|
+
return await start({ job, runId, entries: closed.entries, queueName });
|
|
260
|
+
} catch (e) {
|
|
261
|
+
log("held_settle_failed", { jobId: sanitizeJobId(job?.id ?? ""), code: codeOf(e) });
|
|
262
|
+
return null;
|
|
263
|
+
}
|
|
264
|
+
};
|
|
265
|
+
return settleHeld;
|
|
266
|
+
}
|