@edgehero/pi-dispatch-receiver 1.5.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -0
- package/package.json +7 -4
- package/src/boot-retry.mjs +144 -0
- package/src/cli.mjs +12 -5
- package/src/config.mjs +30 -5
- package/src/filter-azure.mjs +25 -4
- package/src/filter-forgejo.mjs +29 -6
- package/src/filter-gitlab.mjs +29 -6
- package/src/filter.mjs +32 -6
- package/src/poller-config.mjs +6 -1
- package/src/poller.mjs +85 -15
- package/src/receiver.mjs +44 -19
- package/src/start.mjs +234 -85
package/src/filter.mjs
CHANGED
|
@@ -50,7 +50,7 @@ import { escapeRegExp, firstMatchingRule, labelSet, matchedLabel, matchesRule }
|
|
|
50
50
|
|
|
51
51
|
const AUTHOR_ALLOWLIST = new Set(["OWNER", "MEMBER", "COLLABORATOR"]);
|
|
52
52
|
const LABEL_ACTIONS = new Set(["opened", "labeled", "reopened"]);
|
|
53
|
-
const PR_ACTIONS = new Set(["labeled", "opened", "synchronize", "reopened"]);
|
|
53
|
+
export const PR_ACTIONS = new Set(["labeled", "opened", "synchronize", "reopened"]);
|
|
54
54
|
const PR_AUTO_ACTIONS = new Set(["opened", "synchronize", "reopened"]);
|
|
55
55
|
// The triggers.json word for a submitted review, and the raw action GitHub sends on the
|
|
56
56
|
// `pull_request_review` event. They differ on purpose -- see the routing block below.
|
|
@@ -121,7 +121,8 @@ export function filter(eventName, subset, cfg, selfId, deliveryId, closerAuthori
|
|
|
121
121
|
if (!resolved.enqueue) return resolved; // carries the drop reason
|
|
122
122
|
|
|
123
123
|
// (3) Build the job from the INT-WEBHOOK-PAYLOAD-SUBSET fields only. No sender.login (not in the
|
|
124
|
-
// subset),
|
|
124
|
+
// subset), provider/model/maxTurns only when the matched rule named them (absent, the worker fills the
|
|
125
|
+
// deployment default, #502), no field outside the subset --
|
|
125
126
|
// with two trigger-context additions (issue #49): `matched` is harness-computed, the filter's own
|
|
126
127
|
// decision record naming the triggers.json entry that fired (not payload data at all); `comment`,
|
|
127
128
|
// present only on the comment route, carries the invoking comment's body/author_association, both
|
|
@@ -145,6 +146,27 @@ export function filter(eventName, subset, cfg, selfId, deliveryId, closerAuthori
|
|
|
145
146
|
...(resolved.command !== undefined ? { command: resolved.command } : { flow: resolved.flow }),
|
|
146
147
|
...(resolved.packages !== undefined ? { packages: resolved.packages } : {}),
|
|
147
148
|
...(resolved.image !== undefined ? { image: resolved.image } : {}),
|
|
149
|
+
// #227. A SEPARATE spread, and this is the whole of the bug it replaces: folded into the `image`
|
|
150
|
+
// conditional above, a trigger that named a venue but no image had its venue silently dropped here --
|
|
151
|
+
// it loaded, validated, reached this function on the rule, and then ran on the default. That is
|
|
152
|
+
// exactly the destructive absence `validateBackend`'s near-miss sweep refuses a misspelling for,
|
|
153
|
+
// arriving through the plumbing instead of the spelling, one file downstream of the guard.
|
|
154
|
+
...(resolved.backend !== undefined ? { backend: resolved.backend } : {}),
|
|
155
|
+
// #291. A separate spread, the backend line's rule one field over: folded into a neighbour's
|
|
156
|
+
// conditional, a trigger that named only exclusions would have them silently dropped here -- and a
|
|
157
|
+
// dropped exclusion runs the job WITH the tool, the destructive absence the loader's own sweep
|
|
158
|
+
// refuses a misspelling for, arriving through the plumbing instead of the spelling.
|
|
159
|
+
...(resolved.excludeTools !== undefined ? { excludeTools: resolved.excludeTools } : {}),
|
|
160
|
+
// #502. One spread PER FIELD, the excludeTools rule above: folded into one conditional, a trigger
|
|
161
|
+
// that named only a model would have its provider's absence decide whether the model survives,
|
|
162
|
+
// and a dropped model runs the job on the deployment default while the file reads as chosen.
|
|
163
|
+
...(resolved.provider !== undefined ? { provider: resolved.provider } : {}),
|
|
164
|
+
...(resolved.model !== undefined ? { model: resolved.model } : {}),
|
|
165
|
+
...(resolved.maxTurns !== undefined ? { maxTurns: resolved.maxTurns } : {}),
|
|
166
|
+
...(resolved.models !== undefined ? { models: resolved.models } : {}),
|
|
167
|
+
// #501. Its own spread, the rule above: a dropped cap runs the job under the deployment's cap, or none,
|
|
168
|
+
// while the file reads as narrowed.
|
|
169
|
+
...(resolved.maxCostUsd !== undefined ? { maxCostUsd: resolved.maxCostUsd } : {}),
|
|
148
170
|
// The trigger's injected skills dir (REQ-PER-TRIGGER-SKILLS), at JOB level beside image/packages and
|
|
149
171
|
// NEVER inside `trigger`. That placement is sharpest here of all: `trigger` is carried into
|
|
150
172
|
// /job/event.json, and a worker-host path in an agent-readable file is the leak prepare-local's
|
|
@@ -199,7 +221,8 @@ function routeIssueLabel(subset, triggers) {
|
|
|
199
221
|
// A command rule (issue #189) skips flow resolution entirely: the label match IS the dispatch.
|
|
200
222
|
...(rule.command !== undefined ? { command: rule.command } : { flow: rule.flow }),
|
|
201
223
|
packages: rule.packages, // the MATCHED rule's fields -- rules in one file may differ on them
|
|
202
|
-
image: rule.image,
|
|
224
|
+
image: rule.image, backend: rule.backend, excludeTools: rule.excludeTools,
|
|
225
|
+
provider: rule.provider, model: rule.model, maxTurns: rule.maxTurns, models: rule.models, maxCostUsd: rule.maxCostUsd,
|
|
203
226
|
skillsDir: rule.skillsDir,
|
|
204
227
|
secrets: rule.secrets,
|
|
205
228
|
secretsProfile: rule.secretsProfile,
|
|
@@ -259,7 +282,8 @@ function routeComment(subset, triggers, knownFlows) {
|
|
|
259
282
|
// The single comment trigger IS the matched rule here, so its opt-in is the job's. A `<phrase>
|
|
260
283
|
// <flow>` override changes WHICH flow runs, never which triggers.json entry authorized it.
|
|
261
284
|
packages: triggers.comment.packages,
|
|
262
|
-
image: triggers.comment.image,
|
|
285
|
+
image: triggers.comment.image, backend: triggers.comment.backend, excludeTools: triggers.comment.excludeTools,
|
|
286
|
+
provider: triggers.comment.provider, model: triggers.comment.model, maxTurns: triggers.comment.maxTurns, models: triggers.comment.models, maxCostUsd: triggers.comment.maxCostUsd,
|
|
263
287
|
skillsDir: triggers.comment.skillsDir,
|
|
264
288
|
secrets: triggers.comment.secrets,
|
|
265
289
|
secretsProfile: triggers.comment.secretsProfile,
|
|
@@ -357,7 +381,8 @@ function routePullRequest(subset, triggers, action) {
|
|
|
357
381
|
// A command rule (issue #189) skips flow resolution entirely: the rule match IS the dispatch.
|
|
358
382
|
...(rule.command !== undefined ? { command: rule.command } : { flow: rule.flow }),
|
|
359
383
|
packages: rule.packages, // the MATCHED rule's fields -- rules in one file may differ on them
|
|
360
|
-
image: rule.image,
|
|
384
|
+
image: rule.image, backend: rule.backend, excludeTools: rule.excludeTools,
|
|
385
|
+
provider: rule.provider, model: rule.model, maxTurns: rule.maxTurns, models: rule.models, maxCostUsd: rule.maxCostUsd,
|
|
361
386
|
skillsDir: rule.skillsDir,
|
|
362
387
|
secrets: rule.secrets,
|
|
363
388
|
secretsProfile: rule.secretsProfile,
|
|
@@ -473,7 +498,8 @@ function routeClose(rules, number, closerAuthorized, matchedFor, targetFor) {
|
|
|
473
498
|
// A command rule (issue #189) skips flow resolution entirely: the rule match IS the dispatch.
|
|
474
499
|
...(rule.command !== undefined ? { command: rule.command } : { flow: rule.flow }),
|
|
475
500
|
packages: rule.packages, // the MATCHED rule's fields -- rules in one file may differ on them
|
|
476
|
-
image: rule.image,
|
|
501
|
+
image: rule.image, backend: rule.backend, excludeTools: rule.excludeTools,
|
|
502
|
+
provider: rule.provider, model: rule.model, maxTurns: rule.maxTurns, models: rule.models, maxCostUsd: rule.maxCostUsd,
|
|
477
503
|
skillsDir: rule.skillsDir,
|
|
478
504
|
secrets: rule.secrets,
|
|
479
505
|
secretsProfile: rule.secretsProfile,
|
package/src/poller-config.mjs
CHANGED
|
@@ -46,7 +46,8 @@ const POLL_INTERVAL_DEFAULT_SECONDS = 60;
|
|
|
46
46
|
* Parse the poller's config from `env`. Filesystem access is injected (`readFile`, `fileExists`) and
|
|
47
47
|
* forwarded to the receiver loader, so the whole thing is hermetically testable.
|
|
48
48
|
*
|
|
49
|
-
* Returns `{ valkeyUrl, triggers, github, repos, intervalSeconds }` -- and
|
|
49
|
+
* Returns `{ valkeyUrl, triggers, github, repos, intervalSeconds, identityRetryWindowMs }` -- and
|
|
50
|
+
* deliberately nothing else.
|
|
50
51
|
* `webhookSecret`, `port`, `bind` and the other-forge blocks are receiver-only concerns: the poller
|
|
51
52
|
* binds no port and speaks only GitHub (the other forges' producers stay webhook-armed).
|
|
52
53
|
* `repos === null` means "discover from the App installation each boot" and is only reachable when
|
|
@@ -76,6 +77,10 @@ export function loadPollerConfig(env = process.env, { readFile = readFileSync, f
|
|
|
76
77
|
github: base.github, // validated by the shared loadGitHubAuth, exactly as serve validates it
|
|
77
78
|
repos,
|
|
78
79
|
intervalSeconds: Math.max(POLL_INTERVAL_FLOOR_SECONDS, positiveInt(env, "POLL_INTERVAL_SECONDS", POLL_INTERVAL_DEFAULT_SECONDS)),
|
|
80
|
+
// Passed THROUGH the receiver loader rather than re-parsed: one parse, one floor (issue #318).
|
|
81
|
+
// The poller's boot identity gate is the same hard-fail gate serve has, so it retries under the
|
|
82
|
+
// same window -- a pure-polling deployment loses deliveries to a stopped process just as surely.
|
|
83
|
+
identityRetryWindowMs: base.identityRetryWindowMs,
|
|
79
84
|
};
|
|
80
85
|
}
|
|
81
86
|
|
package/src/poller.mjs
CHANGED
|
@@ -108,13 +108,14 @@
|
|
|
108
108
|
import { createSign } from "node:crypto";
|
|
109
109
|
import { readFile as fsReadFile } from "node:fs/promises";
|
|
110
110
|
import { configError } from "@edgehero/pi-dispatch/config";
|
|
111
|
-
import { parseConnection } from "@edgehero/pi-dispatch/connection";
|
|
111
|
+
import { judgeValkeyAtStart, makeRedisClient, parseConnection, valkeyClientContext } from "@edgehero/pi-dispatch/connection";
|
|
112
112
|
import { makeGitHubAuth } from "@edgehero/pi-dispatch/get-token";
|
|
113
113
|
import { enqueueGitHubJob, makeQueue } from "@edgehero/pi-dispatch/queue";
|
|
114
114
|
import { filter, hasCloseTriggers, wantsCloserAuthority } from "./filter.mjs";
|
|
115
115
|
import { makeResolveGitHubAuthority } from "./github-members.mjs";
|
|
116
116
|
import { parseSubset } from "./receiver.mjs";
|
|
117
117
|
import { loadPollerConfig } from "./poller-config.mjs";
|
|
118
|
+
import { retryIdentity } from "./boot-retry.mjs";
|
|
118
119
|
|
|
119
120
|
const API_URL = "https://api.github.com";
|
|
120
121
|
// 35 days: strictly outlives the 31-day gh-* job retention -- see the header's cursor/TTL section.
|
|
@@ -153,9 +154,10 @@ class RateLimited extends Error {
|
|
|
153
154
|
* Boot the poller: config, HARD-FAIL identity, repo set, then the cycle loop. Collaborators are
|
|
154
155
|
* injected with real defaults (start.mjs's convention), so the whole producer is testable offline
|
|
155
156
|
* with no GitHub, no Valkey, and no timers:
|
|
156
|
-
* { fetchFn, redis, queueFn, out, now, random, sleep, selfIdFn, tokenFn, fsDeps }
|
|
157
|
+
* { fetchFn, redis, queueFn, out, now, random, sleep, signals, selfIdFn, tokenFn, fsDeps }
|
|
157
158
|
* Returns `{ stop, done }`: `stop()` ends the loop after the in-flight work; `done` resolves once
|
|
158
|
-
* owned connections are closed
|
|
159
|
+
* owned connections are closed, with the inter-cycle default-sleep timer cleared by then (issue
|
|
160
|
+
* #325) -- after `done`, no timer armed by the poller remains.
|
|
159
161
|
*/
|
|
160
162
|
export async function startPoller(env = process.env, deps = {}) {
|
|
161
163
|
const {
|
|
@@ -164,6 +166,12 @@ export async function startPoller(env = process.env, deps = {}) {
|
|
|
164
166
|
now = Date.now,
|
|
165
167
|
random = Math.random,
|
|
166
168
|
sleep,
|
|
169
|
+
// The signal gate, decoupled from the sleep default (issue #325): `sleep === undefined` used to
|
|
170
|
+
// double as the "real run" flag, which made the REAL default timer unarmable under test without
|
|
171
|
+
// also registering process-wide SIGTERM/SIGINT handlers -- the exact leak the gate exists to
|
|
172
|
+
// prevent. A test that arms the real sleep passes `signals: false`; the one production caller
|
|
173
|
+
// (cli.mjs, no deps) keeps handlers exactly as before through this default.
|
|
174
|
+
signals = sleep === undefined,
|
|
167
175
|
redis,
|
|
168
176
|
queueFn,
|
|
169
177
|
selfIdFn,
|
|
@@ -172,7 +180,11 @@ export async function startPoller(env = process.env, deps = {}) {
|
|
|
172
180
|
fsDeps = {},
|
|
173
181
|
makeAuth = makeGitHubAuth,
|
|
174
182
|
makeQueueFn = makeQueue,
|
|
183
|
+
// Issue #464 (gate round 3): how the poller builds its own Valkey client; a seam, so a test sees none built.
|
|
184
|
+
makeRedisFn = makeRedisClient,
|
|
175
185
|
makeResolveGitHubAuthority: makeResolveGitHubAuthorityFn = makeResolveGitHubAuthority,
|
|
186
|
+
// Issue #464 (gate round 3): the start-time judgement of VALKEY_URL, as the receiver's (start.mjs).
|
|
187
|
+
judgeValkey = (url, opts) => judgeValkeyAtStart(url, valkeyClientContext({ env: opts.env }), { now: opts.now, ...(opts.sleep ? { sleep: opts.sleep } : {}) }),
|
|
176
188
|
} = deps;
|
|
177
189
|
|
|
178
190
|
const cfg = loadPollerConfig(env, fsDeps);
|
|
@@ -181,9 +193,22 @@ export async function startPoller(env = process.env, deps = {}) {
|
|
|
181
193
|
// the bot-loop guard's sole input, and a poller running without it would read the harness's own
|
|
182
194
|
// completion comment next cycle and enqueue it -- an unbounded paid recursion, only slower than the
|
|
183
195
|
// webhook version. No try/catch: an unresolvable identity must prevent the loop from ever starting.
|
|
196
|
+
// A TRANSIENT failure retries in-process under the same window serve uses (issue #318): a
|
|
197
|
+
// pure-polling deployment loses work to a stopped process just as surely as a webhook one.
|
|
198
|
+
//
|
|
199
|
+
// The default sleep sits above the gate because the gate sleeps too (issue #318); the boot retry
|
|
200
|
+
// simply awaits its cancellable promise to completion, so `cancel` is the LOOP's concern alone.
|
|
201
|
+
const sleepFn = sleep ?? cancellableSleep;
|
|
184
202
|
let auth = null;
|
|
185
|
-
|
|
186
|
-
|
|
203
|
+
// `auth ??= await ...` assigns only when the mint RESOLVES, so a retried getAuth re-invokes
|
|
204
|
+
// makeAuth. Caching the PROMISE instead would memoize the first failure and turn the retry loop
|
|
205
|
+
// into a rethrow spinner -- pinned by the poller's retry test.
|
|
206
|
+
// Issue #530: the GitHub client's own warnings go through this JSON log, never Octokit's plain console lines.
|
|
207
|
+
const getAuth = async () => (auth ??= await makeAuth(cfg.github, { log: (event, fields) => out({ event, ...fields }) }));
|
|
208
|
+
const selfId = await retryIdentity(
|
|
209
|
+
() => (selfIdFn ? selfIdFn(cfg.github) : getAuth().then((a) => a.selfId)),
|
|
210
|
+
{ forge: "github", windowMs: cfg.identityRetryWindowMs, log: out, now, sleep: sleepFn },
|
|
211
|
+
);
|
|
187
212
|
out({ event: "self_identity", id: selfId, source: cfg.github.source });
|
|
188
213
|
|
|
189
214
|
// The polling credential comes from the same auth config the worker validates. pat/gh hand back
|
|
@@ -210,9 +235,12 @@ export async function startPoller(env = process.env, deps = {}) {
|
|
|
210
235
|
// queue connection: a long-running producer should survive a Valkey restart.
|
|
211
236
|
let redisClient = redis ?? null;
|
|
212
237
|
let ownRedis = false;
|
|
238
|
+
// Issue #464 (gate round 3): judged before this process builds its own Valkey clients: a refusal exits 2, nothing
|
|
239
|
+
// answering for 20 s exits 1, never a poller running on a queue that cannot connect.
|
|
240
|
+
if (redisClient === null || queueFn === undefined || queueFn === null) await judgeValkey(cfg.valkeyUrl, { env, now, sleep });
|
|
213
241
|
if (redisClient === null) {
|
|
214
|
-
|
|
215
|
-
redisClient =
|
|
242
|
+
// Through connection.mjs, as every Valkey client of this project (issue #464): it judges and pins the address.
|
|
243
|
+
redisClient = makeRedisFn(cfg.valkeyUrl);
|
|
216
244
|
ownRedis = true;
|
|
217
245
|
}
|
|
218
246
|
|
|
@@ -263,7 +291,6 @@ export async function startPoller(env = process.env, deps = {}) {
|
|
|
263
291
|
stopped = true;
|
|
264
292
|
wake();
|
|
265
293
|
};
|
|
266
|
-
const sleepFn = sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
267
294
|
|
|
268
295
|
const done = (async () => {
|
|
269
296
|
let cycleNo = 0;
|
|
@@ -328,14 +355,27 @@ export async function startPoller(env = process.env, deps = {}) {
|
|
|
328
355
|
// in one synchronized stampede).
|
|
329
356
|
let delayMs = Math.max(cfg.intervalSeconds, stats.minDelaySeconds) * 1000;
|
|
330
357
|
if (rateResetMs !== null) delayMs = Math.max(delayMs, rateResetMs - now() + jitterMs(random));
|
|
331
|
-
|
|
358
|
+
// Hold the sleep so the LOSER can be cleared: when stop() wins this race, an uncancelled
|
|
359
|
+
// default timer stays armed for up to a full poll interval after `done` resolves (issue
|
|
360
|
+
// #325) -- masked in production by the signal handler's process.exit(0) below, and real for
|
|
361
|
+
// every caller that stops the poller without exiting. Optional-chained because injected
|
|
362
|
+
// test sleeps are plain promises carrying no cancel; their call count and arguments are
|
|
363
|
+
// untouched, and a rejecting one still propagates through the finally.
|
|
364
|
+
const nap = sleepFn(delayMs);
|
|
365
|
+
try {
|
|
366
|
+
await Promise.race([nap, stopWaker]);
|
|
367
|
+
} finally {
|
|
368
|
+
nap.cancel?.();
|
|
369
|
+
}
|
|
332
370
|
}
|
|
333
371
|
await closeOwned();
|
|
334
372
|
})();
|
|
335
373
|
|
|
336
|
-
// Signal handlers only on a real run (
|
|
337
|
-
// test injection the fakes are per-test, and a process-wide handler would
|
|
338
|
-
|
|
374
|
+
// Signal handlers only on a real run (`signals`, defaulting from the sleep seam) -- start.mjs's
|
|
375
|
+
// rule, same reason: under test injection the fakes are per-test, and a process-wide handler would
|
|
376
|
+
// leak across tests. Its own seam since issue #325, so a test can arm the REAL default sleep
|
|
377
|
+
// without inheriting the handlers.
|
|
378
|
+
if (signals) {
|
|
339
379
|
const shutdown = async (signal) => {
|
|
340
380
|
out({ event: "poller_stopping", signal });
|
|
341
381
|
stop();
|
|
@@ -358,6 +398,27 @@ function jitterMs(random) {
|
|
|
358
398
|
return 1_000 + Math.floor(random() * 29_000);
|
|
359
399
|
}
|
|
360
400
|
|
|
401
|
+
/**
|
|
402
|
+
* The default inter-cycle sleep: a REF'D setTimeout whose promise carries its own `cancel` (issue
|
|
403
|
+
* #325). Ref'd deliberately -- the poller IS its process's main loop and holding the loop through the
|
|
404
|
+
* delay is the point (`DES-RETENTION-SWEEPS-ON-A-TIMER`'s rejected list records the contrast; between
|
|
405
|
+
* cycles this timer is the only thing keeping a pure-poll process alive, cli.mjs's "awaiting done").
|
|
406
|
+
* The defect was never the ref, only survival past stop(): "unref'd is not cleaned up" has a ref'd
|
|
407
|
+
* twin, a cleared-nothing timer that holds the loop for up to a full interval after `done` resolved.
|
|
408
|
+
* A cancelled sleep never resolves, which is safe here because its only awaiter is a race `stopWaker`
|
|
409
|
+
* has already settled, and cancel on an already-fired timer is a no-op, so the winning side's clear
|
|
410
|
+
* costs nothing. EXPORTED for `settleWithin`'s reason (worker/src/start.mjs): the cleared-timer
|
|
411
|
+
* guarantee deserves a deterministic async_hooks pin on the helper, not a census of a full boot.
|
|
412
|
+
*/
|
|
413
|
+
export function cancellableSleep(ms) {
|
|
414
|
+
let timer;
|
|
415
|
+
const p = new Promise((resolve) => {
|
|
416
|
+
timer = setTimeout(resolve, ms);
|
|
417
|
+
});
|
|
418
|
+
p.cancel = () => clearTimeout(timer);
|
|
419
|
+
return p;
|
|
420
|
+
}
|
|
421
|
+
|
|
361
422
|
/** The redis key family for one repo -- see the schema table in the module header. */
|
|
362
423
|
function keyNames(repo) {
|
|
363
424
|
const p = `poll:${repo}`;
|
|
@@ -916,11 +977,20 @@ async function gate(ctx, eventName, payload, deliveryId, stats) {
|
|
|
916
977
|
return;
|
|
917
978
|
}
|
|
918
979
|
const replicas = result.job.replicas ?? 1;
|
|
980
|
+
let created = 0;
|
|
919
981
|
for (let i = 1; i <= replicas; i++) {
|
|
920
|
-
await ctx.enqueue(replicas > 1 ? { ...result.job, replica: i } : result.job);
|
|
982
|
+
const r = await ctx.enqueue(replicas > 1 ? { ...result.job, replica: i } : result.job);
|
|
983
|
+
// Issue #289, the receiver arms' honesty rule at the poller's own seam: a semantic-window swallow
|
|
984
|
+
// gets its own line with the surviving id, `enqueued` counts only what was CREATED, and a bare-id
|
|
985
|
+
// return from an enqueue seam predating the shape counts as created -- the old behaviour exactly.
|
|
986
|
+
if (r && typeof r === "object" && r.deduplicated === true) {
|
|
987
|
+
ctx.out({ event: "deduplicated", delivery: deliveryId, repo: result.job.repo, target: `${result.job.target.type}#${result.job.target.number}`, flow: result.job.flow, jobId: r.jobId, survivingJobId: r.survivingJobId });
|
|
988
|
+
} else {
|
|
989
|
+
created += 1;
|
|
990
|
+
}
|
|
921
991
|
}
|
|
922
|
-
stats.enqueued +=
|
|
923
|
-
ctx.out({ event: "enqueued", delivery: deliveryId, repo: result.job.repo, target: `${result.job.target.type}#${result.job.target.number}`, flow: result.job.flow, replicas });
|
|
992
|
+
stats.enqueued += created;
|
|
993
|
+
if (created > 0) ctx.out({ event: "enqueued", delivery: deliveryId, repo: result.job.repo, target: `${result.job.target.type}#${result.job.target.number}`, flow: result.job.flow, replicas: created });
|
|
924
994
|
}
|
|
925
995
|
|
|
926
996
|
/**
|
package/src/receiver.mjs
CHANGED
|
@@ -205,14 +205,43 @@ function pathOf(url) {
|
|
|
205
205
|
* `enqueue` is a callback because the four arms spell their enqueue differently (a named github/gitlab
|
|
206
206
|
* wrapper, or `enqueueForgeJob` with an explicit kind); the fanout itself is forge-blind.
|
|
207
207
|
*
|
|
208
|
-
* @returns {Promise<number
|
|
208
|
+
* @returns {Promise<{replicas: number, created: number, deduplicated: Array<{jobId, survivingJobId}>}>}
|
|
209
|
+
* what actually happened, per replica (issue #289): `created` counts jobs that now EXIST because of
|
|
210
|
+
* this delivery, and `deduplicated` carries each swallow the semantic window made, with the id that
|
|
211
|
+
* survived. A bare-id return from an enqueue that predates the shape counts as created -- the old
|
|
212
|
+
* behaviour exactly.
|
|
209
213
|
*/
|
|
210
214
|
async function fanout(job, enqueue) {
|
|
211
215
|
const replicas = job.replicas ?? 1;
|
|
216
|
+
let created = 0;
|
|
217
|
+
const deduplicated = [];
|
|
212
218
|
for (let i = 1; i <= replicas; i++) {
|
|
213
|
-
await enqueue(replicas > 1 ? { ...job, replica: i } : job);
|
|
219
|
+
const r = await enqueue(replicas > 1 ? { ...job, replica: i } : job);
|
|
220
|
+
if (r && typeof r === "object" && r.deduplicated === true) deduplicated.push({ jobId: r.jobId, survivingJobId: r.survivingJobId });
|
|
221
|
+
else created += 1;
|
|
214
222
|
}
|
|
215
|
-
return replicas;
|
|
223
|
+
return { replicas, created, deduplicated };
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* One log-and-answer for a fanout outcome, shared by all four arms so a weakened copy cannot hide (the
|
|
228
|
+
* four-copies doctrine above). Issue #289's honesty rule: `enqueued` is logged only when something was
|
|
229
|
+
* CREATED, with `replicas` meaning jobs that now exist; every semantic-window swallow gets its own
|
|
230
|
+
* `deduplicated` line naming the surviving job's id (every field forge- or worker-minted, no payload
|
|
231
|
+
* text); and a delivery that created NOTHING answers `202 {status:"deduplicated"}` -- still a 2xx,
|
|
232
|
+
* because a non-2xx would trigger a redelivery storm for a delivery that was handled, but no longer
|
|
233
|
+
* "queued", because a receiver that logs the swallow while answering success on the wire would be the
|
|
234
|
+
* honest-log/lying-wire split this project refuses. Each arm passes its own target grammar.
|
|
235
|
+
*/
|
|
236
|
+
function respondEnqueueOutcome({ log, res, delivery, repo, target, flow, outcome }) {
|
|
237
|
+
for (const d of outcome.deduplicated) {
|
|
238
|
+
log?.({ event: "deduplicated", delivery, repo, target, flow, jobId: d.jobId, survivingJobId: d.survivingJobId });
|
|
239
|
+
}
|
|
240
|
+
if (outcome.created > 0) {
|
|
241
|
+
log?.({ event: "enqueued", delivery, repo, target, flow, replicas: outcome.created });
|
|
242
|
+
return respond(res, 202, { status: "queued" });
|
|
243
|
+
}
|
|
244
|
+
return respond(res, 202, { status: "deduplicated" });
|
|
216
245
|
}
|
|
217
246
|
|
|
218
247
|
/**
|
|
@@ -266,17 +295,16 @@ function makeGitHubHandler({ queue, routeTo, selfId, cfg, log, resolveAuthority
|
|
|
266
295
|
}
|
|
267
296
|
|
|
268
297
|
// Fanout (REQ-REPLICA-RUNS) lives in `fanout` above; the 202/503 decision stays here, where it always was.
|
|
269
|
-
let
|
|
298
|
+
let outcome;
|
|
270
299
|
try {
|
|
271
|
-
|
|
300
|
+
outcome = await fanout(result.job, async (j) => await enqueueGitHubJob(await routeTo("github", j), j));
|
|
272
301
|
} catch (err) {
|
|
273
302
|
// Own try/catch so a Valkey-down enqueue is a 503 (retryable), not verify's outer 500.
|
|
274
303
|
log?.({ event: "enqueue_failed", delivery, reason: err?.message });
|
|
275
304
|
return respond(res, 503, { error: "enqueue-failed" }); // GitHub redelivers; dedup by GUID coalesces
|
|
276
305
|
}
|
|
277
306
|
|
|
278
|
-
|
|
279
|
-
return respond(res, 202, { status: "queued" });
|
|
307
|
+
return respondEnqueueOutcome({ log, res, delivery, repo: result.job.repo, target: `${result.job.target.type}#${result.job.target.number}`, flow: result.job.flow, outcome });
|
|
280
308
|
});
|
|
281
309
|
}
|
|
282
310
|
|
|
@@ -316,9 +344,9 @@ function makeGitLabHandler({ routeTo, queue, cfg, log, mode, secret, selfId, res
|
|
|
316
344
|
return respond(res, 204);
|
|
317
345
|
}
|
|
318
346
|
|
|
319
|
-
let
|
|
347
|
+
let outcome;
|
|
320
348
|
try {
|
|
321
|
-
|
|
349
|
+
outcome = await fanout(result.job, async (j) => await enqueueGitLabJob(await routeTo("gitlab", j), j));
|
|
322
350
|
} catch (err) {
|
|
323
351
|
log?.({ event: "enqueue_failed", delivery, reason: err?.message });
|
|
324
352
|
return respond(res, 503, { error: "enqueue-failed" }); // GitLab redelivers; dedup by webhook-id coalesces
|
|
@@ -327,8 +355,7 @@ function makeGitLabHandler({ routeTo, queue, cfg, log, mode, secret, selfId, res
|
|
|
327
355
|
// `!` for a merge request, `#` for an issue -- GitLab's own notation, and the same discrimination
|
|
328
356
|
// the semantic dedup key makes, because the two are separate number sequences.
|
|
329
357
|
const sep = result.job.target.type === "pull_request" ? "!" : "#";
|
|
330
|
-
|
|
331
|
-
return respond(res, 202, { status: "queued" });
|
|
358
|
+
return respondEnqueueOutcome({ log, res, delivery, repo: result.job.repo, target: `${result.job.repo}${sep}${result.job.target.number}`, flow: result.job.flow, outcome });
|
|
332
359
|
});
|
|
333
360
|
}
|
|
334
361
|
|
|
@@ -368,16 +395,15 @@ function makeForgejoHandler({ routeTo, queue, cfg, log, secret, selfId, resolveA
|
|
|
368
395
|
return respond(res, 204);
|
|
369
396
|
}
|
|
370
397
|
|
|
371
|
-
let
|
|
398
|
+
let outcome;
|
|
372
399
|
try {
|
|
373
|
-
|
|
400
|
+
outcome = await fanout(result.job, async (j) => await enqueueForgeJob(await routeTo("forgejo", j), "forgejo", j));
|
|
374
401
|
} catch (err) {
|
|
375
402
|
log?.({ event: "enqueue_failed", delivery, reason: err?.message });
|
|
376
403
|
return respond(res, 503, { error: "enqueue-failed" }); // Forgejo redelivers; dedup by GUID coalesces
|
|
377
404
|
}
|
|
378
405
|
|
|
379
|
-
|
|
380
|
-
return respond(res, 202, { status: "queued" });
|
|
406
|
+
return respondEnqueueOutcome({ log, res, delivery, repo: result.job.repo, target: `${result.job.target.type}#${result.job.target.number}`, flow: result.job.flow, outcome });
|
|
381
407
|
});
|
|
382
408
|
}
|
|
383
409
|
|
|
@@ -426,9 +452,9 @@ function makeAzureHandler({ routeTo, queue, cfg, log, mode, secret, headerName,
|
|
|
426
452
|
return respond(res, 204);
|
|
427
453
|
}
|
|
428
454
|
|
|
429
|
-
let
|
|
455
|
+
let outcome;
|
|
430
456
|
try {
|
|
431
|
-
|
|
457
|
+
outcome = await fanout(result.job, async (j) => await enqueueForgeJob(await routeTo("azure", j), "azure", j));
|
|
432
458
|
} catch (err) {
|
|
433
459
|
log?.({ event: "enqueue_failed", delivery, reason: err?.message });
|
|
434
460
|
return respond(res, 503, { error: "enqueue-failed" });
|
|
@@ -437,7 +463,6 @@ function makeAzureHandler({ routeTo, queue, cfg, log, mode, secret, headerName,
|
|
|
437
463
|
// `!` for a pull request, `#` for a work item -- Azure numbers them separately, and this is the same
|
|
438
464
|
// discrimination the semantic dedup key makes.
|
|
439
465
|
const sep = result.job.target.type === "pull_request" ? "!" : "#";
|
|
440
|
-
|
|
441
|
-
return respond(res, 202, { status: "queued" });
|
|
466
|
+
return respondEnqueueOutcome({ log, res, delivery, repo: result.job.repo, target: `${result.job.repo}${sep}${result.job.target.number}`, flow: result.job.flow, outcome });
|
|
442
467
|
});
|
|
443
468
|
}
|