agent-relay 9.2.3 → 9.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/integration-cleanup-journal.d.ts +120 -0
- package/dist/cli/commands/integration-cleanup-journal.d.ts.map +1 -0
- package/dist/cli/commands/integration-cleanup-journal.js +299 -0
- package/dist/cli/commands/integration-cleanup-journal.js.map +1 -0
- package/dist/cli/commands/integration.d.ts +63 -1
- package/dist/cli/commands/integration.d.ts.map +1 -1
- package/dist/cli/commands/integration.js +901 -62
- package/dist/cli/commands/integration.js.map +1 -1
- package/package.json +8 -8
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import { createHash, randomBytes } from 'node:crypto';
|
|
1
|
+
import { createHash, pbkdf2Sync, randomBytes } from 'node:crypto';
|
|
2
|
+
import { hostname } from 'node:os';
|
|
2
3
|
import { getProjectPaths } from '@agent-relay/config';
|
|
4
|
+
import { cleanupEntryKey, fileCleanupJournal, } from './integration-cleanup-journal.js';
|
|
3
5
|
import { addSdkOptions, printJson, runSdk, sdkOptionsFromOpts, withSdkDefaults, } from '../lib/sdk-command.js';
|
|
4
6
|
import { connectProjectBrokerClient } from '../lib/project-broker-client.js';
|
|
5
7
|
import { MIN_RELAYFILE_VERSION, RelayfileControlPlaneClient, RelayfileControlPlaneError, assertRelayfileVersion, } from '@relayfile/client';
|
|
@@ -13,6 +15,7 @@ export { MIN_RELAYFILE_VERSION, assertRelayfileVersion };
|
|
|
13
15
|
* import time. One instance is enough — it negotiates the daemon version once.
|
|
14
16
|
*/
|
|
15
17
|
let sharedClient;
|
|
18
|
+
const RELAYFILE_WEBHOOK_BINDINGS_API_VERSION = 3;
|
|
16
19
|
function controlPlaneClient(options) {
|
|
17
20
|
if (options)
|
|
18
21
|
return new RelayfileControlPlaneClient(options);
|
|
@@ -88,19 +91,28 @@ export function defaultRelayfileBridge(options) {
|
|
|
88
91
|
webhookId: input.webhookId,
|
|
89
92
|
webhookToken: input.webhookToken,
|
|
90
93
|
subscriptionId: input.subscriptionId,
|
|
94
|
+
webhookSubscriptionId: input.webhookSubscriptionId,
|
|
95
|
+
...(input.webhookSubscriptionWorkspaceId
|
|
96
|
+
? { webhookSubscriptionWorkspaceId: input.webhookSubscriptionWorkspaceId }
|
|
97
|
+
: {}),
|
|
91
98
|
});
|
|
92
99
|
},
|
|
93
100
|
async listBindings() {
|
|
94
101
|
const bindings = await client.listBindings();
|
|
95
102
|
// relayfile keys bindings on `pathGlob`; surface it as `resource` so callers
|
|
96
103
|
// (find/unbind) match on the canonical glob.
|
|
97
|
-
return bindings.map((b) =>
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
+
return bindings.map((b) => {
|
|
105
|
+
const { webhookSubscriptionId, webhookSubscriptionWorkspaceId } = b;
|
|
106
|
+
return {
|
|
107
|
+
provider: b.provider ?? '',
|
|
108
|
+
resource: b.pathGlob ?? '',
|
|
109
|
+
channel: b.channel ?? '',
|
|
110
|
+
webhookId: b.webhookId ?? '',
|
|
111
|
+
subscriptionId: b.subscriptionId ?? '',
|
|
112
|
+
webhookSubscriptionId,
|
|
113
|
+
webhookSubscriptionWorkspaceId,
|
|
114
|
+
};
|
|
115
|
+
});
|
|
104
116
|
},
|
|
105
117
|
async unbind(provider, resource) {
|
|
106
118
|
await client.unbind(provider, resource);
|
|
@@ -118,13 +130,17 @@ export function defaultRelayfileBridge(options) {
|
|
|
118
130
|
// Connecting negotiates the daemon version via /v1/hello; a too-old daemon
|
|
119
131
|
// (or one that can't start) throws a typed, actionable error here.
|
|
120
132
|
await client.ensureReady();
|
|
133
|
+
const hello = await client.hello();
|
|
134
|
+
if (!hello.supportedApiVersions?.includes(RELAYFILE_WEBHOOK_BINDINGS_API_VERSION)) {
|
|
135
|
+
throw new RelayfileControlPlaneError('VERSION_INCOMPATIBLE', `relayfile must support control-plane API v${RELAYFILE_WEBHOOK_BINDINGS_API_VERSION} to safely clean up inbound webhook subscriptions. Upgrade relayfile and retry.`);
|
|
136
|
+
}
|
|
121
137
|
},
|
|
122
138
|
async resolveWritebackBinding(channel) {
|
|
123
139
|
try {
|
|
124
|
-
const { url, secret } = await client.writebackSecret(channel);
|
|
140
|
+
const { url, secret, workspaceId } = await client.writebackSecret(channel);
|
|
125
141
|
if (!url?.trim() || !secret?.trim())
|
|
126
142
|
return undefined;
|
|
127
|
-
return { url: url.trim(), secret: secret.trim() };
|
|
143
|
+
return { url: url.trim(), secret: secret.trim(), workspaceId: workspaceId?.trim() || undefined };
|
|
128
144
|
}
|
|
129
145
|
catch (err) {
|
|
130
146
|
// A missing/disconnected writeback secret is a soft miss (caller falls
|
|
@@ -141,8 +157,11 @@ export function defaultRelayfileBridge(options) {
|
|
|
141
157
|
async createWebhookSubscription(input) {
|
|
142
158
|
return client.createWebhookSubscription(input);
|
|
143
159
|
},
|
|
144
|
-
async deleteWebhookSubscription(subscriptionId) {
|
|
145
|
-
await client.deleteWebhookSubscription(subscriptionId);
|
|
160
|
+
async deleteWebhookSubscription(subscriptionId, workspace) {
|
|
161
|
+
await client.deleteWebhookSubscription(subscriptionId, workspace);
|
|
162
|
+
},
|
|
163
|
+
async listWebhookSubscriptions(workspace) {
|
|
164
|
+
return client.listWebhookSubscriptions(workspace);
|
|
146
165
|
},
|
|
147
166
|
};
|
|
148
167
|
}
|
|
@@ -184,6 +203,7 @@ function withIntegrationDefaults(overrides = {}) {
|
|
|
184
203
|
...withSdkDefaults(overrides),
|
|
185
204
|
resolveLocalRelayOptions: resolveLocalBrokerRelayOptions,
|
|
186
205
|
relayfile: defaultRelayfileBridge(),
|
|
206
|
+
cleanupJournal: fileCleanupJournal(),
|
|
187
207
|
isInteractive: () => Boolean(process.stdin.isTTY && process.stdout.isTTY),
|
|
188
208
|
prompt: promptLine,
|
|
189
209
|
...overrides,
|
|
@@ -252,7 +272,7 @@ async function resolveWriteback(deps, commandOpts, channel) {
|
|
|
252
272
|
throw new Error('Could not resolve the writeback signing secret. Ensure relayfile is logged ' +
|
|
253
273
|
'in (relayfile login) and supports `integration writeback-secret`, or pass --bridge-url and --bridge-secret.');
|
|
254
274
|
}
|
|
255
|
-
return { url: binding.url, secret: binding.secret };
|
|
275
|
+
return { url: binding.url, secret: binding.secret, workspaceId: binding.workspaceId };
|
|
256
276
|
}
|
|
257
277
|
async function createRelayfileInboundTarget(commandOpts, local, input) {
|
|
258
278
|
const options = sdkOptionsFromOpts(commandOpts);
|
|
@@ -412,10 +432,444 @@ function webhookNamePrefix(provider, resource) {
|
|
|
412
432
|
function inboundWebhookName(prefix) {
|
|
413
433
|
return `${prefix}:${randomBytes(5).toString('hex')}`;
|
|
414
434
|
}
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
435
|
+
function isAlreadyDeletedWebhookSubscription(err) {
|
|
436
|
+
return err instanceof RelayfileControlPlaneError && err.status === 404;
|
|
437
|
+
}
|
|
438
|
+
/** Relaycast SDK errors expose statusCode/status; a not-found delete means the
|
|
439
|
+
* resource is already gone — the retry must converge, not persist forever. */
|
|
440
|
+
function isNotFoundRelayError(err) {
|
|
441
|
+
if (!err || typeof err !== 'object')
|
|
442
|
+
return false;
|
|
443
|
+
const { statusCode, status } = err;
|
|
444
|
+
return statusCode === 404 || status === 404;
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* Non-secret identity of the relay connection a journal entry may retry
|
|
448
|
+
* through. The workspace key is a credential, so it is run through a
|
|
449
|
+
* purpose-domain PBKDF2 (never a bare hash — CodeQL
|
|
450
|
+
* js/insufficient-password-hash) with the base URL as the salt domain; only
|
|
451
|
+
* the derived digest is ever stored.
|
|
452
|
+
*/
|
|
453
|
+
export function relayCleanupScope(options) {
|
|
454
|
+
let workspaceKey = '';
|
|
455
|
+
try {
|
|
456
|
+
workspaceKey = resolveWorkspaceKey(options);
|
|
457
|
+
}
|
|
458
|
+
catch {
|
|
459
|
+
// No resolvable key: scope on the base URL alone. Entries recorded this
|
|
460
|
+
// way still never leak the key (only a derived digest is stored).
|
|
461
|
+
}
|
|
462
|
+
const digest = pbkdf2Sync(workspaceKey, `agent-relay:cleanup-scope:${resolveBaseUrl(options) ?? ''}`, 10_000, 8, 'sha256').toString('hex');
|
|
463
|
+
return `relay:${digest}`;
|
|
464
|
+
}
|
|
465
|
+
/** The default bridge always talks to the project-local relayfile daemon; the
|
|
466
|
+
* per-entry relayfileWorkspaceId pin (not this scope) guards against active
|
|
467
|
+
* workspace switches between runs. */
|
|
468
|
+
function relayfileCleanupScope() {
|
|
469
|
+
return 'relayfile:project-daemon';
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* Bounded lease window for entry owners, set safely ABOVE every remote
|
|
473
|
+
* request timeout any lifecycle or recovery performs (each spans seconds; a
|
|
474
|
+
* reconciliation at most a few request timeouts). Explicit bounded-lease
|
|
475
|
+
* tradeoff: a process paused (e.g. SIGSTOP) past this window loses its
|
|
476
|
+
* lease and a rival may proceed — accepted deliberately, because the
|
|
477
|
+
* alternative (pure pid probing) lets PID reuse wedge lifecycles FOREVER.
|
|
478
|
+
*/
|
|
479
|
+
/**
|
|
480
|
+
* Lease timing knobs, exported as one mutable object ONLY so tests can run
|
|
481
|
+
* deterministic renewal scenarios with tiny windows; production code never
|
|
482
|
+
* mutates it. renewalMs sits well below leaseMs so an active lifecycle
|
|
483
|
+
* refreshes its lease several times per window.
|
|
484
|
+
*/
|
|
485
|
+
export const OWNER_LEASE_CONFIG = {
|
|
486
|
+
leaseMs: 15 * 60_000,
|
|
487
|
+
renewalMs: 5 * 60_000,
|
|
488
|
+
/** Tolerated forward clock skew. A heartbeat further in the FUTURE than
|
|
489
|
+
* this is malformed or from a skewed clock and counts as expired —
|
|
490
|
+
* otherwise a finite future value (e.g. 9e15) would make its owner live
|
|
491
|
+
* forever and unbound the lease again. */
|
|
492
|
+
skewMs: 2 * 60_000,
|
|
493
|
+
};
|
|
494
|
+
/** A journal entry lease-owner is live ONLY while its bounded lease is
|
|
495
|
+
* fresh AND (same host) its pid probes alive. The heartbeat bound is what
|
|
496
|
+
* makes PID reuse — which can make a dead owner's pid probe pass forever —
|
|
497
|
+
* and unprobeable foreign-host owners an availability delay of at most
|
|
498
|
+
* OWNER_LEASE_MS, never a permanent wedge. Within the window, an alive
|
|
499
|
+
* probe or a foreign host still fails closed. */
|
|
500
|
+
function isOwnerLive(owner) {
|
|
501
|
+
if (!owner)
|
|
502
|
+
return false;
|
|
503
|
+
const age = Date.now() - owner.heartbeatAt;
|
|
504
|
+
if (age > OWNER_LEASE_CONFIG.leaseMs)
|
|
505
|
+
return false; // lease expired
|
|
506
|
+
if (age < -OWNER_LEASE_CONFIG.skewMs)
|
|
507
|
+
return false; // future-dated beyond skew: bounded, not immortal
|
|
508
|
+
if (owner.host !== hostname())
|
|
509
|
+
return true; // cannot prove death — fail closed within the lease
|
|
510
|
+
try {
|
|
511
|
+
process.kill(owner.pid, 0);
|
|
512
|
+
return true;
|
|
513
|
+
}
|
|
514
|
+
catch (err) {
|
|
515
|
+
return err.code !== 'ESRCH';
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
function newCleanupOwner() {
|
|
519
|
+
return {
|
|
520
|
+
pid: process.pid,
|
|
521
|
+
host: hostname(),
|
|
522
|
+
attemptId: randomBytes(8).toString('hex'),
|
|
523
|
+
heartbeatAt: Date.now(),
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Periodic lease renewal for an owned journal record: an unref'd timer
|
|
528
|
+
* atomically re-stamps heartbeatAt on entries owned by exactly this
|
|
529
|
+
* attemptId, so a legitimate lifecycle awaiting external operations past
|
|
530
|
+
* OWNER_LEASE_CONFIG.leaseMs keeps a LIVE reservation (a paused event loop
|
|
531
|
+
* past the bound remains the documented tradeoff). Started only after the
|
|
532
|
+
* reservation is durably acquired; MUST be stopped before the record is
|
|
533
|
+
* cleared on every exit path. A renewal failure latches: assertHealthy()
|
|
534
|
+
* then throws so later lifecycle mutations/commits never proceed blindly on
|
|
535
|
+
* a possibly-lost lease.
|
|
536
|
+
*/
|
|
537
|
+
function startLeaseRenewal(deps, attemptId) {
|
|
538
|
+
let failed;
|
|
539
|
+
let renewing = false;
|
|
540
|
+
let stopped = false;
|
|
541
|
+
const timer = setInterval(() => {
|
|
542
|
+
if (renewing || failed || stopped)
|
|
543
|
+
return;
|
|
544
|
+
renewing = true;
|
|
545
|
+
let matched = 0;
|
|
546
|
+
deps.cleanupJournal
|
|
547
|
+
.update((entries) => entries.map((entry) => {
|
|
548
|
+
if (entry.owner?.attemptId !== attemptId)
|
|
549
|
+
return entry;
|
|
550
|
+
matched += 1;
|
|
551
|
+
return { ...entry, owner: { ...entry.owner, heartbeatAt: Date.now() } };
|
|
552
|
+
}))
|
|
553
|
+
.then(() => {
|
|
554
|
+
// A tick that was already in flight when stop() ran settles as a
|
|
555
|
+
// no-op: after settle removed the record, its zero-match would
|
|
556
|
+
// otherwise latch a FALSE lost-lease warning.
|
|
557
|
+
if (stopped)
|
|
558
|
+
return;
|
|
559
|
+
if (matched === 0) {
|
|
560
|
+
// The owned record is GONE — the lease was lost (e.g. reclaimed);
|
|
561
|
+
// latch so no further external mutation proceeds on it.
|
|
562
|
+
failed = 'reservation record no longer present';
|
|
563
|
+
deps.error('Warning: the lifecycle lease record disappeared during renewal; aborting before any further external mutation.');
|
|
564
|
+
}
|
|
565
|
+
})
|
|
566
|
+
.catch((err) => {
|
|
567
|
+
if (stopped)
|
|
568
|
+
return;
|
|
569
|
+
failed = err instanceof Error ? err.message : String(err);
|
|
570
|
+
deps.error('Warning: could not renew the lifecycle lease; aborting before any further external mutation.');
|
|
571
|
+
})
|
|
572
|
+
.finally(() => {
|
|
573
|
+
renewing = false;
|
|
574
|
+
});
|
|
575
|
+
}, OWNER_LEASE_CONFIG.renewalMs);
|
|
576
|
+
timer.unref?.();
|
|
577
|
+
return {
|
|
578
|
+
stop() {
|
|
579
|
+
// Latch BEFORE clearing so an in-flight tick's settlement observes it.
|
|
580
|
+
stopped = true;
|
|
581
|
+
clearInterval(timer);
|
|
582
|
+
},
|
|
583
|
+
assertHealthy(context) {
|
|
584
|
+
if (failed) {
|
|
585
|
+
throw new Error(`The lifecycle lease for ${context} could not be renewed (${failed}); aborting instead of proceeding on a possibly-lost reservation.`);
|
|
586
|
+
}
|
|
587
|
+
},
|
|
588
|
+
};
|
|
589
|
+
}
|
|
590
|
+
function sameGlobSet(a, b) {
|
|
591
|
+
const left = [...(a ?? [])].sort();
|
|
592
|
+
const right = [...(b ?? [])].sort();
|
|
593
|
+
return left.length === right.length && left.every((glob, i) => glob === right[i]);
|
|
594
|
+
}
|
|
595
|
+
/**
|
|
596
|
+
* Record cleanup work that already exists remotely (the mutation happened; a
|
|
597
|
+
* journal failure here can only be surfaced, not used to abort). Returns
|
|
598
|
+
* whether the entry is durably recorded. Never logs entry urls or ids — the
|
|
599
|
+
* journal file is their only home.
|
|
600
|
+
*/
|
|
601
|
+
async function tryRecordCleanup(deps, entry, context) {
|
|
602
|
+
try {
|
|
603
|
+
await deps.cleanupJournal.update((entries) => entries.some((existing) => cleanupEntryKey(existing) === cleanupEntryKey(entry))
|
|
604
|
+
? entries
|
|
605
|
+
: [...entries, entry]);
|
|
606
|
+
deps.error(`Warning: could not clean up a ${describeCleanupKind(entry.kind)} for ${context}; recorded for retry on a later run.`);
|
|
607
|
+
return true;
|
|
608
|
+
}
|
|
609
|
+
catch (journalErr) {
|
|
610
|
+
deps.error(`Warning: could not clean up a ${describeCleanupKind(entry.kind)} for ${context} AND could not record it for retry (${journalErr instanceof Error ? journalErr.message : String(journalErr)}).`);
|
|
611
|
+
return false;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
function describeCleanupKind(kind) {
|
|
615
|
+
switch (kind) {
|
|
616
|
+
case 'relayfile-webhook-subscription':
|
|
617
|
+
return 'relayfile webhook subscription';
|
|
618
|
+
case 'relay-webhook':
|
|
619
|
+
return 'relay inbound webhook';
|
|
620
|
+
case 'relay-subscription':
|
|
621
|
+
return 'relay event subscription';
|
|
622
|
+
case 'subscribe-attempt':
|
|
623
|
+
return 'subscribe attempt';
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
async function removeCleanupEntry(deps, entry) {
|
|
627
|
+
await deps.cleanupJournal.update((entries) => entries.filter((existing) => cleanupEntryKey(existing) !== cleanupEntryKey(entry)));
|
|
628
|
+
}
|
|
629
|
+
/**
|
|
630
|
+
* Retry every journal entry the current run's connections can act on. Runs on
|
|
631
|
+
* each subscribe/unsubscribe before that run's own lifecycle mutations.
|
|
632
|
+
*
|
|
633
|
+
* Guards: an entry whose lease-owner is live (or unprovable) is untouched —
|
|
634
|
+
* it belongs to an in-flight transaction; entries from other scopes are
|
|
635
|
+
* untouched; ids referenced by an active binding or the caller's keep-set are
|
|
636
|
+
* never deleted; cloud deletes are pinned to the entry's recorded workspace
|
|
637
|
+
* and a cloud 404 only converges when pinned (an unpinned 404 might be the
|
|
638
|
+
* wrong active workspace, so the entry is retained); attempt reconciliation
|
|
639
|
+
* uses the API-v3 list operation. Failures keep the entry. Never logs entry
|
|
640
|
+
* urls or ids.
|
|
641
|
+
*/
|
|
642
|
+
async function sweepPendingCleanups(deps, relay, options) {
|
|
643
|
+
let entries;
|
|
644
|
+
let bindings;
|
|
645
|
+
try {
|
|
646
|
+
[entries, bindings] = await Promise.all([deps.cleanupJournal.list(), deps.relayfile.listBindings()]);
|
|
647
|
+
}
|
|
648
|
+
catch {
|
|
649
|
+
return; // fail closed: unknown journal or binding state sweeps nothing
|
|
650
|
+
}
|
|
651
|
+
if (entries.length === 0)
|
|
652
|
+
return;
|
|
653
|
+
const activeWebhookSubscriptionIds = new Set(bindings.map((b) => b.webhookSubscriptionId).filter((id) => Boolean(id)));
|
|
654
|
+
if (options.keepWebhookSubscriptionId)
|
|
655
|
+
activeWebhookSubscriptionIds.add(options.keepWebhookSubscriptionId);
|
|
656
|
+
const activeWebhookIds = new Set(bindings.map((b) => b.webhookId).filter(Boolean));
|
|
657
|
+
if (options.keepWebhookId)
|
|
658
|
+
activeWebhookIds.add(options.keepWebhookId);
|
|
659
|
+
const activeSubscriptionIds = new Set(bindings.map((b) => b.subscriptionId).filter(Boolean));
|
|
660
|
+
const deleteCloudSubscription = async (id, workspace) => {
|
|
661
|
+
try {
|
|
662
|
+
await deps.relayfile.deleteWebhookSubscription(id, workspace);
|
|
663
|
+
return 'deleted';
|
|
664
|
+
}
|
|
665
|
+
catch (err) {
|
|
666
|
+
if (isAlreadyDeletedWebhookSubscription(err) && workspace)
|
|
667
|
+
return 'deleted';
|
|
668
|
+
if (isAlreadyDeletedWebhookSubscription(err))
|
|
669
|
+
return 'retained'; // unpinned 404: maybe wrong workspace
|
|
670
|
+
throw err;
|
|
671
|
+
}
|
|
672
|
+
};
|
|
673
|
+
for (const entry of entries) {
|
|
674
|
+
if (isOwnerLive(entry.owner))
|
|
675
|
+
continue; // an in-flight transaction's lease
|
|
676
|
+
try {
|
|
677
|
+
switch (entry.kind) {
|
|
678
|
+
case 'relayfile-webhook-subscription': {
|
|
679
|
+
if (entry.scope !== options.relayfileScope || !entry.id)
|
|
680
|
+
continue;
|
|
681
|
+
if (activeWebhookSubscriptionIds.has(entry.id))
|
|
682
|
+
continue;
|
|
683
|
+
if ((await deleteCloudSubscription(entry.id, entry.relayfileWorkspaceId)) === 'retained') {
|
|
684
|
+
continue;
|
|
685
|
+
}
|
|
686
|
+
await removeCleanupEntry(deps, entry);
|
|
687
|
+
deps.log('Retired a relayfile webhook subscription from an earlier failed cleanup.');
|
|
688
|
+
break;
|
|
689
|
+
}
|
|
690
|
+
case 'subscribe-attempt': {
|
|
691
|
+
if (entry.scope !== options.relayfileScope)
|
|
692
|
+
continue;
|
|
693
|
+
if (entry.operation === 'unsubscribe') {
|
|
694
|
+
// An unsubscribe reservation creates nothing; a dead one has
|
|
695
|
+
// nothing to recover (its failures were recorded as ownerless
|
|
696
|
+
// concrete entries) and is simply released.
|
|
697
|
+
await removeCleanupEntry(deps, entry);
|
|
698
|
+
break;
|
|
699
|
+
}
|
|
700
|
+
// CLAIM the stale attempt as this process's live recovery lease
|
|
701
|
+
// BEFORE any remote list/delete. Recovery then holds the same
|
|
702
|
+
// per-resource lease new lifecycles reserve under (prepare aborts
|
|
703
|
+
// on a live owner), so a new subscribe can never create a same-key
|
|
704
|
+
// pre-bind resource while this reconciliation is mid-flight — and
|
|
705
|
+
// if the entry vanished or was claimed meanwhile, recovery skips.
|
|
706
|
+
const recoveryOwner = newCleanupOwner();
|
|
707
|
+
let claimed = false;
|
|
708
|
+
await deps.cleanupJournal.update((fresh) => {
|
|
709
|
+
const index = fresh.findIndex((candidate) => cleanupEntryKey(candidate) === cleanupEntryKey(entry));
|
|
710
|
+
if (index < 0 || isOwnerLive(fresh[index].owner))
|
|
711
|
+
return fresh;
|
|
712
|
+
claimed = true;
|
|
713
|
+
const next = [...fresh];
|
|
714
|
+
next[index] = { ...next[index], owner: recoveryOwner };
|
|
715
|
+
return next;
|
|
716
|
+
});
|
|
717
|
+
if (!claimed)
|
|
718
|
+
break;
|
|
719
|
+
const claimedEntry = { ...entry, owner: recoveryOwner };
|
|
720
|
+
const releaseClaim = () => deps.cleanupJournal
|
|
721
|
+
.update((fresh) => fresh.map((candidate) => cleanupEntryKey(candidate) === cleanupEntryKey(claimedEntry)
|
|
722
|
+
? { ...candidate, owner: entry.owner }
|
|
723
|
+
: candidate))
|
|
724
|
+
.catch(() => undefined);
|
|
725
|
+
try {
|
|
726
|
+
let cloudResolved = false;
|
|
727
|
+
// Cloud side: a concrete id beats list-matching; both are pinned to
|
|
728
|
+
// the attempt's recorded workspace when known.
|
|
729
|
+
if (entry.webhookSubscriptionId &&
|
|
730
|
+
!activeWebhookSubscriptionIds.has(entry.webhookSubscriptionId)) {
|
|
731
|
+
cloudResolved =
|
|
732
|
+
(await deleteCloudSubscription(entry.webhookSubscriptionId, entry.relayfileWorkspaceId)) ===
|
|
733
|
+
'deleted';
|
|
734
|
+
}
|
|
735
|
+
else if (entry.webhookSubscriptionId) {
|
|
736
|
+
cloudResolved = true; // bound and active — nothing to clean
|
|
737
|
+
}
|
|
738
|
+
else if (entry.relayfileWorkspaceId) {
|
|
739
|
+
// Only a workspace-pinned attempt may list-reconcile: an unpinned
|
|
740
|
+
// list would query whatever workspace is active NOW and could
|
|
741
|
+
// falsely settle an attempt whose create landed elsewhere.
|
|
742
|
+
const listed = await deps.relayfile.listWebhookSubscriptions(entry.relayfileWorkspaceId);
|
|
743
|
+
if (listed.workspaceId !== entry.relayfileWorkspaceId) {
|
|
744
|
+
// Fail closed unless the daemon EXPLICITLY confirms it answered
|
|
745
|
+
// for the pinned workspace — a missing echo is not good enough.
|
|
746
|
+
cloudResolved = false;
|
|
747
|
+
}
|
|
748
|
+
else {
|
|
749
|
+
const orphans = listed.subscriptions.filter((subscription) => subscription.url === entry.url &&
|
|
750
|
+
sameGlobSet(subscription.pathGlobs, entry.pathGlobs) &&
|
|
751
|
+
!activeWebhookSubscriptionIds.has(subscription.subscriptionId));
|
|
752
|
+
for (const orphan of orphans) {
|
|
753
|
+
await deleteCloudSubscription(orphan.subscriptionId, entry.relayfileWorkspaceId);
|
|
754
|
+
}
|
|
755
|
+
cloudResolved = true;
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
// Relay side: recover by concrete id or deterministic key.
|
|
759
|
+
const relayResolved = Boolean(relay) && entry.relayScope === options.relayScope;
|
|
760
|
+
if (relay && entry.relayScope === options.relayScope) {
|
|
761
|
+
const webhookIds = new Set();
|
|
762
|
+
if (entry.webhookId) {
|
|
763
|
+
webhookIds.add(entry.webhookId);
|
|
764
|
+
}
|
|
765
|
+
else if (entry.webhookName) {
|
|
766
|
+
// Exact-name match only: a prefix match could hit a DIFFERENT
|
|
767
|
+
// attempt's pre-bind webhook for the same resource.
|
|
768
|
+
const hooks = await relay.webhooks.list();
|
|
769
|
+
for (const hook of hooks) {
|
|
770
|
+
if (hook.name === entry.webhookName && !activeWebhookIds.has(hook.webhookId)) {
|
|
771
|
+
webhookIds.add(hook.webhookId);
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
for (const webhookId of webhookIds) {
|
|
776
|
+
if (activeWebhookIds.has(webhookId))
|
|
777
|
+
continue;
|
|
778
|
+
try {
|
|
779
|
+
await relay.webhooks.delete(webhookId);
|
|
780
|
+
}
|
|
781
|
+
catch (err) {
|
|
782
|
+
if (!isNotFoundRelayError(err))
|
|
783
|
+
throw err;
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
if (entry.subscriptionId) {
|
|
787
|
+
if (!activeSubscriptionIds.has(entry.subscriptionId)) {
|
|
788
|
+
try {
|
|
789
|
+
await relay.webhooks.unsubscribe(entry.subscriptionId);
|
|
790
|
+
}
|
|
791
|
+
catch (err) {
|
|
792
|
+
if (!isNotFoundRelayError(err))
|
|
793
|
+
throw err;
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
else if (entry.writebackUrl) {
|
|
798
|
+
// The attempt's subscription targeted the writeback url with a
|
|
799
|
+
// per-attempt marker appended, so an exact-url match can only be
|
|
800
|
+
// THIS dead attempt's subscription — never another attempt's on
|
|
801
|
+
// the same channel. The active-id guard stays as belt.
|
|
802
|
+
const subscriptions = await relay.webhooks.subscriptions();
|
|
803
|
+
for (const subscription of subscriptions) {
|
|
804
|
+
if (subscription.url === entry.writebackUrl &&
|
|
805
|
+
!activeSubscriptionIds.has(subscription.id)) {
|
|
806
|
+
try {
|
|
807
|
+
await relay.webhooks.unsubscribe(subscription.id);
|
|
808
|
+
}
|
|
809
|
+
catch (err) {
|
|
810
|
+
if (!isNotFoundRelayError(err))
|
|
811
|
+
throw err;
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
if (cloudResolved && relayResolved) {
|
|
818
|
+
await removeCleanupEntry(deps, claimedEntry);
|
|
819
|
+
deps.log(`Reconciled resources left by an interrupted subscribe of ${entry.provider ?? 'unknown'} ${entry.resource ?? ''}.`);
|
|
820
|
+
}
|
|
821
|
+
else {
|
|
822
|
+
// Partial: hand the lease back to its (dead) owner so a later
|
|
823
|
+
// sweep — including one in this same process — can reclaim it.
|
|
824
|
+
await releaseClaim();
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
catch (err) {
|
|
828
|
+
await releaseClaim();
|
|
829
|
+
throw err;
|
|
830
|
+
}
|
|
831
|
+
break;
|
|
832
|
+
}
|
|
833
|
+
case 'relay-webhook': {
|
|
834
|
+
if (!relay || entry.scope !== options.relayScope || !entry.id)
|
|
835
|
+
continue;
|
|
836
|
+
if (activeWebhookIds.has(entry.id))
|
|
837
|
+
continue;
|
|
838
|
+
try {
|
|
839
|
+
await relay.webhooks.delete(entry.id);
|
|
840
|
+
}
|
|
841
|
+
catch (err) {
|
|
842
|
+
// 404: an earlier delete succeeded remotely but its response was
|
|
843
|
+
// lost — the entry has converged and must not be retained forever.
|
|
844
|
+
if (!isNotFoundRelayError(err))
|
|
845
|
+
throw err;
|
|
846
|
+
}
|
|
847
|
+
await removeCleanupEntry(deps, entry);
|
|
848
|
+
deps.log('Retired a relay inbound webhook from an earlier failed cleanup.');
|
|
849
|
+
break;
|
|
850
|
+
}
|
|
851
|
+
case 'relay-subscription': {
|
|
852
|
+
if (!relay || entry.scope !== options.relayScope || !entry.id)
|
|
853
|
+
continue;
|
|
854
|
+
if (activeSubscriptionIds.has(entry.id))
|
|
855
|
+
continue;
|
|
856
|
+
try {
|
|
857
|
+
await relay.webhooks.unsubscribe(entry.id);
|
|
858
|
+
}
|
|
859
|
+
catch (err) {
|
|
860
|
+
if (!isNotFoundRelayError(err))
|
|
861
|
+
throw err;
|
|
862
|
+
}
|
|
863
|
+
await removeCleanupEntry(deps, entry);
|
|
864
|
+
deps.log('Retired a relay event subscription from an earlier failed cleanup.');
|
|
865
|
+
break;
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
catch {
|
|
870
|
+
deps.error(`Warning: a recorded ${describeCleanupKind(entry.kind)} cleanup still failed; it stays recorded for the next run.`);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
419
873
|
}
|
|
420
874
|
/**
|
|
421
875
|
* The relayfile binding currently backing this (provider, pathGlob), if any.
|
|
@@ -442,12 +896,63 @@ async function findExistingBinding(deps, provider, pathGlob) {
|
|
|
442
896
|
* binding references it.
|
|
443
897
|
*/
|
|
444
898
|
async function retireSupersededWebhooks(deps, relay, options) {
|
|
445
|
-
const { provider, prefix, keepWebhookId, priorBinding } = options;
|
|
899
|
+
const { provider, resource, prefix, keepWebhookId, keepWebhookSubscriptionId, priorBinding } = options;
|
|
900
|
+
// The prior binding's ids were pre-recorded in the journal BEFORE bind()
|
|
901
|
+
// overwrote their only other persisted home (see prepareSubscribeIntent), so
|
|
902
|
+
// a failed delete here needs no recording — the entry simply stays; a
|
|
903
|
+
// successful delete clears it.
|
|
446
904
|
if (priorBinding && priorBinding.subscriptionId) {
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
905
|
+
const entry = {
|
|
906
|
+
kind: 'relay-subscription',
|
|
907
|
+
scope: options.relayScope,
|
|
908
|
+
id: priorBinding.subscriptionId,
|
|
909
|
+
};
|
|
910
|
+
try {
|
|
911
|
+
await relay.webhooks.unsubscribe(priorBinding.subscriptionId);
|
|
912
|
+
// A failed remove only leaves a lingering entry; the 404-tolerant sweep
|
|
913
|
+
// self-heals it, so it must not fail the already-successful subscribe.
|
|
914
|
+
await removeCleanupEntry(deps, entry).catch(() => undefined);
|
|
915
|
+
}
|
|
916
|
+
catch (err) {
|
|
917
|
+
if (isNotFoundRelayError(err)) {
|
|
918
|
+
await removeCleanupEntry(deps, entry).catch(() => undefined);
|
|
919
|
+
}
|
|
920
|
+
else {
|
|
921
|
+
deps.error(`Warning: could not retire the prior relay event subscription for ${provider} ${resource}; it stays recorded for a later run.`);
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
const priorWebhookSubId = priorBinding?.webhookSubscriptionId;
|
|
926
|
+
if (priorWebhookSubId && priorWebhookSubId !== keepWebhookSubscriptionId) {
|
|
927
|
+
const priorWorkspaceId = priorBinding?.webhookSubscriptionWorkspaceId;
|
|
928
|
+
const entry = {
|
|
929
|
+
kind: 'relayfile-webhook-subscription',
|
|
930
|
+
scope: options.relayfileScope,
|
|
931
|
+
id: priorWebhookSubId,
|
|
932
|
+
...(priorWorkspaceId ? { relayfileWorkspaceId: priorWorkspaceId } : {}),
|
|
933
|
+
};
|
|
934
|
+
try {
|
|
935
|
+
await deps.relayfile.deleteWebhookSubscription(priorWebhookSubId, priorWorkspaceId);
|
|
936
|
+
await removeCleanupEntry(deps, entry).catch(() => undefined);
|
|
937
|
+
}
|
|
938
|
+
catch (err) {
|
|
939
|
+
// A 404 only converges when the delete was workspace-pinned; an
|
|
940
|
+
// unpinned "not found" might just mean the daemon's active workspace
|
|
941
|
+
// changed. Either way the pre-recorded entry stays until attributable.
|
|
942
|
+
if (isAlreadyDeletedWebhookSubscription(err) && priorWorkspaceId) {
|
|
943
|
+
await removeCleanupEntry(deps, entry).catch(() => undefined);
|
|
944
|
+
}
|
|
945
|
+
else {
|
|
946
|
+
deps.error(`Warning: could not retire the prior relayfile webhook subscription for ${provider} ${resource}; it stays recorded for a later run.`);
|
|
947
|
+
}
|
|
948
|
+
}
|
|
450
949
|
}
|
|
950
|
+
await sweepPendingCleanups(deps, relay, {
|
|
951
|
+
relayScope: options.relayScope,
|
|
952
|
+
relayfileScope: options.relayfileScope,
|
|
953
|
+
keepWebhookSubscriptionId,
|
|
954
|
+
keepWebhookId,
|
|
955
|
+
});
|
|
451
956
|
let webhooks;
|
|
452
957
|
let activeWebhookIds;
|
|
453
958
|
try {
|
|
@@ -458,22 +963,104 @@ async function retireSupersededWebhooks(deps, relay, options) {
|
|
|
458
963
|
catch {
|
|
459
964
|
return; // best-effort hygiene; the new binding is already live
|
|
460
965
|
}
|
|
966
|
+
// Deletion here is strictly ATTRIBUTABLE: the prior binding's exact webhook
|
|
967
|
+
// id and the unbound legacy fixed name. Broad same-prefix orphan deletion
|
|
968
|
+
// was removed — a concurrent attempt's pre-bind webhook shares the prefix
|
|
969
|
+
// and no snapshot ordering can make deleting it provably safe, while every
|
|
970
|
+
// attributable orphan is already covered by the prior id, the journaled
|
|
971
|
+
// exact attempt names, or recorded relay-webhook entries. An unattributable
|
|
972
|
+
// ancient orphan is leaked rather than risked.
|
|
461
973
|
const legacyName = `relayfile:${provider}`;
|
|
462
974
|
for (const hook of webhooks) {
|
|
463
975
|
// Never delete a webhook an active binding references — that includes the
|
|
464
976
|
// one we just bound, and guards against a concurrent re-subscribe that
|
|
465
|
-
// upserted a newer
|
|
977
|
+
// upserted a newer binding between our bind and this sweep.
|
|
466
978
|
if (hook.webhookId === keepWebhookId || activeWebhookIds.has(hook.webhookId))
|
|
467
979
|
continue;
|
|
468
980
|
const isPrior = priorBinding?.webhookId === hook.webhookId;
|
|
469
|
-
const isSameResourceOrphan = hook.name?.startsWith(`${prefix}:`) ?? false;
|
|
470
981
|
const isUnboundLegacy = hook.name === legacyName;
|
|
471
|
-
if (!isPrior && !
|
|
982
|
+
if (!isPrior && !isUnboundLegacy)
|
|
472
983
|
continue;
|
|
984
|
+
// A failed delete here needs no journal entry: this same sweep re-discovers
|
|
985
|
+
// the webhook by name/binding on the next run. Logs stay id-free — the
|
|
986
|
+
// human-meaningful webhook NAME identifies it for operators.
|
|
473
987
|
await relay.webhooks
|
|
474
988
|
.delete(hook.webhookId)
|
|
475
|
-
.then(() => deps.log(`Retired superseded webhook ${hook.
|
|
476
|
-
.catch((
|
|
989
|
+
.then(() => deps.log(`Retired superseded webhook ${hook.name ?? 'unnamed'}.`))
|
|
990
|
+
.catch(() => deps.error(`Warning: failed to retire superseded webhook ${hook.name ?? 'unnamed'}; the next subscribe run retries it.`));
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
/**
|
|
994
|
+
* Enforce the crash-window contract before ANY external create.
|
|
995
|
+
*
|
|
996
|
+
* 1. Retry all recorded cleanup work first (every subscribe run sweeps).
|
|
997
|
+
* 2. A retained attempt with the same (scope, provider, resource, url, glob
|
|
998
|
+
* set) after the sweep is either a LIVE concurrent transaction (its
|
|
999
|
+
* lease-owner is alive — abort rather than interleave with it) or an
|
|
1000
|
+
* interrupted one that could not be reconciled because the daemon or cloud
|
|
1001
|
+
* was unreachable — abort rather than double-subscribe).
|
|
1002
|
+
* 3. Journal, in ONE atomic write, this run's owner-leased attempt record
|
|
1003
|
+
* (whose deterministic keys can recover all three creates after a crash)
|
|
1004
|
+
* AND the prior binding's non-rediscoverable ids — bind() is about to
|
|
1005
|
+
* overwrite their only other persisted home. Until then the sweep's
|
|
1006
|
+
* active-binding guard keeps the pre-recorded ids untouchable. Any journal
|
|
1007
|
+
* failure aborts the run (nothing external has been created yet).
|
|
1008
|
+
*/
|
|
1009
|
+
async function prepareSubscribeAttempt(deps, relay, attempt, channel, scopes) {
|
|
1010
|
+
// Ownership conflicts match on the durable binding identity ONLY — a
|
|
1011
|
+
// regenerated inbound-target url or glob spelling must not let two
|
|
1012
|
+
// same-resource lifecycles run concurrently. (url/globs remain the cloud
|
|
1013
|
+
// RECOVERY keys, not the ownership key.)
|
|
1014
|
+
const matchesAttemptKey = (entry) => entry.kind === 'subscribe-attempt' &&
|
|
1015
|
+
entry.scope === attempt.scope &&
|
|
1016
|
+
entry.provider === attempt.provider &&
|
|
1017
|
+
entry.resource === attempt.resource;
|
|
1018
|
+
await sweepPendingCleanups(deps, relay, scopes);
|
|
1019
|
+
// Pin the exact relayfile workspace BEFORE any cloud create: a crash after
|
|
1020
|
+
// the create reaches the server but before its response would otherwise
|
|
1021
|
+
// leave the attempt unpinned, and an unpinned list/delete after an
|
|
1022
|
+
// active-workspace switch could falsely settle it. The pin is MANDATORY on
|
|
1023
|
+
// every path — no pin, no create. writeback-secret resolution provides it
|
|
1024
|
+
// normally; with --bridge-url/--bridge-secret overrides the control plane is
|
|
1025
|
+
// still consulted for the workspace alone, then list subscriptions provides
|
|
1026
|
+
// the fallback. A run that cannot obtain a pin aborts here.
|
|
1027
|
+
if (!attempt.relayfileWorkspaceId) {
|
|
1028
|
+
// Even with explicit --bridge-url/--bridge-secret overrides, the control
|
|
1029
|
+
// plane is still consulted for the workspace identity alone.
|
|
1030
|
+
const resolved = await deps.relayfile.resolveWritebackBinding(channel).catch(() => undefined);
|
|
1031
|
+
if (resolved?.workspaceId)
|
|
1032
|
+
attempt.relayfileWorkspaceId = resolved.workspaceId;
|
|
1033
|
+
}
|
|
1034
|
+
if (!attempt.relayfileWorkspaceId) {
|
|
1035
|
+
const { workspaceId } = await deps.relayfile.listWebhookSubscriptions();
|
|
1036
|
+
if (workspaceId)
|
|
1037
|
+
attempt.relayfileWorkspaceId = workspaceId;
|
|
1038
|
+
}
|
|
1039
|
+
if (!attempt.relayfileWorkspaceId) {
|
|
1040
|
+
// No pin, no create: an unpinned cloud subscription would be unretryable
|
|
1041
|
+
// across an active-workspace switch.
|
|
1042
|
+
throw new Error(`Could not determine the active relayfile workspace before subscribing ${attempt.provider} ${attempt.resource}; upgrade the relayfile daemon (>= control-plane API v3 with workspace identity) and re-run.`);
|
|
1043
|
+
}
|
|
1044
|
+
// Check-and-reserve happens INSIDE one journal update, under its exclusive
|
|
1045
|
+
// lock — two racing prepares cannot both observe "no matching attempt" and
|
|
1046
|
+
// both append their own reservation (check-then-write TOCTOU). The prior
|
|
1047
|
+
// binding is read (and its ids pre-recorded) only AFTER this lease is held,
|
|
1048
|
+
// making the lease the linearization point for the whole lifecycle.
|
|
1049
|
+
let conflict;
|
|
1050
|
+
await deps.cleanupJournal.update((entries) => {
|
|
1051
|
+
const retained = entries.filter(matchesAttemptKey);
|
|
1052
|
+
if (retained.length > 0) {
|
|
1053
|
+
conflict = retained.some((entry) => isOwnerLive(entry.owner)) ? 'live' : 'unreconciled';
|
|
1054
|
+
return entries;
|
|
1055
|
+
}
|
|
1056
|
+
const keys = new Set(entries.map(cleanupEntryKey));
|
|
1057
|
+
return keys.has(cleanupEntryKey(attempt)) ? entries : [...entries, attempt];
|
|
1058
|
+
});
|
|
1059
|
+
if (conflict === 'live') {
|
|
1060
|
+
throw new Error(`Another subscribe of ${attempt.provider} ${attempt.resource} appears to be in progress. Wait for it to finish (or for its process to exit) and re-run.`);
|
|
1061
|
+
}
|
|
1062
|
+
if (conflict === 'unreconciled') {
|
|
1063
|
+
throw new Error(`Could not reconcile the resources left by an interrupted subscribe of ${attempt.provider} ${attempt.resource}. Aborting before creating another subscription; re-run once relayfile-cloud is reachable.`);
|
|
477
1064
|
}
|
|
478
1065
|
}
|
|
479
1066
|
async function runSubscribe(deps, providerArg, opts) {
|
|
@@ -503,7 +1090,8 @@ async function runSubscribe(deps, providerArg, opts) {
|
|
|
503
1090
|
deps.error(`Warning: ${resolved.warning}`);
|
|
504
1091
|
}
|
|
505
1092
|
const relayOptions = sdkOptionsFromOpts(opts);
|
|
506
|
-
const
|
|
1093
|
+
const effectiveRelayOptions = local && !explicitWorkspaceKey(opts) ? localRetryOptions(relayOptions, local) : relayOptions;
|
|
1094
|
+
const relay = deps.createAgentRelay(effectiveRelayOptions);
|
|
507
1095
|
await ensureRecipient(relay, provider, to, opts);
|
|
508
1096
|
const channel = targetChannel(to);
|
|
509
1097
|
const events = commaList(opts.events);
|
|
@@ -515,69 +1103,223 @@ async function runSubscribe(deps, providerArg, opts) {
|
|
|
515
1103
|
});
|
|
516
1104
|
const prefix = webhookNamePrefix(provider, pathGlob);
|
|
517
1105
|
const name = inboundWebhookName(prefix);
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
//
|
|
521
|
-
//
|
|
522
|
-
|
|
1106
|
+
const relayScope = relayCleanupScope(effectiveRelayOptions);
|
|
1107
|
+
const relayfileScope = relayfileCleanupScope();
|
|
1108
|
+
// The attempt record is this transaction's lease AND its crash-recovery
|
|
1109
|
+
// anchor: written before any create, upgraded with each server-assigned id,
|
|
1110
|
+
// removed only on settle. Its deterministic keys — the inbound (url, glob
|
|
1111
|
+
// set) for the cloud subscription, the resource-scoped webhook name prefix,
|
|
1112
|
+
// and the per-channel writeback url — let a later run reconcile whichever
|
|
1113
|
+
// creates landed if this process dies at ANY point in between.
|
|
1114
|
+
const owner = newCleanupOwner();
|
|
1115
|
+
// The relay event subscription targets the per-channel writeback url with a
|
|
1116
|
+
// NONSECRET per-attempt marker appended. relayfile-cloud's writeback route
|
|
1117
|
+
// ignores the query string and its HMAC covers headers+body only, so
|
|
1118
|
+
// delivery is unaffected — but the exact url now uniquely identifies THIS
|
|
1119
|
+
// attempt's subscription, making post-create/pre-id crash recovery
|
|
1120
|
+
// deterministic with no cross-resource deletion risk on shared channels.
|
|
1121
|
+
const subscriptionTargetUrl = `${writeback.url}${writeback.url.includes('?') ? '&' : '?'}relaySubscribeAttempt=${owner.attemptId}`;
|
|
1122
|
+
const attempt = {
|
|
1123
|
+
kind: 'subscribe-attempt',
|
|
1124
|
+
scope: relayfileScope,
|
|
1125
|
+
provider,
|
|
1126
|
+
resource: pathGlob,
|
|
1127
|
+
url: inboundTarget.url,
|
|
1128
|
+
pathGlobs: [pathGlob],
|
|
1129
|
+
webhookName: name,
|
|
1130
|
+
writebackUrl: subscriptionTargetUrl,
|
|
1131
|
+
relayScope,
|
|
1132
|
+
...(writeback.workspaceId ? { relayfileWorkspaceId: writeback.workspaceId } : {}),
|
|
1133
|
+
owner,
|
|
1134
|
+
};
|
|
1135
|
+
const upgradeAttempt = async (patch) => {
|
|
1136
|
+
Object.assign(attempt, patch);
|
|
1137
|
+
// Best-effort: if the write fails, the previous attempt state (already
|
|
1138
|
+
// durable) still recovers this create through its deterministic key.
|
|
1139
|
+
await deps.cleanupJournal
|
|
1140
|
+
.update((entries) => entries.map((entry) => entry.owner?.attemptId === attempt.owner?.attemptId && entry.kind === 'subscribe-attempt'
|
|
1141
|
+
? { ...entry, ...patch }
|
|
1142
|
+
: entry))
|
|
1143
|
+
.catch(() => deps.error('Warning: could not update the subscribe attempt record; recovery falls back to its deterministic keys.'));
|
|
1144
|
+
};
|
|
1145
|
+
await prepareSubscribeAttempt(deps, relay, attempt, channel, { relayScope, relayfileScope });
|
|
1146
|
+
// The reservation is durable — keep its lease fresh for as long as this
|
|
1147
|
+
// transaction runs, however long the external operations below take.
|
|
1148
|
+
const lease = startLeaseRenewal(deps, owner.attemptId);
|
|
1149
|
+
// With the lease held, capture the binding we're about to replace
|
|
1150
|
+
// (create-first: the per-attempt nonce in `name` means createInbound never
|
|
1151
|
+
// collides on the unique (workspace, name) index, so the replacement can
|
|
1152
|
+
// exist alongside the old one) and pre-record its non-rediscoverable ids
|
|
1153
|
+
// BEFORE any create/bind. A crash between the lease and this pre-record is
|
|
1154
|
+
// safe — nothing external has mutated; any failure here releases the lease
|
|
1155
|
+
// and aborts before mutations.
|
|
1156
|
+
let priorBinding;
|
|
1157
|
+
try {
|
|
1158
|
+
priorBinding = await findExistingBinding(deps, provider, pathGlob);
|
|
1159
|
+
const priorEntries = [];
|
|
1160
|
+
if (priorBinding?.subscriptionId) {
|
|
1161
|
+
priorEntries.push({ kind: 'relay-subscription', scope: relayScope, id: priorBinding.subscriptionId });
|
|
1162
|
+
}
|
|
1163
|
+
if (priorBinding?.webhookSubscriptionId) {
|
|
1164
|
+
priorEntries.push({
|
|
1165
|
+
kind: 'relayfile-webhook-subscription',
|
|
1166
|
+
scope: relayfileScope,
|
|
1167
|
+
id: priorBinding.webhookSubscriptionId,
|
|
1168
|
+
...(priorBinding.webhookSubscriptionWorkspaceId
|
|
1169
|
+
? { relayfileWorkspaceId: priorBinding.webhookSubscriptionWorkspaceId }
|
|
1170
|
+
: {}),
|
|
1171
|
+
});
|
|
1172
|
+
}
|
|
1173
|
+
if (priorEntries.length > 0) {
|
|
1174
|
+
await deps.cleanupJournal.update((entries) => {
|
|
1175
|
+
const keys = new Set(entries.map(cleanupEntryKey));
|
|
1176
|
+
return [...entries, ...priorEntries.filter((entry) => !keys.has(cleanupEntryKey(entry)))];
|
|
1177
|
+
});
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
catch (err) {
|
|
1181
|
+
lease.stop();
|
|
1182
|
+
await removeCleanupEntry(deps, attempt).catch(() => undefined);
|
|
1183
|
+
throw err;
|
|
1184
|
+
}
|
|
523
1185
|
let webhook;
|
|
524
1186
|
let subscription;
|
|
525
1187
|
let relayfileWebhook;
|
|
1188
|
+
let bindingInput;
|
|
526
1189
|
try {
|
|
1190
|
+
lease.assertHealthy(`${provider} ${pathGlob}`);
|
|
527
1191
|
webhook = await relay.webhooks.createInbound({ channel, name });
|
|
528
|
-
|
|
1192
|
+
await upgradeAttempt({ webhookId: webhook.webhookId });
|
|
1193
|
+
lease.assertHealthy(`${provider} ${pathGlob}`);
|
|
1194
|
+
const createdRelayfileWebhook = await deps.relayfile.createWebhookSubscription({
|
|
529
1195
|
url: inboundTarget.url,
|
|
530
1196
|
pathGlobs: [pathGlob],
|
|
531
1197
|
secret: inboundTarget.secret,
|
|
1198
|
+
// Pin the create to the journaled workspace: an active-workspace switch
|
|
1199
|
+
// between writeback-secret resolution and this POST must not land the
|
|
1200
|
+
// subscription in a workspace other than the recorded pin.
|
|
1201
|
+
...(attempt.relayfileWorkspaceId ? { workspace: attempt.relayfileWorkspaceId } : {}),
|
|
1202
|
+
});
|
|
1203
|
+
relayfileWebhook = createdRelayfileWebhook;
|
|
1204
|
+
if (attempt.relayfileWorkspaceId &&
|
|
1205
|
+
createdRelayfileWebhook.workspaceId &&
|
|
1206
|
+
createdRelayfileWebhook.workspaceId !== attempt.relayfileWorkspaceId) {
|
|
1207
|
+
// The daemon created the subscription somewhere other than the pinned
|
|
1208
|
+
// workspace. Never overwrite the durable pin silently — fail into the
|
|
1209
|
+
// rollback (which deletes/records using the CREATE's actual echo).
|
|
1210
|
+
throw new Error(`relayfile created the webhook subscription in a different workspace than the one pinned for ${provider} ${pathGlob}; aborting and rolling back.`);
|
|
1211
|
+
}
|
|
1212
|
+
await upgradeAttempt({
|
|
1213
|
+
webhookSubscriptionId: createdRelayfileWebhook.subscriptionId,
|
|
1214
|
+
...(createdRelayfileWebhook.workspaceId
|
|
1215
|
+
? { relayfileWorkspaceId: createdRelayfileWebhook.workspaceId }
|
|
1216
|
+
: {}),
|
|
532
1217
|
});
|
|
1218
|
+
lease.assertHealthy(`${provider} ${pathGlob}`);
|
|
533
1219
|
subscription = await relay.integrations.subscriptions.create({
|
|
534
1220
|
event: events.length === 1 ? events[0] : 'message.created',
|
|
535
1221
|
events: events.length ? events : ['message.created', 'thread.reply'],
|
|
536
1222
|
filter: { channel },
|
|
537
|
-
url:
|
|
1223
|
+
url: subscriptionTargetUrl,
|
|
538
1224
|
secret: writeback.secret,
|
|
539
1225
|
});
|
|
540
|
-
await
|
|
1226
|
+
await upgradeAttempt({ subscriptionId: subscription.id });
|
|
1227
|
+
lease.assertHealthy(`${provider} ${pathGlob}`);
|
|
1228
|
+
bindingInput = {
|
|
541
1229
|
provider,
|
|
542
1230
|
resource: pathGlob,
|
|
543
1231
|
channel,
|
|
544
1232
|
webhookId: webhook.webhookId,
|
|
545
1233
|
webhookToken: webhook.token,
|
|
546
1234
|
subscriptionId: subscription.id,
|
|
547
|
-
|
|
1235
|
+
webhookSubscriptionId: createdRelayfileWebhook.subscriptionId,
|
|
1236
|
+
...(attempt.relayfileWorkspaceId || createdRelayfileWebhook.workspaceId
|
|
1237
|
+
? {
|
|
1238
|
+
webhookSubscriptionWorkspaceId: attempt.relayfileWorkspaceId ?? createdRelayfileWebhook.workspaceId,
|
|
1239
|
+
}
|
|
1240
|
+
: {}),
|
|
1241
|
+
};
|
|
1242
|
+
await deps.relayfile.bind(bindingInput);
|
|
548
1243
|
}
|
|
549
1244
|
catch (err) {
|
|
550
|
-
// The new binding never fully landed: roll back only what we just created
|
|
551
|
-
//
|
|
1245
|
+
// The new binding never fully landed: roll back only what we just created,
|
|
1246
|
+
// leave any prior working binding untouched, and keep/record a durable
|
|
1247
|
+
// entry for every id whose rollback delete fails. Successful deletes
|
|
1248
|
+
// resolve; anything unresolved keeps the attempt record as its anchor.
|
|
1249
|
+
let unresolved = false;
|
|
552
1250
|
if (subscription) {
|
|
553
|
-
|
|
554
|
-
.delete(subscription.id)
|
|
555
|
-
|
|
1251
|
+
try {
|
|
1252
|
+
await relay.integrations.subscriptions.delete(subscription.id);
|
|
1253
|
+
}
|
|
1254
|
+
catch (cleanupErr) {
|
|
1255
|
+
if (!isNotFoundRelayError(cleanupErr)) {
|
|
1256
|
+
const recorded = await tryRecordCleanup(deps, { kind: 'relay-subscription', scope: relayScope, id: subscription.id }, `${provider} ${pathGlob}`);
|
|
1257
|
+
unresolved ||= !recorded;
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
556
1260
|
}
|
|
557
1261
|
if (webhook) {
|
|
558
|
-
|
|
559
|
-
.delete(webhook.webhookId)
|
|
560
|
-
|
|
1262
|
+
try {
|
|
1263
|
+
await relay.webhooks.delete(webhook.webhookId);
|
|
1264
|
+
}
|
|
1265
|
+
catch (cleanupErr) {
|
|
1266
|
+
if (!isNotFoundRelayError(cleanupErr)) {
|
|
1267
|
+
const recorded = await tryRecordCleanup(deps, { kind: 'relay-webhook', scope: relayScope, id: webhook.webhookId }, `${provider} ${pathGlob}`);
|
|
1268
|
+
unresolved ||= !recorded;
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
561
1271
|
}
|
|
562
1272
|
if (relayfileWebhook) {
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
1273
|
+
const rollbackWorkspaceId = relayfileWebhook.workspaceId ?? attempt.relayfileWorkspaceId;
|
|
1274
|
+
try {
|
|
1275
|
+
await deps.relayfile.deleteWebhookSubscription(relayfileWebhook.subscriptionId, rollbackWorkspaceId);
|
|
1276
|
+
}
|
|
1277
|
+
catch (cleanupErr) {
|
|
1278
|
+
if (!isAlreadyDeletedWebhookSubscription(cleanupErr) || !rollbackWorkspaceId) {
|
|
1279
|
+
const recorded = await tryRecordCleanup(deps, {
|
|
1280
|
+
kind: 'relayfile-webhook-subscription',
|
|
1281
|
+
scope: relayfileScope,
|
|
1282
|
+
id: relayfileWebhook.subscriptionId,
|
|
1283
|
+
...(rollbackWorkspaceId ? { relayfileWorkspaceId: rollbackWorkspaceId } : {}),
|
|
1284
|
+
}, `${provider} ${pathGlob}`);
|
|
1285
|
+
unresolved ||= !recorded;
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
lease.stop();
|
|
1290
|
+
if (!unresolved) {
|
|
1291
|
+
// Everything we created is deleted or durably recorded as an ownerless
|
|
1292
|
+
// entry — the attempt record has served its purpose. If it cannot be
|
|
1293
|
+
// removed it is only over-cautious: a later run reconciles it.
|
|
1294
|
+
await removeCleanupEntry(deps, attempt).catch(() => undefined);
|
|
566
1295
|
}
|
|
567
1296
|
throw err;
|
|
568
1297
|
}
|
|
569
|
-
// New binding is live (relayfile.bind upserts on (provider, pathGlob))
|
|
570
|
-
//
|
|
1298
|
+
// New binding is live (relayfile.bind upserts on (provider, pathGlob)); its
|
|
1299
|
+
// record now persists all three ids, so the attempt record and the
|
|
1300
|
+
// pre-recorded prior-id entries it superseded are resolved. Then retire
|
|
1301
|
+
// what the binding replaced and sweep.
|
|
1302
|
+
lease.stop();
|
|
1303
|
+
await deps.cleanupJournal
|
|
1304
|
+
.update((entries) => {
|
|
1305
|
+
const resolved = new Set([cleanupEntryKey(attempt)]);
|
|
1306
|
+
return entries.filter((entry) => !resolved.has(cleanupEntryKey(entry)));
|
|
1307
|
+
})
|
|
1308
|
+
.catch(() => deps.error('Warning: could not clear the completed subscribe attempt record; a later run will reconcile it (guarded by the active binding).'));
|
|
571
1309
|
await retireSupersededWebhooks(deps, relay, {
|
|
572
1310
|
provider,
|
|
1311
|
+
resource: pathGlob,
|
|
573
1312
|
prefix,
|
|
574
1313
|
keepWebhookId: webhook.webhookId,
|
|
1314
|
+
keepWebhookSubscriptionId: bindingInput.webhookSubscriptionId,
|
|
575
1315
|
priorBinding,
|
|
1316
|
+
relayScope,
|
|
1317
|
+
relayfileScope,
|
|
576
1318
|
});
|
|
577
1319
|
// Show the native resource the user typed, plus the resolved glob when they differ.
|
|
578
1320
|
const boundLabel = pathGlob === resource ? resource : `${resource} (${pathGlob})`;
|
|
579
1321
|
deps.log(`✓ ${provider} ${boundLabel} bound -> ${to}`);
|
|
580
|
-
deps.log(
|
|
1322
|
+
deps.log('✓ Server-side inbound webhook subscription created.');
|
|
581
1323
|
deps.log('✓ Listening. Replies will post back in-thread.');
|
|
582
1324
|
}
|
|
583
1325
|
async function runUnsubscribe(deps, provider, opts) {
|
|
@@ -589,27 +1331,124 @@ async function runUnsubscribe(deps, provider, opts) {
|
|
|
589
1331
|
// Resolve native -> glob: relayfile keys bindings on the glob, so the user's
|
|
590
1332
|
// native `--resource` (e.g. owner/repo) must be canonicalized to match.
|
|
591
1333
|
const { pathGlob } = await deps.relayfile.resolveResourcePath(provider, resource);
|
|
592
|
-
const bindings = await deps.relayfile.listBindings();
|
|
593
|
-
const binding = bindings.find((item) => item.provider === provider && item.resource === pathGlob);
|
|
594
|
-
if (!binding) {
|
|
595
|
-
throw new Error(`No binding found for ${provider} ${resource}.`);
|
|
596
|
-
}
|
|
597
1334
|
const local = await deps.resolveLocalRelayOptions();
|
|
598
1335
|
const relayOptions = sdkOptionsFromOpts(opts);
|
|
599
|
-
const
|
|
1336
|
+
const effectiveRelayOptions = local && !explicitWorkspaceKey(opts) ? localRetryOptions(relayOptions, local) : relayOptions;
|
|
1337
|
+
const relay = deps.createAgentRelay(effectiveRelayOptions);
|
|
1338
|
+
const relayScope = relayCleanupScope(effectiveRelayOptions);
|
|
1339
|
+
const relayfileScope = relayfileCleanupScope();
|
|
1340
|
+
// Retry recorded cleanup work before this run's own lifecycle mutations.
|
|
1341
|
+
await sweepPendingCleanups(deps, relay, { relayScope, relayfileScope });
|
|
1342
|
+
// Unsubscribe shares the per-(scope, provider, resource) lifecycle lease
|
|
1343
|
+
// with subscribe: without it, a stale unsubscribe could finish its deletes
|
|
1344
|
+
// AFTER a concurrent re-subscribe bound a fresh replacement and then
|
|
1345
|
+
// unbind() would discard that new binding, orphaning its resources.
|
|
1346
|
+
const reservation = {
|
|
1347
|
+
kind: 'subscribe-attempt',
|
|
1348
|
+
operation: 'unsubscribe',
|
|
1349
|
+
scope: relayfileScope,
|
|
1350
|
+
provider,
|
|
1351
|
+
resource: pathGlob,
|
|
1352
|
+
owner: newCleanupOwner(),
|
|
1353
|
+
};
|
|
1354
|
+
let lifecycleConflict = false;
|
|
1355
|
+
await deps.cleanupJournal.update((entries) => {
|
|
1356
|
+
const conflicting = entries.some((entry) => entry.kind === 'subscribe-attempt' &&
|
|
1357
|
+
entry.scope === relayfileScope &&
|
|
1358
|
+
entry.provider === provider &&
|
|
1359
|
+
entry.resource === pathGlob &&
|
|
1360
|
+
isOwnerLive(entry.owner));
|
|
1361
|
+
if (conflicting) {
|
|
1362
|
+
lifecycleConflict = true;
|
|
1363
|
+
return entries;
|
|
1364
|
+
}
|
|
1365
|
+
return [...entries, reservation];
|
|
1366
|
+
});
|
|
1367
|
+
if (lifecycleConflict) {
|
|
1368
|
+
throw new Error(`Another subscribe/unsubscribe of ${provider} ${resource} appears to be in progress. Wait for it to finish and re-run.`);
|
|
1369
|
+
}
|
|
1370
|
+
const unsubscribeLease = startLeaseRenewal(deps, reservation.owner.attemptId);
|
|
1371
|
+
const releaseReservation = () => {
|
|
1372
|
+
unsubscribeLease.stop();
|
|
1373
|
+
return removeCleanupEntry(deps, reservation).catch(() => undefined);
|
|
1374
|
+
};
|
|
1375
|
+
// The binding is read only AFTER the lease is held — the lease is the
|
|
1376
|
+
// linearization point, so these ids cannot be a stale snapshot from before
|
|
1377
|
+
// a concurrent replacement completed.
|
|
1378
|
+
let binding;
|
|
600
1379
|
try {
|
|
601
|
-
await
|
|
1380
|
+
const bindings = await deps.relayfile.listBindings();
|
|
1381
|
+
binding = bindings.find((item) => item.provider === provider && item.resource === pathGlob);
|
|
602
1382
|
}
|
|
603
1383
|
catch (err) {
|
|
604
|
-
|
|
1384
|
+
await releaseReservation();
|
|
1385
|
+
throw err;
|
|
605
1386
|
}
|
|
1387
|
+
if (!binding) {
|
|
1388
|
+
await releaseReservation();
|
|
1389
|
+
throw new Error(`No binding found for ${provider} ${resource}.`);
|
|
1390
|
+
}
|
|
1391
|
+
// The binding record about to be unbound holds the only persisted copy of
|
|
1392
|
+
// these ids. Any delete failure below must be durably journaled BEFORE
|
|
1393
|
+
// unbind, or the run aborts with the binding (and its ids) intact. The
|
|
1394
|
+
// whole post-reservation section always releases the lease — an abort that
|
|
1395
|
+
// left it live would wedge later lifecycles until this pid exits.
|
|
606
1396
|
try {
|
|
607
|
-
|
|
1397
|
+
const abortRetainingBinding = (what) => {
|
|
1398
|
+
throw new Error(`Failed to remove the ${what} for ${provider} ${resource} and could not record it for retry. The binding was left in place — re-run \`integration unsubscribe\`.`);
|
|
1399
|
+
};
|
|
1400
|
+
const webhookSubId = binding.webhookSubscriptionId;
|
|
1401
|
+
if (webhookSubId) {
|
|
1402
|
+
unsubscribeLease.assertHealthy(`${provider} ${resource}`);
|
|
1403
|
+
const bindingWorkspaceId = binding.webhookSubscriptionWorkspaceId;
|
|
1404
|
+
const entry = {
|
|
1405
|
+
kind: 'relayfile-webhook-subscription',
|
|
1406
|
+
scope: relayfileScope,
|
|
1407
|
+
id: webhookSubId,
|
|
1408
|
+
...(bindingWorkspaceId ? { relayfileWorkspaceId: bindingWorkspaceId } : {}),
|
|
1409
|
+
};
|
|
1410
|
+
try {
|
|
1411
|
+
await deps.relayfile.deleteWebhookSubscription(webhookSubId, bindingWorkspaceId);
|
|
1412
|
+
}
|
|
1413
|
+
catch (err) {
|
|
1414
|
+
// A pinned 404 proves the subscription is gone. An UNPINNED 404 might
|
|
1415
|
+
// just be the wrong active workspace — record the id (retained until a
|
|
1416
|
+
// matching-workspace delete succeeds) before discarding the binding.
|
|
1417
|
+
if (!isAlreadyDeletedWebhookSubscription(err) || !bindingWorkspaceId) {
|
|
1418
|
+
const recorded = await tryRecordCleanup(deps, entry, `${provider} ${resource}`);
|
|
1419
|
+
if (!recorded)
|
|
1420
|
+
abortRetainingBinding('relayfile webhook subscription');
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
unsubscribeLease.assertHealthy(`${provider} ${resource}`);
|
|
1425
|
+
try {
|
|
1426
|
+
await relay.webhooks.delete(binding.webhookId);
|
|
1427
|
+
}
|
|
1428
|
+
catch (err) {
|
|
1429
|
+
if (!isNotFoundRelayError(err)) {
|
|
1430
|
+
const recorded = await tryRecordCleanup(deps, { kind: 'relay-webhook', scope: relayScope, id: binding.webhookId }, `${provider} ${resource}`);
|
|
1431
|
+
if (!recorded)
|
|
1432
|
+
abortRetainingBinding('relay inbound webhook');
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
unsubscribeLease.assertHealthy(`${provider} ${resource}`);
|
|
1436
|
+
try {
|
|
1437
|
+
await relay.webhooks.unsubscribe(binding.subscriptionId);
|
|
1438
|
+
}
|
|
1439
|
+
catch (err) {
|
|
1440
|
+
if (!isNotFoundRelayError(err)) {
|
|
1441
|
+
const recorded = await tryRecordCleanup(deps, { kind: 'relay-subscription', scope: relayScope, id: binding.subscriptionId }, `${provider} ${resource}`);
|
|
1442
|
+
if (!recorded)
|
|
1443
|
+
abortRetainingBinding('relay event subscription');
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
unsubscribeLease.assertHealthy(`${provider} ${resource}`);
|
|
1447
|
+
await deps.relayfile.unbind(provider, pathGlob);
|
|
608
1448
|
}
|
|
609
|
-
|
|
610
|
-
|
|
1449
|
+
finally {
|
|
1450
|
+
await releaseReservation();
|
|
611
1451
|
}
|
|
612
|
-
await deps.relayfile.unbind(provider, pathGlob);
|
|
613
1452
|
deps.log(`Unsubscribed ${provider} ${resource}.`);
|
|
614
1453
|
}
|
|
615
1454
|
export function registerIntegrationCommands(program, overrides = {}) {
|