@songsid/agend 2.1.7-beta.3 → 2.1.7-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-endpoint.d.ts +2 -0
- package/dist/agent-endpoint.js +47 -5
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/codex.d.ts +18 -0
- package/dist/backend/codex.js +68 -3
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/types.d.ts +9 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/mcp-operation-response.d.ts +3 -0
- package/dist/channel/mcp-operation-response.js +21 -0
- package/dist/channel/mcp-operation-response.js.map +1 -0
- package/dist/channel/mcp-operation-response.test.d.ts +1 -0
- package/dist/channel/mcp-operation-response.test.js +23 -0
- package/dist/channel/mcp-operation-response.test.js.map +1 -0
- package/dist/channel/mcp-server.js +20 -5
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/daemon.d.ts +23 -0
- package/dist/daemon.js +248 -24
- package/dist/daemon.js.map +1 -1
- package/dist/delivery-outbox.d.ts +90 -0
- package/dist/delivery-outbox.js +562 -0
- package/dist/delivery-outbox.js.map +1 -0
- package/dist/delivery-outbox.test.d.ts +1 -0
- package/dist/delivery-outbox.test.js +343 -0
- package/dist/delivery-outbox.test.js.map +1 -0
- package/dist/fleet-manager.d.ts +38 -1
- package/dist/fleet-manager.js +302 -36
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +5 -0
- package/dist/instance-lifecycle.js +38 -0
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.js +2 -0
- package/dist/instructions.js.map +1 -1
- package/dist/outbound-handlers.d.ts +22 -1
- package/dist/outbound-handlers.js +81 -7
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/remote-credential-check.d.ts +39 -0
- package/dist/remote-credential-check.js +81 -0
- package/dist/remote-credential-check.js.map +1 -0
- package/dist/usage/providers.d.ts +11 -6
- package/dist/usage/providers.js +17 -10
- package/dist/usage/providers.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { EventEmitter } from "node:events";
|
|
2
|
+
export declare const DURABLE_DELIVERY_MAX_ATTEMPTS = 8;
|
|
3
|
+
export declare const DURABLE_DELIVERY_MAX_AGE_MS: number;
|
|
4
|
+
export declare const DURABLE_DELIVERY_ABORT_BACKOFF_MS = 5000;
|
|
5
|
+
export interface DaemonDeliveryPort {
|
|
6
|
+
begin(deliveryId: string, targetBootId: string, attemptNo: number): "begun" | "duplicate" | "stale";
|
|
7
|
+
abort(deliveryId: string, targetBootId: string, attemptNo: number, reason: string): boolean;
|
|
8
|
+
complete(deliveryId: string, targetBootId: string, attemptNo: number, outcome: "delivered" | "failed" | "uncertain", evidence?: string): boolean;
|
|
9
|
+
retryBeforeBegin(deliveryId: string, targetBootId: string, attemptNo: number, reason: string, delayMs?: number): boolean;
|
|
10
|
+
}
|
|
11
|
+
export type OutboxState = "queued" | "delivering" | "submission_started" | "retry_wait" | "delivered" | "failed" | "uncertain" | "cancelled";
|
|
12
|
+
export interface NewOutboxDelivery {
|
|
13
|
+
operationId: string;
|
|
14
|
+
sourceKey: string;
|
|
15
|
+
sourceInstance: string;
|
|
16
|
+
sourceDaemonBootId: string;
|
|
17
|
+
targetInstance: string;
|
|
18
|
+
targetSession?: string;
|
|
19
|
+
kind: string;
|
|
20
|
+
correlationId?: string;
|
|
21
|
+
payload: Record<string, unknown>;
|
|
22
|
+
}
|
|
23
|
+
export interface OutboxDelivery {
|
|
24
|
+
deliveryId: string;
|
|
25
|
+
operationId: string;
|
|
26
|
+
sourceKey: string;
|
|
27
|
+
sourceInstance: string;
|
|
28
|
+
sourceDaemonBootId: string;
|
|
29
|
+
targetInstance: string;
|
|
30
|
+
targetSession: string | null;
|
|
31
|
+
targetDaemonBootId: string | null;
|
|
32
|
+
kind: string;
|
|
33
|
+
correlationId: string | null;
|
|
34
|
+
payload: Record<string, unknown>;
|
|
35
|
+
state: OutboxState;
|
|
36
|
+
attemptNo: number;
|
|
37
|
+
createdSeq: number;
|
|
38
|
+
managerBootId: string | null;
|
|
39
|
+
responseDeliveredAt: string | null;
|
|
40
|
+
nextAttemptAt: string | null;
|
|
41
|
+
lastError: string | null;
|
|
42
|
+
}
|
|
43
|
+
export interface ClaimedOutboxDelivery extends OutboxDelivery {
|
|
44
|
+
targetDaemonBootId: string;
|
|
45
|
+
}
|
|
46
|
+
/** Durable admission and fenced state transitions for the fleet delivery queue. */
|
|
47
|
+
export declare class DeliveryOutbox extends EventEmitter {
|
|
48
|
+
private readonly db;
|
|
49
|
+
readonly managerBootId: string;
|
|
50
|
+
constructor(dbPath: string, managerBootId?: string);
|
|
51
|
+
/** Commit admission before returning success. Duplicate source keys return their original row. */
|
|
52
|
+
admit(input: NewOutboxDelivery): {
|
|
53
|
+
delivery: OutboxDelivery;
|
|
54
|
+
inserted: boolean;
|
|
55
|
+
};
|
|
56
|
+
get(deliveryId: string): OutboxDelivery | undefined;
|
|
57
|
+
getByOperation(sourceInstance: string, operationId: string): OutboxDelivery[];
|
|
58
|
+
listPending(): OutboxDelivery[];
|
|
59
|
+
nextRetryAt(): string | null;
|
|
60
|
+
/** Claim the earliest eligible target row; one caller owns each target lane. */
|
|
61
|
+
claimNext(managerBootId: string, targetBootIdFor: (target: string) => string | null, blockedTargets: ReadonlySet<string>): ClaimedOutboxDelivery | undefined;
|
|
62
|
+
/** Idempotent begin permit, committed immediately before the pane side effect. */
|
|
63
|
+
begin(deliveryId: string, targetBootId: string, attemptNo: number): "begun" | "duplicate" | "stale";
|
|
64
|
+
/** Safe only when the caller can prove no pane write occurred for this attempt. */
|
|
65
|
+
abort(deliveryId: string, targetBootId: string, attemptNo: number, reason: string): boolean;
|
|
66
|
+
complete(deliveryId: string, targetBootId: string, attemptNo: number, outcome: "delivered" | "failed" | "uncertain", evidence?: string): boolean;
|
|
67
|
+
/** Transient pre-submit failure: state is retryable and the same row keeps its FIFO position. */
|
|
68
|
+
retryBeforeBegin(deliveryId: string, targetBootId: string, attemptNo: number, reason: string, delayMs?: number): boolean;
|
|
69
|
+
failBeforeBegin(deliveryId: string, reason: string): boolean;
|
|
70
|
+
/** Expire unavailable/pre-submit rows instead of leaving them queued forever. */
|
|
71
|
+
expireStale(nowMs?: number, maxAgeMs?: number): number;
|
|
72
|
+
/** Next queued/retry expiry for bounded maintenance scheduling. */
|
|
73
|
+
nextExpiryAt(): string | null;
|
|
74
|
+
markResponseDelivered(sourceInstance: string, operationId: string): number;
|
|
75
|
+
/** Old process leases are recoverable; an old submission attempt needs evidence reconciliation. */
|
|
76
|
+
recoverForBoot(managerBootId: string): {
|
|
77
|
+
queued: number;
|
|
78
|
+
uncertain: number;
|
|
79
|
+
};
|
|
80
|
+
/** Atomically create a post-restart notice for an accepted operation whose response was not recorded. */
|
|
81
|
+
admitPostRestartOutcomeNotice(parent: OutboxDelivery, currentSourceBootId: string): OutboxDelivery | undefined;
|
|
82
|
+
recoverTargetGeneration(targetInstance: string, currentTargetBootId: string): {
|
|
83
|
+
queued: number;
|
|
84
|
+
uncertain: number;
|
|
85
|
+
};
|
|
86
|
+
getUnansweredAccepted(sourceInstance: string, currentBootId: string): OutboxDelivery[];
|
|
87
|
+
close(): void;
|
|
88
|
+
/** Must be called inside the same SQLite transaction as the terminal transition. */
|
|
89
|
+
private insertFailureNotice;
|
|
90
|
+
}
|
|
@@ -0,0 +1,562 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { EventEmitter } from "node:events";
|
|
4
|
+
import { chmodSync, mkdirSync } from "node:fs";
|
|
5
|
+
import { dirname } from "node:path";
|
|
6
|
+
export const DURABLE_DELIVERY_MAX_ATTEMPTS = 8;
|
|
7
|
+
export const DURABLE_DELIVERY_MAX_AGE_MS = 24 * 60 * 60_000;
|
|
8
|
+
export const DURABLE_DELIVERY_ABORT_BACKOFF_MS = 5_000;
|
|
9
|
+
function mapRow(row) {
|
|
10
|
+
return {
|
|
11
|
+
deliveryId: row.delivery_id,
|
|
12
|
+
operationId: row.operation_id,
|
|
13
|
+
sourceKey: row.source_key,
|
|
14
|
+
sourceInstance: row.source_instance,
|
|
15
|
+
sourceDaemonBootId: row.source_daemon_boot_id,
|
|
16
|
+
targetInstance: row.target_instance,
|
|
17
|
+
targetSession: row.target_session,
|
|
18
|
+
targetDaemonBootId: row.target_daemon_boot_id,
|
|
19
|
+
kind: row.kind,
|
|
20
|
+
correlationId: row.correlation_id,
|
|
21
|
+
payload: JSON.parse(row.payload_json),
|
|
22
|
+
state: row.state,
|
|
23
|
+
attemptNo: row.attempt_no,
|
|
24
|
+
createdSeq: row.created_seq,
|
|
25
|
+
managerBootId: row.manager_boot_id,
|
|
26
|
+
responseDeliveredAt: row.response_delivered_at,
|
|
27
|
+
nextAttemptAt: row.next_attempt_at,
|
|
28
|
+
lastError: row.last_error,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
/** Durable admission and fenced state transitions for the fleet delivery queue. */
|
|
32
|
+
export class DeliveryOutbox extends EventEmitter {
|
|
33
|
+
db;
|
|
34
|
+
managerBootId;
|
|
35
|
+
constructor(dbPath, managerBootId = randomUUID()) {
|
|
36
|
+
super();
|
|
37
|
+
this.managerBootId = managerBootId;
|
|
38
|
+
mkdirSync(dirname(dbPath), { recursive: true, mode: 0o700 });
|
|
39
|
+
this.db = new Database(dbPath);
|
|
40
|
+
chmodSync(dbPath, 0o600);
|
|
41
|
+
this.db.pragma("journal_mode = WAL");
|
|
42
|
+
this.db.pragma("synchronous = NORMAL");
|
|
43
|
+
this.db.pragma("busy_timeout = 1000");
|
|
44
|
+
this.db.pragma("foreign_keys = ON");
|
|
45
|
+
this.db.exec(`
|
|
46
|
+
CREATE TABLE IF NOT EXISTS deliveries (
|
|
47
|
+
delivery_id TEXT PRIMARY KEY,
|
|
48
|
+
operation_id TEXT NOT NULL,
|
|
49
|
+
source_key TEXT NOT NULL UNIQUE,
|
|
50
|
+
source_instance TEXT NOT NULL,
|
|
51
|
+
source_daemon_boot_id TEXT NOT NULL,
|
|
52
|
+
target_instance TEXT NOT NULL,
|
|
53
|
+
target_session TEXT,
|
|
54
|
+
target_daemon_boot_id TEXT,
|
|
55
|
+
kind TEXT NOT NULL,
|
|
56
|
+
correlation_id TEXT,
|
|
57
|
+
payload_json TEXT NOT NULL,
|
|
58
|
+
state TEXT NOT NULL CHECK (state IN (
|
|
59
|
+
'queued','delivering','submission_started','retry_wait',
|
|
60
|
+
'delivered','failed','uncertain','cancelled'
|
|
61
|
+
)),
|
|
62
|
+
attempt_no INTEGER NOT NULL DEFAULT 0,
|
|
63
|
+
created_seq INTEGER NOT NULL UNIQUE,
|
|
64
|
+
manager_boot_id TEXT,
|
|
65
|
+
response_delivered_at TEXT,
|
|
66
|
+
next_attempt_at TEXT,
|
|
67
|
+
last_error TEXT,
|
|
68
|
+
created_at TEXT NOT NULL,
|
|
69
|
+
updated_at TEXT NOT NULL,
|
|
70
|
+
accepted_at TEXT NOT NULL,
|
|
71
|
+
submitted_at TEXT,
|
|
72
|
+
finished_at TEXT
|
|
73
|
+
);
|
|
74
|
+
CREATE INDEX IF NOT EXISTS idx_delivery_state_seq
|
|
75
|
+
ON deliveries(state, created_seq);
|
|
76
|
+
CREATE INDEX IF NOT EXISTS idx_delivery_target_state_seq
|
|
77
|
+
ON deliveries(target_instance, state, created_seq);
|
|
78
|
+
CREATE INDEX IF NOT EXISTS idx_delivery_operation_source
|
|
79
|
+
ON deliveries(source_instance, operation_id);
|
|
80
|
+
|
|
81
|
+
CREATE TABLE IF NOT EXISTS delivery_attempts (
|
|
82
|
+
delivery_id TEXT NOT NULL REFERENCES deliveries(delivery_id),
|
|
83
|
+
target_daemon_boot_id TEXT NOT NULL,
|
|
84
|
+
attempt_no INTEGER NOT NULL,
|
|
85
|
+
state TEXT NOT NULL CHECK (state IN ('begun','aborted','delivered','failed','uncertain')),
|
|
86
|
+
begin_ack_at TEXT,
|
|
87
|
+
abort_at TEXT,
|
|
88
|
+
finished_at TEXT,
|
|
89
|
+
evidence TEXT,
|
|
90
|
+
PRIMARY KEY (delivery_id, target_daemon_boot_id, attempt_no)
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
CREATE TABLE IF NOT EXISTS outcome_notices (
|
|
94
|
+
source_instance TEXT NOT NULL,
|
|
95
|
+
source_daemon_boot_id TEXT NOT NULL,
|
|
96
|
+
operation_id TEXT NOT NULL,
|
|
97
|
+
target_instance TEXT NOT NULL,
|
|
98
|
+
notice_delivery_id TEXT NOT NULL,
|
|
99
|
+
created_at TEXT NOT NULL,
|
|
100
|
+
PRIMARY KEY (source_instance, source_daemon_boot_id, operation_id, target_instance)
|
|
101
|
+
);
|
|
102
|
+
CREATE TABLE IF NOT EXISTS failure_notices (
|
|
103
|
+
parent_delivery_id TEXT PRIMARY KEY,
|
|
104
|
+
notice_delivery_id TEXT NOT NULL UNIQUE,
|
|
105
|
+
created_at TEXT NOT NULL
|
|
106
|
+
);
|
|
107
|
+
PRAGMA user_version = 1;
|
|
108
|
+
`);
|
|
109
|
+
}
|
|
110
|
+
/** Commit admission before returning success. Duplicate source keys return their original row. */
|
|
111
|
+
admit(input) {
|
|
112
|
+
const now = new Date().toISOString();
|
|
113
|
+
const deliveryId = randomUUID();
|
|
114
|
+
const insert = this.db.prepare(`
|
|
115
|
+
INSERT OR IGNORE INTO deliveries (
|
|
116
|
+
delivery_id, operation_id, source_key, source_instance, source_daemon_boot_id,
|
|
117
|
+
target_instance, target_session, kind, correlation_id, payload_json, state,
|
|
118
|
+
attempt_no, created_seq, manager_boot_id, created_at, updated_at, accepted_at
|
|
119
|
+
) VALUES (
|
|
120
|
+
@delivery_id, @operation_id, @source_key, @source_instance, @source_daemon_boot_id,
|
|
121
|
+
@target_instance, @target_session, @kind, @correlation_id, @payload_json, 'queued',
|
|
122
|
+
0, (SELECT COALESCE(MAX(created_seq), 0) + 1 FROM deliveries), NULL,
|
|
123
|
+
@created_at, @updated_at, @accepted_at
|
|
124
|
+
)
|
|
125
|
+
`);
|
|
126
|
+
const select = this.db.prepare("SELECT * FROM deliveries WHERE source_key = ?");
|
|
127
|
+
const transaction = this.db.transaction(() => {
|
|
128
|
+
const result = insert.run({
|
|
129
|
+
delivery_id: deliveryId,
|
|
130
|
+
operation_id: input.operationId,
|
|
131
|
+
source_key: input.sourceKey,
|
|
132
|
+
source_instance: input.sourceInstance,
|
|
133
|
+
source_daemon_boot_id: input.sourceDaemonBootId,
|
|
134
|
+
target_instance: input.targetInstance,
|
|
135
|
+
target_session: input.targetSession ?? null,
|
|
136
|
+
kind: input.kind,
|
|
137
|
+
correlation_id: input.correlationId ?? null,
|
|
138
|
+
payload_json: JSON.stringify(input.payload),
|
|
139
|
+
created_at: now,
|
|
140
|
+
updated_at: now,
|
|
141
|
+
accepted_at: now,
|
|
142
|
+
});
|
|
143
|
+
const row = select.get(input.sourceKey);
|
|
144
|
+
if (!row)
|
|
145
|
+
throw new Error("outbox admission committed without a readable row");
|
|
146
|
+
return { delivery: mapRow(row), inserted: result.changes === 1 };
|
|
147
|
+
});
|
|
148
|
+
return transaction();
|
|
149
|
+
}
|
|
150
|
+
get(deliveryId) {
|
|
151
|
+
const row = this.db.prepare("SELECT * FROM deliveries WHERE delivery_id = ?").get(deliveryId);
|
|
152
|
+
return row ? mapRow(row) : undefined;
|
|
153
|
+
}
|
|
154
|
+
getByOperation(sourceInstance, operationId) {
|
|
155
|
+
const rows = this.db.prepare(`
|
|
156
|
+
SELECT * FROM deliveries WHERE source_instance = ? AND operation_id = ? ORDER BY created_seq
|
|
157
|
+
`).all(sourceInstance, operationId);
|
|
158
|
+
return rows.map(mapRow);
|
|
159
|
+
}
|
|
160
|
+
listPending() {
|
|
161
|
+
const rows = this.db.prepare(`
|
|
162
|
+
SELECT * FROM deliveries
|
|
163
|
+
WHERE state IN ('queued','delivering','submission_started','retry_wait')
|
|
164
|
+
ORDER BY created_seq
|
|
165
|
+
`).all();
|
|
166
|
+
return rows.map(mapRow);
|
|
167
|
+
}
|
|
168
|
+
nextRetryAt() {
|
|
169
|
+
const row = this.db.prepare(`
|
|
170
|
+
SELECT MIN(next_attempt_at) AS next_attempt_at FROM deliveries
|
|
171
|
+
WHERE state='retry_wait' AND next_attempt_at IS NOT NULL
|
|
172
|
+
`).get();
|
|
173
|
+
return row.next_attempt_at;
|
|
174
|
+
}
|
|
175
|
+
/** Claim the earliest eligible target row; one caller owns each target lane. */
|
|
176
|
+
claimNext(managerBootId, targetBootIdFor, blockedTargets) {
|
|
177
|
+
const now = new Date().toISOString();
|
|
178
|
+
const rows = this.db.prepare(`
|
|
179
|
+
SELECT * FROM deliveries WHERE state IN ('queued','retry_wait')
|
|
180
|
+
ORDER BY created_seq
|
|
181
|
+
`).all();
|
|
182
|
+
// The oldest pending row owns its target lane even while it is in retry
|
|
183
|
+
// backoff or the target is temporarily unavailable. Later rows must not
|
|
184
|
+
// pass it merely because they happen to be eligible first.
|
|
185
|
+
const visitedTargets = new Set();
|
|
186
|
+
for (const row of rows) {
|
|
187
|
+
if (visitedTargets.has(row.target_instance))
|
|
188
|
+
continue;
|
|
189
|
+
visitedTargets.add(row.target_instance);
|
|
190
|
+
if (blockedTargets.has(row.target_instance))
|
|
191
|
+
continue;
|
|
192
|
+
if (row.state === "retry_wait" && row.next_attempt_at && row.next_attempt_at > now)
|
|
193
|
+
continue;
|
|
194
|
+
const targetBootId = targetBootIdFor(row.target_instance);
|
|
195
|
+
if (!targetBootId)
|
|
196
|
+
continue;
|
|
197
|
+
const claimedAt = new Date().toISOString();
|
|
198
|
+
const result = this.db.prepare(`
|
|
199
|
+
UPDATE deliveries
|
|
200
|
+
SET state='delivering', target_daemon_boot_id=?, manager_boot_id=?,
|
|
201
|
+
attempt_no=attempt_no+1, updated_at=?, last_error=NULL,next_attempt_at=NULL
|
|
202
|
+
WHERE delivery_id=? AND state IN ('queued','retry_wait')
|
|
203
|
+
`).run(targetBootId, managerBootId, claimedAt, row.delivery_id);
|
|
204
|
+
if (result.changes !== 1)
|
|
205
|
+
continue;
|
|
206
|
+
const claimed = this.get(row.delivery_id);
|
|
207
|
+
if (claimed)
|
|
208
|
+
return { ...claimed, targetDaemonBootId: targetBootId };
|
|
209
|
+
}
|
|
210
|
+
return undefined;
|
|
211
|
+
}
|
|
212
|
+
/** Idempotent begin permit, committed immediately before the pane side effect. */
|
|
213
|
+
begin(deliveryId, targetBootId, attemptNo) {
|
|
214
|
+
const now = new Date().toISOString();
|
|
215
|
+
const transaction = this.db.transaction(() => {
|
|
216
|
+
const row = this.db.prepare("SELECT * FROM deliveries WHERE delivery_id=?").get(deliveryId);
|
|
217
|
+
if (!row || row.target_daemon_boot_id !== targetBootId || row.attempt_no !== attemptNo) {
|
|
218
|
+
return { outcome: "stale", failed: false };
|
|
219
|
+
}
|
|
220
|
+
const existing = this.db.prepare(`
|
|
221
|
+
SELECT state FROM delivery_attempts
|
|
222
|
+
WHERE delivery_id=? AND target_daemon_boot_id=? AND attempt_no=?
|
|
223
|
+
`).get(deliveryId, targetBootId, attemptNo);
|
|
224
|
+
if (existing)
|
|
225
|
+
return { outcome: existing.state === "begun" ? "duplicate" : "stale", failed: false };
|
|
226
|
+
if (row.state !== "delivering")
|
|
227
|
+
return { outcome: "stale", failed: false };
|
|
228
|
+
const started = this.db.prepare(`
|
|
229
|
+
SELECT COUNT(*) AS count FROM delivery_attempts WHERE delivery_id=?
|
|
230
|
+
`).get(deliveryId);
|
|
231
|
+
if (started.count >= DURABLE_DELIVERY_MAX_ATTEMPTS) {
|
|
232
|
+
const update = this.db.prepare(`
|
|
233
|
+
UPDATE deliveries SET state='failed',updated_at=?,finished_at=?,last_error='delivery submission attempt limit reached'
|
|
234
|
+
WHERE delivery_id=? AND target_daemon_boot_id=? AND attempt_no=? AND state='delivering'
|
|
235
|
+
`).run(now, now, deliveryId, targetBootId, attemptNo);
|
|
236
|
+
if (update.changes !== 1)
|
|
237
|
+
return { outcome: "stale", failed: false };
|
|
238
|
+
this.insertFailureNotice(row, "failed", "delivery submission attempt limit reached", now);
|
|
239
|
+
return { outcome: "stale", failed: true };
|
|
240
|
+
}
|
|
241
|
+
this.db.prepare(`
|
|
242
|
+
INSERT INTO delivery_attempts(delivery_id,target_daemon_boot_id,attempt_no,state,begin_ack_at)
|
|
243
|
+
VALUES (?,?,?,'begun',?)
|
|
244
|
+
`).run(deliveryId, targetBootId, attemptNo, now);
|
|
245
|
+
this.db.prepare(`
|
|
246
|
+
UPDATE deliveries SET state='submission_started', submitted_at=?, updated_at=?
|
|
247
|
+
WHERE delivery_id=? AND state='delivering'
|
|
248
|
+
`).run(now, now, deliveryId);
|
|
249
|
+
return { outcome: "begun", failed: false };
|
|
250
|
+
});
|
|
251
|
+
const result = transaction();
|
|
252
|
+
if (result.outcome === "begun")
|
|
253
|
+
this.emit("state", { deliveryId, state: "submission_started" });
|
|
254
|
+
else if (result.failed)
|
|
255
|
+
this.emit("state", { deliveryId, state: "failed" });
|
|
256
|
+
return result.outcome;
|
|
257
|
+
}
|
|
258
|
+
/** Safe only when the caller can prove no pane write occurred for this attempt. */
|
|
259
|
+
abort(deliveryId, targetBootId, attemptNo, reason) {
|
|
260
|
+
const now = new Date().toISOString();
|
|
261
|
+
const transaction = this.db.transaction(() => {
|
|
262
|
+
const attempt = this.db.prepare(`
|
|
263
|
+
SELECT state FROM delivery_attempts
|
|
264
|
+
WHERE delivery_id=? AND target_daemon_boot_id=? AND attempt_no=?
|
|
265
|
+
`).get(deliveryId, targetBootId, attemptNo);
|
|
266
|
+
if (attempt?.state === "aborted")
|
|
267
|
+
return true;
|
|
268
|
+
if (attempt?.state !== "begun")
|
|
269
|
+
return false;
|
|
270
|
+
const delivery = this.db.prepare(`
|
|
271
|
+
SELECT state,target_daemon_boot_id,attempt_no FROM deliveries WHERE delivery_id=?
|
|
272
|
+
`).get(deliveryId);
|
|
273
|
+
if (delivery?.state !== "submission_started"
|
|
274
|
+
|| delivery.target_daemon_boot_id !== targetBootId
|
|
275
|
+
|| delivery.attempt_no !== attemptNo)
|
|
276
|
+
return false;
|
|
277
|
+
const updateAttempt = this.db.prepare(`
|
|
278
|
+
UPDATE delivery_attempts SET state='aborted',abort_at=?,evidence=?
|
|
279
|
+
WHERE delivery_id=? AND target_daemon_boot_id=? AND attempt_no=? AND state='begun'
|
|
280
|
+
`).run(now, reason.slice(0, 300), deliveryId, targetBootId, attemptNo);
|
|
281
|
+
if (updateAttempt.changes !== 1)
|
|
282
|
+
return false;
|
|
283
|
+
const updateDelivery = this.db.prepare(`
|
|
284
|
+
UPDATE deliveries SET state='retry_wait',updated_at=?,last_error=?,next_attempt_at=?
|
|
285
|
+
WHERE delivery_id=? AND state='submission_started' AND target_daemon_boot_id=? AND attempt_no=?
|
|
286
|
+
`).run(now, reason.slice(0, 300), new Date(Date.now() + DURABLE_DELIVERY_ABORT_BACKOFF_MS).toISOString(), deliveryId, targetBootId, attemptNo);
|
|
287
|
+
if (updateDelivery.changes !== 1)
|
|
288
|
+
throw new Error("outbox abort lost its delivery fence");
|
|
289
|
+
return true;
|
|
290
|
+
});
|
|
291
|
+
const changed = transaction();
|
|
292
|
+
if (changed)
|
|
293
|
+
this.emit("state", { deliveryId, state: "retry_wait" });
|
|
294
|
+
return changed;
|
|
295
|
+
}
|
|
296
|
+
complete(deliveryId, targetBootId, attemptNo, outcome, evidence) {
|
|
297
|
+
const now = new Date().toISOString();
|
|
298
|
+
const transaction = this.db.transaction(() => {
|
|
299
|
+
const parent = this.db.prepare("SELECT * FROM deliveries WHERE delivery_id=?").get(deliveryId);
|
|
300
|
+
const result = this.db.prepare(`
|
|
301
|
+
UPDATE deliveries SET state=?,updated_at=?,finished_at=?,last_error=?
|
|
302
|
+
WHERE delivery_id=? AND target_daemon_boot_id=? AND attempt_no=? AND state='submission_started'
|
|
303
|
+
`).run(outcome, now, now, outcome === "delivered" ? null : (evidence ?? outcome).slice(0, 300), deliveryId, targetBootId, attemptNo);
|
|
304
|
+
if (result.changes !== 1) {
|
|
305
|
+
const existing = this.db.prepare("SELECT state FROM deliveries WHERE delivery_id=?").get(deliveryId);
|
|
306
|
+
return { changed: false, accepted: existing?.state === outcome };
|
|
307
|
+
}
|
|
308
|
+
this.db.prepare(`
|
|
309
|
+
UPDATE delivery_attempts SET state=?,finished_at=?,evidence=?
|
|
310
|
+
WHERE delivery_id=? AND target_daemon_boot_id=? AND attempt_no=? AND state='begun'
|
|
311
|
+
`).run(outcome, now, evidence?.slice(0, 300) ?? null, deliveryId, targetBootId, attemptNo);
|
|
312
|
+
if (parent && (outcome === "failed" || outcome === "uncertain")) {
|
|
313
|
+
this.insertFailureNotice(parent, outcome, evidence ?? outcome, now);
|
|
314
|
+
}
|
|
315
|
+
return { changed: true, accepted: true };
|
|
316
|
+
});
|
|
317
|
+
const result = transaction();
|
|
318
|
+
if (result.changed)
|
|
319
|
+
this.emit("state", { deliveryId, state: outcome });
|
|
320
|
+
return result.accepted;
|
|
321
|
+
}
|
|
322
|
+
/** Transient pre-submit failure: state is retryable and the same row keeps its FIFO position. */
|
|
323
|
+
retryBeforeBegin(deliveryId, targetBootId, attemptNo, reason, delayMs = 30_000) {
|
|
324
|
+
const now = new Date().toISOString();
|
|
325
|
+
const next = new Date(Date.now() + Math.max(0, delayMs)).toISOString();
|
|
326
|
+
const transaction = this.db.transaction(() => {
|
|
327
|
+
const row = this.db.prepare("SELECT * FROM deliveries WHERE delivery_id=?").get(deliveryId);
|
|
328
|
+
if (!row || row.target_daemon_boot_id !== targetBootId || row.attempt_no !== attemptNo || row.state !== "delivering") {
|
|
329
|
+
return undefined;
|
|
330
|
+
}
|
|
331
|
+
const ageMs = Date.now() - Date.parse(row.created_at);
|
|
332
|
+
// Readiness and idle deferrals occur before begin and do not consume the
|
|
333
|
+
// submission-attempt budget. The age limit remains the visible bound for
|
|
334
|
+
// work that never reaches a pane-side-effect attempt.
|
|
335
|
+
const exhausted = ageMs >= DURABLE_DELIVERY_MAX_AGE_MS;
|
|
336
|
+
const outcome = exhausted ? "failed" : "retry_wait";
|
|
337
|
+
const update = this.db.prepare(`
|
|
338
|
+
UPDATE deliveries SET state=?,updated_at=?,finished_at=?,last_error=?,next_attempt_at=?
|
|
339
|
+
WHERE delivery_id=? AND target_daemon_boot_id=? AND attempt_no=? AND state='delivering'
|
|
340
|
+
`).run(outcome, now, exhausted ? now : null, reason.slice(0, 300), exhausted ? null : next, deliveryId, targetBootId, attemptNo);
|
|
341
|
+
if (update.changes !== 1)
|
|
342
|
+
return undefined;
|
|
343
|
+
if (exhausted)
|
|
344
|
+
this.insertFailureNotice(row, "failed", `${reason}; delivery age limit reached`, now);
|
|
345
|
+
return outcome;
|
|
346
|
+
});
|
|
347
|
+
const outcome = transaction();
|
|
348
|
+
if (outcome)
|
|
349
|
+
this.emit("state", { deliveryId, state: outcome });
|
|
350
|
+
return outcome !== undefined;
|
|
351
|
+
}
|
|
352
|
+
failBeforeBegin(deliveryId, reason) {
|
|
353
|
+
const now = new Date().toISOString();
|
|
354
|
+
const transaction = this.db.transaction(() => {
|
|
355
|
+
const row = this.db.prepare("SELECT * FROM deliveries WHERE delivery_id=?").get(deliveryId);
|
|
356
|
+
if (!row)
|
|
357
|
+
return false;
|
|
358
|
+
const changed = this.db.prepare(`
|
|
359
|
+
UPDATE deliveries SET state='failed',updated_at=?,finished_at=?,last_error=?,next_attempt_at=NULL
|
|
360
|
+
WHERE delivery_id=? AND state IN ('queued','delivering','retry_wait')
|
|
361
|
+
`).run(now, now, reason.slice(0, 300), deliveryId).changes === 1;
|
|
362
|
+
if (changed)
|
|
363
|
+
this.insertFailureNotice(row, "failed", reason, now);
|
|
364
|
+
return changed;
|
|
365
|
+
});
|
|
366
|
+
const changed = transaction();
|
|
367
|
+
if (changed)
|
|
368
|
+
this.emit("state", { deliveryId, state: "failed" });
|
|
369
|
+
return changed;
|
|
370
|
+
}
|
|
371
|
+
/** Expire unavailable/pre-submit rows instead of leaving them queued forever. */
|
|
372
|
+
expireStale(nowMs = Date.now(), maxAgeMs = DURABLE_DELIVERY_MAX_AGE_MS) {
|
|
373
|
+
const cutoff = new Date(nowMs - maxAgeMs).toISOString();
|
|
374
|
+
const now = new Date(nowMs).toISOString();
|
|
375
|
+
const transaction = this.db.transaction(() => {
|
|
376
|
+
const stale = this.db.prepare(`
|
|
377
|
+
SELECT * FROM deliveries
|
|
378
|
+
WHERE state IN ('queued','retry_wait') AND created_at<=?
|
|
379
|
+
ORDER BY created_seq
|
|
380
|
+
`).all(cutoff);
|
|
381
|
+
const update = this.db.prepare(`
|
|
382
|
+
UPDATE deliveries SET state='failed',updated_at=?,finished_at=?,last_error='delivery TTL expired before target became available',next_attempt_at=NULL
|
|
383
|
+
WHERE delivery_id=? AND state IN ('queued','retry_wait')
|
|
384
|
+
`);
|
|
385
|
+
let changed = 0;
|
|
386
|
+
for (const row of stale) {
|
|
387
|
+
if (update.run(now, now, row.delivery_id).changes !== 1)
|
|
388
|
+
continue;
|
|
389
|
+
this.insertFailureNotice(row, "failed", "delivery TTL expired before target became available", now);
|
|
390
|
+
changed++;
|
|
391
|
+
}
|
|
392
|
+
return changed;
|
|
393
|
+
});
|
|
394
|
+
const changed = transaction();
|
|
395
|
+
if (changed)
|
|
396
|
+
this.emit("expired", { count: changed });
|
|
397
|
+
return changed;
|
|
398
|
+
}
|
|
399
|
+
/** Next queued/retry expiry for bounded maintenance scheduling. */
|
|
400
|
+
nextExpiryAt() {
|
|
401
|
+
const row = this.db.prepare(`
|
|
402
|
+
SELECT MIN(created_at) AS created_at FROM deliveries WHERE state IN ('queued','retry_wait')
|
|
403
|
+
`).get();
|
|
404
|
+
return row.created_at ? new Date(Date.parse(row.created_at) + DURABLE_DELIVERY_MAX_AGE_MS).toISOString() : null;
|
|
405
|
+
}
|
|
406
|
+
markResponseDelivered(sourceInstance, operationId) {
|
|
407
|
+
const now = new Date().toISOString();
|
|
408
|
+
return this.db.prepare(`
|
|
409
|
+
UPDATE deliveries SET response_delivered_at=?,updated_at=?
|
|
410
|
+
WHERE source_instance=? AND operation_id=? AND response_delivered_at IS NULL
|
|
411
|
+
`).run(now, now, sourceInstance, operationId).changes;
|
|
412
|
+
}
|
|
413
|
+
/** Old process leases are recoverable; an old submission attempt needs evidence reconciliation. */
|
|
414
|
+
recoverForBoot(managerBootId) {
|
|
415
|
+
const now = new Date().toISOString();
|
|
416
|
+
const transaction = this.db.transaction(() => {
|
|
417
|
+
const uncertainRows = this.db.prepare(`
|
|
418
|
+
SELECT * FROM deliveries WHERE manager_boot_id IS NOT NULL AND manager_boot_id<>? AND state='submission_started'
|
|
419
|
+
`).all(managerBootId);
|
|
420
|
+
const queued = this.db.prepare(`
|
|
421
|
+
UPDATE deliveries SET state='queued',manager_boot_id=NULL,target_daemon_boot_id=NULL,next_attempt_at=NULL,updated_at=?
|
|
422
|
+
WHERE manager_boot_id IS NOT NULL AND manager_boot_id<>? AND state IN ('delivering','retry_wait')
|
|
423
|
+
`).run(now, managerBootId).changes;
|
|
424
|
+
// Until transcript reconciliation is enabled, never blindly replay a row
|
|
425
|
+
// whose pane side effect may already have happened.
|
|
426
|
+
const uncertain = this.db.prepare(`
|
|
427
|
+
UPDATE deliveries SET state='uncertain',updated_at=?,finished_at=?,last_error='process restarted during submission; reconciliation required'
|
|
428
|
+
WHERE manager_boot_id IS NOT NULL AND manager_boot_id<>? AND state='submission_started'
|
|
429
|
+
`).run(now, now, managerBootId).changes;
|
|
430
|
+
for (const row of uncertainRows) {
|
|
431
|
+
this.insertFailureNotice(row, "uncertain", "process restarted during submission; reconciliation required", now);
|
|
432
|
+
}
|
|
433
|
+
return { queued, uncertain };
|
|
434
|
+
});
|
|
435
|
+
return transaction();
|
|
436
|
+
}
|
|
437
|
+
/** Atomically create a post-restart notice for an accepted operation whose response was not recorded. */
|
|
438
|
+
admitPostRestartOutcomeNotice(parent, currentSourceBootId) {
|
|
439
|
+
const sourceKey = `post-restart:${parent.sourceInstance}:${parent.sourceDaemonBootId}:${parent.operationId}:${parent.targetInstance}`;
|
|
440
|
+
const operationId = `notice:${parent.sourceDaemonBootId}:${parent.operationId}:${parent.targetInstance}`;
|
|
441
|
+
const now = new Date().toISOString();
|
|
442
|
+
const deliveryId = randomUUID();
|
|
443
|
+
const noticeText = `[system:delivery-outcome] A previous tool operation was durably accepted before restart. operation_id=${parent.operationId}; target=${parent.targetInstance}; current_state=${parent.state}. Do not resend this operation. Ask the operator to inspect its delivery status.`;
|
|
444
|
+
const payload = {
|
|
445
|
+
type: "fleet_inbound",
|
|
446
|
+
content: noticeText,
|
|
447
|
+
meta: {
|
|
448
|
+
user: "AgEnD recovery",
|
|
449
|
+
user_id: "agend-system",
|
|
450
|
+
message_id: `delivery-notice-${parent.operationId}-${parent.targetInstance}`,
|
|
451
|
+
chat_id: "",
|
|
452
|
+
thread_id: "",
|
|
453
|
+
source: "delivery-outbox",
|
|
454
|
+
},
|
|
455
|
+
};
|
|
456
|
+
const transaction = this.db.transaction(() => {
|
|
457
|
+
const prior = this.db.prepare(`
|
|
458
|
+
SELECT d.* FROM outcome_notices n JOIN deliveries d ON d.delivery_id=n.notice_delivery_id
|
|
459
|
+
WHERE n.source_instance=? AND n.source_daemon_boot_id=? AND n.operation_id=? AND n.target_instance=?
|
|
460
|
+
`).get(parent.sourceInstance, parent.sourceDaemonBootId, parent.operationId, parent.targetInstance);
|
|
461
|
+
if (prior)
|
|
462
|
+
return mapRow(prior);
|
|
463
|
+
const terminalNotice = this.db.prepare(`
|
|
464
|
+
SELECT d.* FROM failure_notices n JOIN deliveries d ON d.delivery_id=n.notice_delivery_id
|
|
465
|
+
WHERE n.parent_delivery_id=?
|
|
466
|
+
`).get(parent.deliveryId);
|
|
467
|
+
if (terminalNotice)
|
|
468
|
+
return mapRow(terminalNotice);
|
|
469
|
+
const reserved = this.db.prepare(`
|
|
470
|
+
INSERT OR IGNORE INTO outcome_notices
|
|
471
|
+
(source_instance,source_daemon_boot_id,operation_id,target_instance,notice_delivery_id,created_at)
|
|
472
|
+
VALUES (?,?,?,?,?,?)
|
|
473
|
+
`).run(parent.sourceInstance, parent.sourceDaemonBootId, parent.operationId, parent.targetInstance, deliveryId, now);
|
|
474
|
+
if (reserved.changes !== 1)
|
|
475
|
+
return undefined;
|
|
476
|
+
this.db.prepare(`
|
|
477
|
+
INSERT INTO deliveries (
|
|
478
|
+
delivery_id,operation_id,source_key,source_instance,source_daemon_boot_id,
|
|
479
|
+
target_instance,target_session,kind,correlation_id,payload_json,state,
|
|
480
|
+
attempt_no,created_seq,manager_boot_id,created_at,updated_at,accepted_at
|
|
481
|
+
) VALUES (?,?,?,?,?,?,NULL,'post_restart_outcome_notice',NULL,?,'queued',0,
|
|
482
|
+
(SELECT COALESCE(MAX(created_seq),0)+1 FROM deliveries),NULL,?,?,?)
|
|
483
|
+
`).run(deliveryId, operationId, sourceKey, "agend-system", currentSourceBootId, parent.sourceInstance, JSON.stringify(payload), now, now, now);
|
|
484
|
+
return mapRow(this.db.prepare("SELECT * FROM deliveries WHERE delivery_id=?").get(deliveryId));
|
|
485
|
+
});
|
|
486
|
+
const notice = transaction();
|
|
487
|
+
if (notice)
|
|
488
|
+
this.emit("admitted", notice);
|
|
489
|
+
return notice;
|
|
490
|
+
}
|
|
491
|
+
recoverTargetGeneration(targetInstance, currentTargetBootId) {
|
|
492
|
+
const now = new Date().toISOString();
|
|
493
|
+
const transaction = this.db.transaction(() => {
|
|
494
|
+
const uncertainRows = this.db.prepare(`
|
|
495
|
+
SELECT * FROM deliveries WHERE target_instance=? AND target_daemon_boot_id IS NOT NULL
|
|
496
|
+
AND target_daemon_boot_id<>? AND state='submission_started'
|
|
497
|
+
`).all(targetInstance, currentTargetBootId);
|
|
498
|
+
const queued = this.db.prepare(`
|
|
499
|
+
UPDATE deliveries SET state='queued',manager_boot_id=NULL,target_daemon_boot_id=NULL,next_attempt_at=NULL,updated_at=?
|
|
500
|
+
WHERE target_instance=? AND target_daemon_boot_id IS NOT NULL AND target_daemon_boot_id<>?
|
|
501
|
+
AND state IN ('delivering','retry_wait')
|
|
502
|
+
`).run(now, targetInstance, currentTargetBootId).changes;
|
|
503
|
+
const uncertain = this.db.prepare(`
|
|
504
|
+
UPDATE deliveries SET state='uncertain',updated_at=?,finished_at=?,last_error='target daemon generation changed during submission; reconciliation required'
|
|
505
|
+
WHERE target_instance=? AND target_daemon_boot_id IS NOT NULL AND target_daemon_boot_id<>?
|
|
506
|
+
AND state='submission_started'
|
|
507
|
+
`).run(now, now, targetInstance, currentTargetBootId).changes;
|
|
508
|
+
for (const row of uncertainRows) {
|
|
509
|
+
this.insertFailureNotice(row, "uncertain", "target daemon generation changed during submission; reconciliation required", now);
|
|
510
|
+
}
|
|
511
|
+
return { queued, uncertain };
|
|
512
|
+
});
|
|
513
|
+
const result = transaction();
|
|
514
|
+
if (result.queued + result.uncertain > 0)
|
|
515
|
+
this.emit("generation_recovered", { targetInstance, ...result });
|
|
516
|
+
return result;
|
|
517
|
+
}
|
|
518
|
+
getUnansweredAccepted(sourceInstance, currentBootId) {
|
|
519
|
+
const rows = this.db.prepare(`
|
|
520
|
+
SELECT * FROM deliveries
|
|
521
|
+
WHERE source_instance=? AND source_daemon_boot_id<>? AND response_delivered_at IS NULL
|
|
522
|
+
ORDER BY created_seq
|
|
523
|
+
`).all(sourceInstance, currentBootId);
|
|
524
|
+
return rows.map(mapRow);
|
|
525
|
+
}
|
|
526
|
+
close() {
|
|
527
|
+
this.db.close();
|
|
528
|
+
}
|
|
529
|
+
/** Must be called inside the same SQLite transaction as the terminal transition. */
|
|
530
|
+
insertFailureNotice(parent, outcome, reason, now) {
|
|
531
|
+
if (parent.kind === "delivery_outcome_notice" || parent.kind === "post_restart_outcome_notice")
|
|
532
|
+
return;
|
|
533
|
+
const noticeId = randomUUID();
|
|
534
|
+
const operationId = `notice:${parent.operation_id}:${parent.delivery_id}`;
|
|
535
|
+
const payload = {
|
|
536
|
+
type: "fleet_inbound",
|
|
537
|
+
content: `[system:delivery-outcome] Cross-instance operation outcome: operation_id=${parent.operation_id}; delivery_id=${parent.delivery_id}; target=${parent.target_instance}; state=${outcome}; detail=${reason.slice(0, 200)}. ${outcome === "uncertain" ? "Do not resend until an operator reconciles this delivery." : "The delivery failed after bounded retries; report this outcome to the operator."}`,
|
|
538
|
+
meta: {
|
|
539
|
+
user: "AgEnD delivery outbox",
|
|
540
|
+
user_id: "agend-system",
|
|
541
|
+
message_id: `delivery-outcome-${parent.delivery_id}`,
|
|
542
|
+
chat_id: "",
|
|
543
|
+
thread_id: "",
|
|
544
|
+
source: "delivery-outbox",
|
|
545
|
+
},
|
|
546
|
+
};
|
|
547
|
+
const reserved = this.db.prepare(`
|
|
548
|
+
INSERT OR IGNORE INTO failure_notices(parent_delivery_id,notice_delivery_id,created_at) VALUES (?,?,?)
|
|
549
|
+
`).run(parent.delivery_id, noticeId, now);
|
|
550
|
+
if (reserved.changes !== 1)
|
|
551
|
+
return;
|
|
552
|
+
this.db.prepare(`
|
|
553
|
+
INSERT INTO deliveries (
|
|
554
|
+
delivery_id,operation_id,source_key,source_instance,source_daemon_boot_id,
|
|
555
|
+
target_instance,target_session,target_daemon_boot_id,kind,correlation_id,payload_json,state,
|
|
556
|
+
attempt_no,created_seq,manager_boot_id,created_at,updated_at,accepted_at
|
|
557
|
+
) VALUES (?,?,?,?,?,?,NULL,NULL,'delivery_outcome_notice',?,?,'queued',0,
|
|
558
|
+
(SELECT COALESCE(MAX(created_seq),0)+1 FROM deliveries),NULL,?,?,?)
|
|
559
|
+
`).run(noticeId, operationId, `failure-notice:${parent.delivery_id}`, "agend-system", this.managerBootId, parent.source_instance, parent.correlation_id, JSON.stringify(payload), now, now, now);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
//# sourceMappingURL=delivery-outbox.js.map
|