@agentpulselabs/cli-plugin-pulse 1.1.2 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +3 -2
- package/src/commands/pulse/cutover.js +34 -10
- package/src/daemon/main.js +1 -0
- package/src/hub/core/db.mjs +21 -5
- package/src/hub/workers/manager.mjs +44 -6
- package/src/lib/contract.sha256 +1 -1
- package/src/lib/cutover.js +51 -16
- package/src/lib/cutoverRequest.js +485 -0
- package/src/lib/cutoverWiring.js +6 -3
- package/src/lib/daemonProcess.js +43 -6
- package/src/lib/daemonRuntime.js +25 -7
- package/src/lib/hubDbReader.js +6 -0
- package/src/lib/hubScan.js +1 -0
- package/src/lib/ledger.js +21 -12
- package/src/lib/lifecycle.js +3 -1
- package/src/lib/runnerStore.js +61 -49
- package/src/lib/serviceManager.js +3 -1
- package/src/runner/bridge.js +8 -3
- package/src/runner/capture.js +83 -20
- package/src/runner/channelDiscipline.js +48 -14
- package/src/runner/channelWalk.js +42 -0
- package/src/runner/deliveryStates.js +22 -9
- package/src/runner/directoryCache.js +34 -0
- package/src/runner/facade.js +71 -14
- package/src/runner/hubCapture.js +105 -56
- package/src/runner/loopMonitor.js +89 -0
- package/src/runner/runnerHub.js +3 -1
- package/src/runner/runnerProcess.js +58 -16
- package/src/runner/runnerService.js +29 -4
- package/src/runner/serial.js +5 -0
- package/src/runner/spool.js +46 -11
package/src/lib/daemonRuntime.js
CHANGED
|
@@ -152,6 +152,7 @@ export class DaemonRuntime {
|
|
|
152
152
|
attemptNo: attempt.attemptNo,
|
|
153
153
|
fenceToken: attempt.fenceToken,
|
|
154
154
|
events,
|
|
155
|
+
itemRefusals: true,
|
|
155
156
|
});
|
|
156
157
|
} catch (error) {
|
|
157
158
|
if (machineWideError(error)) throw error;
|
|
@@ -164,15 +165,32 @@ export class DaemonRuntime {
|
|
|
164
165
|
this.log('events_discarded', { ...trace, discarded });
|
|
165
166
|
continue;
|
|
166
167
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
168
|
+
await this.#settleUpload(attempt, events, response);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Apply the cloud's answer to one uploaded batch. A stored event comes back with its `cursor` (`cloudCursor` is read
|
|
174
|
+
* when that is the only one) and leaves the outbox. A refused one ({ refused }) is dropped on its own, so the rest of
|
|
175
|
+
* the batch is kept. An event the answer leaves out stays pending and is sent again.
|
|
176
|
+
*/
|
|
177
|
+
async #settleUpload(attempt, events, response) {
|
|
178
|
+
if (!Array.isArray(response?.events)) return;
|
|
179
|
+
const sent = new Set(events.map((event) => event.sourceEventId));
|
|
180
|
+
const accepted = [];
|
|
181
|
+
const refused = new Map();
|
|
182
|
+
for (const item of response.events) {
|
|
183
|
+
if (!sent.has(item?.sourceEventId)) continue;
|
|
184
|
+
if (item.refused) refused.set(item.sourceEventId, item.refused.code || 'HUB_EVENT_REFUSED');
|
|
185
|
+
else if (Number.isSafeInteger(item.cursor ?? item.cloudCursor)) {
|
|
186
|
+
accepted.push({ sourceEventId: item.sourceEventId, cursor: item.cursor ?? item.cloudCursor });
|
|
174
187
|
}
|
|
175
188
|
}
|
|
189
|
+
if (accepted.length) await this.ledger.acknowledgeEvents(accepted);
|
|
190
|
+
if (!refused.size) return;
|
|
191
|
+
const codes = [...new Set(refused.values())].sort().join(',');
|
|
192
|
+
const { discarded } = await this.ledger.discardEvents(attempt.jobId, codes, { sourceEventIds: [...refused.keys()] });
|
|
193
|
+
this.log('events_refused', { jobId: attempt.jobId, type: attempt.jobType, attemptNo: attempt.attemptNo, code: codes, discarded });
|
|
176
194
|
}
|
|
177
195
|
|
|
178
196
|
#compose(grant) {
|
package/src/lib/hubDbReader.js
CHANGED
|
@@ -129,4 +129,10 @@ export class HubSnapshot {
|
|
|
129
129
|
const statement = this.db.prepare('SELECT kind, key, payload, updated_at FROM facts WHERE kind = ? ORDER BY key');
|
|
130
130
|
return kinds.flatMap((kind) => statement.all(kind).map(plain));
|
|
131
131
|
}
|
|
132
|
+
|
|
133
|
+
/** The `{ kind, key }` of the same facts in the same order, without their payloads. */
|
|
134
|
+
factKeys(kinds) {
|
|
135
|
+
const statement = this.db.prepare('SELECT kind, key FROM facts WHERE kind = ? ORDER BY key');
|
|
136
|
+
return kinds.flatMap((kind) => statement.all(kind).map(plain));
|
|
137
|
+
}
|
|
132
138
|
}
|
package/src/lib/hubScan.js
CHANGED
|
@@ -17,6 +17,7 @@ export const SKIPPED_FACT_KINDS = Object.freeze({
|
|
|
17
17
|
run: 'machine_runtime_state',
|
|
18
18
|
wake: 'machine_runtime_state',
|
|
19
19
|
manager: 'machine_runtime_state',
|
|
20
|
+
'manager-restart': 'machine_runtime_state',
|
|
20
21
|
totals: 'phase1_deviation_totals',
|
|
21
22
|
});
|
|
22
23
|
export const REMOVABLE_KINDS = Object.freeze(['delivery', 'stream', 'agent-row', 'request', 'status_override']);
|
package/src/lib/ledger.js
CHANGED
|
@@ -226,6 +226,7 @@ export class DurableLedger {
|
|
|
226
226
|
return clone(this.data.outbox.slice(0, Math.max(0, Math.min(100, limit))));
|
|
227
227
|
}
|
|
228
228
|
|
|
229
|
+
/** Events the cloud stored, as { sourceEventId, cursor } with the cloud cursor it gave each, the wire's names. */
|
|
229
230
|
async acknowledgeEvents(acknowledgements) {
|
|
230
231
|
return this.#mutate((state) => {
|
|
231
232
|
if (!Array.isArray(acknowledgements) || acknowledgements.length > 100) {
|
|
@@ -237,15 +238,15 @@ export class DurableLedger {
|
|
|
237
238
|
if (
|
|
238
239
|
!pendingIds.has(item.sourceEventId) ||
|
|
239
240
|
seen.has(item.sourceEventId) ||
|
|
240
|
-
!Number.isSafeInteger(item.
|
|
241
|
-
item.
|
|
241
|
+
!Number.isSafeInteger(item.cursor) ||
|
|
242
|
+
item.cursor < 1
|
|
242
243
|
) {
|
|
243
244
|
throw new HubError('HUB_EVENT_ACK_INVALID', 'Event acknowledgement does not match the sent batch');
|
|
244
245
|
}
|
|
245
246
|
seen.add(item.sourceEventId);
|
|
246
247
|
}
|
|
247
248
|
const accepted = new Map(
|
|
248
|
-
acknowledgements.map((item) => [item.sourceEventId, Number(item.
|
|
249
|
+
acknowledgements.map((item) => [item.sourceEventId, Number(item.cursor)])
|
|
249
250
|
);
|
|
250
251
|
for (const event of state.outbox) {
|
|
251
252
|
if (!accepted.has(event.sourceEventId) || !TERMINAL_STATES.has(event.terminalState)) continue;
|
|
@@ -465,19 +466,27 @@ export class DurableLedger {
|
|
|
465
466
|
}
|
|
466
467
|
|
|
467
468
|
/**
|
|
468
|
-
* Drop a job's pending events the cloud refused for good (its fence is gone or the batch is invalid)
|
|
469
|
-
*
|
|
469
|
+
* Drop a job's pending events the cloud refused for good: all of them (its fence is gone or the batch is invalid),
|
|
470
|
+
* or only sourceEventIds (the cloud refused those items and stored the rest). They never count toward
|
|
471
|
+
* acceptedLocalCursor. A terminal attempt has nothing left to upload once its terminal event is gone.
|
|
470
472
|
*/
|
|
471
|
-
async discardEvents(jobId, code) {
|
|
473
|
+
async discardEvents(jobId, code, { sourceEventIds = null } = {}) {
|
|
474
|
+
const only = sourceEventIds ? new Set(sourceEventIds) : null;
|
|
472
475
|
return this.#mutate((state) => {
|
|
473
|
-
const
|
|
474
|
-
|
|
476
|
+
const dropped = (event) => event.jobId === jobId && (!only || only.has(event.sourceEventId));
|
|
477
|
+
const gone = state.outbox.filter(dropped);
|
|
478
|
+
state.outbox = state.outbox.filter((event) => !dropped(event));
|
|
475
479
|
const attempt = state.attempts[jobId];
|
|
476
|
-
if (attempt &&
|
|
477
|
-
attempt.eventsDiscarded = {
|
|
478
|
-
|
|
480
|
+
if (attempt && gone.length) {
|
|
481
|
+
attempt.eventsDiscarded = {
|
|
482
|
+
count: (only ? attempt.eventsDiscarded?.count || 0 : 0) + gone.length,
|
|
483
|
+
code,
|
|
484
|
+
at: new Date().toISOString(),
|
|
485
|
+
};
|
|
486
|
+
const terminalGone = !only || gone.some((event) => event.terminalState === attempt.state);
|
|
487
|
+
if (TERMINAL_STATES.has(attempt.state) && terminalGone) attempt.terminalUploaded = true;
|
|
479
488
|
}
|
|
480
|
-
return { discarded };
|
|
489
|
+
return { discarded: gone.length };
|
|
481
490
|
});
|
|
482
491
|
}
|
|
483
492
|
|
package/src/lib/lifecycle.js
CHANGED
|
@@ -3,10 +3,12 @@ import { HubError } from './errors.js';
|
|
|
3
3
|
import { validateJobResult } from './intents.js';
|
|
4
4
|
import { redactText } from './redaction.js';
|
|
5
5
|
|
|
6
|
+
// Hex, so the id always starts with a letter or digit: Pulse clouds before 1.1.3 refuse a sourceEventId starting with
|
|
7
|
+
// '-' or '_', which a base64url digest does about 3% of the time.
|
|
6
8
|
function eventId(tuple, type) {
|
|
7
9
|
return createHash('sha256')
|
|
8
10
|
.update(`${tuple.jobId}\0${tuple.attemptNo}\0${tuple.fenceToken}\0${type}`)
|
|
9
|
-
.digest('
|
|
11
|
+
.digest('hex');
|
|
10
12
|
}
|
|
11
13
|
|
|
12
14
|
export class AttemptCoordinator {
|
package/src/lib/runnerStore.js
CHANGED
|
@@ -91,6 +91,8 @@ export class RunnerStore {
|
|
|
91
91
|
}
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
+
#statements = new Map();
|
|
95
|
+
|
|
94
96
|
constructor({ db, file, now, maxPending }) {
|
|
95
97
|
this.db = db;
|
|
96
98
|
this.file = file;
|
|
@@ -103,20 +105,30 @@ export class RunnerStore {
|
|
|
103
105
|
this.db.close();
|
|
104
106
|
}
|
|
105
107
|
|
|
108
|
+
/** The statement for this SQL, prepared once per connection: the scanners call the store once or more per row. */
|
|
109
|
+
sql(text) {
|
|
110
|
+
let statement = this.#statements.get(text);
|
|
111
|
+
if (!statement) {
|
|
112
|
+
statement = this.db.prepare(text);
|
|
113
|
+
this.#statements.set(text, statement);
|
|
114
|
+
}
|
|
115
|
+
return statement;
|
|
116
|
+
}
|
|
117
|
+
|
|
106
118
|
/**
|
|
107
119
|
* Run `fn` in one immediate transaction: everything it enqueues and every cursor it moves commit together. Inside a
|
|
108
120
|
* transaction already open on this connection (the runner's capture) it joins that one.
|
|
109
121
|
*/
|
|
110
122
|
transaction(fn) {
|
|
111
123
|
if (this.depth > 0 || this.db.isTransaction) return fn();
|
|
112
|
-
this.
|
|
124
|
+
this.sql('BEGIN IMMEDIATE').run();
|
|
113
125
|
this.depth += 1;
|
|
114
126
|
try {
|
|
115
127
|
const out = fn();
|
|
116
|
-
this.
|
|
128
|
+
this.sql('COMMIT').run();
|
|
117
129
|
return out;
|
|
118
130
|
} catch (error) {
|
|
119
|
-
this.
|
|
131
|
+
this.sql('ROLLBACK').run();
|
|
120
132
|
throw error;
|
|
121
133
|
} finally {
|
|
122
134
|
this.depth -= 1;
|
|
@@ -124,34 +136,34 @@ export class RunnerStore {
|
|
|
124
136
|
}
|
|
125
137
|
|
|
126
138
|
meta(key) {
|
|
127
|
-
const row = this.
|
|
139
|
+
const row = this.sql('SELECT value FROM meta WHERE key = ?').get(key);
|
|
128
140
|
return row ? JSON.parse(row.value) : null;
|
|
129
141
|
}
|
|
130
142
|
|
|
131
143
|
setMeta(key, value) {
|
|
132
|
-
this.
|
|
144
|
+
this.sql('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value')
|
|
133
145
|
.run(key, JSON.stringify(value));
|
|
134
146
|
}
|
|
135
147
|
|
|
136
148
|
cursor(source) {
|
|
137
|
-
const row = this.
|
|
149
|
+
const row = this.sql('SELECT position FROM cursor WHERE source = ?').get(source);
|
|
138
150
|
return row ? JSON.parse(row.position) : null;
|
|
139
151
|
}
|
|
140
152
|
|
|
141
153
|
setCursor(source, position) {
|
|
142
|
-
this.
|
|
154
|
+
this.sql('INSERT INTO cursor (source, position) VALUES (?, ?) ON CONFLICT (source) DO UPDATE SET position = excluded.position')
|
|
143
155
|
.run(source, JSON.stringify(position));
|
|
144
156
|
}
|
|
145
157
|
|
|
146
158
|
/** Whether a key was ever enqueued: pending, acknowledged or dead. */
|
|
147
159
|
known(key) {
|
|
148
|
-
return Boolean(this.
|
|
160
|
+
return Boolean(this.sql(
|
|
149
161
|
'SELECT 1 AS found FROM outbox WHERE key = ? UNION ALL SELECT 1 FROM sent WHERE key = ? UNION ALL SELECT 1 FROM dead WHERE key = ? LIMIT 1',
|
|
150
162
|
).get(key, key, key));
|
|
151
163
|
}
|
|
152
164
|
|
|
153
165
|
pendingCount() {
|
|
154
|
-
return Number(this.
|
|
166
|
+
return Number(this.sql('SELECT COUNT(*) AS n FROM outbox').get().n);
|
|
155
167
|
}
|
|
156
168
|
|
|
157
169
|
/**
|
|
@@ -160,7 +172,7 @@ export class RunnerStore {
|
|
|
160
172
|
*/
|
|
161
173
|
enqueue(entries) {
|
|
162
174
|
return this.transaction(() => {
|
|
163
|
-
const insert = this.
|
|
175
|
+
const insert = this.sql(
|
|
164
176
|
`INSERT INTO outbox (key, grp, type, payload, source, enqueued_at)
|
|
165
177
|
SELECT ?, ?, ?, ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM sent WHERE key = ?) AND NOT EXISTS (SELECT 1 FROM dead WHERE key = ?)
|
|
166
178
|
ON CONFLICT (key) DO NOTHING`,
|
|
@@ -214,40 +226,40 @@ export class RunnerStore {
|
|
|
214
226
|
/** The cloud stored (or already had) this key. */
|
|
215
227
|
acked(key, { seq = null, duplicate = false } = {}) {
|
|
216
228
|
this.transaction(() => {
|
|
217
|
-
const row = this.
|
|
229
|
+
const row = this.sql('SELECT source, type FROM outbox WHERE key = ?').get(key);
|
|
218
230
|
if (!row) return;
|
|
219
|
-
this.
|
|
231
|
+
this.sql('INSERT INTO sent (key, source, type, seq, duplicate, at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (key) DO NOTHING')
|
|
220
232
|
.run(key, row.source, row.type, seq === null ? null : String(seq), duplicate ? 1 : 0, this.now());
|
|
221
|
-
this.
|
|
233
|
+
this.sql('DELETE FROM outbox WHERE key = ?').run(key);
|
|
222
234
|
});
|
|
223
235
|
}
|
|
224
236
|
|
|
225
237
|
/** A retryable refusal: the row stays first in its group and is due again after `delayMs`. */
|
|
226
238
|
retry(key, { code, delayMs }) {
|
|
227
|
-
this.
|
|
239
|
+
this.sql('UPDATE outbox SET attempts = attempts + 1, next_at = ?, last_error = ? WHERE key = ?').run(this.now() + delayMs, code, key);
|
|
228
240
|
}
|
|
229
241
|
|
|
230
242
|
/** Held behind an earlier command of its group: due again after `delayMs`, without counting an attempt. */
|
|
231
243
|
hold(key, { code, delayMs }) {
|
|
232
|
-
this.
|
|
244
|
+
this.sql('UPDATE outbox SET next_at = ?, last_error = ? WHERE key = ?').run(this.now() + delayMs, code, key);
|
|
233
245
|
}
|
|
234
246
|
|
|
235
247
|
/** A refusal that repeating cannot fix: the row leaves the outbox with its reason. */
|
|
236
248
|
dead(key, { code, message = null }) {
|
|
237
249
|
this.transaction(() => {
|
|
238
|
-
const row = this.
|
|
250
|
+
const row = this.sql('SELECT grp, type, payload, source FROM outbox WHERE key = ?').get(key);
|
|
239
251
|
if (!row) return;
|
|
240
|
-
this.
|
|
252
|
+
this.sql(
|
|
241
253
|
`INSERT INTO dead (key, grp, type, payload, source, code, message, at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
242
254
|
ON CONFLICT (key) DO UPDATE SET code = excluded.code, message = excluded.message, at = excluded.at`,
|
|
243
255
|
).run(key, row.grp, row.type, row.payload, row.source, code, message, this.now());
|
|
244
|
-
this.
|
|
256
|
+
this.sql('DELETE FROM outbox WHERE key = ?').run(key);
|
|
245
257
|
});
|
|
246
258
|
}
|
|
247
259
|
|
|
248
260
|
/** A command refused before it was queued (it never reaches the cloud), recorded with its reason. */
|
|
249
261
|
refuse(entry, { code, message = null }) {
|
|
250
|
-
this.
|
|
262
|
+
this.sql(
|
|
251
263
|
`INSERT INTO dead (key, grp, type, payload, source, code, message, at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
252
264
|
ON CONFLICT (key) DO NOTHING`,
|
|
253
265
|
).run(entry.key, entry.group, entry.type, JSON.stringify(entry.payload ?? null), entry.source, code, message, this.now());
|
|
@@ -255,38 +267,38 @@ export class RunnerStore {
|
|
|
255
267
|
|
|
256
268
|
/** The code a key was refused or dead-lettered with, or null. */
|
|
257
269
|
deadCode(key) {
|
|
258
|
-
return this.
|
|
270
|
+
return this.sql('SELECT code FROM dead WHERE key = ?').get(key)?.code ?? null;
|
|
259
271
|
}
|
|
260
272
|
|
|
261
273
|
/** Every queued command in queue order, payload parsed (a reconcile reads a scratch store's whole scan this way). */
|
|
262
274
|
planned() {
|
|
263
|
-
return this.
|
|
275
|
+
return this.sql('SELECT key, grp, type, payload, source FROM outbox ORDER BY ord').all()
|
|
264
276
|
.map((row) => ({ key: row.key, group: row.grp, type: row.type, payload: JSON.parse(row.payload), source: row.source }));
|
|
265
277
|
}
|
|
266
278
|
|
|
267
279
|
/** Every set-aside key with its code (all of them; deadRows is the newest few). */
|
|
268
280
|
deadKeys() {
|
|
269
|
-
return this.
|
|
281
|
+
return this.sql('SELECT key, type, source, code FROM dead ORDER BY key').all().map((row) => ({ ...row }));
|
|
270
282
|
}
|
|
271
283
|
|
|
272
284
|
deadRows({ limit = 20 } = {}) {
|
|
273
|
-
return this.
|
|
285
|
+
return this.sql('SELECT key, type, source, code, message, at FROM dead ORDER BY at DESC LIMIT ?').all(limit).map((row) => ({ ...row }));
|
|
274
286
|
}
|
|
275
287
|
|
|
276
288
|
/** Counts per source: pending, sent (written or duplicate) and dead (by code). */
|
|
277
289
|
counts() {
|
|
278
290
|
const out = {};
|
|
279
291
|
const at = (source) => (out[source] ||= { pending: 0, written: 0, duplicate: 0, dead: {} });
|
|
280
|
-
for (const row of this.
|
|
281
|
-
for (const row of this.
|
|
292
|
+
for (const row of this.sql('SELECT source, COUNT(*) AS n FROM outbox GROUP BY source').all()) at(row.source).pending = Number(row.n);
|
|
293
|
+
for (const row of this.sql('SELECT source, duplicate, COUNT(*) AS n FROM sent GROUP BY source, duplicate').all()) {
|
|
282
294
|
at(row.source)[row.duplicate ? 'duplicate' : 'written'] = Number(row.n);
|
|
283
295
|
}
|
|
284
|
-
for (const row of this.
|
|
296
|
+
for (const row of this.sql('SELECT source, code, COUNT(*) AS n FROM dead GROUP BY source, code').all()) at(row.source).dead[row.code] = Number(row.n);
|
|
285
297
|
return out;
|
|
286
298
|
}
|
|
287
299
|
|
|
288
300
|
version(entity) {
|
|
289
|
-
const row = this.
|
|
301
|
+
const row = this.sql('SELECT sha, raw_sha, static_sha, key, at FROM version WHERE entity = ?').get(entity);
|
|
290
302
|
return row ? { sha: row.sha, rawSha: row.raw_sha, staticSha: row.static_sha, key: row.key, at: Number(row.at) } : null;
|
|
291
303
|
}
|
|
292
304
|
|
|
@@ -295,7 +307,7 @@ export class RunnerStore {
|
|
|
295
307
|
* `<prefix><sep><sha>.<k>` when it returns after another version (A→B→A), so the return is a new event that applies.
|
|
296
308
|
*/
|
|
297
309
|
versionKey(entity, prefix, sha, separator = ':') {
|
|
298
|
-
const seen = this.
|
|
310
|
+
const seen = this.sql('SELECT count FROM version_seen WHERE entity = ? AND sha = ?').get(entity, sha);
|
|
299
311
|
const count = seen ? Number(seen.count) : 0;
|
|
300
312
|
return count === 0 ? `${prefix}${separator}${sha}` : `${prefix}${separator}${sha}.${count}`;
|
|
301
313
|
}
|
|
@@ -304,12 +316,12 @@ export class RunnerStore {
|
|
|
304
316
|
setVersion(entity, { sha, rawSha = null, staticSha = null, key }) {
|
|
305
317
|
const last = this.version(entity);
|
|
306
318
|
if (last?.sha !== sha) {
|
|
307
|
-
this.
|
|
319
|
+
this.sql(
|
|
308
320
|
`INSERT INTO version_seen (entity, sha, count) VALUES (?, ?, 1)
|
|
309
321
|
ON CONFLICT (entity, sha) DO UPDATE SET count = count + 1`,
|
|
310
322
|
).run(entity, sha);
|
|
311
323
|
}
|
|
312
|
-
this.
|
|
324
|
+
this.sql(
|
|
313
325
|
`INSERT INTO version (entity, sha, raw_sha, static_sha, key, at) VALUES (?, ?, ?, ?, ?, ?)
|
|
314
326
|
ON CONFLICT (entity) DO UPDATE SET sha = excluded.sha, raw_sha = excluded.raw_sha, static_sha = excluded.static_sha,
|
|
315
327
|
key = excluded.key, at = excluded.at`,
|
|
@@ -318,67 +330,67 @@ export class RunnerStore {
|
|
|
318
330
|
|
|
319
331
|
/** The raw form last read, without changing what was sent (an unchanged row skips redaction next time). */
|
|
320
332
|
touchVersion(entity, rawSha) {
|
|
321
|
-
this.
|
|
333
|
+
this.sql('UPDATE version SET raw_sha = ? WHERE entity = ?').run(rawSha, entity);
|
|
322
334
|
}
|
|
323
335
|
|
|
324
336
|
dropVersion(entity) {
|
|
325
|
-
this.
|
|
337
|
+
this.sql('DELETE FROM version WHERE entity = ?').run(entity);
|
|
326
338
|
}
|
|
327
339
|
|
|
328
340
|
entities(prefix) {
|
|
329
|
-
return this.
|
|
341
|
+
return this.sql("SELECT entity FROM version WHERE entity >= ? AND entity < ? || char(1114111)").all(prefix, prefix).map((row) => row.entity);
|
|
330
342
|
}
|
|
331
343
|
|
|
332
344
|
/** The receipt state last queued for each recipient row still able to change (not acked or expired). */
|
|
333
345
|
openReceipts() {
|
|
334
|
-
return this.
|
|
346
|
+
return this.sql("SELECT message_id, recipient, state FROM receipt_state WHERE state NOT IN ('acked', 'expired')").all()
|
|
335
347
|
.map((row) => ({ messageId: row.message_id, recipient: row.recipient, state: row.state }));
|
|
336
348
|
}
|
|
337
349
|
|
|
338
350
|
receiptState(messageId, recipient) {
|
|
339
|
-
return this.
|
|
351
|
+
return this.sql('SELECT state FROM receipt_state WHERE message_id = ? AND recipient = ?').get(messageId, recipient)?.state ?? null;
|
|
340
352
|
}
|
|
341
353
|
|
|
342
354
|
setReceiptState(messageId, recipient, state) {
|
|
343
|
-
this.
|
|
355
|
+
this.sql(
|
|
344
356
|
`INSERT INTO receipt_state (message_id, recipient, state) VALUES (?, ?, ?)
|
|
345
357
|
ON CONFLICT (message_id, recipient) DO UPDATE SET state = excluded.state`,
|
|
346
358
|
).run(messageId, recipient, state);
|
|
347
359
|
}
|
|
348
360
|
|
|
349
361
|
dropReceipts(messageId) {
|
|
350
|
-
this.
|
|
362
|
+
this.sql('DELETE FROM receipt_state WHERE message_id = ?').run(messageId);
|
|
351
363
|
}
|
|
352
364
|
|
|
353
365
|
rememberMessage(id, createdAt, group) {
|
|
354
|
-
this.
|
|
366
|
+
this.sql('INSERT INTO recent_message (id, created_at, grp) VALUES (?, ?, ?) ON CONFLICT (id) DO NOTHING').run(id, createdAt, group);
|
|
355
367
|
}
|
|
356
368
|
|
|
357
369
|
recentMessages(sinceMs) {
|
|
358
|
-
return this.
|
|
370
|
+
return this.sql('SELECT id, created_at, grp FROM recent_message WHERE created_at >= ? ORDER BY created_at').all(sinceMs)
|
|
359
371
|
.map((row) => ({ id: row.id, createdAt: Number(row.created_at), group: row.grp }));
|
|
360
372
|
}
|
|
361
373
|
|
|
362
374
|
forgetMessage(id) {
|
|
363
|
-
this.
|
|
375
|
+
this.sql('DELETE FROM recent_message WHERE id = ?').run(id);
|
|
364
376
|
}
|
|
365
377
|
|
|
366
378
|
pruneRecentMessages(beforeMs) {
|
|
367
|
-
this.
|
|
379
|
+
this.sql('DELETE FROM recent_message WHERE created_at < ?').run(beforeMs);
|
|
368
380
|
}
|
|
369
381
|
|
|
370
382
|
ingested(channel) {
|
|
371
|
-
return this.
|
|
383
|
+
return this.sql('SELECT ordinal, sha, post_key, at FROM ingested WHERE channel = ? ORDER BY ordinal').all(channel)
|
|
372
384
|
.map((row) => ({ ordinal: Number(row.ordinal), sha: row.sha, postKey: row.post_key, at: Number(row.at) }));
|
|
373
385
|
}
|
|
374
386
|
|
|
375
387
|
/** The key a channel's block (its digest and occurrence) was first sent under, or null. */
|
|
376
388
|
ingestedKey(channel, sha) {
|
|
377
|
-
return this.
|
|
389
|
+
return this.sql('SELECT post_key FROM ingested WHERE channel = ? AND sha = ?').get(channel, sha)?.post_key ?? null;
|
|
378
390
|
}
|
|
379
391
|
|
|
380
392
|
setIngested(channel, { ordinal, sha, postKey, at }) {
|
|
381
|
-
this.
|
|
393
|
+
this.sql(
|
|
382
394
|
`INSERT INTO ingested (channel, ordinal, sha, post_key, at) VALUES (?, ?, ?, ?, ?)
|
|
383
395
|
ON CONFLICT (channel, ordinal) DO UPDATE SET sha = excluded.sha, post_key = excluded.post_key, at = excluded.at`,
|
|
384
396
|
).run(channel, ordinal, sha, postKey, at);
|
|
@@ -387,18 +399,18 @@ export class RunnerStore {
|
|
|
387
399
|
/** What one channel file holds now, by position, replacing what was recorded before. */
|
|
388
400
|
replaceIngested(channel, rows) {
|
|
389
401
|
this.transaction(() => {
|
|
390
|
-
this.
|
|
402
|
+
this.sql('DELETE FROM ingested WHERE channel = ?').run(channel);
|
|
391
403
|
for (const row of rows) this.setIngested(channel, row);
|
|
392
404
|
});
|
|
393
405
|
}
|
|
394
406
|
|
|
395
407
|
presenceSent(actorId) {
|
|
396
|
-
const row = this.
|
|
408
|
+
const row = this.sql('SELECT last_seen, sent_at FROM presence_sent WHERE actor_id = ?').get(actorId);
|
|
397
409
|
return row ? { lastSeen: Number(row.last_seen), sentAt: Number(row.sent_at) } : null;
|
|
398
410
|
}
|
|
399
411
|
|
|
400
412
|
setPresenceSent(actorId, lastSeen) {
|
|
401
|
-
this.
|
|
413
|
+
this.sql(
|
|
402
414
|
`INSERT INTO presence_sent (actor_id, last_seen, sent_at) VALUES (?, ?, ?)
|
|
403
415
|
ON CONFLICT (actor_id) DO UPDATE SET last_seen = excluded.last_seen, sent_at = excluded.sent_at`,
|
|
404
416
|
).run(actorId, lastSeen, this.now());
|
|
@@ -406,12 +418,12 @@ export class RunnerStore {
|
|
|
406
418
|
|
|
407
419
|
/** Whether the cloud acknowledged `key` as a command of `type`. */
|
|
408
420
|
sentAs(key, type) {
|
|
409
|
-
return Boolean(this.
|
|
421
|
+
return Boolean(this.sql('SELECT 1 AS found FROM sent WHERE key = ? AND type = ?').get(key, type));
|
|
410
422
|
}
|
|
411
423
|
|
|
412
424
|
/** Acknowledged keys older than `beforeMs` go; the cloud's dedupe is the backstop for anything resent after that. */
|
|
413
425
|
pruneSent(beforeMs) {
|
|
414
|
-
return Number(this.
|
|
426
|
+
return Number(this.sql('DELETE FROM sent WHERE at < ?').run(beforeMs).changes);
|
|
415
427
|
}
|
|
416
428
|
}
|
|
417
429
|
|
|
@@ -10,7 +10,9 @@ import {
|
|
|
10
10
|
} from './storage.js';
|
|
11
11
|
import { execFileBounded as execFile } from './subprocess.js';
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
/** The Pulse daemon's own service label; the cutover that runs inside it must never retire it. */
|
|
14
|
+
export const SERVICE_LABEL = 'com.agentpulse.hub';
|
|
15
|
+
const LABEL = SERVICE_LABEL;
|
|
14
16
|
const SYSTEMD_UNIT = 'agentpulse-hub.service';
|
|
15
17
|
const START_ATTEMPTS = 6;
|
|
16
18
|
const START_RETRY_MS = 250;
|
package/src/runner/bridge.js
CHANGED
|
@@ -24,6 +24,8 @@ const PRESENCE_CURSOR = 'presence_cursor';
|
|
|
24
24
|
export const SYNC_FAILING_AFTER = 3;
|
|
25
25
|
/** SCOPE-02: while blocked for a scope or grant, the sync is retried this often with the sign-in read again, so a re-login clears it. */
|
|
26
26
|
export const BLOCK_RETRY_MS = 60_000;
|
|
27
|
+
/** Changes capture refused to send because they hold secret-like content (hubScan): set aside by design, not a fault. */
|
|
28
|
+
const WITHHELD_CODE = 'content_withheld';
|
|
27
29
|
const ERROR_CODES = new Set(['HUB_LOGIN_REQUIRED', 'HUB_SCOPE_DENIED', 'capability_denied', 'cloud_first_off', 'HUB_MACHINE_REVOKED']);
|
|
28
30
|
|
|
29
31
|
/**
|
|
@@ -63,6 +65,7 @@ export class RunnerBridge {
|
|
|
63
65
|
if (this.running) return;
|
|
64
66
|
this.running = true;
|
|
65
67
|
this.runner.on('wrote', () => this.kick());
|
|
68
|
+
this.runner.on('loop_stall', (message) => this.log('loop_stall', message.details || {}));
|
|
66
69
|
this.syncing = this.#syncLoop();
|
|
67
70
|
this.flushing = this.#flushLoop();
|
|
68
71
|
if (this.leaseConfig) this.leasing = this.#leaseLoop();
|
|
@@ -254,10 +257,12 @@ export class RunnerBridge {
|
|
|
254
257
|
const now = this.now();
|
|
255
258
|
const pending = this.store.pendingCount();
|
|
256
259
|
const dead = Number(this.store.db.prepare('SELECT COUNT(*) AS n FROM dead').get().n);
|
|
260
|
+
const withheld = Number(this.store.db.prepare('SELECT COUNT(*) AS n FROM dead WHERE code = ?').get(WITHHELD_CODE).n);
|
|
261
|
+
const setAside = dead - withheld;
|
|
257
262
|
const quarantined = Number(this.store.db.prepare('SELECT COUNT(*) AS n FROM pulse_quarantine').get()?.n || 0);
|
|
258
263
|
const applied = this.applied();
|
|
259
264
|
const lag = Math.max(0, this.state.head - applied);
|
|
260
|
-
const outbox = { pending, dead, capacity: OUTBOX_MAX_PENDING };
|
|
265
|
+
const outbox = { pending, dead, withheld, capacity: OUTBOX_MAX_PENDING };
|
|
261
266
|
const base = { lastSyncAt: this.state.lastSyncAt, applied, head: this.state.head, lag, outbox, quarantined,
|
|
262
267
|
staleSeconds: this.state.lastSyncAt ? (now - this.state.lastSyncAt) / 1000 : null };
|
|
263
268
|
const { block } = this.state.error || {};
|
|
@@ -276,8 +281,8 @@ export class RunnerBridge {
|
|
|
276
281
|
return { ...base, state: 'offline', reason: this.state.lastContactAt ? 'no contact with Pulse' : 'not connected yet' };
|
|
277
282
|
}
|
|
278
283
|
if (lag > LAG_EVENTS || (pending > 0 && this.state.flushError)) return { ...base, state: 'catching_up', reason: `${lag} events and ${pending} changes to go` };
|
|
279
|
-
if (
|
|
280
|
-
return { ...base, state: 'degraded', reason: `${
|
|
284
|
+
if (setAside || quarantined || pending > 0.8 * OUTBOX_MAX_PENDING) {
|
|
285
|
+
return { ...base, state: 'degraded', reason: `${setAside} change${setAside === 1 ? '' : 's'} and ${quarantined} event${quarantined === 1 ? '' : 's'} set aside` };
|
|
281
286
|
}
|
|
282
287
|
if (!this.state.lastSyncAt || now - this.state.lastSyncAt > ONLINE_MS) return { ...base, state: 'catching_up', reason: 'waiting for a sync' };
|
|
283
288
|
return { ...base, state: 'online', reason: 'in sync' };
|