@pdpp/local-collector 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -0
- package/dist/local-collector/bin/pdpp-local-collector.js +347 -0
- package/dist/local-collector/src/errors.d.ts +12 -0
- package/dist/local-collector/src/errors.js +20 -0
- package/dist/local-collector/src/runner.d.ts +16 -0
- package/dist/local-collector/src/runner.js +59 -0
- package/dist/polyfill-connectors/connectors/claude_code/index.js +806 -0
- package/dist/polyfill-connectors/connectors/claude_code/parsers.js +224 -0
- package/dist/polyfill-connectors/connectors/claude_code/schemas.js +120 -0
- package/dist/polyfill-connectors/connectors/claude_code/types.js +1 -0
- package/dist/polyfill-connectors/connectors/codex/index.js +880 -0
- package/dist/polyfill-connectors/connectors/codex/parsers.js +159 -0
- package/dist/polyfill-connectors/connectors/codex/schemas.js +118 -0
- package/dist/polyfill-connectors/connectors/codex/types.js +1 -0
- package/dist/polyfill-connectors/src/auth.js +76 -0
- package/dist/polyfill-connectors/src/browser-handoff.js +197 -0
- package/dist/polyfill-connectors/src/collector-protocol.d.ts +2 -0
- package/dist/polyfill-connectors/src/collector-protocol.js +2 -0
- package/dist/polyfill-connectors/src/collector-runner.d.ts +139 -0
- package/dist/polyfill-connectors/src/collector-runner.js +1084 -0
- package/dist/polyfill-connectors/src/connector-runtime-protocol.d.ts +191 -0
- package/dist/polyfill-connectors/src/connector-runtime-protocol.js +1 -0
- package/dist/polyfill-connectors/src/connector-runtime.js +879 -0
- package/dist/polyfill-connectors/src/fixture-capture.js +237 -0
- package/dist/polyfill-connectors/src/is-main-module.d.ts +1 -0
- package/dist/polyfill-connectors/src/is-main-module.js +17 -0
- package/dist/polyfill-connectors/src/local-device-client.d.ts +126 -0
- package/dist/polyfill-connectors/src/local-device-client.js +132 -0
- package/dist/polyfill-connectors/src/local-device-envelope.d.ts +26 -0
- package/dist/polyfill-connectors/src/local-device-envelope.js +43 -0
- package/dist/polyfill-connectors/src/local-device-outbox.d.ts +115 -0
- package/dist/polyfill-connectors/src/local-device-outbox.js +509 -0
- package/dist/polyfill-connectors/src/local-device-queue.d.ts +34 -0
- package/dist/polyfill-connectors/src/local-device-queue.js +133 -0
- package/dist/polyfill-connectors/src/local-source-inventory.js +119 -0
- package/dist/polyfill-connectors/src/pdpp-safe-text.js +13 -0
- package/dist/polyfill-connectors/src/runner/index.d.ts +11 -0
- package/dist/polyfill-connectors/src/runner/index.js +10 -0
- package/dist/polyfill-connectors/src/runtime-capabilities.d.ts +40 -0
- package/dist/polyfill-connectors/src/runtime-capabilities.js +59 -0
- package/dist/polyfill-connectors/src/safe-emit.d.ts +3 -0
- package/dist/polyfill-connectors/src/safe-emit.js +30 -0
- package/dist/polyfill-connectors/src/safe-text-preview.js +156 -0
- package/dist/polyfill-connectors/src/schema-registry.js +17 -0
- package/dist/polyfill-connectors/src/scope-filters.d.ts +38 -0
- package/dist/polyfill-connectors/src/scope-filters.js +80 -0
- package/dist/polyfill-connectors/src/shutdown-hook.js +51 -0
- package/dist/polyfill-connectors/src/streaming-target-registration.js +161 -0
- package/package.json +63 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
export type LocalDeviceOutboxKind = "record_batch" | "checkpoint" | "gap" | "blob_upload";
|
|
2
|
+
export type LocalDeviceOutboxStatus = "ready" | "leased" | "succeeded" | "dead_letter";
|
|
3
|
+
export interface LocalDeviceOutboxItem {
|
|
4
|
+
acknowledged_at: string | null;
|
|
5
|
+
attempt_count: number;
|
|
6
|
+
body_hash: string;
|
|
7
|
+
created_at: string;
|
|
8
|
+
id: string;
|
|
9
|
+
insert_order: number;
|
|
10
|
+
kind: LocalDeviceOutboxKind;
|
|
11
|
+
last_error: string | null;
|
|
12
|
+
lease_epoch: number;
|
|
13
|
+
lease_holder: string | null;
|
|
14
|
+
lease_until: string | null;
|
|
15
|
+
next_attempt_at: string;
|
|
16
|
+
payload: unknown;
|
|
17
|
+
source_instance_id: string;
|
|
18
|
+
status: LocalDeviceOutboxStatus;
|
|
19
|
+
updated_at: string;
|
|
20
|
+
}
|
|
21
|
+
export interface LocalDeviceOutboxSummary {
|
|
22
|
+
deadLetter: number;
|
|
23
|
+
leased: number;
|
|
24
|
+
oldestReadyAt: string | null;
|
|
25
|
+
ready: number;
|
|
26
|
+
retrying: number;
|
|
27
|
+
staleLeases: number;
|
|
28
|
+
succeeded: number;
|
|
29
|
+
total: number;
|
|
30
|
+
}
|
|
31
|
+
export interface LocalDeviceOutboxOptions {
|
|
32
|
+
clock?: () => Date;
|
|
33
|
+
path: string;
|
|
34
|
+
}
|
|
35
|
+
export interface LocalDeviceOutboxEnqueueInput {
|
|
36
|
+
id: string;
|
|
37
|
+
kind: LocalDeviceOutboxKind;
|
|
38
|
+
nextAttemptAt?: Date;
|
|
39
|
+
payload: unknown;
|
|
40
|
+
sourceInstanceId: string;
|
|
41
|
+
}
|
|
42
|
+
export interface BuildLocalDeviceOutboxIdInput {
|
|
43
|
+
kind: LocalDeviceOutboxKind;
|
|
44
|
+
parts: readonly unknown[];
|
|
45
|
+
sourceInstanceId: string;
|
|
46
|
+
}
|
|
47
|
+
export interface LocalDeviceOutboxClaimInput {
|
|
48
|
+
excludeKinds?: readonly LocalDeviceOutboxKind[];
|
|
49
|
+
holder: string;
|
|
50
|
+
leaseMs: number;
|
|
51
|
+
limit?: number;
|
|
52
|
+
sourceInstanceId?: string;
|
|
53
|
+
}
|
|
54
|
+
export interface LocalDeviceOutboxLeaseInput {
|
|
55
|
+
holder: string;
|
|
56
|
+
id: string;
|
|
57
|
+
leaseEpoch: number;
|
|
58
|
+
}
|
|
59
|
+
export interface LocalDeviceOutboxFailInput extends LocalDeviceOutboxLeaseInput {
|
|
60
|
+
error: string;
|
|
61
|
+
retryBackoffMs: number;
|
|
62
|
+
}
|
|
63
|
+
export interface LocalDeviceOutboxDeadLetterInput extends LocalDeviceOutboxLeaseInput {
|
|
64
|
+
error: string;
|
|
65
|
+
}
|
|
66
|
+
export interface LocalDeviceOutboxRenewInput extends LocalDeviceOutboxLeaseInput {
|
|
67
|
+
leaseMs: number;
|
|
68
|
+
}
|
|
69
|
+
export declare class LocalDeviceOutbox {
|
|
70
|
+
#private;
|
|
71
|
+
constructor(options: LocalDeviceOutboxOptions);
|
|
72
|
+
close(): void;
|
|
73
|
+
enqueue(input: LocalDeviceOutboxEnqueueInput): LocalDeviceOutboxItem;
|
|
74
|
+
claimReady(input: LocalDeviceOutboxClaimInput): LocalDeviceOutboxItem[];
|
|
75
|
+
peekReady(input?: {
|
|
76
|
+
sourceInstanceId?: string;
|
|
77
|
+
}): LocalDeviceOutboxItem | null;
|
|
78
|
+
acknowledge(input: LocalDeviceOutboxLeaseInput): void;
|
|
79
|
+
failRetryable(input: LocalDeviceOutboxFailInput): void;
|
|
80
|
+
deadLetter(input: LocalDeviceOutboxDeadLetterInput): void;
|
|
81
|
+
renewLease(input: LocalDeviceOutboxRenewInput): LocalDeviceOutboxItem;
|
|
82
|
+
recoverExpiredLeases(input?: {
|
|
83
|
+
sourceInstanceId?: string;
|
|
84
|
+
}): number;
|
|
85
|
+
get(id: string): LocalDeviceOutboxItem | null;
|
|
86
|
+
deleteSucceeded(id: string): boolean;
|
|
87
|
+
hasNonSucceededWork(input: {
|
|
88
|
+
excludeKinds?: readonly LocalDeviceOutboxKind[];
|
|
89
|
+
kinds?: readonly LocalDeviceOutboxKind[];
|
|
90
|
+
sourceInstanceId: string;
|
|
91
|
+
}): boolean;
|
|
92
|
+
hasNonSucceededPredecessor(input: {
|
|
93
|
+
beforeInsertOrder: number;
|
|
94
|
+
kinds: readonly LocalDeviceOutboxKind[];
|
|
95
|
+
sourceInstanceId: string;
|
|
96
|
+
}): boolean;
|
|
97
|
+
countOpenGaps(input: {
|
|
98
|
+
sourceInstanceId: string;
|
|
99
|
+
}): number;
|
|
100
|
+
listByKind(input: {
|
|
101
|
+
kind: LocalDeviceOutboxKind;
|
|
102
|
+
sourceInstanceId: string;
|
|
103
|
+
statuses?: readonly LocalDeviceOutboxStatus[];
|
|
104
|
+
}): LocalDeviceOutboxItem[];
|
|
105
|
+
maxRecordBatchSeq(input: {
|
|
106
|
+
sourceInstanceId: string;
|
|
107
|
+
}): number;
|
|
108
|
+
list(input?: {
|
|
109
|
+
sourceInstanceId?: string;
|
|
110
|
+
}): LocalDeviceOutboxItem[];
|
|
111
|
+
summary(input?: {
|
|
112
|
+
sourceInstanceId?: string;
|
|
113
|
+
}): LocalDeviceOutboxSummary;
|
|
114
|
+
}
|
|
115
|
+
export declare function buildLocalDeviceOutboxId(input: BuildLocalDeviceOutboxIdInput): string;
|
|
@@ -0,0 +1,509 @@
|
|
|
1
|
+
import { mkdirSync } from "node:fs";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
import { DatabaseSync } from "node:sqlite";
|
|
4
|
+
import { hashCanonicalJson } from "./local-device-envelope.js";
|
|
5
|
+
const CURRENT_SCHEMA_VERSION = 1;
|
|
6
|
+
export class LocalDeviceOutbox {
|
|
7
|
+
#clock;
|
|
8
|
+
#db;
|
|
9
|
+
constructor(options) {
|
|
10
|
+
this.#clock = options.clock ?? (() => new Date());
|
|
11
|
+
if (options.path !== ":memory:") {
|
|
12
|
+
mkdirSync(dirname(options.path), { recursive: true });
|
|
13
|
+
}
|
|
14
|
+
this.#db = new DatabaseSync(options.path);
|
|
15
|
+
this.#initialize();
|
|
16
|
+
}
|
|
17
|
+
close() {
|
|
18
|
+
this.#db.close();
|
|
19
|
+
}
|
|
20
|
+
enqueue(input) {
|
|
21
|
+
const now = this.#now();
|
|
22
|
+
const payloadJson = JSON.stringify(input.payload);
|
|
23
|
+
const bodyHash = hashCanonicalJson(input.payload);
|
|
24
|
+
const existing = this.get(input.id);
|
|
25
|
+
if (existing) {
|
|
26
|
+
if (existing.body_hash !== bodyHash ||
|
|
27
|
+
existing.kind !== input.kind ||
|
|
28
|
+
existing.source_instance_id !== input.sourceInstanceId) {
|
|
29
|
+
throw new Error(`local outbox id collision with different payload: ${input.id}`);
|
|
30
|
+
}
|
|
31
|
+
return existing;
|
|
32
|
+
}
|
|
33
|
+
const row = {
|
|
34
|
+
acknowledged_at: null,
|
|
35
|
+
attempt_count: 0,
|
|
36
|
+
body_hash: bodyHash,
|
|
37
|
+
created_at: now,
|
|
38
|
+
id: input.id,
|
|
39
|
+
insert_order: 0,
|
|
40
|
+
kind: input.kind,
|
|
41
|
+
last_error: null,
|
|
42
|
+
lease_epoch: 0,
|
|
43
|
+
lease_holder: null,
|
|
44
|
+
lease_until: null,
|
|
45
|
+
next_attempt_at: (input.nextAttemptAt ?? this.#clock()).toISOString(),
|
|
46
|
+
payload_json: payloadJson,
|
|
47
|
+
source_instance_id: input.sourceInstanceId,
|
|
48
|
+
status: "ready",
|
|
49
|
+
updated_at: now,
|
|
50
|
+
};
|
|
51
|
+
this.#db
|
|
52
|
+
.prepare(`INSERT INTO local_device_outbox (
|
|
53
|
+
id,
|
|
54
|
+
source_instance_id,
|
|
55
|
+
kind,
|
|
56
|
+
status,
|
|
57
|
+
payload_json,
|
|
58
|
+
body_hash,
|
|
59
|
+
attempt_count,
|
|
60
|
+
next_attempt_at,
|
|
61
|
+
lease_holder,
|
|
62
|
+
lease_epoch,
|
|
63
|
+
lease_until,
|
|
64
|
+
last_error,
|
|
65
|
+
acknowledged_at,
|
|
66
|
+
created_at,
|
|
67
|
+
updated_at
|
|
68
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
69
|
+
.run(row.id, row.source_instance_id, row.kind, row.status, row.payload_json, row.body_hash, row.attempt_count, row.next_attempt_at, row.lease_holder, row.lease_epoch, row.lease_until, row.last_error, row.acknowledged_at, row.created_at, row.updated_at);
|
|
70
|
+
const inserted = this.get(row.id);
|
|
71
|
+
if (!inserted) {
|
|
72
|
+
throw new Error(`local outbox insert disappeared before readback: ${row.id}`);
|
|
73
|
+
}
|
|
74
|
+
return inserted;
|
|
75
|
+
}
|
|
76
|
+
claimReady(input) {
|
|
77
|
+
const now = this.#now();
|
|
78
|
+
const leaseUntil = new Date(this.#clock().getTime() + input.leaseMs).toISOString();
|
|
79
|
+
const limit = Math.max(1, input.limit ?? 1);
|
|
80
|
+
const candidates = this.#selectReady(input.sourceInstanceId, now, limit, input.excludeKinds);
|
|
81
|
+
const claimed = [];
|
|
82
|
+
for (const candidate of candidates) {
|
|
83
|
+
const nextEpoch = candidate.lease_epoch + 1;
|
|
84
|
+
const result = this.#db
|
|
85
|
+
.prepare(`UPDATE local_device_outbox
|
|
86
|
+
SET status = 'leased',
|
|
87
|
+
lease_holder = ?,
|
|
88
|
+
lease_epoch = ?,
|
|
89
|
+
lease_until = ?,
|
|
90
|
+
updated_at = ?
|
|
91
|
+
WHERE id = ?
|
|
92
|
+
AND status = 'ready'`)
|
|
93
|
+
.run(input.holder, nextEpoch, leaseUntil, now, candidate.id);
|
|
94
|
+
if (result.changes !== 1) {
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
const next = this.get(candidate.id);
|
|
98
|
+
if (next) {
|
|
99
|
+
claimed.push(next);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return claimed;
|
|
103
|
+
}
|
|
104
|
+
peekReady(input = {}) {
|
|
105
|
+
const [candidate] = this.#selectReady(input.sourceInstanceId, this.#now(), 1);
|
|
106
|
+
return candidate ? rowToItem(candidate) : null;
|
|
107
|
+
}
|
|
108
|
+
acknowledge(input) {
|
|
109
|
+
const now = this.#now();
|
|
110
|
+
const result = this.#db
|
|
111
|
+
.prepare(`UPDATE local_device_outbox
|
|
112
|
+
SET status = 'succeeded',
|
|
113
|
+
acknowledged_at = ?,
|
|
114
|
+
lease_holder = NULL,
|
|
115
|
+
lease_until = NULL,
|
|
116
|
+
updated_at = ?
|
|
117
|
+
WHERE id = ?
|
|
118
|
+
AND status = 'leased'
|
|
119
|
+
AND lease_holder = ?
|
|
120
|
+
AND lease_epoch = ?
|
|
121
|
+
AND lease_until > ?`)
|
|
122
|
+
.run(now, now, input.id, input.holder, input.leaseEpoch, now);
|
|
123
|
+
assertOneChange(Number(result.changes), `local outbox lease not current for acknowledge: ${input.id}`);
|
|
124
|
+
}
|
|
125
|
+
failRetryable(input) {
|
|
126
|
+
const now = this.#now();
|
|
127
|
+
const nextAttemptAt = new Date(this.#clock().getTime() + input.retryBackoffMs).toISOString();
|
|
128
|
+
const result = this.#db
|
|
129
|
+
.prepare(`UPDATE local_device_outbox
|
|
130
|
+
SET status = 'ready',
|
|
131
|
+
attempt_count = attempt_count + 1,
|
|
132
|
+
next_attempt_at = ?,
|
|
133
|
+
lease_holder = NULL,
|
|
134
|
+
lease_until = NULL,
|
|
135
|
+
last_error = ?,
|
|
136
|
+
updated_at = ?
|
|
137
|
+
WHERE id = ?
|
|
138
|
+
AND status = 'leased'
|
|
139
|
+
AND lease_holder = ?
|
|
140
|
+
AND lease_epoch = ?
|
|
141
|
+
AND lease_until > ?`)
|
|
142
|
+
.run(nextAttemptAt, input.error, now, input.id, input.holder, input.leaseEpoch, now);
|
|
143
|
+
assertOneChange(Number(result.changes), `local outbox lease not current for retry: ${input.id}`);
|
|
144
|
+
}
|
|
145
|
+
deadLetter(input) {
|
|
146
|
+
const now = this.#now();
|
|
147
|
+
const result = this.#db
|
|
148
|
+
.prepare(`UPDATE local_device_outbox
|
|
149
|
+
SET status = 'dead_letter',
|
|
150
|
+
attempt_count = attempt_count + 1,
|
|
151
|
+
lease_holder = NULL,
|
|
152
|
+
lease_until = NULL,
|
|
153
|
+
last_error = ?,
|
|
154
|
+
updated_at = ?
|
|
155
|
+
WHERE id = ?
|
|
156
|
+
AND status = 'leased'
|
|
157
|
+
AND lease_holder = ?
|
|
158
|
+
AND lease_epoch = ?
|
|
159
|
+
AND lease_until > ?`)
|
|
160
|
+
.run(input.error, now, input.id, input.holder, input.leaseEpoch, now);
|
|
161
|
+
assertOneChange(Number(result.changes), `local outbox lease not current for dead-letter: ${input.id}`);
|
|
162
|
+
}
|
|
163
|
+
renewLease(input) {
|
|
164
|
+
const now = this.#now();
|
|
165
|
+
const leaseUntil = new Date(this.#clock().getTime() + input.leaseMs).toISOString();
|
|
166
|
+
const result = this.#db
|
|
167
|
+
.prepare(`UPDATE local_device_outbox
|
|
168
|
+
SET lease_until = ?,
|
|
169
|
+
updated_at = ?
|
|
170
|
+
WHERE id = ?
|
|
171
|
+
AND status = 'leased'
|
|
172
|
+
AND lease_holder = ?
|
|
173
|
+
AND lease_epoch = ?
|
|
174
|
+
AND lease_until > ?`)
|
|
175
|
+
.run(leaseUntil, now, input.id, input.holder, input.leaseEpoch, now);
|
|
176
|
+
assertOneChange(Number(result.changes), `local outbox lease not current for renew: ${input.id}`);
|
|
177
|
+
const item = this.get(input.id);
|
|
178
|
+
if (!item) {
|
|
179
|
+
throw new Error(`local outbox item missing after renew: ${input.id}`);
|
|
180
|
+
}
|
|
181
|
+
return item;
|
|
182
|
+
}
|
|
183
|
+
recoverExpiredLeases(input = {}) {
|
|
184
|
+
const now = this.#now();
|
|
185
|
+
const sql = `UPDATE local_device_outbox
|
|
186
|
+
SET status = 'ready',
|
|
187
|
+
lease_holder = NULL,
|
|
188
|
+
lease_until = NULL,
|
|
189
|
+
last_error = COALESCE(last_error, 'lease expired before acknowledgement'),
|
|
190
|
+
updated_at = ?
|
|
191
|
+
WHERE status = 'leased'
|
|
192
|
+
AND lease_until IS NOT NULL
|
|
193
|
+
AND lease_until <= ?`;
|
|
194
|
+
const result = input.sourceInstanceId
|
|
195
|
+
? this.#db.prepare(`${sql} AND source_instance_id = ?`).run(now, now, input.sourceInstanceId)
|
|
196
|
+
: this.#db.prepare(sql).run(now, now);
|
|
197
|
+
return Number(result.changes);
|
|
198
|
+
}
|
|
199
|
+
get(id) {
|
|
200
|
+
const row = this.#db.prepare("SELECT *, rowid AS insert_order FROM local_device_outbox WHERE id = ?").get(id);
|
|
201
|
+
return row ? rowToItem(row) : null;
|
|
202
|
+
}
|
|
203
|
+
deleteSucceeded(id) {
|
|
204
|
+
const result = this.#db.prepare("DELETE FROM local_device_outbox WHERE id = ? AND status = 'succeeded'").run(id);
|
|
205
|
+
return Number(result.changes) === 1;
|
|
206
|
+
}
|
|
207
|
+
hasNonSucceededWork(input) {
|
|
208
|
+
const clauses = ["source_instance_id = ?", "status != 'succeeded'"];
|
|
209
|
+
const params = [input.sourceInstanceId];
|
|
210
|
+
if (input.kinds && input.kinds.length > 0) {
|
|
211
|
+
clauses.push(`kind IN (${input.kinds.map(() => "?").join(", ")})`);
|
|
212
|
+
params.push(...input.kinds);
|
|
213
|
+
}
|
|
214
|
+
if (input.excludeKinds && input.excludeKinds.length > 0) {
|
|
215
|
+
clauses.push(`kind NOT IN (${input.excludeKinds.map(() => "?").join(", ")})`);
|
|
216
|
+
params.push(...input.excludeKinds);
|
|
217
|
+
}
|
|
218
|
+
const row = this.#db
|
|
219
|
+
.prepare(`SELECT 1 AS found FROM local_device_outbox WHERE ${clauses.join(" AND ")} LIMIT 1`)
|
|
220
|
+
.get(...params);
|
|
221
|
+
return Boolean(row);
|
|
222
|
+
}
|
|
223
|
+
hasNonSucceededPredecessor(input) {
|
|
224
|
+
if (input.kinds.length === 0) {
|
|
225
|
+
return false;
|
|
226
|
+
}
|
|
227
|
+
const row = this.#db
|
|
228
|
+
.prepare(`SELECT 1 AS found FROM local_device_outbox
|
|
229
|
+
WHERE source_instance_id = ?
|
|
230
|
+
AND rowid < ?
|
|
231
|
+
AND status != 'succeeded'
|
|
232
|
+
AND kind IN (${input.kinds.map(() => "?").join(", ")})
|
|
233
|
+
LIMIT 1`)
|
|
234
|
+
.get(input.sourceInstanceId, input.beforeInsertOrder, ...input.kinds);
|
|
235
|
+
return Boolean(row);
|
|
236
|
+
}
|
|
237
|
+
countOpenGaps(input) {
|
|
238
|
+
const row = this.#db
|
|
239
|
+
.prepare(`SELECT COUNT(*) AS total FROM local_device_outbox
|
|
240
|
+
WHERE source_instance_id = ?
|
|
241
|
+
AND kind = 'gap'
|
|
242
|
+
AND status IN ('ready', 'leased')`)
|
|
243
|
+
.get(input.sourceInstanceId);
|
|
244
|
+
return isRecord(row) ? numberFrom(row.total) : 0;
|
|
245
|
+
}
|
|
246
|
+
listByKind(input) {
|
|
247
|
+
const clauses = ["source_instance_id = ?", "kind = ?"];
|
|
248
|
+
const params = [input.sourceInstanceId, input.kind];
|
|
249
|
+
if (input.statuses && input.statuses.length > 0) {
|
|
250
|
+
clauses.push(`status IN (${input.statuses.map(() => "?").join(", ")})`);
|
|
251
|
+
params.push(...input.statuses);
|
|
252
|
+
}
|
|
253
|
+
const rows = this.#db
|
|
254
|
+
.prepare(`SELECT *, rowid AS insert_order FROM local_device_outbox
|
|
255
|
+
WHERE ${clauses.join(" AND ")}
|
|
256
|
+
ORDER BY insert_order`)
|
|
257
|
+
.all(...params);
|
|
258
|
+
return rows.map((row) => rowToItem(row));
|
|
259
|
+
}
|
|
260
|
+
maxRecordBatchSeq(input) {
|
|
261
|
+
const row = this.#db
|
|
262
|
+
.prepare(`SELECT COALESCE(MAX(CAST(json_extract(payload_json, '$.batchSeq') AS INTEGER)), 0) AS max_seq
|
|
263
|
+
FROM local_device_outbox
|
|
264
|
+
WHERE source_instance_id = ?
|
|
265
|
+
AND kind = 'record_batch'`)
|
|
266
|
+
.get(input.sourceInstanceId);
|
|
267
|
+
return isRecord(row) ? numberFrom(row.max_seq) : 0;
|
|
268
|
+
}
|
|
269
|
+
list(input = {}) {
|
|
270
|
+
const rows = input.sourceInstanceId
|
|
271
|
+
? this.#db
|
|
272
|
+
.prepare(`SELECT *, rowid AS insert_order FROM local_device_outbox
|
|
273
|
+
WHERE source_instance_id = ?
|
|
274
|
+
ORDER BY source_instance_id, insert_order`)
|
|
275
|
+
.all(input.sourceInstanceId)
|
|
276
|
+
: this.#db
|
|
277
|
+
.prepare(`SELECT *, rowid AS insert_order FROM local_device_outbox
|
|
278
|
+
ORDER BY source_instance_id, insert_order`)
|
|
279
|
+
.all();
|
|
280
|
+
return rows.map((row) => rowToItem(row));
|
|
281
|
+
}
|
|
282
|
+
summary(input = {}) {
|
|
283
|
+
const now = this.#now();
|
|
284
|
+
const summary = {
|
|
285
|
+
deadLetter: 0,
|
|
286
|
+
leased: 0,
|
|
287
|
+
oldestReadyAt: null,
|
|
288
|
+
ready: 0,
|
|
289
|
+
retrying: 0,
|
|
290
|
+
staleLeases: 0,
|
|
291
|
+
succeeded: 0,
|
|
292
|
+
total: 0,
|
|
293
|
+
};
|
|
294
|
+
const aggregateSql = `
|
|
295
|
+
SELECT
|
|
296
|
+
status,
|
|
297
|
+
COUNT(*) AS total,
|
|
298
|
+
SUM(CASE WHEN status = 'ready' AND next_attempt_at > ? THEN 1 ELSE 0 END) AS retrying,
|
|
299
|
+
SUM(CASE WHEN status = 'leased' AND lease_until IS NOT NULL AND lease_until <= ? THEN 1 ELSE 0 END) AS stale_leases,
|
|
300
|
+
MIN(CASE WHEN status = 'ready' THEN created_at ELSE NULL END) AS oldest_ready
|
|
301
|
+
FROM local_device_outbox
|
|
302
|
+
${input.sourceInstanceId ? "WHERE source_instance_id = ?" : ""}
|
|
303
|
+
GROUP BY status`;
|
|
304
|
+
const statement = this.#db.prepare(aggregateSql);
|
|
305
|
+
const rows = input.sourceInstanceId ? statement.all(now, now, input.sourceInstanceId) : statement.all(now, now);
|
|
306
|
+
for (const rowLike of rows) {
|
|
307
|
+
if (!isRecord(rowLike)) {
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
310
|
+
const status = rowLike.status;
|
|
311
|
+
const total = numberFrom(rowLike.total);
|
|
312
|
+
summary.total += total;
|
|
313
|
+
if (status === "ready") {
|
|
314
|
+
summary.ready = total;
|
|
315
|
+
summary.retrying = numberFrom(rowLike.retrying);
|
|
316
|
+
const oldest = rowLike.oldest_ready;
|
|
317
|
+
if (typeof oldest === "string") {
|
|
318
|
+
summary.oldestReadyAt = oldest;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
else if (status === "leased") {
|
|
322
|
+
summary.leased = total;
|
|
323
|
+
summary.staleLeases = numberFrom(rowLike.stale_leases);
|
|
324
|
+
}
|
|
325
|
+
else if (status === "succeeded") {
|
|
326
|
+
summary.succeeded = total;
|
|
327
|
+
}
|
|
328
|
+
else if (status === "dead_letter") {
|
|
329
|
+
summary.deadLetter = total;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return summary;
|
|
333
|
+
}
|
|
334
|
+
#initialize() {
|
|
335
|
+
this.#db.exec(`
|
|
336
|
+
PRAGMA journal_mode = WAL;
|
|
337
|
+
PRAGMA busy_timeout = 5000;
|
|
338
|
+
PRAGMA foreign_keys = ON;
|
|
339
|
+
`);
|
|
340
|
+
const version = this.#schemaVersion();
|
|
341
|
+
if (version > CURRENT_SCHEMA_VERSION) {
|
|
342
|
+
throw new Error(`local outbox schema version ${version} is newer than supported version ${CURRENT_SCHEMA_VERSION}`);
|
|
343
|
+
}
|
|
344
|
+
if (version < 1) {
|
|
345
|
+
this.#applySchemaV1();
|
|
346
|
+
this.#db.exec(`PRAGMA user_version = ${CURRENT_SCHEMA_VERSION}`);
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
this.#applySchemaV1();
|
|
350
|
+
}
|
|
351
|
+
#applySchemaV1() {
|
|
352
|
+
this.#db.exec(`
|
|
353
|
+
CREATE TABLE IF NOT EXISTS local_device_outbox (
|
|
354
|
+
id TEXT PRIMARY KEY,
|
|
355
|
+
source_instance_id TEXT NOT NULL,
|
|
356
|
+
kind TEXT NOT NULL CHECK (kind IN ('record_batch', 'checkpoint', 'gap', 'blob_upload')),
|
|
357
|
+
status TEXT NOT NULL CHECK (status IN ('ready', 'leased', 'succeeded', 'dead_letter')),
|
|
358
|
+
payload_json TEXT NOT NULL,
|
|
359
|
+
body_hash TEXT NOT NULL,
|
|
360
|
+
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
361
|
+
next_attempt_at TEXT NOT NULL,
|
|
362
|
+
lease_holder TEXT,
|
|
363
|
+
lease_epoch INTEGER NOT NULL DEFAULT 0,
|
|
364
|
+
lease_until TEXT,
|
|
365
|
+
last_error TEXT,
|
|
366
|
+
acknowledged_at TEXT,
|
|
367
|
+
created_at TEXT NOT NULL,
|
|
368
|
+
updated_at TEXT NOT NULL
|
|
369
|
+
);
|
|
370
|
+
CREATE INDEX IF NOT EXISTS local_device_outbox_ready_idx
|
|
371
|
+
ON local_device_outbox (status, next_attempt_at, source_instance_id, created_at);
|
|
372
|
+
CREATE INDEX IF NOT EXISTS local_device_outbox_lease_idx
|
|
373
|
+
ON local_device_outbox (status, lease_until);
|
|
374
|
+
CREATE INDEX IF NOT EXISTS local_device_outbox_source_idx
|
|
375
|
+
ON local_device_outbox (source_instance_id, status);
|
|
376
|
+
`);
|
|
377
|
+
}
|
|
378
|
+
#selectReady(sourceInstanceId, now, limit, excludeKinds = []) {
|
|
379
|
+
const kindClause = excludeKinds.length > 0 ? `AND kind NOT IN (${excludeKinds.map(() => "?").join(", ")})` : "";
|
|
380
|
+
if (sourceInstanceId) {
|
|
381
|
+
return this.#db
|
|
382
|
+
.prepare(`SELECT *, rowid AS insert_order FROM local_device_outbox
|
|
383
|
+
WHERE status = 'ready'
|
|
384
|
+
AND source_instance_id = ?
|
|
385
|
+
AND next_attempt_at <= ?
|
|
386
|
+
${kindClause}
|
|
387
|
+
ORDER BY insert_order
|
|
388
|
+
LIMIT ?`)
|
|
389
|
+
.all(sourceInstanceId, now, ...excludeKinds, limit)
|
|
390
|
+
.map(asOutboxRow);
|
|
391
|
+
}
|
|
392
|
+
return this.#db
|
|
393
|
+
.prepare(`SELECT *, rowid AS insert_order FROM local_device_outbox
|
|
394
|
+
WHERE status = 'ready'
|
|
395
|
+
AND next_attempt_at <= ?
|
|
396
|
+
${kindClause}
|
|
397
|
+
ORDER BY source_instance_id, insert_order
|
|
398
|
+
LIMIT ?`)
|
|
399
|
+
.all(now, ...excludeKinds, limit)
|
|
400
|
+
.map(asOutboxRow);
|
|
401
|
+
}
|
|
402
|
+
#now() {
|
|
403
|
+
return this.#clock().toISOString();
|
|
404
|
+
}
|
|
405
|
+
#schemaVersion() {
|
|
406
|
+
const row = this.#db.prepare("PRAGMA user_version").get();
|
|
407
|
+
if (!isRecord(row)) {
|
|
408
|
+
return 0;
|
|
409
|
+
}
|
|
410
|
+
const version = row.user_version;
|
|
411
|
+
return typeof version === "bigint" || typeof version === "number" ? Number(version) : 0;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
export function buildLocalDeviceOutboxId(input) {
|
|
415
|
+
return `local-outbox:${hashCanonicalJson({
|
|
416
|
+
kind: input.kind,
|
|
417
|
+
parts: input.parts,
|
|
418
|
+
source_instance_id: input.sourceInstanceId,
|
|
419
|
+
})}`;
|
|
420
|
+
}
|
|
421
|
+
function rowToItem(rowLike) {
|
|
422
|
+
const row = asOutboxRow(rowLike);
|
|
423
|
+
return {
|
|
424
|
+
acknowledged_at: row.acknowledged_at,
|
|
425
|
+
attempt_count: row.attempt_count,
|
|
426
|
+
body_hash: row.body_hash,
|
|
427
|
+
created_at: row.created_at,
|
|
428
|
+
id: row.id,
|
|
429
|
+
insert_order: row.insert_order,
|
|
430
|
+
kind: row.kind,
|
|
431
|
+
last_error: row.last_error,
|
|
432
|
+
lease_epoch: row.lease_epoch,
|
|
433
|
+
lease_holder: row.lease_holder,
|
|
434
|
+
lease_until: row.lease_until,
|
|
435
|
+
next_attempt_at: row.next_attempt_at,
|
|
436
|
+
payload: JSON.parse(row.payload_json),
|
|
437
|
+
source_instance_id: row.source_instance_id,
|
|
438
|
+
status: row.status,
|
|
439
|
+
updated_at: row.updated_at,
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
function assertOneChange(changes, message) {
|
|
443
|
+
if (changes !== 1) {
|
|
444
|
+
throw new Error(message);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
function asOutboxRow(row) {
|
|
448
|
+
if (!isRecord(row)) {
|
|
449
|
+
throw new Error("local outbox query returned a non-object row");
|
|
450
|
+
}
|
|
451
|
+
const kind = row.kind;
|
|
452
|
+
const status = row.status;
|
|
453
|
+
if (typeof row.acknowledged_at !== "string" && row.acknowledged_at !== null) {
|
|
454
|
+
throw new Error("local outbox row has invalid acknowledged_at");
|
|
455
|
+
}
|
|
456
|
+
if (typeof row.attempt_count !== "number" ||
|
|
457
|
+
typeof row.body_hash !== "string" ||
|
|
458
|
+
typeof row.created_at !== "string" ||
|
|
459
|
+
typeof row.id !== "string" ||
|
|
460
|
+
(typeof row.insert_order !== "number" && typeof row.insert_order !== "bigint") ||
|
|
461
|
+
!isOutboxKind(kind) ||
|
|
462
|
+
typeof row.lease_epoch !== "number" ||
|
|
463
|
+
(typeof row.lease_holder !== "string" && row.lease_holder !== null) ||
|
|
464
|
+
(typeof row.lease_until !== "string" && row.lease_until !== null) ||
|
|
465
|
+
(typeof row.last_error !== "string" && row.last_error !== null) ||
|
|
466
|
+
typeof row.next_attempt_at !== "string" ||
|
|
467
|
+
typeof row.payload_json !== "string" ||
|
|
468
|
+
typeof row.source_instance_id !== "string" ||
|
|
469
|
+
!isOutboxStatus(status) ||
|
|
470
|
+
typeof row.updated_at !== "string") {
|
|
471
|
+
throw new Error("local outbox row has invalid shape");
|
|
472
|
+
}
|
|
473
|
+
return {
|
|
474
|
+
acknowledged_at: row.acknowledged_at,
|
|
475
|
+
attempt_count: row.attempt_count,
|
|
476
|
+
body_hash: row.body_hash,
|
|
477
|
+
created_at: row.created_at,
|
|
478
|
+
id: row.id,
|
|
479
|
+
insert_order: numberFrom(row.insert_order),
|
|
480
|
+
kind,
|
|
481
|
+
last_error: row.last_error,
|
|
482
|
+
lease_epoch: row.lease_epoch,
|
|
483
|
+
lease_holder: row.lease_holder,
|
|
484
|
+
lease_until: row.lease_until,
|
|
485
|
+
next_attempt_at: row.next_attempt_at,
|
|
486
|
+
payload_json: row.payload_json,
|
|
487
|
+
source_instance_id: row.source_instance_id,
|
|
488
|
+
status,
|
|
489
|
+
updated_at: row.updated_at,
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
function isRecord(value) {
|
|
493
|
+
return typeof value === "object" && value !== null;
|
|
494
|
+
}
|
|
495
|
+
function numberFrom(value) {
|
|
496
|
+
if (typeof value === "number") {
|
|
497
|
+
return value;
|
|
498
|
+
}
|
|
499
|
+
if (typeof value === "bigint") {
|
|
500
|
+
return Number(value);
|
|
501
|
+
}
|
|
502
|
+
return 0;
|
|
503
|
+
}
|
|
504
|
+
function isOutboxKind(value) {
|
|
505
|
+
return value === "record_batch" || value === "checkpoint" || value === "gap" || value === "blob_upload";
|
|
506
|
+
}
|
|
507
|
+
function isOutboxStatus(value) {
|
|
508
|
+
return value === "ready" || value === "leased" || value === "succeeded" || value === "dead_letter";
|
|
509
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { LocalDeviceRecordEnvelope } from "./local-device-envelope.js";
|
|
2
|
+
export type LocalDeviceQueueStatus = "pending" | "in_flight" | "sent" | "permanent_failure";
|
|
3
|
+
export interface LocalDeviceQueueItem {
|
|
4
|
+
available_at: string;
|
|
5
|
+
batch_id: string;
|
|
6
|
+
batch_seq: number;
|
|
7
|
+
created_at: string;
|
|
8
|
+
last_error?: string;
|
|
9
|
+
records: LocalDeviceRecordEnvelope[];
|
|
10
|
+
retry_count: number;
|
|
11
|
+
source_instance_id: string;
|
|
12
|
+
status: LocalDeviceQueueStatus;
|
|
13
|
+
updated_at: string;
|
|
14
|
+
}
|
|
15
|
+
export interface LocalDeviceQueueOptions {
|
|
16
|
+
clock?: () => Date;
|
|
17
|
+
path: string;
|
|
18
|
+
retryBackoffMs?: (retryCount: number) => number;
|
|
19
|
+
}
|
|
20
|
+
export declare class LocalDeviceQueue {
|
|
21
|
+
#private;
|
|
22
|
+
constructor(options: LocalDeviceQueueOptions);
|
|
23
|
+
enqueue(input: {
|
|
24
|
+
batchId: string;
|
|
25
|
+
batchSeq: number;
|
|
26
|
+
records: LocalDeviceRecordEnvelope[];
|
|
27
|
+
sourceInstanceId: string;
|
|
28
|
+
}): Promise<LocalDeviceQueueItem>;
|
|
29
|
+
dequeueReady(): Promise<LocalDeviceQueueItem | null>;
|
|
30
|
+
markSent(batchId: string): Promise<void>;
|
|
31
|
+
markRetry(batchId: string, error: string): Promise<void>;
|
|
32
|
+
markPermanentFailure(batchId: string, error: string): Promise<void>;
|
|
33
|
+
list(): Promise<LocalDeviceQueueItem[]>;
|
|
34
|
+
}
|