@indigoai-us/hq-cloud 6.15.77 → 6.15.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-worker.d.ts +3 -0
- package/dist/bin/sync-runner-worker.d.ts.map +1 -0
- package/dist/bin/sync-runner-worker.js +88 -0
- package/dist/bin/sync-runner-worker.js.map +1 -0
- package/dist/bin/sync-runner.d.ts +3 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +31 -13
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +85 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/index.test.js +11 -1
- package/dist/index.test.js.map +1 -1
- package/dist/journal.d.ts +1 -0
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +4 -0
- package/dist/journal.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +93 -0
- package/dist/sync/area-ledger.d.ts.map +1 -0
- package/dist/sync/area-ledger.js +460 -0
- package/dist/sync/area-ledger.js.map +1 -0
- package/dist/sync/area-ledger.test.d.ts +2 -0
- package/dist/sync/area-ledger.test.d.ts.map +1 -0
- package/dist/sync/area-ledger.test.js +199 -0
- package/dist/sync/area-ledger.test.js.map +1 -0
- package/dist/sync/child-process-sync-worker.d.ts +53 -0
- package/dist/sync/child-process-sync-worker.d.ts.map +1 -0
- package/dist/sync/child-process-sync-worker.js +256 -0
- package/dist/sync/child-process-sync-worker.js.map +1 -0
- package/dist/sync/child-process-sync-worker.test.d.ts +2 -0
- package/dist/sync/child-process-sync-worker.test.d.ts.map +1 -0
- package/dist/sync/child-process-sync-worker.test.js +262 -0
- package/dist/sync/child-process-sync-worker.test.js.map +1 -0
- package/dist/sync/event-sync.d.ts +14 -4
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +177 -61
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +140 -10
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/index.d.ts +4 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +2 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +7 -2
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +22 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/state-store.d.ts +46 -1
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +263 -11
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync/state-store.test.js +125 -2
- package/dist/sync/state-store.test.js.map +1 -1
- package/dist/sync/sync-work.d.ts +2 -0
- package/dist/sync/sync-work.d.ts.map +1 -1
- package/dist/sync/sync-work.js.map +1 -1
- package/dist/sync/sync-worker-port.d.ts +59 -0
- package/dist/sync/sync-worker-port.d.ts.map +1 -0
- package/dist/sync/sync-worker-port.js +67 -0
- package/dist/sync/sync-worker-port.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import { execFileSync, fork, spawn } from "node:child_process";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as os from "node:os";
|
|
4
|
+
import * as path from "node:path";
|
|
5
|
+
import { afterEach, beforeAll, describe, expect, it } from "vitest";
|
|
6
|
+
import { DefaultParallelSyncCoordinator } from "./parallel-sync-coordinator.js";
|
|
7
|
+
import { ChildProcessSyncWorkerAdapter } from "./child-process-sync-worker.js";
|
|
8
|
+
import { StateStore } from "./state-store.js";
|
|
9
|
+
import { toWorkerExecuteMessage } from "./sync-worker-port.js";
|
|
10
|
+
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
|
11
|
+
const workerScript = path.join(repoRoot, "dist/bin/sync-runner-worker.js");
|
|
12
|
+
const adapters = [];
|
|
13
|
+
const roots = [];
|
|
14
|
+
beforeAll(() => {
|
|
15
|
+
execFileSync("pnpm", ["run", "build"], { cwd: repoRoot, stdio: "pipe" });
|
|
16
|
+
}, 120_000);
|
|
17
|
+
afterEach(async () => {
|
|
18
|
+
await Promise.all(adapters.splice(0).map((adapter) => adapter.shutdown()));
|
|
19
|
+
for (const root of roots.splice(0))
|
|
20
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
21
|
+
});
|
|
22
|
+
function worker(workerClass, env = {}, onProgress, options = {}) {
|
|
23
|
+
const adapter = new ChildProcessSyncWorkerAdapter({
|
|
24
|
+
workerClass,
|
|
25
|
+
workerScript: options.workerScript ?? workerScript,
|
|
26
|
+
env: { HQ_SYNC_WORKER_TEST_MODE: "1", ...env },
|
|
27
|
+
maxRestarts: options.maxRestarts,
|
|
28
|
+
readyTimeoutMs: options.readyTimeoutMs,
|
|
29
|
+
restartBackoffMs: options.restartBackoffMs ?? 10,
|
|
30
|
+
shutdownGraceMs: 100,
|
|
31
|
+
onProgress,
|
|
32
|
+
});
|
|
33
|
+
adapters.push(adapter);
|
|
34
|
+
return adapter;
|
|
35
|
+
}
|
|
36
|
+
function slice(id, pathName = `notes/${id}`) {
|
|
37
|
+
return {
|
|
38
|
+
id,
|
|
39
|
+
signal: new AbortController().signal,
|
|
40
|
+
work: { class: "realtime", source: "watcher", direction: "push", paths: [pathName], causality: `cause-${id}` },
|
|
41
|
+
paths: [{ canonicalPath: pathName, targetUid: "target-personal", areaId: "root:notes", epoch: Number(id.replace(/\D/g, "")) || 1 }],
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
function temporaryRoot() {
|
|
45
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), "hq-child-worker-"));
|
|
46
|
+
roots.push(root);
|
|
47
|
+
return root;
|
|
48
|
+
}
|
|
49
|
+
function waitFor(predicate, timeoutMs = 2_000) {
|
|
50
|
+
const deadline = Date.now() + timeoutMs;
|
|
51
|
+
return new Promise((resolve, reject) => {
|
|
52
|
+
const tick = () => {
|
|
53
|
+
if (predicate())
|
|
54
|
+
return resolve();
|
|
55
|
+
if (Date.now() >= deadline)
|
|
56
|
+
return reject(new Error("timed out waiting for child process state"));
|
|
57
|
+
setTimeout(tick, 10);
|
|
58
|
+
};
|
|
59
|
+
tick();
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
function isAlive(pid) {
|
|
63
|
+
try {
|
|
64
|
+
process.kill(pid, 0);
|
|
65
|
+
return true;
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function forkWorker() {
|
|
72
|
+
return fork(workerScript, [], { env: { ...process.env, HQ_SYNC_WORKER_TEST_MODE: "1" }, stdio: ["ignore", "ignore", "ignore", "ipc"] });
|
|
73
|
+
}
|
|
74
|
+
function workerFixture(source) {
|
|
75
|
+
const script = path.join(temporaryRoot(), "fixture.cjs");
|
|
76
|
+
fs.writeFileSync(script, source);
|
|
77
|
+
return script;
|
|
78
|
+
}
|
|
79
|
+
function waitForReady(child) {
|
|
80
|
+
return new Promise((resolve, reject) => {
|
|
81
|
+
child.once("error", reject);
|
|
82
|
+
child.on("message", (message) => {
|
|
83
|
+
if (typeof message === "object" && message !== null && message.type === "ready") {
|
|
84
|
+
resolve(message);
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
function waitForOutcome(child, requestId) {
|
|
90
|
+
return new Promise((resolve, reject) => {
|
|
91
|
+
const onMessage = (message) => {
|
|
92
|
+
if (typeof message !== "object" || message === null)
|
|
93
|
+
return;
|
|
94
|
+
const value = message;
|
|
95
|
+
if (value.type !== "outcome" || value.requestId !== requestId || value.outcome === undefined)
|
|
96
|
+
return;
|
|
97
|
+
child.removeListener("message", onMessage);
|
|
98
|
+
child.removeListener("error", onError);
|
|
99
|
+
resolve({ outcome: value.outcome });
|
|
100
|
+
};
|
|
101
|
+
const onError = (error) => {
|
|
102
|
+
child.removeListener("message", onMessage);
|
|
103
|
+
reject(error);
|
|
104
|
+
};
|
|
105
|
+
child.on("message", onMessage);
|
|
106
|
+
child.once("error", onError);
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
describe("ChildProcessSyncWorkerAdapter", () => {
|
|
110
|
+
it("starts isolated realtime and chore processes", async () => {
|
|
111
|
+
const realtime = worker("realtime");
|
|
112
|
+
const chore = worker("chore");
|
|
113
|
+
const [realtimeIdentity, choreIdentity] = await Promise.all([realtime.start(), chore.start()]);
|
|
114
|
+
expect(realtimeIdentity.pid).not.toBe(choreIdentity.pid);
|
|
115
|
+
expect(realtimeIdentity.moduleStateId).not.toBe(choreIdentity.moduleStateId);
|
|
116
|
+
await expect(realtime.execute(slice("1"))).resolves.toMatchObject({ paths: [{ status: "durable", epoch: 1 }] });
|
|
117
|
+
await expect(chore.execute(slice("2"))).resolves.toMatchObject({ paths: [{ status: "durable", epoch: 2 }] });
|
|
118
|
+
});
|
|
119
|
+
it("keeps the parent heartbeat and realtime process responsive while a chore parks", async () => {
|
|
120
|
+
let parked = false;
|
|
121
|
+
const chore = worker("chore", { HQ_SYNC_WORKER_TEST_PARK: "1" }, () => { parked = true; });
|
|
122
|
+
const realtime = worker("realtime");
|
|
123
|
+
const parkedWork = chore.execute(slice("3"));
|
|
124
|
+
await waitFor(() => parked);
|
|
125
|
+
const heartbeat = await new Promise((resolve) => setTimeout(() => resolve(Date.now()), 20));
|
|
126
|
+
const started = Date.now();
|
|
127
|
+
await expect(realtime.execute(slice("4"))).resolves.toMatchObject({ paths: [{ status: "durable" }] });
|
|
128
|
+
expect(Date.now() - started).toBeLessThan(500);
|
|
129
|
+
expect(heartbeat).toBeGreaterThan(0);
|
|
130
|
+
void parkedWork.catch(() => undefined);
|
|
131
|
+
});
|
|
132
|
+
it("returns a cancelled outcome without dispatching work aborted during child readiness", async () => {
|
|
133
|
+
const capture = path.join(temporaryRoot(), "dispatches.ndjson");
|
|
134
|
+
const delayedWorker = workerFixture([
|
|
135
|
+
"const fs=require('node:fs');",
|
|
136
|
+
"setTimeout(()=>process.send?.({protocol:1,type:'ready',pid:process.pid,moduleStateId:'delayed'}),60);",
|
|
137
|
+
"process.on('message',(message)=>{if(message.type!=='execute')return;fs.appendFileSync(process.env.HQ_CAPTURE_PATH,JSON.stringify(message)+'\\n');process.send?.({protocol:1,type:'outcome',requestId:message.requestId,outcome:{paths:message.slice.paths.map((path)=>({path:path.canonicalPath,epoch:path.epoch,status:'durable'}))}});});",
|
|
138
|
+
"setInterval(()=>undefined,1_000);",
|
|
139
|
+
].join(""));
|
|
140
|
+
const adapter = worker("realtime", { HQ_CAPTURE_PATH: capture }, undefined, { workerScript: delayedWorker });
|
|
141
|
+
const controller = new AbortController();
|
|
142
|
+
const pending = adapter.execute({ ...slice("cancelled-during-ready"), signal: controller.signal });
|
|
143
|
+
controller.abort();
|
|
144
|
+
await expect(pending).resolves.toMatchObject({ paths: [{ status: "retryable", reason: "worker cancelled before dispatch" }] });
|
|
145
|
+
expect(fs.existsSync(capture)).toBe(false);
|
|
146
|
+
});
|
|
147
|
+
it("fails a child startup that times out or sends an invalid protocol handshake", async () => {
|
|
148
|
+
const pidPath = path.join(temporaryRoot(), "handshake.pid");
|
|
149
|
+
const silentWorker = workerFixture([
|
|
150
|
+
"require('node:fs').writeFileSync(process.env.HQ_PID_PATH,String(process.pid));",
|
|
151
|
+
"setInterval(()=>undefined,1_000);",
|
|
152
|
+
].join(""));
|
|
153
|
+
const silent = worker("realtime", { HQ_PID_PATH: pidPath }, undefined, { workerScript: silentWorker, readyTimeoutMs: 50 });
|
|
154
|
+
await expect(silent.start()).rejects.toThrow("worker protocol handshake timed out after 50ms");
|
|
155
|
+
const silentPid = Number(fs.readFileSync(pidPath, "utf8"));
|
|
156
|
+
await waitFor(() => !isAlive(silentPid));
|
|
157
|
+
for (const message of [
|
|
158
|
+
"{protocol:2,type:'ready',pid:process.pid,moduleStateId:'wrong-version'}",
|
|
159
|
+
"{protocol:1,type:'ready',pid:process.pid}",
|
|
160
|
+
]) {
|
|
161
|
+
const invalidWorker = workerFixture(`process.send?.(${message});setInterval(()=>undefined,1_000);`);
|
|
162
|
+
const invalid = worker("realtime", {}, undefined, { workerScript: invalidWorker, readyTimeoutMs: 500 });
|
|
163
|
+
await expect(invalid.start()).rejects.toThrow("invalid ready message or protocol version skew (expected protocol 1)");
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
it.each(["pre-mutation", "post-mutation", "pre-result"])("treats SIGKILL at %s as retryable rather than implicit success", async (phase) => {
|
|
167
|
+
const crashing = worker("realtime", { HQ_SYNC_WORKER_TEST_CRASH_PHASE: phase });
|
|
168
|
+
const coordinator = new DefaultParallelSyncCoordinator({
|
|
169
|
+
realtimeWorker: crashing,
|
|
170
|
+
choreWorker: worker("chore"),
|
|
171
|
+
identifyPath: () => ({ targetUid: "target-personal", areaId: "root:notes" }),
|
|
172
|
+
planChorePaths: () => [],
|
|
173
|
+
});
|
|
174
|
+
await expect(coordinator.submit({ class: "realtime", source: "watcher", direction: "push", paths: ["notes/kill"], causality: `kill-${phase}` }))
|
|
175
|
+
.resolves.toMatchObject({ paths: [{ status: "retryable" }] });
|
|
176
|
+
expect(coordinator.inspect().leases).toEqual([]);
|
|
177
|
+
await coordinator.shutdown();
|
|
178
|
+
});
|
|
179
|
+
it("exits when its IPC parent disconnects and when that parent dies", async () => {
|
|
180
|
+
const direct = forkWorker();
|
|
181
|
+
const directReady = await waitForReady(direct);
|
|
182
|
+
direct.disconnect();
|
|
183
|
+
await waitFor(() => !isAlive(directReady.pid));
|
|
184
|
+
const parent = spawn(process.execPath, ["-e", [
|
|
185
|
+
"const {fork}=require('node:child_process');",
|
|
186
|
+
"const child=fork(process.env.HQ_WORKER_SCRIPT,[],{env:{...process.env,HQ_SYNC_WORKER_TEST_MODE:'1'},stdio:['ignore','pipe','pipe','ipc']});",
|
|
187
|
+
"child.once('message',(message)=>{console.log(message.pid);process.exit(0);});",
|
|
188
|
+
].join("")], { env: { ...process.env, HQ_WORKER_SCRIPT: workerScript }, stdio: ["ignore", "pipe", "pipe"] });
|
|
189
|
+
const pid = await new Promise((resolve, reject) => {
|
|
190
|
+
let stdout = "";
|
|
191
|
+
parent.stdout.on("data", (chunk) => { stdout += chunk.toString("utf8"); });
|
|
192
|
+
parent.once("error", reject);
|
|
193
|
+
parent.once("exit", () => resolve(Number(stdout.trim())));
|
|
194
|
+
});
|
|
195
|
+
expect(Number.isSafeInteger(pid)).toBe(true);
|
|
196
|
+
await waitFor(() => !isAlive(pid));
|
|
197
|
+
});
|
|
198
|
+
it("removes a dead child append lock through the existing PID stale-lock protocol", async () => {
|
|
199
|
+
const root = temporaryRoot();
|
|
200
|
+
const store = StateStore.open({
|
|
201
|
+
rootDir: root,
|
|
202
|
+
scopeId: "worker-lock-cleanup",
|
|
203
|
+
initialState: { values: [] },
|
|
204
|
+
reduce: (state, record) => ({ values: [...state.values, String(record.payload)] }),
|
|
205
|
+
});
|
|
206
|
+
const lockPath = path.join(store.stateDir, "append.lock");
|
|
207
|
+
const crashing = worker("chore", { HQ_SYNC_WORKER_TEST_LOCK_PATH: lockPath, HQ_SYNC_WORKER_TEST_CRASH_PHASE: "pre-result" });
|
|
208
|
+
await expect(crashing.execute(slice("5"))).rejects.toThrow("worker exited before outcome");
|
|
209
|
+
expect(fs.existsSync(lockPath)).toBe(true);
|
|
210
|
+
store.append(1, "recovered");
|
|
211
|
+
expect(fs.existsSync(lockPath)).toBe(false);
|
|
212
|
+
});
|
|
213
|
+
it("restarts within its bound, drains high-volume IPC, and leaves no child orphan", async () => {
|
|
214
|
+
const capture = path.join(temporaryRoot(), "messages.ndjson");
|
|
215
|
+
const adapter = worker("realtime", { HQ_SYNC_WORKER_TEST_CAPTURE_PATH: capture });
|
|
216
|
+
const first = await adapter.start();
|
|
217
|
+
process.kill(first.pid, "SIGKILL");
|
|
218
|
+
await waitFor(() => adapter.inspect()?.pid !== undefined && adapter.inspect()?.pid !== first.pid);
|
|
219
|
+
const outcomes = await Promise.all(Array.from({ length: 512 }, (_, index) => adapter.execute(slice(`load-${index + 10}`))));
|
|
220
|
+
expect(outcomes).toHaveLength(512);
|
|
221
|
+
expect(outcomes.every((outcome) => outcome.paths[0]?.status === "durable")).toBe(true);
|
|
222
|
+
const messages = fs.readFileSync(capture, "utf8");
|
|
223
|
+
expect(messages).toContain('"target-personal"');
|
|
224
|
+
expect(messages).toContain('"epoch"');
|
|
225
|
+
expect(messages).not.toContain("env-only-secret");
|
|
226
|
+
const finalPid = adapter.inspect().pid;
|
|
227
|
+
await adapter.shutdown();
|
|
228
|
+
await waitFor(() => !isAlive(finalPid));
|
|
229
|
+
});
|
|
230
|
+
it("applies restart backoff and exhaustion to queued work after a crash", async () => {
|
|
231
|
+
const adapter = worker("realtime", {}, undefined, { maxRestarts: 1, restartBackoffMs: 100 });
|
|
232
|
+
const first = await adapter.start();
|
|
233
|
+
process.kill(first.pid, "SIGKILL");
|
|
234
|
+
await waitFor(() => adapter.inspect() === undefined);
|
|
235
|
+
const queued = adapter.execute(slice("backoff"));
|
|
236
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
237
|
+
expect(adapter.inspect()).toBeUndefined();
|
|
238
|
+
await expect(queued).resolves.toMatchObject({ paths: [{ status: "durable" }] });
|
|
239
|
+
const restarted = adapter.inspect();
|
|
240
|
+
process.kill(restarted.pid, "SIGKILL");
|
|
241
|
+
await waitFor(() => adapter.inspect() === undefined);
|
|
242
|
+
await expect(adapter.execute(slice("exhausted"))).rejects.toThrow("worker restart limit exhausted after 1 restart(s)");
|
|
243
|
+
});
|
|
244
|
+
it("ignores late cancellation for a completed request ID", async () => {
|
|
245
|
+
const direct = forkWorker();
|
|
246
|
+
const ready = await waitForReady(direct);
|
|
247
|
+
try {
|
|
248
|
+
const first = toWorkerExecuteMessage(slice("reused-id"));
|
|
249
|
+
direct.send(first);
|
|
250
|
+
await expect(waitForOutcome(direct, first.requestId)).resolves.toMatchObject({ outcome: { paths: [{ status: "durable" }] } });
|
|
251
|
+
direct.send({ protocol: 1, type: "cancel", requestId: first.requestId });
|
|
252
|
+
const repeated = toWorkerExecuteMessage(slice("reused-id"));
|
|
253
|
+
direct.send(repeated);
|
|
254
|
+
await expect(waitForOutcome(direct, repeated.requestId)).resolves.toMatchObject({ outcome: { paths: [{ status: "durable" }] } });
|
|
255
|
+
}
|
|
256
|
+
finally {
|
|
257
|
+
direct.disconnect();
|
|
258
|
+
await waitFor(() => !isAlive(ready.pid));
|
|
259
|
+
}
|
|
260
|
+
});
|
|
261
|
+
});
|
|
262
|
+
//# sourceMappingURL=child-process-sync-worker.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"child-process-sync-worker.test.js","sourceRoot":"","sources":["../../src/sync/child-process-sync-worker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAqB,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACpE,OAAO,EAAE,8BAA8B,EAAE,MAAM,gCAAgC,CAAC;AAChF,OAAO,EAAE,6BAA6B,EAAsC,MAAM,gCAAgC,CAAC;AACnH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAG/D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAC5D,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,gCAAgC,CAAC,CAAC;AAC3E,MAAM,QAAQ,GAAoC,EAAE,CAAC;AACrD,MAAM,KAAK,GAAa,EAAE,CAAC;AAE3B,SAAS,CAAC,GAAG,EAAE;IACb,YAAY,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;AAC3E,CAAC,EAAE,OAAO,CAAC,CAAC;AAEZ,SAAS,CAAC,KAAK,IAAI,EAAE;IACnB,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAC3E,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACxF,CAAC,CAAC,CAAC;AAEH,SAAS,MAAM,CACb,WAAiC,EACjC,MAAyB,EAAE,EAC3B,UAAuB,EACvB,UAAuH,EAAE;IAEzH,MAAM,OAAO,GAAG,IAAI,6BAA6B,CAAC;QAChD,WAAW;QACX,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,YAAY;QAClD,GAAG,EAAE,EAAE,wBAAwB,EAAE,GAAG,EAAE,GAAG,GAAG,EAAE;QAC9C,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,EAAE;QAChD,eAAe,EAAE,GAAG;QACpB,UAAU;KACX,CAAC,CAAC;IACH,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,KAAK,CAAC,EAAU,EAAE,QAAQ,GAAG,SAAS,EAAE,EAAE;IACjD,OAAO;QACL,EAAE;QACF,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC,MAAM;QACpC,IAAI,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE;QAC9G,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;KACpI,CAAC;AACJ,CAAC;AAED,SAAS,aAAa;IACpB,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;IACxE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,OAAO,CAAC,SAAwB,EAAE,SAAS,GAAG,KAAK;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACxC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,IAAI,GAAG,GAAS,EAAE;YACtB,IAAI,SAAS,EAAE;gBAAE,OAAO,OAAO,EAAE,CAAC;YAClC,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ;gBAAE,OAAO,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC,CAAC;YAClG,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACvB,CAAC,CAAC;QACF,IAAI,EAAE,CAAC;IACT,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,OAAO,CAAC,GAAW;IAC1B,IAAI,CAAC;QAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAAC,OAAO,IAAI,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,KAAK,CAAC;IAAC,CAAC;AACpE,CAAC;AAED,SAAS,UAAU;IACjB,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,wBAAwB,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;AAC1I,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,aAAa,CAAC,CAAC;IACzD,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,KAAmB;IACvC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAgB,EAAE,EAAE;YACvC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAK,OAA8B,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBACxG,OAAO,CAAC,OAA0B,CAAC,CAAC;YACtC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,KAAmB,EAAE,SAAiB;IAC5D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,SAAS,GAAG,CAAC,OAAgB,EAAQ,EAAE;YAC3C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI;gBAAE,OAAO;YAC5D,MAAM,KAAK,GAAG,OAAkG,CAAC;YACjH,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;gBAAE,OAAO;YACrG,KAAK,CAAC,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC3C,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACvC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,CAAC,KAAY,EAAQ,EAAE;YACrC,KAAK,CAAC,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC3C,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC;QACF,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9B,MAAM,CAAC,gBAAgB,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAE/F,MAAM,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACzD,MAAM,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;QAC7E,MAAM,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAChH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC/G,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gFAAgF,EAAE,KAAK,IAAI,EAAE;QAC9F,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,EAAE,EAAE,wBAAwB,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3F,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpG,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACtG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,CAAC,SAAS,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qFAAqF,EAAE,KAAK,IAAI,EAAE;QACnG,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,mBAAmB,CAAC,CAAC;QAChE,MAAM,aAAa,GAAG,aAAa,CAAC;YAClC,8BAA8B;YAC9B,uGAAuG;YACvG,6UAA6U;YAC7U,mCAAmC;SACpC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACZ,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,CAAC;QAC7G,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,CAAC,wBAAwB,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACnG,UAAU,CAAC,KAAK,EAAE,CAAC;QAEnB,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,kCAAkC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC/H,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,eAAe,CAAC,CAAC;QAC5D,MAAM,YAAY,GAAG,aAAa,CAAC;YACjC,gFAAgF;YAChF,mCAAmC;SACpC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACZ,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3H,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC;QAC/F,MAAM,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QAC3D,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAEzC,KAAK,MAAM,OAAO,IAAI;YACpB,yEAAyE;YACzE,2CAA2C;SAC5C,EAAE,CAAC;YACF,MAAM,aAAa,GAAG,aAAa,CAAC,kBAAkB,OAAO,qCAAqC,CAAC,CAAC;YACpG,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,CAAC;YACxG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,sEAAsE,CAAC,CAAC;QACxH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,eAAe,EAAE,YAAY,CAAU,CAAC,CAAC,gEAAgE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QAClJ,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,EAAE,EAAE,+BAA+B,EAAE,KAAK,EAAE,CAAC,CAAC;QAChF,MAAM,WAAW,GAAG,IAAI,8BAA8B,CAAC;YACrD,cAAc,EAAE,QAAQ;YACxB,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC;YAC5B,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;YAC5E,cAAc,EAAE,GAAG,EAAE,CAAC,EAAE;SACzB,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,QAAQ,KAAK,EAAE,EAAE,CAAC,CAAC;aAC7I,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACjD,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QAC/E,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,CAAC,UAAU,EAAE,CAAC;QACpB,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE;gBAC5C,6CAA6C;gBAC7C,6IAA6I;gBAC7I,+EAA+E;aAChF,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,gBAAgB,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAC7G,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxD,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnF,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;QAC7F,MAAM,IAAI,GAAG,aAAa,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAuB;YAClD,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,qBAAqB;YAC9B,YAAY,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;SACnF,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,EAAE,EAAE,6BAA6B,EAAE,QAAQ,EAAE,+BAA+B,EAAE,YAAY,EAAE,CAAC,CAAC;QAC7H,MAAM,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;QAC3F,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QAC7B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;QAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,iBAAiB,CAAC,CAAC;QAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,EAAE,gCAAgC,EAAE,OAAO,EAAE,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACnC,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,GAAG,KAAK,SAAS,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,GAAG,KAAK,KAAK,CAAC,GAAG,CAAC,CAAC;QAClG,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,KAAK,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5H,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvF,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAClD,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAChD,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,EAAG,CAAC,GAAG,CAAC;QACxC,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;QACzB,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACnF,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC,CAAC;QAC7F,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACnC,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,SAAS,CAAC,CAAC;QAErD,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;QACjD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,aAAa,EAAE,CAAC;QAC1C,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QAEhF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,EAAG,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACvC,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,SAAS,CAAC,CAAC;QACrD,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,mDAAmD,CAAC,CAAC;IACzH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,sBAAsB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,MAAM,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAE9H,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;YACzE,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;YAC5D,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,MAAM,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACnI,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -150,6 +150,12 @@ export interface StartEventSyncOptions {
|
|
|
150
150
|
now?: () => number;
|
|
151
151
|
/** Clock seam for deferred publish cleanup tests. */
|
|
152
152
|
clock?: Clock;
|
|
153
|
+
/** Test seam for event-sync startup retry scheduling. */
|
|
154
|
+
retryInitialMs?: number;
|
|
155
|
+
/** Test seam for the capped event-sync startup retry delay. */
|
|
156
|
+
retryMaxMs?: number;
|
|
157
|
+
/** Test seam for full-jitter event-sync startup retries. */
|
|
158
|
+
retryRandom?: () => number;
|
|
153
159
|
/** Override the per-path trailing-publish interval. */
|
|
154
160
|
publishMinIntervalMs?: number;
|
|
155
161
|
/** Existing runner state directory for restart-durable publish suppression. */
|
|
@@ -174,6 +180,9 @@ export interface EventSyncHandles {
|
|
|
174
180
|
/** Tear down transport + receiver (runner shutdown path). */
|
|
175
181
|
dispose: () => Promise<void>;
|
|
176
182
|
}
|
|
183
|
+
/** Startup retry defaults: full jitter over a 30s → 10min exponential cap. */
|
|
184
|
+
export declare const DEFAULT_EVENT_SYNC_RETRY_INITIAL_MS = 30000;
|
|
185
|
+
export declare const DEFAULT_EVENT_SYNC_RETRY_MAX_MS: number;
|
|
177
186
|
/** Every v2 signal has identical correctness semantics: schedule one drain. */
|
|
178
187
|
export type RealtimeDrainSignal = "watcher" | "wake" | "high-water" | "reconnect" | "retry";
|
|
179
188
|
export interface UnifiedRealtimeSchedulerOptions {
|
|
@@ -207,10 +216,11 @@ export interface UnifiedRealtimeScheduler {
|
|
|
207
216
|
*/
|
|
208
217
|
export declare function createUnifiedRealtimeScheduler(options: UnifiedRealtimeSchedulerOptions): UnifiedRealtimeScheduler;
|
|
209
218
|
/**
|
|
210
|
-
* Bring up the publish + receive legs.
|
|
211
|
-
*
|
|
212
|
-
*
|
|
219
|
+
* Bring up the publish + receive legs. A failed initial subscribe deliberately
|
|
220
|
+
* keeps the watch runner poll-only, but returns an offline handle that retries
|
|
221
|
+
* the complete startup sequence forever. Once it succeeds, the same handle
|
|
222
|
+
* forwards publishes and disposes the live legs just like an immediate start.
|
|
213
223
|
*/
|
|
214
|
-
export declare function startEventSync(opts: StartEventSyncOptions): Promise<EventSyncHandles
|
|
224
|
+
export declare function startEventSync(opts: StartEventSyncOptions): Promise<EventSyncHandles>;
|
|
215
225
|
export {};
|
|
216
226
|
//# sourceMappingURL=event-sync.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event-sync.d.ts","sourceRoot":"","sources":["../../src/sync/event-sync.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EACL,SAAS,EAIV,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,iBAAiB,EAAE,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAGL,KAAK,YAAY,EAEjB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,
|
|
1
|
+
{"version":3,"file":"event-sync.d.ts","sourceRoot":"","sources":["../../src/sync/event-sync.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EACL,SAAS,EAIV,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,iBAAiB,EAAE,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAGL,KAAK,YAAY,EAEjB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAGL,KAAK,KAAK,EACV,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACrB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,KAAK,qBAAqB,EAC3B,MAAM,cAAc,CAAC;AAGtB;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAEtE;AAID,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,oDAAoD;IACpD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,WAAW,EAAE,wBAAwB,CAAC;IACtC,oFAAoF;IACpF,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,qBAAqB,CAAC;KACpC,CAAC;CACH;AAED,0EAA0E;AAC1E,KAAK,SAAS,GAAG,CACf,GAAG,EAAE,MAAM,EACX,IAAI,EAAE;IACJ,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,KACE,OAAO,CAAC;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;CAAE,CAAC,CAAC;AA6CvE;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,eAAe,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAwDjC;AAID;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,GAC9B,aAAa,CAsCf;AAwBD,MAAM,WAAW,0BAA0B;IACzC,mEAAmE;IACnE,OAAO,EAAE,qBAAqB,CAAC;IAC/B,+EAA+E;IAC/E,SAAS,EAAE,MAAM,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAChD;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,qBAAqB,KAAK,aAAa,CAAC;IAC1D,8CAA8C;IAC9C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAgBD;;;;;;;;;;;GAWG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,0BAA0B,GAC/B,aAAa,CA2Cf;AAID,yEAAyE;AACzE,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;AAErD,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,SAAS,EAAE,eAAe,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,eAAe,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACvC;;;OAGG;IACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC,gFAAgF;IAChF,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,gFAAgF;IAChF,GAAG,CAAC,EAAE,YAAY,CAAC;IAEnB,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACjE,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,qBAAqB,KAAK,aAAa,CAAC;IAC1D,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,eAAe,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAC9C,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IACzC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,yDAAyD;IACzD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,MAAM,CAAC;IAC3B,uDAAuD;IACvD,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,+EAA+E;IAC/E,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC,+EAA+E;IAC/E,6BAA6B,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACjE,wEAAwE;IACxE,iBAAiB,CAAC,EAAE,CAAC,WAAW,EAAE,OAAO,KAAK,IAAI,CAAC;CACpD;AAED,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,YAAY,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,CAAC;IAC/C,2CAA2C;IAC3C,QAAQ,EAAE,YAAY,CAAC;IACvB,4EAA4E;IAC5E,WAAW,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B;AAED,8EAA8E;AAC9E,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAC1D,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAI3D,+EAA+E;AAC/E,MAAM,MAAM,mBAAmB,GAC3B,SAAS,GACT,MAAM,GACN,YAAY,GACZ,WAAW,GACX,OAAO,CAAC;AAEZ,MAAM,WAAW,+BAA+B;IAC9C;;;OAGG;IACH,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1D,qEAAqE;IACrE,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IACxC,wEAAwE;IACxE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,mBAAmB,EAAE,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;CACjE;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAAC;IAC1C,4EAA4E;IAC5E,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,oFAAoF;IACpF,eAAe,CAAC,mBAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxF,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,+BAA+B,GACvC,wBAAwB,CAiI1B;AA8ID;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,gBAAgB,CAAC,CAgH3B"}
|
package/dist/sync/event-sync.js
CHANGED
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
import { SQSClient, ReceiveMessageCommand, DeleteMessageCommand, ChangeMessageVisibilityCommand, } from "@aws-sdk/client-sqs";
|
|
33
33
|
import { HttpPushTransport } from "./push-transport.js";
|
|
34
34
|
import { SqsPushReceiver, RealtimeWakeReceiver, } from "./push-receiver.js";
|
|
35
|
-
import { PushEventEmitter } from "../watcher.js";
|
|
35
|
+
import { PushEventEmitter, systemClock, } from "../watcher.js";
|
|
36
36
|
import { emitCloudTelemetry, } from "../telemetry-events.js";
|
|
37
37
|
import { createSyncLatencyMetricPublisher, } from "./metrics.js";
|
|
38
38
|
import { isEventSyncLocallyDisabled } from "./feature-flags.js";
|
|
@@ -246,6 +246,9 @@ export function createRefreshingSqsClient(opts) {
|
|
|
246
246
|
changeMessageVisibility: (args) => withRetry(() => inner.changeMessageVisibility(args)),
|
|
247
247
|
};
|
|
248
248
|
}
|
|
249
|
+
/** Startup retry defaults: full jitter over a 30s → 10min exponential cap. */
|
|
250
|
+
export const DEFAULT_EVENT_SYNC_RETRY_INITIAL_MS = 30_000;
|
|
251
|
+
export const DEFAULT_EVENT_SYNC_RETRY_MAX_MS = 10 * 60_000;
|
|
249
252
|
/**
|
|
250
253
|
* Coalesces all v2 notifications into a single drain bit. The cursor carried
|
|
251
254
|
* by a wake is intentionally ignored: duplicate, stale, and reordered wakes
|
|
@@ -394,35 +397,48 @@ export function createUnifiedRealtimeScheduler(options) {
|
|
|
394
397
|
},
|
|
395
398
|
};
|
|
396
399
|
}
|
|
397
|
-
/**
|
|
398
|
-
|
|
399
|
-
* on ANY startup failure — subscribe 5xx, tenant resolution failure, etc. —
|
|
400
|
-
* after logging the reason. The caller treats `null` as "today's behavior".
|
|
401
|
-
*/
|
|
402
|
-
export async function startEventSync(opts) {
|
|
400
|
+
/** Build the real publish + receive legs once a subscribe attempt succeeds. */
|
|
401
|
+
async function startLiveEventSync(opts) {
|
|
403
402
|
const log = opts.log ?? ((m) => console.error(m));
|
|
403
|
+
const tenantId = await opts.resolveTenantId();
|
|
404
|
+
const deviceId = opts.deviceId;
|
|
405
|
+
// The subscription response is the authenticated legacy inventory/grant.
|
|
406
|
+
// No email, environment, tenant, or queued hint can reach the live path
|
|
407
|
+
// without this server-issued queue and credential material.
|
|
408
|
+
const subscribe = opts.subscribe ??
|
|
409
|
+
((dev) => subscribeSyncReceive({
|
|
410
|
+
apiUrl: opts.apiUrl,
|
|
411
|
+
authToken: opts.authToken,
|
|
412
|
+
deviceId: dev,
|
|
413
|
+
}));
|
|
414
|
+
const initial = await subscribe(deviceId);
|
|
415
|
+
const syncBatchFn = opts.syncBatchFn ?? (opts.syncFn
|
|
416
|
+
? async ({ events, signal }) => {
|
|
417
|
+
for (const event of events)
|
|
418
|
+
await opts.syncFn({ event, signal });
|
|
419
|
+
}
|
|
420
|
+
: null);
|
|
421
|
+
if (!syncBatchFn) {
|
|
422
|
+
throw new Error("event-sync: syncBatchFn is required");
|
|
423
|
+
}
|
|
424
|
+
// ── publish leg (US-017) ──
|
|
425
|
+
const transport = opts.transport ??
|
|
426
|
+
new HttpPushTransport({
|
|
427
|
+
apiUrl: opts.apiUrl,
|
|
428
|
+
pushPath: "/v1/sync/push",
|
|
429
|
+
authToken: opts.authToken,
|
|
430
|
+
});
|
|
404
431
|
try {
|
|
405
|
-
const tenantId = await opts.resolveTenantId();
|
|
406
|
-
const deviceId = opts.deviceId;
|
|
407
|
-
// The subscription response is the authenticated legacy inventory/grant.
|
|
408
|
-
// No email, environment, tenant, or queued hint can reach the live path
|
|
409
|
-
// without this server-issued queue and credential material.
|
|
410
|
-
const subscribe = opts.subscribe ??
|
|
411
|
-
((dev) => subscribeSyncReceive({
|
|
412
|
-
apiUrl: opts.apiUrl,
|
|
413
|
-
authToken: opts.authToken,
|
|
414
|
-
deviceId: dev,
|
|
415
|
-
}));
|
|
416
|
-
const initial = await subscribe(deviceId);
|
|
417
|
-
// ── publish leg (US-017) ──
|
|
418
|
-
const transport = opts.transport ??
|
|
419
|
-
new HttpPushTransport({
|
|
420
|
-
apiUrl: opts.apiUrl,
|
|
421
|
-
pushPath: "/v1/sync/push",
|
|
422
|
-
authToken: opts.authToken,
|
|
423
|
-
});
|
|
424
432
|
await transport.start();
|
|
425
|
-
|
|
433
|
+
}
|
|
434
|
+
catch (startupError) {
|
|
435
|
+
await transport.dispose().catch(() => undefined);
|
|
436
|
+
throw startupError;
|
|
437
|
+
}
|
|
438
|
+
let emitter = null;
|
|
439
|
+
let receiver = null;
|
|
440
|
+
try {
|
|
441
|
+
emitter = new PushEventEmitter({
|
|
426
442
|
originTenantId: tenantId,
|
|
427
443
|
originDeviceId: deviceId,
|
|
428
444
|
transport,
|
|
@@ -446,15 +462,6 @@ export async function startEventSync(opts) {
|
|
|
446
462
|
initialPublishedContentHashes: opts.initialPublishedContentHashes,
|
|
447
463
|
});
|
|
448
464
|
// ── receive leg (US-018) ──
|
|
449
|
-
const syncBatchFn = opts.syncBatchFn ?? (opts.syncFn
|
|
450
|
-
? async ({ events, signal }) => {
|
|
451
|
-
for (const event of events)
|
|
452
|
-
await opts.syncFn({ event, signal });
|
|
453
|
-
}
|
|
454
|
-
: null);
|
|
455
|
-
if (!syncBatchFn) {
|
|
456
|
-
throw new Error("event-sync: syncBatchFn is required");
|
|
457
|
-
}
|
|
458
465
|
const publishMetric = initial.cloudWatch
|
|
459
466
|
? createSyncLatencyMetricPublisher(initial.cloudWatch)
|
|
460
467
|
: async () => undefined;
|
|
@@ -464,7 +471,7 @@ export async function startEventSync(opts) {
|
|
|
464
471
|
buildSqs: opts.buildSqs,
|
|
465
472
|
now: opts.now,
|
|
466
473
|
});
|
|
467
|
-
|
|
474
|
+
receiver = new SqsPushReceiver({
|
|
468
475
|
tenantId,
|
|
469
476
|
queueUrl: initial.queueUrl,
|
|
470
477
|
sqs,
|
|
@@ -476,30 +483,54 @@ export async function startEventSync(opts) {
|
|
|
476
483
|
onReceiveActivity: opts.onReceiveActivity,
|
|
477
484
|
});
|
|
478
485
|
await receiver.start();
|
|
479
|
-
log(`event-sync: live (tenant=${tenantId} device=${deviceId} queue=${initial.queueUrl})`);
|
|
480
|
-
void emitCloudTelemetry(opts.telemetryClient, {
|
|
481
|
-
eventName: "event_sync_live",
|
|
482
|
-
source: "event-sync",
|
|
483
|
-
properties: {
|
|
484
|
-
deviceId,
|
|
485
|
-
startupResult: "live",
|
|
486
|
-
receiveState: "enabled",
|
|
487
|
-
publishState: "enabled",
|
|
488
|
-
},
|
|
489
|
-
}, { claims: opts.telemetryClaims });
|
|
490
|
-
return {
|
|
491
|
-
publishBatch: (batch) => void emitter.emitForBatch(batch),
|
|
492
|
-
receiver,
|
|
493
|
-
ownDeviceId: deviceId,
|
|
494
|
-
dispose: async () => {
|
|
495
|
-
await receiver.dispose().catch(() => undefined);
|
|
496
|
-
emitter.dispose();
|
|
497
|
-
await transport.dispose().catch(() => undefined);
|
|
498
|
-
},
|
|
499
|
-
};
|
|
500
486
|
}
|
|
501
|
-
catch (
|
|
502
|
-
|
|
487
|
+
catch (startupError) {
|
|
488
|
+
await receiver?.dispose().catch(() => undefined);
|
|
489
|
+
emitter?.dispose();
|
|
490
|
+
await transport.dispose().catch(() => undefined);
|
|
491
|
+
throw startupError;
|
|
492
|
+
}
|
|
493
|
+
if (!emitter || !receiver) {
|
|
494
|
+
throw new Error("event-sync: live startup completed without handles");
|
|
495
|
+
}
|
|
496
|
+
log(`event-sync: live (tenant=${tenantId} device=${deviceId} queue=${initial.queueUrl})`);
|
|
497
|
+
void emitCloudTelemetry(opts.telemetryClient, {
|
|
498
|
+
eventName: "event_sync_live",
|
|
499
|
+
source: "event-sync",
|
|
500
|
+
properties: {
|
|
501
|
+
deviceId,
|
|
502
|
+
startupResult: "live",
|
|
503
|
+
receiveState: "enabled",
|
|
504
|
+
publishState: "enabled",
|
|
505
|
+
},
|
|
506
|
+
}, { claims: opts.telemetryClaims });
|
|
507
|
+
return {
|
|
508
|
+
publishBatch: (batch) => void emitter.emitForBatch(batch),
|
|
509
|
+
receiver,
|
|
510
|
+
ownDeviceId: deviceId,
|
|
511
|
+
dispose: async () => {
|
|
512
|
+
await receiver.dispose().catch(() => undefined);
|
|
513
|
+
emitter.dispose();
|
|
514
|
+
await transport.dispose().catch(() => undefined);
|
|
515
|
+
},
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
function describeStartupError(error) {
|
|
519
|
+
return error instanceof Error ? error.message : String(error);
|
|
520
|
+
}
|
|
521
|
+
/**
|
|
522
|
+
* Bring up the publish + receive legs. A failed initial subscribe deliberately
|
|
523
|
+
* keeps the watch runner poll-only, but returns an offline handle that retries
|
|
524
|
+
* the complete startup sequence forever. Once it succeeds, the same handle
|
|
525
|
+
* forwards publishes and disposes the live legs just like an immediate start.
|
|
526
|
+
*/
|
|
527
|
+
export async function startEventSync(opts) {
|
|
528
|
+
const log = opts.log ?? ((m) => console.error(m));
|
|
529
|
+
try {
|
|
530
|
+
return await startLiveEventSync(opts);
|
|
531
|
+
}
|
|
532
|
+
catch (error) {
|
|
533
|
+
log(`event-sync: startup failed, continuing poll-only: ${describeStartupError(error)}`);
|
|
503
534
|
void emitCloudTelemetry(opts.telemetryClient, {
|
|
504
535
|
eventName: "event_sync_live",
|
|
505
536
|
source: "event-sync",
|
|
@@ -510,7 +541,92 @@ export async function startEventSync(opts) {
|
|
|
510
541
|
publishState: "disabled",
|
|
511
542
|
},
|
|
512
543
|
}, { claims: opts.telemetryClaims });
|
|
513
|
-
|
|
544
|
+
const clock = opts.clock ?? systemClock;
|
|
545
|
+
const initialDelayMs = opts.retryInitialMs ?? DEFAULT_EVENT_SYNC_RETRY_INITIAL_MS;
|
|
546
|
+
const maxDelayMs = opts.retryMaxMs ?? DEFAULT_EVENT_SYNC_RETRY_MAX_MS;
|
|
547
|
+
if (!Number.isSafeInteger(initialDelayMs) || initialDelayMs < 0) {
|
|
548
|
+
throw new Error("event-sync: retryInitialMs must be a non-negative safe integer");
|
|
549
|
+
}
|
|
550
|
+
if (!Number.isSafeInteger(maxDelayMs) || maxDelayMs < initialDelayMs) {
|
|
551
|
+
throw new Error("event-sync: retryMaxMs must be a safe integer no smaller than retryInitialMs");
|
|
552
|
+
}
|
|
553
|
+
let disposed = false;
|
|
554
|
+
let live = null;
|
|
555
|
+
let retryTimer = null;
|
|
556
|
+
let retryAttempt = null;
|
|
557
|
+
let disposePromise = null;
|
|
558
|
+
let failures = 0;
|
|
559
|
+
const jitteredDelay = () => {
|
|
560
|
+
const ceiling = Math.min(maxDelayMs, initialDelayMs * 2 ** failures);
|
|
561
|
+
const random = Math.min(1, Math.max(0, (opts.retryRandom ?? Math.random)()));
|
|
562
|
+
return Math.floor(ceiling * random);
|
|
563
|
+
};
|
|
564
|
+
const retry = async () => {
|
|
565
|
+
if (disposed || live)
|
|
566
|
+
return;
|
|
567
|
+
try {
|
|
568
|
+
const handles = await startLiveEventSync(opts);
|
|
569
|
+
if (disposed) {
|
|
570
|
+
await handles.dispose();
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
live = handles;
|
|
574
|
+
}
|
|
575
|
+
catch (retryError) {
|
|
576
|
+
scheduleRetry(retryError);
|
|
577
|
+
}
|
|
578
|
+
};
|
|
579
|
+
const scheduleRetry = (retryError, initial = false) => {
|
|
580
|
+
if (disposed || live || retryTimer !== null)
|
|
581
|
+
return;
|
|
582
|
+
const delayMs = jitteredDelay();
|
|
583
|
+
failures += 1;
|
|
584
|
+
log(`event-sync: startup retry ${initial ? "scheduled" : "failed"} (attempt=${failures} reason=${describeStartupError(retryError)}); next retry in ${delayMs}ms`);
|
|
585
|
+
retryTimer = clock.setTimeout(() => {
|
|
586
|
+
retryTimer = null;
|
|
587
|
+
const attempt = retry();
|
|
588
|
+
retryAttempt = attempt;
|
|
589
|
+
void attempt.then(() => {
|
|
590
|
+
if (retryAttempt === attempt)
|
|
591
|
+
retryAttempt = null;
|
|
592
|
+
}, () => {
|
|
593
|
+
if (retryAttempt === attempt)
|
|
594
|
+
retryAttempt = null;
|
|
595
|
+
});
|
|
596
|
+
}, delayMs);
|
|
597
|
+
retryTimer.unref?.();
|
|
598
|
+
};
|
|
599
|
+
scheduleRetry(error, true);
|
|
600
|
+
const dispose = async () => {
|
|
601
|
+
if (disposePromise !== null)
|
|
602
|
+
return disposePromise;
|
|
603
|
+
disposed = true;
|
|
604
|
+
disposePromise = (async () => {
|
|
605
|
+
if (retryTimer !== null)
|
|
606
|
+
clock.clearTimeout(retryTimer);
|
|
607
|
+
retryTimer = null;
|
|
608
|
+
await retryAttempt;
|
|
609
|
+
const active = live;
|
|
610
|
+
live = null;
|
|
611
|
+
if (active)
|
|
612
|
+
await active.dispose();
|
|
613
|
+
})();
|
|
614
|
+
return disposePromise;
|
|
615
|
+
};
|
|
616
|
+
return {
|
|
617
|
+
publishBatch: (batch) => live?.publishBatch(batch),
|
|
618
|
+
receiver: {
|
|
619
|
+
start: async () => {
|
|
620
|
+
await live?.receiver.start();
|
|
621
|
+
},
|
|
622
|
+
dispose,
|
|
623
|
+
get connected() {
|
|
624
|
+
return live?.receiver.connected ?? false;
|
|
625
|
+
},
|
|
626
|
+
},
|
|
627
|
+
ownDeviceId: opts.deviceId,
|
|
628
|
+
dispose,
|
|
629
|
+
};
|
|
514
630
|
}
|
|
515
631
|
}
|
|
516
632
|
//# sourceMappingURL=event-sync.js.map
|