@opendatalabs/personal-server-ts-core 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/index.d.ts +64 -2
- package/dist/api/index.d.ts.map +1 -1
- package/dist/api/index.js +189 -45
- package/dist/api/index.js.map +1 -1
- package/dist/contracts/data.d.ts +16 -1
- package/dist/contracts/data.d.ts.map +1 -1
- package/dist/contracts/data.js +71 -19
- package/dist/contracts/data.js.map +1 -1
- package/dist/derivatives/api.d.ts +33 -0
- package/dist/derivatives/api.d.ts.map +1 -0
- package/dist/derivatives/api.js +216 -0
- package/dist/derivatives/api.js.map +1 -0
- package/dist/derivatives/compute.d.ts +106 -0
- package/dist/derivatives/compute.d.ts.map +1 -0
- package/dist/derivatives/compute.js +373 -0
- package/dist/derivatives/compute.js.map +1 -0
- package/dist/derivatives/index.d.ts +9 -0
- package/dist/derivatives/index.d.ts.map +1 -0
- package/dist/derivatives/index.js +9 -0
- package/dist/derivatives/index.js.map +1 -0
- package/dist/derivatives/inference.d.ts +102 -0
- package/dist/derivatives/inference.d.ts.map +1 -0
- package/dist/derivatives/inference.js +159 -0
- package/dist/derivatives/inference.js.map +1 -0
- package/dist/derivatives/prompt.d.ts +64 -0
- package/dist/derivatives/prompt.d.ts.map +1 -0
- package/dist/derivatives/prompt.js +186 -0
- package/dist/derivatives/prompt.js.map +1 -0
- package/dist/derivatives/registration.d.ts +53 -0
- package/dist/derivatives/registration.d.ts.map +1 -0
- package/dist/derivatives/registration.js +163 -0
- package/dist/derivatives/registration.js.map +1 -0
- package/dist/derivatives/scheduler.d.ts +56 -0
- package/dist/derivatives/scheduler.d.ts.map +1 -0
- package/dist/derivatives/scheduler.js +151 -0
- package/dist/derivatives/scheduler.js.map +1 -0
- package/dist/derivatives/store.d.ts +14 -0
- package/dist/derivatives/store.d.ts.map +1 -0
- package/dist/derivatives/store.js +75 -0
- package/dist/derivatives/store.js.map +1 -0
- package/dist/derivatives/types.d.ts +72 -0
- package/dist/derivatives/types.d.ts.map +1 -0
- package/dist/derivatives/types.js +25 -0
- package/dist/derivatives/types.js.map +1 -0
- package/dist/errors/catalog.d.ts +26 -0
- package/dist/errors/catalog.d.ts.map +1 -1
- package/dist/errors/catalog.js +39 -1
- package/dist/errors/catalog.js.map +1 -1
- package/dist/logging/access-log.d.ts +7 -1
- package/dist/logging/access-log.d.ts.map +1 -1
- package/dist/mcp/read-client.d.ts.map +1 -1
- package/dist/mcp/read-client.js +25 -1
- package/dist/mcp/read-client.js.map +1 -1
- package/dist/mcp/session.d.ts.map +1 -1
- package/dist/mcp/session.js +10 -1
- package/dist/mcp/session.js.map +1 -1
- package/dist/ports/index.d.ts +125 -0
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/index.js.map +1 -1
- package/dist/schemas/server-config.d.ts +12 -0
- package/dist/schemas/server-config.d.ts.map +1 -1
- package/dist/schemas/server-config.js +29 -0
- package/dist/schemas/server-config.js.map +1 -1
- package/dist/storage/adapters/index.d.ts +1 -1
- package/dist/storage/adapters/index.d.ts.map +1 -1
- package/dist/storage/adapters/index.js +1 -1
- package/dist/storage/adapters/index.js.map +1 -1
- package/dist/storage/adapters/vana.d.ts +2 -0
- package/dist/storage/adapters/vana.d.ts.map +1 -1
- package/dist/storage/adapters/vana.js +5 -1
- package/dist/storage/adapters/vana.js.map +1 -1
- package/dist/storage/index/types.d.ts +1 -0
- package/dist/storage/index/types.d.ts.map +1 -1
- package/dist/sync/data-point-feed.d.ts +35 -0
- package/dist/sync/data-point-feed.d.ts.map +1 -0
- package/dist/sync/data-point-feed.js +122 -0
- package/dist/sync/data-point-feed.js.map +1 -0
- package/dist/sync/delete-data-port.d.ts +50 -0
- package/dist/sync/delete-data-port.d.ts.map +1 -0
- package/dist/sync/delete-data-port.js +244 -0
- package/dist/sync/delete-data-port.js.map +1 -0
- package/dist/sync/engine/sync-manager.d.ts +14 -4
- package/dist/sync/engine/sync-manager.d.ts.map +1 -1
- package/dist/sync/engine/sync-manager.js +53 -8
- package/dist/sync/engine/sync-manager.js.map +1 -1
- package/dist/sync/index.d.ts +7 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +7 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/pending-blob-deletions.d.ts +29 -0
- package/dist/sync/pending-blob-deletions.d.ts.map +1 -0
- package/dist/sync/pending-blob-deletions.js +116 -0
- package/dist/sync/pending-blob-deletions.js.map +1 -0
- package/dist/sync/scope-deletions.d.ts +155 -0
- package/dist/sync/scope-deletions.d.ts.map +1 -0
- package/dist/sync/scope-deletions.js +225 -0
- package/dist/sync/scope-deletions.js.map +1 -0
- package/dist/sync/tombstone.d.ts +27 -0
- package/dist/sync/tombstone.d.ts.map +1 -0
- package/dist/sync/tombstone.js +28 -0
- package/dist/sync/tombstone.js.map +1 -0
- package/dist/sync/workers/delete.d.ts +152 -31
- package/dist/sync/workers/delete.d.ts.map +1 -1
- package/dist/sync/workers/delete.js +461 -27
- package/dist/sync/workers/delete.js.map +1 -1
- package/dist/sync/workers/download.d.ts +52 -1
- package/dist/sync/workers/download.d.ts.map +1 -1
- package/dist/sync/workers/download.js +129 -2
- package/dist/sync/workers/download.js.map +1 -1
- package/dist/sync/workers/upload.d.ts +31 -1
- package/dist/sync/workers/upload.d.ts.map +1 -1
- package/dist/sync/workers/upload.js +108 -4
- package/dist/sync/workers/upload.js.map +1 -1
- package/dist/test-utils/index.d.ts +1 -0
- package/dist/test-utils/index.d.ts.map +1 -1
- package/dist/test-utils/index.js +1 -0
- package/dist/test-utils/index.js.map +1 -1
- package/dist/test-utils/memory-storage.d.ts +12 -0
- package/dist/test-utils/memory-storage.d.ts.map +1 -0
- package/dist/test-utils/memory-storage.js +146 -0
- package/dist/test-utils/memory-storage.js.map +1 -0
- package/package.json +5 -1
|
@@ -1,40 +1,474 @@
|
|
|
1
|
+
import { computeDataPointId } from "../data-point-id.js";
|
|
2
|
+
import { deletionTimestamp, isEntryCoveredByTombstone, tombstoneVersion, } from "../scope-deletions.js";
|
|
1
3
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* in place until DPv2 grows a deletion API.
|
|
4
|
+
* Exact blob deletes sent per pass. vana-storage allows 30 DELETE requests
|
|
5
|
+
* per owner per 60 seconds. A DELETE request's own pass and the next sync
|
|
6
|
+
* cycle's retry pass can land inside one window, so each pass sends at most
|
|
7
|
+
* half the budget; whatever does not fit is queued and drained on later
|
|
8
|
+
* cycles, and a 429 simply re-queues its key.
|
|
9
|
+
*/
|
|
10
|
+
export const BLOB_DELETE_BATCH_SIZE = 15;
|
|
11
|
+
/**
|
|
12
|
+
* The storage keys a tombstone covers for `scope`. A re-add registered after
|
|
13
|
+
* the tombstone lives under a higher version key, so it is never in this set
|
|
14
|
+
* and survives by construction. With no tombstone version (never registered,
|
|
15
|
+
* or the gateway did not report one) only the local entries' keys can be
|
|
16
|
+
* enumerated.
|
|
16
17
|
*/
|
|
17
|
-
export
|
|
18
|
-
const {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
const PAGE_SIZE =
|
|
18
|
+
export function planBlobDeletions(storage, scope, tombstoneVersionValue) {
|
|
19
|
+
const tombstone = { version: tombstoneVersionValue };
|
|
20
|
+
const last = tombstoneVersionValue === null ? null : BigInt(tombstoneVersionValue);
|
|
21
|
+
const keys = new Set();
|
|
22
|
+
const PAGE_SIZE = 500;
|
|
22
23
|
for (let offset = 0;; offset += PAGE_SIZE) {
|
|
23
24
|
const entries = storage.listVersions(scope, { limit: PAGE_SIZE, offset });
|
|
24
|
-
|
|
25
|
+
for (const entry of entries) {
|
|
26
|
+
if (!isEntryCoveredByTombstone(entry, tombstone))
|
|
27
|
+
continue;
|
|
28
|
+
const version = BigInt(entry.version);
|
|
29
|
+
if (last !== null && version >= 1n && version <= last)
|
|
30
|
+
continue;
|
|
31
|
+
keys.add(version.toString());
|
|
32
|
+
}
|
|
25
33
|
if (entries.length < PAGE_SIZE)
|
|
26
34
|
break;
|
|
27
35
|
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
36
|
+
return {
|
|
37
|
+
keys: [...keys].sort((a, b) => (BigInt(a) < BigInt(b) ? -1 : 1)),
|
|
38
|
+
range: last === null || last < 1n ? null : { from: "1", to: last.toString() },
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/** Take up to `count` versions from the head of a range; `rest` is what remains. */
|
|
42
|
+
function takeFromRange(range, count) {
|
|
43
|
+
const from = BigInt(range.from);
|
|
44
|
+
const to = BigInt(range.to);
|
|
45
|
+
const versions = [];
|
|
46
|
+
let cursor = from;
|
|
47
|
+
while (cursor <= to && versions.length < count) {
|
|
48
|
+
versions.push(cursor.toString());
|
|
49
|
+
cursor += 1n;
|
|
32
50
|
}
|
|
33
51
|
return {
|
|
52
|
+
versions,
|
|
53
|
+
rest: cursor <= to ? { from: cursor.toString(), to: range.to } : null,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
function rangeSize(range) {
|
|
57
|
+
return BigInt(range.to) - BigInt(range.from) + 1n;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Count of blob keys a marker list still stands for, clamped for reporting.
|
|
61
|
+
* An unexpanded marker (range not known yet) counts as one.
|
|
62
|
+
*/
|
|
63
|
+
function countPendingKeys(markers) {
|
|
64
|
+
let total = 0n;
|
|
65
|
+
for (const marker of markers) {
|
|
66
|
+
if (marker.version !== null)
|
|
67
|
+
total += 1n;
|
|
68
|
+
else if (marker.range)
|
|
69
|
+
total += rangeSize(marker.range);
|
|
70
|
+
else
|
|
71
|
+
total += 1n;
|
|
72
|
+
}
|
|
73
|
+
return total > BigInt(Number.MAX_SAFE_INTEGER)
|
|
74
|
+
? Number.MAX_SAFE_INTEGER
|
|
75
|
+
: Number(total);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Durable scope deletion, in the only order that cannot resurrect data:
|
|
79
|
+
*
|
|
80
|
+
* 1. gateway tombstone (owner-signed AddData at current + 1; the fact)
|
|
81
|
+
* 2. storage blobs (exact keys of every version the tombstone covers)
|
|
82
|
+
* 3. local copy (index rows + files + sidecars)
|
|
83
|
+
*
|
|
84
|
+
* The local delete never runs before the gateway has acknowledged: until the
|
|
85
|
+
* registry says "deleted", the next sync cycle would re-pull the point from
|
|
86
|
+
* the still-live row. Step 2 deletes exact per-version keys, never the scope
|
|
87
|
+
* prefix, so a re-add another replica lands at any time keeps its ciphertext
|
|
88
|
+
* (it is registered above the tombstone version and stored under that key).
|
|
89
|
+
* Keys that could not be deleted in this pass are queued as exact retry
|
|
90
|
+
* markers; step 3 does not wait for them because the tombstone already makes
|
|
91
|
+
* the deletion stick.
|
|
92
|
+
*
|
|
93
|
+
* Not serialised here: the sync manager runs this under the same lock as its
|
|
94
|
+
* upload/download cycles so a delete cannot interleave with an upload of the
|
|
95
|
+
* same scope on this replica (see engine/sync-manager.ts). Uploads racing on
|
|
96
|
+
* another replica are handled by the upload worker's deletion guard and the
|
|
97
|
+
* download worker's reconcile.
|
|
98
|
+
*/
|
|
99
|
+
export async function deleteScope(deps, scope) {
|
|
100
|
+
const { storage, deleteData, pendingBlobDeletions, scopeDeletions, logger } = deps;
|
|
101
|
+
const now = deps.now ?? (() => new Date());
|
|
102
|
+
const dataPointId = deps.serverOwner
|
|
103
|
+
? computeDataPointId(deps.serverOwner, scope)
|
|
104
|
+
: null;
|
|
105
|
+
const result = {
|
|
34
106
|
scope,
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
107
|
+
dataPointId,
|
|
108
|
+
durable: false,
|
|
109
|
+
steps: {
|
|
110
|
+
gateway: { status: "skipped", reason: "sync-disabled" },
|
|
111
|
+
storage: { status: "skipped", reason: "sync-disabled" },
|
|
112
|
+
local: { status: "skipped" },
|
|
113
|
+
},
|
|
114
|
+
pendingBlobDeletion: false,
|
|
115
|
+
};
|
|
116
|
+
if (deleteData) {
|
|
117
|
+
// 1. Gateway tombstone: the durable fact. Nothing else happens if it
|
|
118
|
+
// fails, so the owner gets a clear "nothing was deleted" instead of a
|
|
119
|
+
// local delete that sync silently undoes.
|
|
120
|
+
let tombstoneVersionValue = null;
|
|
121
|
+
let tombstoneKnown = false;
|
|
122
|
+
try {
|
|
123
|
+
let outcome = await deleteData.tombstone(scope);
|
|
124
|
+
if (outcome.status === "not-registered") {
|
|
125
|
+
// "Nothing to tombstone" is only durable if the registry still has
|
|
126
|
+
// no row. A registration racing our lookup would otherwise survive
|
|
127
|
+
// the local delete and sync would restore the scope: re-read, and
|
|
128
|
+
// tombstone the row that appeared. If it keeps appearing, or the
|
|
129
|
+
// registry cannot be asked, refuse rather than delete on a guess.
|
|
130
|
+
let registry = await registryState(deps, scope);
|
|
131
|
+
if (registry.status === "live") {
|
|
132
|
+
outcome = await deleteData.tombstone(scope);
|
|
133
|
+
registry = await registryState(deps, scope);
|
|
134
|
+
}
|
|
135
|
+
if (outcome.status === "not-registered" &&
|
|
136
|
+
registry.status !== "deleted-or-absent") {
|
|
137
|
+
throw registry.status === "unknown"
|
|
138
|
+
? registry.error
|
|
139
|
+
: new Error("Scope was registered concurrently while it was being deleted; retry the delete");
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
if (outcome.status === "not-registered") {
|
|
143
|
+
result.steps.gateway = { status: "skipped", reason: "not-registered" };
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
tombstoneKnown = true;
|
|
147
|
+
tombstoneVersionValue =
|
|
148
|
+
outcome.version === null
|
|
149
|
+
? null
|
|
150
|
+
: tombstoneVersion({ expectedVersion: outcome.version });
|
|
151
|
+
result.steps.gateway = {
|
|
152
|
+
status: "ok",
|
|
153
|
+
...(outcome.status === "already-deleted" && {
|
|
154
|
+
reason: "already-deleted",
|
|
155
|
+
}),
|
|
156
|
+
version: outcome.version,
|
|
157
|
+
deletedAt: outcome.deletedAt,
|
|
158
|
+
};
|
|
159
|
+
// Reads on this replica must refuse the scope from this point on,
|
|
160
|
+
// without waiting for the feed to echo the tombstone back. A missing
|
|
161
|
+
// `deletedAt` (gateway did not echo one) is taken as "now": later
|
|
162
|
+
// than any local entry the deletion covers, so nothing slips through.
|
|
163
|
+
scopeDeletions?.markDeleted(scope, {
|
|
164
|
+
deletedAt: outcome.deletedAt ?? now().toISOString(),
|
|
165
|
+
version: outcome.version,
|
|
166
|
+
}, "local-delete");
|
|
167
|
+
}
|
|
168
|
+
result.durable = true;
|
|
169
|
+
}
|
|
170
|
+
catch (err) {
|
|
171
|
+
const message = errorMessage(err);
|
|
172
|
+
result.steps.gateway = { status: "failed", error: message };
|
|
173
|
+
result.steps.storage = { status: "skipped", reason: "gateway-failed" };
|
|
174
|
+
result.steps.local = { status: "skipped", reason: "gateway-failed" };
|
|
175
|
+
logger.error({ scope, dataPointId, error: message }, "Gateway tombstone failed; scope NOT deleted (local copy kept so sync cannot resurrect a half-deleted scope)");
|
|
176
|
+
return result;
|
|
177
|
+
}
|
|
178
|
+
// 2. Storage blobs: exact keys, one pass of at most a batch, the rest
|
|
179
|
+
// queued. A tombstone whose version the gateway did not report cannot be
|
|
180
|
+
// expanded into registry keys yet; a version-less marker asks the retry
|
|
181
|
+
// to expand it once the registry answers.
|
|
182
|
+
const plan = planBlobDeletions(storage, scope, tombstoneVersionValue);
|
|
183
|
+
const batch = plan.keys.slice(0, BLOB_DELETE_BATCH_SIZE);
|
|
184
|
+
const leftovers = plan.keys
|
|
185
|
+
.slice(BLOB_DELETE_BATCH_SIZE)
|
|
186
|
+
.map((version) => ({ scope, version }));
|
|
187
|
+
if (plan.range) {
|
|
188
|
+
const taken = takeFromRange(plan.range, BLOB_DELETE_BATCH_SIZE - batch.length);
|
|
189
|
+
batch.push(...taken.versions);
|
|
190
|
+
if (taken.rest)
|
|
191
|
+
leftovers.push({ scope, version: null, range: taken.rest });
|
|
192
|
+
}
|
|
193
|
+
if (tombstoneKnown && tombstoneVersionValue === null) {
|
|
194
|
+
leftovers.push({ scope, version: null });
|
|
195
|
+
}
|
|
196
|
+
const storageStep = await deleteBlobKeys({ deleteData, pendingBlobDeletions, logger }, scope, batch, leftovers);
|
|
197
|
+
result.steps.storage = storageStep.step;
|
|
198
|
+
result.pendingBlobDeletion = storageStep.pending > 0;
|
|
199
|
+
}
|
|
200
|
+
// 3. Local copy. Safe now: the registry row is a tombstone (or the point
|
|
201
|
+
// was never registered / sync is off), so nothing can re-pull it.
|
|
202
|
+
try {
|
|
203
|
+
const deletedCount = await storage.deleteScope(scope);
|
|
204
|
+
result.steps.local = { status: "ok", deletedCount };
|
|
205
|
+
}
|
|
206
|
+
catch (err) {
|
|
207
|
+
const message = errorMessage(err);
|
|
208
|
+
result.steps.local = { status: "failed", error: message };
|
|
209
|
+
logger.error({ scope, error: message }, "Local scope deletion failed");
|
|
210
|
+
}
|
|
211
|
+
logger.info({
|
|
212
|
+
scope,
|
|
213
|
+
dataPointId,
|
|
214
|
+
durable: result.durable,
|
|
215
|
+
gateway: result.steps.gateway.status,
|
|
216
|
+
storage: result.steps.storage.status,
|
|
217
|
+
local: result.steps.local.status,
|
|
218
|
+
}, "Scope deletion finished");
|
|
219
|
+
return result;
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Delete one pass worth of exact keys and reconcile the retry markers:
|
|
223
|
+
* completed keys (deleted or already absent) leave the store, failed and
|
|
224
|
+
* not-yet-attempted keys enter it.
|
|
225
|
+
*/
|
|
226
|
+
async function deleteBlobKeys(deps, scope, batch, leftovers) {
|
|
227
|
+
const { deleteData, pendingBlobDeletions, logger } = deps;
|
|
228
|
+
let outcome;
|
|
229
|
+
try {
|
|
230
|
+
outcome =
|
|
231
|
+
batch.length > 0 && deleteData
|
|
232
|
+
? await deleteData.deleteBlobVersions(scope, batch)
|
|
233
|
+
: { deleted: [], missing: [], failed: [] };
|
|
234
|
+
}
|
|
235
|
+
catch (err) {
|
|
236
|
+
// The port reports per-key failures; a thrown error means none of the
|
|
237
|
+
// batch can be trusted as done.
|
|
238
|
+
outcome = {
|
|
239
|
+
deleted: [],
|
|
240
|
+
missing: [],
|
|
241
|
+
failed: batch.map((version) => ({ version, error: errorMessage(err) })),
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
const completed = [...outcome.deleted, ...outcome.missing].map((version) => ({ scope, version }));
|
|
245
|
+
const remaining = [
|
|
246
|
+
...outcome.failed.map(({ version }) => ({ scope, version })),
|
|
247
|
+
...leftovers,
|
|
248
|
+
];
|
|
249
|
+
const remainingKeys = countPendingKeys(remaining);
|
|
250
|
+
let recorded = remainingKeys;
|
|
251
|
+
if (pendingBlobDeletions) {
|
|
252
|
+
try {
|
|
253
|
+
if (completed.length > 0)
|
|
254
|
+
await pendingBlobDeletions.remove(completed);
|
|
255
|
+
if (remaining.length > 0)
|
|
256
|
+
await pendingBlobDeletions.add(remaining);
|
|
257
|
+
}
|
|
258
|
+
catch (markerErr) {
|
|
259
|
+
recorded = 0;
|
|
260
|
+
logger.error({ scope, error: errorMessage(markerErr), keys: remainingKeys }, "Could not record pending blob deletion markers");
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
else if (remaining.length > 0) {
|
|
264
|
+
recorded = 0;
|
|
265
|
+
logger.error({ scope, keys: remainingKeys }, "Blob deletions left unfinished with no marker store to retry them");
|
|
266
|
+
}
|
|
267
|
+
const counts = {
|
|
268
|
+
blobsDeleted: outcome.deleted.length,
|
|
269
|
+
blobsMissing: outcome.missing.length,
|
|
270
|
+
blobsPending: remainingKeys,
|
|
271
|
+
};
|
|
272
|
+
if (outcome.failed.length > 0) {
|
|
273
|
+
const first = outcome.failed[0];
|
|
274
|
+
logger.warn({
|
|
275
|
+
scope,
|
|
276
|
+
failed: outcome.failed.length,
|
|
277
|
+
pending: recorded,
|
|
278
|
+
error: first.error,
|
|
279
|
+
}, "Storage blob deletion failed for some keys after gateway tombstone; will retry");
|
|
280
|
+
return {
|
|
281
|
+
step: {
|
|
282
|
+
status: "failed",
|
|
283
|
+
error: `${outcome.failed.length} blob delete(s) failed: ${first.error}`,
|
|
284
|
+
...counts,
|
|
285
|
+
},
|
|
286
|
+
pending: recorded,
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
if (remaining.length > 0) {
|
|
290
|
+
logger.info({ scope, ...counts }, "Storage blob deletion continues on later sync cycles (rate-limited batch)");
|
|
291
|
+
return { step: { status: "deferred", ...counts }, pending: recorded };
|
|
292
|
+
}
|
|
293
|
+
return { step: { status: "ok", ...counts }, pending: 0 };
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Finish blob deletions whose tombstone landed but whose storage deletes did
|
|
297
|
+
* not: one batch of exact keys per sync cycle, taken from exact-key markers
|
|
298
|
+
* first and then from the head of range markers (which advance in place),
|
|
299
|
+
* each key retried until storage acknowledges (2xx or 404). Unexpanded
|
|
300
|
+
* markers (the gateway did not report a tombstone version at delete time)
|
|
301
|
+
* are expanded from the registry first; if the registry shows the scope live
|
|
302
|
+
* again the old tombstone's version is unknowable, so the marker is dropped
|
|
303
|
+
* and the pre-tombstone ciphertext stays (owner-decryptable only) until the
|
|
304
|
+
* next delete of that scope.
|
|
305
|
+
*/
|
|
306
|
+
export async function retryPendingBlobDeletions(deps) {
|
|
307
|
+
const { deleteData, pendingBlobDeletions, logger } = deps;
|
|
308
|
+
const result = {
|
|
309
|
+
completed: [],
|
|
310
|
+
superseded: [],
|
|
311
|
+
failed: [],
|
|
312
|
+
remaining: 0,
|
|
313
|
+
};
|
|
314
|
+
if (!deleteData || !pendingBlobDeletions)
|
|
315
|
+
return result;
|
|
316
|
+
let markers = await pendingBlobDeletions.list();
|
|
317
|
+
if (markers.length === 0)
|
|
318
|
+
return result;
|
|
319
|
+
// Expand markers whose tombstone version was unknown at delete time.
|
|
320
|
+
for (const marker of markers.filter((key) => key.version === null && !key.range)) {
|
|
321
|
+
const registry = await registryState(deps, marker.scope);
|
|
322
|
+
if (registry.status === "unknown") {
|
|
323
|
+
result.failed.push({
|
|
324
|
+
scope: marker.scope,
|
|
325
|
+
version: null,
|
|
326
|
+
error: registry.error.message,
|
|
327
|
+
});
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
330
|
+
if (registry.status === "live") {
|
|
331
|
+
await pendingBlobDeletions.remove([marker]);
|
|
332
|
+
result.superseded.push(marker.scope);
|
|
333
|
+
logger.warn({
|
|
334
|
+
scope: marker.scope,
|
|
335
|
+
dataPointId: registry.record.id,
|
|
336
|
+
version: registry.record.expectedVersion,
|
|
337
|
+
}, "Scope was re-added after its tombstone; dropping the unexpanded blob deletion marker so the live version's ciphertext survives");
|
|
338
|
+
continue;
|
|
339
|
+
}
|
|
340
|
+
const plan = planBlobDeletions(deps.storage ?? { listVersions: () => [] }, marker.scope, registry.status === "deleted" ? registry.version : null);
|
|
341
|
+
const expanded = plan.keys.map((version) => ({
|
|
342
|
+
scope: marker.scope,
|
|
343
|
+
version,
|
|
344
|
+
}));
|
|
345
|
+
if (plan.range) {
|
|
346
|
+
expanded.push({ scope: marker.scope, version: null, range: plan.range });
|
|
347
|
+
}
|
|
348
|
+
await pendingBlobDeletions.remove([marker]);
|
|
349
|
+
await pendingBlobDeletions.add(expanded);
|
|
350
|
+
}
|
|
351
|
+
markers = await pendingBlobDeletions.list();
|
|
352
|
+
// One rate-limited batch in stored order: exact keys, then range heads.
|
|
353
|
+
let budget = BLOB_DELETE_BATCH_SIZE;
|
|
354
|
+
const byScope = new Map();
|
|
355
|
+
const exactOrigin = new Set();
|
|
356
|
+
const advancedRanges = [];
|
|
357
|
+
const enqueue = (scope, version) => {
|
|
358
|
+
const versions = byScope.get(scope) ?? [];
|
|
359
|
+
versions.push(version);
|
|
360
|
+
byScope.set(scope, versions);
|
|
38
361
|
};
|
|
362
|
+
for (const marker of markers) {
|
|
363
|
+
if (budget === 0)
|
|
364
|
+
break;
|
|
365
|
+
if (marker.version !== null) {
|
|
366
|
+
enqueue(marker.scope, marker.version);
|
|
367
|
+
exactOrigin.add(`${marker.scope}\u0000${marker.version}`);
|
|
368
|
+
budget -= 1;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
for (const marker of markers) {
|
|
372
|
+
if (budget === 0)
|
|
373
|
+
break;
|
|
374
|
+
if (marker.version === null && marker.range) {
|
|
375
|
+
const taken = takeFromRange(marker.range, budget);
|
|
376
|
+
for (const version of taken.versions)
|
|
377
|
+
enqueue(marker.scope, version);
|
|
378
|
+
budget -= taken.versions.length;
|
|
379
|
+
advancedRanges.push({
|
|
380
|
+
old: marker,
|
|
381
|
+
next: taken.rest
|
|
382
|
+
? { scope: marker.scope, version: null, range: taken.rest }
|
|
383
|
+
: null,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
for (const [scope, versions] of byScope) {
|
|
388
|
+
let outcome;
|
|
389
|
+
try {
|
|
390
|
+
outcome = await deleteData.deleteBlobVersions(scope, versions);
|
|
391
|
+
}
|
|
392
|
+
catch (err) {
|
|
393
|
+
const message = errorMessage(err);
|
|
394
|
+
outcome = {
|
|
395
|
+
deleted: [],
|
|
396
|
+
missing: [],
|
|
397
|
+
failed: versions.map((version) => ({ version, error: message })),
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
const completed = [...outcome.deleted, ...outcome.missing]
|
|
401
|
+
.filter((version) => exactOrigin.has(`${scope}\u0000${version}`))
|
|
402
|
+
.map((version) => ({ scope, version }));
|
|
403
|
+
if (completed.length > 0)
|
|
404
|
+
await pendingBlobDeletions.remove(completed);
|
|
405
|
+
result.completed.push(...[...outcome.deleted, ...outcome.missing].map((version) => ({ scope, version })));
|
|
406
|
+
// A failed key that came from a range becomes an exact marker; a failed
|
|
407
|
+
// exact marker simply stays.
|
|
408
|
+
const failedFromRange = outcome.failed
|
|
409
|
+
.filter(({ version }) => !exactOrigin.has(`${scope}\u0000${version}`))
|
|
410
|
+
.map(({ version }) => ({ scope, version }));
|
|
411
|
+
if (failedFromRange.length > 0) {
|
|
412
|
+
await pendingBlobDeletions.add(failedFromRange);
|
|
413
|
+
}
|
|
414
|
+
for (const failure of outcome.failed) {
|
|
415
|
+
result.failed.push({ scope, ...failure });
|
|
416
|
+
}
|
|
417
|
+
if (outcome.deleted.length + outcome.missing.length > 0) {
|
|
418
|
+
logger.info({
|
|
419
|
+
scope,
|
|
420
|
+
deleted: outcome.deleted.length,
|
|
421
|
+
missing: outcome.missing.length,
|
|
422
|
+
}, "Completed pending blob deletions");
|
|
423
|
+
}
|
|
424
|
+
if (outcome.failed.length > 0) {
|
|
425
|
+
logger.warn({
|
|
426
|
+
scope,
|
|
427
|
+
failed: outcome.failed.length,
|
|
428
|
+
error: outcome.failed[0].error,
|
|
429
|
+
}, "Pending blob deletion failed again");
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
// Advance the ranges only now: a crash mid-pass re-sends already-deleted
|
|
433
|
+
// keys (404, harmless) rather than skipping keys that were never sent.
|
|
434
|
+
for (const { old, next } of advancedRanges) {
|
|
435
|
+
await pendingBlobDeletions.remove([old]);
|
|
436
|
+
if (next)
|
|
437
|
+
await pendingBlobDeletions.add([next]);
|
|
438
|
+
}
|
|
439
|
+
result.remaining = (await pendingBlobDeletions.list()).length;
|
|
440
|
+
return result;
|
|
441
|
+
}
|
|
442
|
+
/**
|
|
443
|
+
* What the registry currently says about `scope`. "live" means a row exists
|
|
444
|
+
* that is not a tombstone; "deleted" carries the tombstone's version;
|
|
445
|
+
* "deleted-or-absent" is used when nothing can be asked (no feed or owner
|
|
446
|
+
* wired) or the row is gone.
|
|
447
|
+
*/
|
|
448
|
+
async function registryState(deps, scope) {
|
|
449
|
+
if (!deps.dataPointFeed || !deps.serverOwner) {
|
|
450
|
+
return { status: "deleted-or-absent" };
|
|
451
|
+
}
|
|
452
|
+
let record;
|
|
453
|
+
try {
|
|
454
|
+
record = await deps.dataPointFeed.getDataPoint({
|
|
455
|
+
ownerAddress: deps.serverOwner,
|
|
456
|
+
scope,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
catch (err) {
|
|
460
|
+
return {
|
|
461
|
+
status: "unknown",
|
|
462
|
+
error: err instanceof Error ? err : new Error(String(err)),
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
if (record === null)
|
|
466
|
+
return { status: "deleted-or-absent" };
|
|
467
|
+
if (deletionTimestamp(record) === null)
|
|
468
|
+
return { status: "live", record };
|
|
469
|
+
return { status: "deleted", version: tombstoneVersion(record) };
|
|
470
|
+
}
|
|
471
|
+
function errorMessage(err) {
|
|
472
|
+
return err instanceof Error ? err.message : String(err);
|
|
39
473
|
}
|
|
40
474
|
//# sourceMappingURL=delete.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"delete.js","sourceRoot":"","sources":["../../../src/sync/workers/delete.ts"],"names":[],"mappings":"AAwBA;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAsB,EACtB,KAAa;IAEb,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAEjC,6EAA6E;IAC7E,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,CAAC;IACvB,KAAK,IAAI,MAAM,GAAG,CAAC,GAAI,MAAM,IAAI,SAAS,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1E,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QACvE,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS;YAAE,MAAM;IACxC,CAAC;IAED,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CACT,EAAE,KAAK,EAAE,cAAc,EAAE,EACzB,iEAAiE;YAC/D,mEAAmE;YACnE,6CAA6C,CAChD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK;QACL,iBAAiB,EAAE,CAAC;QACpB,YAAY,EAAE,CAAC;QACf,MAAM,EAAE,EAAE;KACX,CAAC;AACJ,CAAC"}
|
|
1
|
+
{"version":3,"file":"delete.js","sourceRoot":"","sources":["../../../src/sync/workers/delete.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EACL,iBAAiB,EACjB,yBAAyB,EACzB,gBAAgB,GAEjB,MAAM,uBAAuB,CAAC;AAE/B;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AA8FzC;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAA8C,EAC9C,KAAa,EACb,qBAAoC;IAEpC,MAAM,SAAS,GAAG,EAAE,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACrD,MAAM,IAAI,GACR,qBAAqB,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,SAAS,GAAG,GAAG,CAAC;IACtB,KAAK,IAAI,MAAM,GAAG,CAAC,GAAI,MAAM,IAAI,SAAS,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1E,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,SAAS,CAAC;gBAAE,SAAS;YAC3D,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,IAAI,IAAI;gBAAE,SAAS;YAChE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS;YAAE,MAAM;IACxC,CAAC;IACD,OAAO;QACL,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChE,KAAK,EACH,IAAI,KAAK,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE;KACzE,CAAC;AACJ,CAAC;AAED,oFAAoF;AACpF,SAAS,aAAa,CACpB,KAAmC,EACnC,KAAa;IAEb,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,OAAO,MAAM,IAAI,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjC,MAAM,IAAI,EAAE,CAAC;IACf,CAAC;IACD,OAAO;QACL,QAAQ;QACR,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI;KACtE,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAmC;IACpD,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;AACpD,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,OAA8B;IACtD,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI;YAAE,KAAK,IAAI,EAAE,CAAC;aACpC,IAAI,MAAM,CAAC,KAAK;YAAE,KAAK,IAAI,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;;YACnD,KAAK,IAAI,EAAE,CAAC;IACnB,CAAC;IACD,OAAO,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC;QAC5C,CAAC,CAAC,MAAM,CAAC,gBAAgB;QACzB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAqB,EACrB,KAAa;IAEb,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,EAAE,GACzE,IAAI,CAAC;IACP,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW;QAClC,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC;QAC7C,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,MAAM,GAAsB;QAChC,KAAK;QACL,WAAW;QACX,OAAO,EAAE,KAAK;QACd,KAAK,EAAE;YACL,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE;YACvD,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE;YACvD,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE;SAC7B;QACD,mBAAmB,EAAE,KAAK;KAC3B,CAAC;IAEF,IAAI,UAAU,EAAE,CAAC;QACf,qEAAqE;QACrE,sEAAsE;QACtE,0CAA0C;QAC1C,IAAI,qBAAqB,GAAkB,IAAI,CAAC;QAChD,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAChD,IAAI,OAAO,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;gBACxC,mEAAmE;gBACnE,mEAAmE;gBACnE,kEAAkE;gBAClE,iEAAiE;gBACjE,kEAAkE;gBAClE,IAAI,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAChD,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;oBAC/B,OAAO,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;oBAC5C,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC9C,CAAC;gBACD,IACE,OAAO,CAAC,MAAM,KAAK,gBAAgB;oBACnC,QAAQ,CAAC,MAAM,KAAK,mBAAmB,EACvC,CAAC;oBACD,MAAM,QAAQ,CAAC,MAAM,KAAK,SAAS;wBACjC,CAAC,CAAC,QAAQ,CAAC,KAAK;wBAChB,CAAC,CAAC,IAAI,KAAK,CACP,gFAAgF,CACjF,CAAC;gBACR,CAAC;YACH,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;gBACxC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;YACzE,CAAC;iBAAM,CAAC;gBACN,cAAc,GAAG,IAAI,CAAC;gBACtB,qBAAqB;oBACnB,OAAO,CAAC,OAAO,KAAK,IAAI;wBACtB,CAAC,CAAC,IAAI;wBACN,CAAC,CAAC,gBAAgB,CAAC,EAAE,eAAe,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC7D,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG;oBACrB,MAAM,EAAE,IAAI;oBACZ,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,iBAAiB,IAAI;wBAC1C,MAAM,EAAE,iBAAiB;qBAC1B,CAAC;oBACF,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,SAAS,EAAE,OAAO,CAAC,SAAS;iBAC7B,CAAC;gBACF,kEAAkE;gBAClE,qEAAqE;gBACrE,kEAAkE;gBAClE,sEAAsE;gBACtE,cAAc,EAAE,WAAW,CACzB,KAAK,EACL;oBACE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE;oBACnD,OAAO,EAAE,OAAO,CAAC,OAAO;iBACzB,EACD,cAAc,CACf,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;YAC5D,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;YACvE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;YACrE,MAAM,CAAC,KAAK,CACV,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,EACtC,6GAA6G,CAC9G,CAAC;YACF,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,sEAAsE;QACtE,yEAAyE;QACzE,wEAAwE;QACxE,0CAA0C;QAC1C,MAAM,IAAI,GAAG,iBAAiB,CAAC,OAAO,EAAE,KAAK,EAAE,qBAAqB,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACzD,MAAM,SAAS,GAA0B,IAAI,CAAC,IAAI;aAC/C,KAAK,CAAC,sBAAsB,CAAC;aAC7B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC1C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,aAAa,CACzB,IAAI,CAAC,KAAK,EACV,sBAAsB,GAAG,KAAK,CAAC,MAAM,CACtC,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC9B,IAAI,KAAK,CAAC,IAAI;gBACZ,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,cAAc,IAAI,qBAAqB,KAAK,IAAI,EAAE,CAAC;YACrD,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,cAAc,CACtC,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,EAAE,EAC5C,KAAK,EACL,KAAK,EACL,SAAS,CACV,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;QACxC,MAAM,CAAC,mBAAmB,GAAG,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC;IACvD,CAAC;IAED,yEAAyE;IACzE,kEAAkE;IAClE,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACtD,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;IACtD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC1D,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,6BAA6B,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,IAAI,CACT;QACE,KAAK;QACL,WAAW;QACX,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM;QACpC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM;QACpC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM;KACjC,EACD,yBAAyB,CAC1B,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAC3B,IAA6E,EAC7E,KAAa,EACb,KAAe,EACf,SAAgC;IAEhC,MAAM,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC1D,IAAI,OAAkE,CAAC;IACvE,IAAI,CAAC;QACH,OAAO;YACL,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU;gBAC5B,CAAC,CAAC,MAAM,UAAU,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC;gBACnD,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACjD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,sEAAsE;QACtE,gCAAgC;QAChC,OAAO,GAAG;YACR,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;SACxE,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAC5D,CAAC,OAAO,EAAuB,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CACvD,CAAC;IACF,MAAM,SAAS,GAA0B;QACvC,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAC5D,GAAG,SAAS;KACb,CAAC;IACF,MAAM,aAAa,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAClD,IAAI,QAAQ,GAAG,aAAa,CAAC;IAC7B,IAAI,oBAAoB,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,oBAAoB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACvE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACtE,CAAC;QAAC,OAAO,SAAS,EAAE,CAAC;YACnB,QAAQ,GAAG,CAAC,CAAC;YACb,MAAM,CAAC,KAAK,CACV,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAC9D,gDAAgD,CACjD,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,QAAQ,GAAG,CAAC,CAAC;QACb,MAAM,CAAC,KAAK,CACV,EAAE,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,EAC9B,mEAAmE,CACpE,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG;QACb,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;QACpC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;QACpC,YAAY,EAAE,aAAa;KAC5B,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,IAAI,CACT;YACE,KAAK;YACL,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;YAC7B,OAAO,EAAE,QAAQ;YACjB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,EACD,gFAAgF,CACjF,CAAC;QACF,OAAO;YACL,IAAI,EAAE;gBACJ,MAAM,EAAE,QAAQ;gBAChB,KAAK,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,2BAA2B,KAAK,CAAC,KAAK,EAAE;gBACvE,GAAG,MAAM;aACV;YACD,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,CAAC,IAAI,CACT,EAAE,KAAK,EAAE,GAAG,MAAM,EAAE,EACpB,2EAA2E,CAC5E,CAAC;QACF,OAAO,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IACxE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;AAC3D,CAAC;AAYD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAUC;IAED,MAAM,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC1D,MAAM,MAAM,GAAoC;QAC9C,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;QACd,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,CAAC;KACb,CAAC;IACF,IAAI,CAAC,UAAU,IAAI,CAAC,oBAAoB;QAAE,OAAO,MAAM,CAAC;IAExD,IAAI,OAAO,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,CAAC;IAChD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAExC,qEAAqE;IACrE,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,CACjC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAC5C,EAAE,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACzD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;gBACjB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO;aAC9B,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC/B,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,IAAI,CACT;gBACE,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE;gBAC/B,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,eAAe;aACzC,EACD,gIAAgI,CACjI,CAAC;YACF,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,iBAAiB,CAC5B,IAAI,CAAC,OAAO,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAC1C,MAAM,CAAC,KAAK,EACZ,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACxD,CAAC;QACF,MAAM,QAAQ,GAA0B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAClE,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO;SACR,CAAC,CAAC,CAAC;QACJ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5C,MAAM,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,CAAC;IAE5C,wEAAwE;IACxE,IAAI,MAAM,GAAG,sBAAsB,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,cAAc,GAGf,EAAE,CAAC;IACR,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,OAAe,EAAE,EAAE;QACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC1C,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC/B,CAAC,CAAC;IACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,KAAK,CAAC;YAAE,MAAM;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YACtC,WAAW,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,SAAS,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YAC1D,MAAM,IAAI,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,KAAK,CAAC;YAAE,MAAM;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAClD,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ;gBAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACrE,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAChC,cAAc,CAAC,IAAI,CAAC;gBAClB,GAAG,EAAE,MAAM;gBACX,IAAI,EAAE,KAAK,CAAC,IAAI;oBACd,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;oBAC3D,CAAC,CAAC,IAAI;aACT,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,OAAO,EAAE,CAAC;QACxC,IAAI,OAAkE,CAAC;QACvE,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAClC,OAAO,GAAG;gBACR,OAAO,EAAE,EAAE;gBACX,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;aACjE,CAAC;QACJ,CAAC;QACD,MAAM,SAAS,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;aACvD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,OAAO,EAAE,CAAC,CAAC;aAChE,GAAG,CAAC,CAAC,OAAO,EAAuB,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC/D,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,oBAAoB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACvE,MAAM,CAAC,SAAS,CAAC,IAAI,CACnB,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAC7C,CAAC,OAAO,EAAuB,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CACvD,CACF,CAAC;QACF,wEAAwE;QACxE,6BAA6B;QAC7B,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM;aACnC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,OAAO,EAAE,CAAC,CAAC;aACrE,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAuB,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACnE,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,oBAAoB,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAClD,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CACT;gBACE,KAAK;gBACL,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;gBAC/B,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;aAChC,EACD,kCAAkC,CACnC,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CACT;gBACE,KAAK;gBACL,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;gBAC7B,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;aAC/B,EACD,oCAAoC,CACrC,CAAC;QACJ,CAAC;IACH,CAAC;IACD,yEAAyE;IACzE,uEAAuE;IACvE,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,cAAc,EAAE,CAAC;QAC3C,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzC,IAAI,IAAI;YAAE,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,MAAM,CAAC,SAAS,GAAG,CAAC,MAAM,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC9D,OAAO,MAAM,CAAC;AAChB,CAAC;AAQD;;;;;GAKG;AACH,KAAK,UAAU,aAAa,CAC1B,IAA4D,EAC5D,KAAa;IAEb,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7C,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IACzC,CAAC;IACD,IAAI,MAAkC,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC;YAC7C,YAAY,EAAE,IAAI,CAAC,WAAW;YAC9B,KAAK;SACN,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;SAC3D,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC5D,IAAI,iBAAiB,CAAC,MAAM,CAAC,KAAK,IAAI;QAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC1E,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IAChC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC"}
|
|
@@ -2,7 +2,8 @@ import type { StorageAdapter } from "../../storage/adapters/interface.js";
|
|
|
2
2
|
import { type DataPointRecord, type GatewayClient } from "@opendatalabs/vana-sdk/browser";
|
|
3
3
|
import type { SyncCursor } from "../cursor.js";
|
|
4
4
|
import type { Logger } from "../../logger/index.js";
|
|
5
|
-
import type { DataStoragePort } from "../../ports/index.js";
|
|
5
|
+
import type { DataPointFeedPort, DataPointFeedRecord, DataStoragePort, PendingBlobDeletionStore } from "../../ports/index.js";
|
|
6
|
+
import { type ScopeDeletionTracker } from "../scope-deletions.js";
|
|
6
7
|
import { type DownloadRetryMemory } from "../retry-memory.js";
|
|
7
8
|
/**
|
|
8
9
|
* Minimal diagnostics hook — keeps core free of lite-specific imports.
|
|
@@ -36,6 +37,48 @@ export interface DownloadWorkerDeps {
|
|
|
36
37
|
logger: Logger;
|
|
37
38
|
/** Optional diagnostics hook — omit to disable instrumentation. */
|
|
38
39
|
diagnostics?: DownloadDiagnosticsHook;
|
|
40
|
+
/**
|
|
41
|
+
* Deletion-aware registry listing (`includeDeleted=true`). When present,
|
|
42
|
+
* tombstoned points reach this replica and their local copies are removed
|
|
43
|
+
* instead of re-pulled. Falls back to the SDK client listing, which only
|
|
44
|
+
* sees deletions the gateway volunteers without being asked.
|
|
45
|
+
*/
|
|
46
|
+
dataPointFeed?: DataPointFeedPort;
|
|
47
|
+
/**
|
|
48
|
+
* Read-side deletion memory. Every listed row is recorded here (deleted or
|
|
49
|
+
* live) and a listing with no further pages marks the memory complete, so
|
|
50
|
+
* reads can refuse tombstoned scopes without their own gateway lookups.
|
|
51
|
+
*/
|
|
52
|
+
scopeDeletions?: ScopeDeletionTracker;
|
|
53
|
+
/**
|
|
54
|
+
* Exact-key cleanup queue. An unsynced local entry a tombstone covers may
|
|
55
|
+
* have had its ciphertext uploaded under a version key the registry never
|
|
56
|
+
* recorded (crashed before registering); reconciling drops the entry and
|
|
57
|
+
* queues that exact key so the sync cycle's rate-limited pass removes it.
|
|
58
|
+
*/
|
|
59
|
+
pendingBlobDeletions?: PendingBlobDeletionStore;
|
|
60
|
+
/**
|
|
61
|
+
* Called after a downloaded data point is written and indexed locally
|
|
62
|
+
* (a new local version of `scope`). The derivative compute layer marks
|
|
63
|
+
* questions that read the scope stale. Best-effort: a throwing hook is
|
|
64
|
+
* logged and never fails the download.
|
|
65
|
+
*/
|
|
66
|
+
onDataPointIndexed?: (event: {
|
|
67
|
+
scope: string;
|
|
68
|
+
dataPointId: string;
|
|
69
|
+
version: number;
|
|
70
|
+
collectedAt: string;
|
|
71
|
+
/** The record's stamped `$lineage.sources`, when it is a derivative. */
|
|
72
|
+
lineageSources?: string[];
|
|
73
|
+
}) => void;
|
|
74
|
+
}
|
|
75
|
+
export interface DeletionReconcileResult {
|
|
76
|
+
scope: string;
|
|
77
|
+
deletedAt: string;
|
|
78
|
+
/** Local versions removed because the tombstone covers them. */
|
|
79
|
+
removed: number;
|
|
80
|
+
/** Local unsynced versions ingested after the deletion and kept. */
|
|
81
|
+
kept: number;
|
|
39
82
|
}
|
|
40
83
|
export interface DownloadResult {
|
|
41
84
|
dataPointId: string;
|
|
@@ -82,6 +125,14 @@ export declare function downloadOne(deps: DownloadWorkerDeps, record: DataPointR
|
|
|
82
125
|
* each, and advance the cursor only when the page fully succeeds.
|
|
83
126
|
*/
|
|
84
127
|
export declare function downloadAll(deps: DownloadWorkerDeps, options?: DownloadAllOptions): Promise<DownloadResult[]>;
|
|
128
|
+
/**
|
|
129
|
+
* Apply a gateway tombstone to the local index: remove every local version
|
|
130
|
+
* the tombstone covers (synced rows at or below its version, and unsynced
|
|
131
|
+
* rows ingested without knowledge of it), keep unsynced rows ingested on top
|
|
132
|
+
* of it (a legitimate re-add that the upload worker registers after the
|
|
133
|
+
* tombstone version). See `isEntryCoveredByTombstone`.
|
|
134
|
+
*/
|
|
135
|
+
export declare function reconcileDeletedDataPoint(deps: Pick<DownloadWorkerDeps, "storage" | "logger" | "pendingBlobDeletions">, record: Pick<DataPointFeedRecord, "id" | "scope" | "expectedVersion">, deletedAt: string): Promise<DeletionReconcileResult>;
|
|
85
136
|
export declare function repairLocalMissingBlockSidecars(deps: Pick<DownloadWorkerDeps, "storage" | "logger" | "diagnostics">): Promise<LocalMissingBlockRepairSummary>;
|
|
86
137
|
export {};
|
|
87
138
|
//# sourceMappingURL=download.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/sync/workers/download.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,aAAa,EACnB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/sync/workers/download.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,aAAa,EACnB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EAEf,wBAAwB,EACzB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,uBAAuB,CAAC;AAQ/B,OAAO,EAAoB,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGhF;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/C,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAClD,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9C,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAChE,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9C,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9D,oBAAoB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACtD,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACpD,oBAAoB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,eAAe,CAAC;IACzB,cAAc,EAAE,cAAc,CAAC;IAC/B,OAAO,EAAE,aAAa,CAAC;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,UAAU,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,WAAW,CAAC,EAAE,uBAAuB,CAAC;IACtC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC;;;;OAIG;IACH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,wBAAwB,CAAC;IAChD;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE;QAC3B,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;QACpB,wEAAwE;QACxE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;KAC3B,KAAK,IAAI,CAAC;CACZ;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,gEAAgE;IAChE,OAAO,EAAE,MAAM,CAAC;IAChB,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,mBAAmB,CAAC;CACnC;AAYD,UAAU,8BAA8B;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAYD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAoNhC;AAED;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,kBAAkB,EACxB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,cAAc,EAAE,CAAC,CAuK3B;AAED;;;;;;GAMG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,SAAS,GAAG,QAAQ,GAAG,sBAAsB,CAAC,EAC7E,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,GAAG,OAAO,GAAG,iBAAiB,CAAC,EACrE,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,uBAAuB,CAAC,CAsDlC;AAED,wBAAsB,+BAA+B,CACnD,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,SAAS,GAAG,QAAQ,GAAG,aAAa,CAAC,GACnE,OAAO,CAAC,8BAA8B,CAAC,CAyDzC"}
|