@substrat-run/control-plane-api 0.140.0 → 0.142.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +3 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +498 -97
- package/dist/api.js.map +1 -1
- package/dist/cf-observability.d.ts.map +1 -1
- package/dist/cf-observability.js +40 -3
- package/dist/cf-observability.js.map +1 -1
- package/dist/field-coverage-read.d.ts +44 -0
- package/dist/field-coverage-read.d.ts.map +1 -0
- package/dist/field-coverage-read.js +45 -0
- package/dist/field-coverage-read.js.map +1 -0
- package/dist/field-coverage-tally.d.ts +12 -0
- package/dist/field-coverage-tally.d.ts.map +1 -1
- package/dist/field-coverage-tally.js +55 -14
- package/dist/field-coverage-tally.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/observability.d.ts +13 -0
- package/dist/observability.d.ts.map +1 -1
- package/dist/platform-entry.d.ts +4 -4
- package/dist/platform-entry.generated.d.ts +1 -1
- package/dist/platform-entry.generated.d.ts.map +1 -1
- package/dist/platform-entry.generated.js +3 -3
- package/dist/platform-entry.generated.js.map +1 -1
- package/dist/platform-entry.js +6 -6
- package/dist/platform-entry.js.map +1 -1
- package/dist/provider-error.d.ts +34 -5
- package/dist/provider-error.d.ts.map +1 -1
- package/dist/provider-error.js +60 -5
- package/dist/provider-error.js.map +1 -1
- package/dist/scope-copy-backfill.d.ts +106 -0
- package/dist/scope-copy-backfill.d.ts.map +1 -0
- package/dist/scope-copy-backfill.js +418 -0
- package/dist/scope-copy-backfill.js.map +1 -0
- package/dist/scope-copy-cleanup.d.ts +86 -0
- package/dist/scope-copy-cleanup.d.ts.map +1 -0
- package/dist/scope-copy-cleanup.js +234 -0
- package/dist/scope-copy-cleanup.js.map +1 -0
- package/dist/vertical-client.d.ts +5 -1
- package/dist/vertical-client.d.ts.map +1 -1
- package/dist/vertical-client.js +20 -5
- package/dist/vertical-client.js.map +1 -1
- package/package.json +10 -10
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
import { BACKFILL_MOVE_ID } from '@substrat-run/kernel';
|
|
2
|
+
import { hasCarriedAwayTombstone, listAllScopeScriptCopies, routeOfScope } from './scope-copy-cleanup.js';
|
|
3
|
+
/**
|
|
4
|
+
* The backfill of copies made before the directory's copy ledger existed (#1722).
|
|
5
|
+
*
|
|
6
|
+
* A copy move records both its scripts in the ledger before it writes, so reap and erasure reach
|
|
7
|
+
* every copy made since. A scope's data lived in other scripts before that, and nothing names
|
|
8
|
+
* those copies to reap or erasure. The admin log is never swept, so a scope's own rows say where
|
|
9
|
+
* it was routed, and so where its store was. This walks the log, oldest first, one page per
|
|
10
|
+
* call, and records each such script the ledger does not know yet as `retained`.
|
|
11
|
+
*
|
|
12
|
+
* **Where a scope's data lived** is derived from the scope's whole timeline, never from one row:
|
|
13
|
+
* - **A pin.** Each `setScopeServingRef` row names the serving script before and after; while
|
|
14
|
+
* pinned, the scope's route was that script. The pin at birth is the first such row's `before`,
|
|
15
|
+
* or the directory's current pin when the scope has none. An unpin routes the scope to what it
|
|
16
|
+
* falls back on, the version bound then (or its slug, never bound), which is a home too.
|
|
17
|
+
* - **A bind while unpinned.** A `bindScopeVersion` row names a version (and a promote's row the
|
|
18
|
+
* version before it), whose registry row names its script. It is a home only while the scope was
|
|
19
|
+
* not pinned: a pinned scope's bind (a private vertical's promote moves the pointer of every
|
|
20
|
+
* served install) routes nothing there, and its script never held the scope's store.
|
|
21
|
+
* - **Birth by slug.** A non-preview scope born unpinned was provisioned through its vertical's
|
|
22
|
+
* slug, into the script of the version on `prod` at that instant (`listChannelHistory`), and no
|
|
23
|
+
* later row names that script unless a bind happens to. A preview is born by its own bind.
|
|
24
|
+
* - **A fork** is born into the script its source was routed to at that instant, whatever pin its
|
|
25
|
+
* own row took, derived the same way from the source's timeline (the source's slug resolves to
|
|
26
|
+
* its vertical's serving script when that vertical served in place then).
|
|
27
|
+
*
|
|
28
|
+
* What cannot be derived is a **failure**: reported in every response, written as an ops record on
|
|
29
|
+
* a real run (`scope.copy-backfill`), never marked clean, and reported again on a re-run. That is a
|
|
30
|
+
* row naming no version, a version the registry no longer holds or that names no script, a scope
|
|
31
|
+
* or fork source whose directory row is gone, a scope with no `provisionScope` row (its other rows
|
|
32
|
+
* are still derived), a slug with no `prod` version at the time or one that changed while the store
|
|
33
|
+
* may have been born (from `BIRTH_WINDOW_MS` before the birth row to the scope's activation: the
|
|
34
|
+
* row can come before or after the birth, see `slugAt`), a script no deployment answers for, and a
|
|
35
|
+
* store that cannot be read.
|
|
36
|
+
*
|
|
37
|
+
* - **Never a wipe, never a phantom.** `retained` is reached by reap and erasure and is never swept.
|
|
38
|
+
* A real run reads one thing, the derived home's `_substrat_meta`, for the tombstone that proves
|
|
39
|
+
* an earlier wipe ran. A dry run touches no store at all: it resolves each script's deployment and
|
|
40
|
+
* reports what a real run would check.
|
|
41
|
+
* - **Taken over by the ledger.** Once a move's own entry for the same script settles `done`, the
|
|
42
|
+
* backfilled entry settles with it (`COPY_BACKFILL_SUPERSEDE_SQL`).
|
|
43
|
+
* - **Idempotent and resumable.** `backfillScopeScriptCopy` writes nothing for a script the ledger
|
|
44
|
+
* already names for the scope; `nextCursor` is the last row read. Admin-log ids are minted before
|
|
45
|
+
* their row is written, so a row in flight while a page is read can sort below the cursor and be
|
|
46
|
+
* skipped. That is harmless here: the rows this exists for were written before the ledger existed.
|
|
47
|
+
* - **No race with a live move.** The entry is never pending, so no move's confirmation and no lease
|
|
48
|
+
* sweep acts on it; a reap claim refuses it, and an erasure that read the inventory first refuses
|
|
49
|
+
* to finalize on the changed count and is retried.
|
|
50
|
+
* - **Erasures that came first.** Each recorded entry is audited as `backfillScopeCopy` with the
|
|
51
|
+
* erasure epoch it was written under, and every erasure records its own (`ErasureEpochStamp`). A
|
|
52
|
+
* subject whose last erasure did not reach the latest recorded copy (`erasedBeforeBackfill`) is
|
|
53
|
+
* reported by every run, and a real run writes an ops record for each scope (`erased-before`).
|
|
54
|
+
* Erasing them again is the operator's call.
|
|
55
|
+
*/
|
|
56
|
+
export const BACKFILL_PAGE_MAX = 200;
|
|
57
|
+
/**
|
|
58
|
+
* How long BEFORE its `provisionScope` row a store may have been born. `POST /verticals/:slug/instances`
|
|
59
|
+
* provisions in the vertical first and writes the directory row after, inside one request, whose
|
|
60
|
+
* `retryTransient` waits about three seconds at most: fifteen minutes is a generous margin over
|
|
61
|
+
* that, not a measured bound. The other side of the window has no constant (`birthEnd`).
|
|
62
|
+
*/
|
|
63
|
+
export const BIRTH_WINDOW_MS = 15 * 60_000;
|
|
64
|
+
/** The rows a page walks: each can name a home. */
|
|
65
|
+
const TIMELINE = ['provisionScope', 'bindScopeVersion', 'setScopeServingRef'];
|
|
66
|
+
/** A scope's own rows: those, and its activation, which bounds its birth. */
|
|
67
|
+
const SCOPE_TIMELINE = [...TIMELINE, 'activateScope'];
|
|
68
|
+
const objectOf = (v) => v !== null && typeof v === 'object' && !Array.isArray(v) ? v : null;
|
|
69
|
+
const stringOf = (v) => typeof v === 'string' && v.length > 0 ? v : null;
|
|
70
|
+
/** Every row an admin-log filter matches, oldest first. */
|
|
71
|
+
async function allRows(admin, actor, filter) {
|
|
72
|
+
const out = [];
|
|
73
|
+
for (;;) {
|
|
74
|
+
const page = await admin.auditLog(actor, { ...filter, order: 'asc', limit: BACKFILL_PAGE_MAX,
|
|
75
|
+
...(out.length ? { cursor: out[out.length - 1].id } : {}) });
|
|
76
|
+
out.push(...page);
|
|
77
|
+
if (page.length < BACKFILL_PAGE_MAX)
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* The scope's pin just before log row `id`: a script, null when unpinned, undefined when nothing
|
|
83
|
+
* says (a deleted scope that never moved its pin: its birth pin was on the row that is gone).
|
|
84
|
+
*/
|
|
85
|
+
function pinnedBefore(tl, id) {
|
|
86
|
+
const serving = tl.rows.filter((r) => r.action === 'setScopeServingRef');
|
|
87
|
+
const last = serving.filter((r) => r.id < id).at(-1);
|
|
88
|
+
if (last)
|
|
89
|
+
return stringOf(objectOf(last.after)?.servingRef);
|
|
90
|
+
if (serving.length)
|
|
91
|
+
return stringOf(objectOf(serving[0].before)?.servingRef);
|
|
92
|
+
return tl.exists ? tl.servingRefNow : undefined;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* The latest a scope's store can have been born: its first activation after its birth row, which a
|
|
96
|
+
* provision writes once the store exists, else its next bind or pin (a move that needed the store),
|
|
97
|
+
* else unknown (`null`, any later time).
|
|
98
|
+
*/
|
|
99
|
+
function birthEnd(tl, provision) {
|
|
100
|
+
const after = tl.rows.filter((r) => r.id > provision.id);
|
|
101
|
+
return (after.find((r) => r.action === 'activateScope') ?? after.find((r) => r.action !== 'provisionScope'))?.at ?? null;
|
|
102
|
+
}
|
|
103
|
+
class Deriver {
|
|
104
|
+
input;
|
|
105
|
+
versions = new Map();
|
|
106
|
+
timelines = new Map();
|
|
107
|
+
prodHistory = new Map();
|
|
108
|
+
servingHistory;
|
|
109
|
+
constructor(input) {
|
|
110
|
+
this.input = input;
|
|
111
|
+
}
|
|
112
|
+
versionOf(id, vertical) {
|
|
113
|
+
const key = `${vertical}/${id}`;
|
|
114
|
+
if (!this.versions.has(key))
|
|
115
|
+
this.versions.set(key, this.input.admin.getVersion(this.input.actor, id, vertical));
|
|
116
|
+
return this.versions.get(key);
|
|
117
|
+
}
|
|
118
|
+
timeline(tenantId, scopeId) {
|
|
119
|
+
const key = `${tenantId}/${scopeId}`;
|
|
120
|
+
if (!this.timelines.has(key))
|
|
121
|
+
this.timelines.set(key, this.load(tenantId, scopeId));
|
|
122
|
+
return this.timelines.get(key);
|
|
123
|
+
}
|
|
124
|
+
async load(tenantId, scopeId) {
|
|
125
|
+
const { admin, actor } = this.input;
|
|
126
|
+
const [scope, rows, copies] = await Promise.all([
|
|
127
|
+
admin.getScopeRecord(actor, tenantId, scopeId),
|
|
128
|
+
allRows(admin, actor, { tenantId, scopeId, action: SCOPE_TIMELINE }),
|
|
129
|
+
listAllScopeScriptCopies(admin, actor, tenantId, scopeId),
|
|
130
|
+
]);
|
|
131
|
+
const provision = rows.find((r) => r.action === 'provisionScope');
|
|
132
|
+
return {
|
|
133
|
+
tenantId, scopeId,
|
|
134
|
+
exists: Boolean(scope),
|
|
135
|
+
servingRefNow: scope?.servingRef ?? null,
|
|
136
|
+
route: scope ? await routeOfScope(this.input, scope, (id, vertical) => this.versionOf(id, vertical)) : null,
|
|
137
|
+
rows,
|
|
138
|
+
provision,
|
|
139
|
+
vertical: (provision ? stringOf(objectOf(provision.after)?.vertical) ?? provision.vertical : null) ?? scope?.vertical ?? null,
|
|
140
|
+
known: new Set(copies.map((c) => c.scriptRef)),
|
|
141
|
+
backfilled: new Set(copies.filter((c) => c.moveId === BACKFILL_MOVE_ID && c.state !== 'done').map((c) => c.scriptRef)),
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
/** A version's script, or why it has none. */
|
|
145
|
+
async scriptOf(id, vertical) {
|
|
146
|
+
const v = await this.versionOf(id, vertical);
|
|
147
|
+
if (!v)
|
|
148
|
+
return { failure: `version ${id} of '${vertical}' is not in the registry` };
|
|
149
|
+
if (!v.deploymentRef)
|
|
150
|
+
return { failure: `version ${id} of '${vertical}' names no deployment script` };
|
|
151
|
+
return { scriptRef: v.deploymentRef };
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Where `vertical`'s slug resolved for a store born around `at`: its serving script when it served
|
|
155
|
+
* in place, else prod's. Ordered against other tables by time alone, and the birth is not at `at`
|
|
156
|
+
* itself. The install path writes the directory row after the vertical provisioned the store
|
|
157
|
+
* (`BIRTH_WINDOW_MS` before it); the dashboard's connected install writes it first and provisions
|
|
158
|
+
* after, and a retried install provisions again with no new row. So the birth lies anywhere from
|
|
159
|
+
* `BIRTH_WINDOW_MS` before `at` to `until`, the first evidence the store exists (`birthEnd`), or
|
|
160
|
+
* any time after when there is none, and a slug change in that window is ambiguous, never a guess.
|
|
161
|
+
*/
|
|
162
|
+
async slugAt(vertical, at, until) {
|
|
163
|
+
const { admin, actor } = this.input;
|
|
164
|
+
this.servingHistory ??= allRows(admin, actor, { action: 'setVerticalServing' });
|
|
165
|
+
const from = new Date(Date.parse(at) - BIRTH_WINDOW_MS).toISOString();
|
|
166
|
+
const inWindow = (when) => when >= from && (until === null || when <= until);
|
|
167
|
+
const ambiguous = {
|
|
168
|
+
failure: `'${vertical}' changed where its slug resolves between ${from} and ${until ?? 'any later time'}, ` +
|
|
169
|
+
'where the store was born, so the script it was born in is not known',
|
|
170
|
+
};
|
|
171
|
+
const servingRows = (await this.servingHistory).filter((r) => r.vertical === vertical || objectOf(r.after)?.vertical === vertical);
|
|
172
|
+
if (servingRows.some((r) => inWindow(r.at)))
|
|
173
|
+
return ambiguous;
|
|
174
|
+
const serving = servingRows.filter((r) => r.at < at).at(-1);
|
|
175
|
+
const servingRef = serving ? stringOf(objectOf(serving.after)?.ref) : null;
|
|
176
|
+
if (servingRef)
|
|
177
|
+
return { scriptRef: servingRef };
|
|
178
|
+
if (!this.prodHistory.has(vertical))
|
|
179
|
+
this.prodHistory.set(vertical, (async () => {
|
|
180
|
+
const out = [];
|
|
181
|
+
for (;;) {
|
|
182
|
+
const page = await admin.listChannelHistory(actor, vertical, 'prod', { order: 'asc', limit: BACKFILL_PAGE_MAX, ...(out.length ? { cursor: out[out.length - 1].id } : {}) });
|
|
183
|
+
out.push(...page);
|
|
184
|
+
if (page.length < BACKFILL_PAGE_MAX)
|
|
185
|
+
return out;
|
|
186
|
+
}
|
|
187
|
+
})());
|
|
188
|
+
const history = await this.prodHistory.get(vertical);
|
|
189
|
+
if (history.some((e) => inWindow(e.at)))
|
|
190
|
+
return ambiguous;
|
|
191
|
+
const prod = history.filter((e) => e.at < at).at(-1);
|
|
192
|
+
if (!prod)
|
|
193
|
+
return { failure: `'${vertical}' had no prod version at ${at}, so the script its slug resolved to is not known` };
|
|
194
|
+
return this.scriptOf(prod.versionId, vertical);
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* The script `tl`'s scope was routed to just before log row `id` was written at `at`. When that is
|
|
198
|
+
* its slug, the resolution is read over `at` to `until`, as `slugAt` says.
|
|
199
|
+
*/
|
|
200
|
+
async routeAt(tl, id, at, until) {
|
|
201
|
+
const pin = pinnedBefore(tl, id);
|
|
202
|
+
if (pin === undefined)
|
|
203
|
+
return { failure: `scope ${tl.scopeId} is gone and its pin at the time is not in the log` };
|
|
204
|
+
return pin ? { scriptRef: pin } : this.unpinnedRouteAt(tl, id, at, until);
|
|
205
|
+
}
|
|
206
|
+
/** Where `tl`'s scope routed just before row `id` with no pin: its bound version, else its slug. */
|
|
207
|
+
async unpinnedRouteAt(tl, id, at, until = at) {
|
|
208
|
+
const bind = tl.rows.filter((r) => r.action === 'bindScopeVersion' && r.id < id).at(-1);
|
|
209
|
+
if (bind) {
|
|
210
|
+
const after = objectOf(bind.after);
|
|
211
|
+
const versionId = stringOf(after?.versionId);
|
|
212
|
+
const vertical = stringOf(after?.vertical) ?? bind.vertical;
|
|
213
|
+
if (!versionId || !vertical)
|
|
214
|
+
return { failure: `bind row ${bind.id} names no version or vertical` };
|
|
215
|
+
return this.scriptOf(versionId, vertical);
|
|
216
|
+
}
|
|
217
|
+
if (!tl.vertical)
|
|
218
|
+
return { failure: 'its vertical is not in the log or the directory' };
|
|
219
|
+
return this.slugAt(tl.vertical, at, until);
|
|
220
|
+
}
|
|
221
|
+
/** The script one timeline row names as a home of its scope, or why it cannot be derived. */
|
|
222
|
+
async candidatesOf(tl, row) {
|
|
223
|
+
const before = objectOf(row.before);
|
|
224
|
+
const after = objectOf(row.after);
|
|
225
|
+
if (row.action === 'setScopeServingRef') {
|
|
226
|
+
const pins = [stringOf(before?.servingRef), stringOf(after?.servingRef)]
|
|
227
|
+
.filter((ref) => ref !== null)
|
|
228
|
+
.map((scriptRef) => ({ scriptRef }));
|
|
229
|
+
// An unpin routes the scope to what it falls back on: the version bound then, or its slug.
|
|
230
|
+
return stringOf(after?.servingRef) ? pins : [...pins, await this.unpinnedRouteAt(tl, row.id, row.at)];
|
|
231
|
+
}
|
|
232
|
+
if (row.action === 'bindScopeVersion') {
|
|
233
|
+
// Pinned, the bind moved a pointer and routed nothing: its script never held the store.
|
|
234
|
+
if (pinnedBefore(tl, row.id))
|
|
235
|
+
return [];
|
|
236
|
+
const vertical = stringOf(after?.vertical) ?? row.vertical;
|
|
237
|
+
const versionId = stringOf(after?.versionId);
|
|
238
|
+
if (!versionId || !vertical)
|
|
239
|
+
return [{ failure: 'the bind row names no version or vertical' }];
|
|
240
|
+
const ids = [versionId, stringOf(before?.versionId)].filter((v) => v !== null);
|
|
241
|
+
return Promise.all(ids.map((id) => this.scriptOf(id, vertical)));
|
|
242
|
+
}
|
|
243
|
+
// provisionScope: where the store was born, when no other row names it.
|
|
244
|
+
if (row.id !== tl.provision?.id)
|
|
245
|
+
return [];
|
|
246
|
+
if (after?.kind === 'preview')
|
|
247
|
+
return []; // born by its bind
|
|
248
|
+
const vertical = stringOf(after?.vertical) ?? row.vertical;
|
|
249
|
+
if (!vertical)
|
|
250
|
+
return [];
|
|
251
|
+
// A fork's bytes went into its source's route, whatever pin its own row took: before #2130 its
|
|
252
|
+
// directory row inherited the vertical's serving ref like any install's (review r2).
|
|
253
|
+
const forkedFrom = stringOf(after?.forkedFrom);
|
|
254
|
+
if (forkedFrom) {
|
|
255
|
+
const source = await this.timeline(tl.tenantId, forkedFrom);
|
|
256
|
+
if (!source.exists && !source.rows.length)
|
|
257
|
+
return [{ failure: `the fork's source ${forkedFrom} is not in the directory or the log` }];
|
|
258
|
+
return [await this.routeAt(source, row.id, row.at, birthEnd(tl, row))];
|
|
259
|
+
}
|
|
260
|
+
if (pinnedBefore(tl, row.id))
|
|
261
|
+
return []; // born on its pin, which its rows or its route name
|
|
262
|
+
return [await this.slugAt(vertical, row.at, birthEnd(tl, row))];
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
export async function backfillScopeScriptCopies(input, opts) {
|
|
266
|
+
const { admin, actor } = input;
|
|
267
|
+
const limit = Math.min(Math.max(opts.limit ?? 100, 1), BACKFILL_PAGE_MAX);
|
|
268
|
+
const rows = await admin.auditLog(actor, { action: TIMELINE, order: 'asc', limit, ...(opts.cursor ? { cursor: opts.cursor } : {}) });
|
|
269
|
+
const derive = new Deriver(input);
|
|
270
|
+
const entries = [];
|
|
271
|
+
const seen = new Map();
|
|
272
|
+
const unborn = new Set();
|
|
273
|
+
for (const row of rows) {
|
|
274
|
+
const tenantId = row.tenantId;
|
|
275
|
+
const scopeId = row.scopeId;
|
|
276
|
+
const base = { fromLogId: row.id, tenantId, scopeId };
|
|
277
|
+
const failure = (scriptRef, reason) => entries.push({ ...base, scriptRef, outcome: 'failure', reason });
|
|
278
|
+
if (!tenantId || !scopeId) {
|
|
279
|
+
failure(null, 'the row names no tenant or scope');
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
const tl = await derive.timeline(tenantId, scopeId);
|
|
283
|
+
seen.set(`${tenantId}/${scopeId}`, tl);
|
|
284
|
+
if (!tl.exists) {
|
|
285
|
+
failure(null, 'the scope has no directory row; a copy it left has no ledger anchor');
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
// No birth row: the birth is a failure, once per scope on the page; its binds and pins still count.
|
|
289
|
+
if (!tl.provision && !unborn.has(`${tenantId}/${scopeId}`)) {
|
|
290
|
+
unborn.add(`${tenantId}/${scopeId}`);
|
|
291
|
+
failure(null, 'the scope has no provisionScope row; the script it was born in is not in the log');
|
|
292
|
+
}
|
|
293
|
+
for (const c of await derive.candidatesOf(tl, row)) {
|
|
294
|
+
if ('failure' in c) {
|
|
295
|
+
failure(null, c.failure);
|
|
296
|
+
continue;
|
|
297
|
+
}
|
|
298
|
+
const scriptRef = c.scriptRef;
|
|
299
|
+
const at = { ...base, scriptRef };
|
|
300
|
+
if (tl.route === scriptRef) {
|
|
301
|
+
entries.push({ ...at, outcome: 'route' });
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
if (tl.known.has(scriptRef)) {
|
|
305
|
+
entries.push({ ...at, outcome: 'ledgered' });
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
const holder = await input.resolveRef(scriptRef).catch(() => undefined);
|
|
309
|
+
if (!holder) {
|
|
310
|
+
failure(scriptRef, 'no deployment resolves for this script');
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
if (opts.dryRun) {
|
|
314
|
+
entries.push({ ...at, outcome: 'would-record' });
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
try {
|
|
318
|
+
if (await hasCarriedAwayTombstone(holder, scopeId)) {
|
|
319
|
+
tl.known.add(scriptRef); // a later row naming it is answered from here
|
|
320
|
+
entries.push({ ...at, outcome: 'wiped' });
|
|
321
|
+
continue;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
catch (e) {
|
|
325
|
+
failure(scriptRef, `the store could not be read (${e instanceof Error ? e.message : String(e)})`);
|
|
326
|
+
continue;
|
|
327
|
+
}
|
|
328
|
+
const result = await admin.backfillScopeScriptCopy(actor, tenantId, scopeId, scriptRef, { fromLogId: row.id });
|
|
329
|
+
if (result === 'reaping') {
|
|
330
|
+
failure(scriptRef, 'the scope is being reaped; the reap does not reach this copy');
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
if (result === 'missing') {
|
|
334
|
+
failure(scriptRef, 'the scope has no directory row');
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
tl.known.add(scriptRef);
|
|
338
|
+
if (result === 'recorded')
|
|
339
|
+
tl.backfilled.add(scriptRef);
|
|
340
|
+
entries.push({ ...at, outcome: result });
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
const erasedBefore = (await Promise.all([...seen.values()].filter((tl) => tl.backfilled.size > 0)
|
|
344
|
+
.map(async (tl) => ({ tenantId: tl.tenantId, scopeId: tl.scopeId, subjects: await erasedBeforeBackfill(admin, actor, tl) }))))
|
|
345
|
+
.filter((s) => s.subjects.length > 0);
|
|
346
|
+
if (!opts.dryRun) {
|
|
347
|
+
await Promise.all([
|
|
348
|
+
...entries.filter((e) => e.outcome === 'failure').map((e) => admin.recordOpsFailure({
|
|
349
|
+
actor, operation: 'scope.copy-backfill', stage: 'unresolved',
|
|
350
|
+
tenantId: e.tenantId, scopeId: e.scopeId,
|
|
351
|
+
message: `admin-log row ${e.fromLogId}${e.scriptRef ? ` (script '${e.scriptRef}')` : ''}: ${e.reason} — ` +
|
|
352
|
+
'a copy this may name is not in the copy ledger, so reap and erasure do not reach it (#1722)',
|
|
353
|
+
})),
|
|
354
|
+
...erasedBefore.map((s) => admin.recordOpsFailure({
|
|
355
|
+
actor, operation: 'scope.copy-backfill', stage: 'erased-before',
|
|
356
|
+
tenantId: s.tenantId, scopeId: s.scopeId,
|
|
357
|
+
message: `${s.subjects.length} subject(s) erased in scope ${s.scopeId} before its historic copies were ledgered: ` +
|
|
358
|
+
'those erasures never reached the copies, so each must be erased again (#1722)',
|
|
359
|
+
})),
|
|
360
|
+
]);
|
|
361
|
+
}
|
|
362
|
+
const counts = { 'would-record': 0, recorded: 0, ledgered: 0, route: 0, wiped: 0, failure: 0 };
|
|
363
|
+
for (const e of entries)
|
|
364
|
+
counts[e.outcome]++;
|
|
365
|
+
return {
|
|
366
|
+
dryRun: opts.dryRun,
|
|
367
|
+
nextCursor: rows.at(-1)?.id ?? opts.cursor ?? null,
|
|
368
|
+
done: rows.length < limit,
|
|
369
|
+
rowsRead: rows.length,
|
|
370
|
+
entries,
|
|
371
|
+
counts,
|
|
372
|
+
erasedBefore,
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* The subjects whose last erasure in the scope never reached its latest backfilled copy. Every row
|
|
377
|
+
* carries the erasure epoch it ran under (`before.erasureEpoch`): the backfill the epoch read in its
|
|
378
|
+
* insert's own transaction, an erasure an `ErasureEpochStamp`.
|
|
379
|
+
* - An **orchestrated** erasure claimed the copy inventory at its epoch. A claim that came first
|
|
380
|
+
* moved the epoch past what it compared, so it is below the backfill's epoch even when its audit
|
|
381
|
+
* row landed later, and missed the copy; one that came after counted the copy and redacted it.
|
|
382
|
+
* - A **direct** shred took no claim and redacted only the scope's own store. It never reaches a
|
|
383
|
+
* script copy, and a backfilled entry is always one, so it is missed whatever its epoch.
|
|
384
|
+
* - A row with **no stamp** (written before the stamps) cannot be ordered, so it is missed: the
|
|
385
|
+
* re-erasure that asks for sees the open copy and goes orchestrated, which converges.
|
|
386
|
+
* Each backfill row commits with its entry, in the insert's own unit, so a backfilled entry always
|
|
387
|
+
* has one; a scope found without any still counts every erased subject, the safe answer.
|
|
388
|
+
*/
|
|
389
|
+
async function erasedBeforeBackfill(admin, actor, tl) {
|
|
390
|
+
const filter = { tenantId: tl.tenantId, scopeId: tl.scopeId };
|
|
391
|
+
const [shreds, backfills] = await Promise.all([
|
|
392
|
+
allRows(admin, actor, { ...filter, action: 'shredSubject' }),
|
|
393
|
+
allRows(admin, actor, { ...filter, action: 'backfillScopeCopy' }),
|
|
394
|
+
]);
|
|
395
|
+
const before = (r) => objectOf(r.before);
|
|
396
|
+
const epochOf = (r) => {
|
|
397
|
+
const v = before(r)?.erasureEpoch;
|
|
398
|
+
return typeof v === 'number' ? v : null;
|
|
399
|
+
};
|
|
400
|
+
const last = new Map();
|
|
401
|
+
for (const r of shreds) {
|
|
402
|
+
const subject = stringOf(objectOf(r.after)?.subjectId);
|
|
403
|
+
if (subject)
|
|
404
|
+
last.set(subject, r);
|
|
405
|
+
}
|
|
406
|
+
// The latest copy is the one recorded at the highest epoch, not the last row by id: two runs that
|
|
407
|
+
// overlap can land their rows out of epoch order (on Cloudflare the row follows the insert).
|
|
408
|
+
const epochs = backfills.map(epochOf).filter((e) => e !== null);
|
|
409
|
+
const theirs = epochs.length ? Math.max(...epochs) : null;
|
|
410
|
+
const missed = (r) => {
|
|
411
|
+
const mine = epochOf(r);
|
|
412
|
+
if (mine === null || theirs === null || before(r)?.path !== 'orchestrated')
|
|
413
|
+
return true;
|
|
414
|
+
return mine < theirs;
|
|
415
|
+
};
|
|
416
|
+
return [...last].filter(([, r]) => missed(r)).map(([subject]) => subject).sort();
|
|
417
|
+
}
|
|
418
|
+
//# sourceMappingURL=scope-copy-backfill.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scope-copy-backfill.js","sourceRoot":"","sources":["../src/scope-copy-backfill.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAkB,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,YAAY,EAAyB,MAAM,yBAAyB,CAAC;AAEjI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAErC;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,GAAG,MAAM,CAAC;AA4C3C,mDAAmD;AACnD,MAAM,QAAQ,GAAkB,CAAC,gBAAgB,EAAE,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;AAC7F,6EAA6E;AAC7E,MAAM,cAAc,GAAkB,CAAC,GAAG,QAAQ,EAAE,eAAe,CAAC,CAAC;AAErE,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAkC,EAAE,CAC9D,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAA4B,CAAC,CAAC,CAAC,IAAI,CAAC;AACjG,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAiB,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAEjG,2DAA2D;AAC3D,KAAK,UAAU,OAAO,CAAC,KAAgB,EAAE,KAAsB,EAAE,MAA4C;IAC3G,MAAM,GAAG,GAAoB,EAAE,CAAC;IAChC,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,iBAAiB;YAC1F,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAChE,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QAClB,IAAI,IAAI,CAAC,MAAM,GAAG,iBAAiB;YAAE,OAAO,GAAG,CAAC;IAClD,CAAC;AACH,CAAC;AAmBD;;;GAGG;AACH,SAAS,YAAY,CAAC,EAAY,EAAE,EAAU;IAC5C,MAAM,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,oBAAoB,CAAC,CAAC;IACzE,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,IAAI,IAAI;QAAE,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,MAAM;QAAE,OAAO,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC,CAAC;IAC9E,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;AAClD,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,EAAY,EAAE,SAAwB;IACtD,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IACzD,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,eAAe,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,CAAC,CAAC,EAAE,EAAE,IAAI,IAAI,CAAC;AAC3H,CAAC;AAED,MAAM,OAAO;IAMkB,KAAK;IALjB,QAAQ,GAAG,IAAI,GAAG,EAA4B,CAAC;IAC/C,SAAS,GAAG,IAAI,GAAG,EAA6B,CAAC;IACjD,WAAW,GAAG,IAAI,GAAG,EAA0C,CAAC;IACzE,cAAc,CAAuC;IAE7D,YAA6B,KAAuB;qBAAvB,KAAK;IAAqB,CAAC;IAExD,SAAS,CAAC,EAAU,EAAE,QAAgB;QACpC,MAAM,GAAG,GAAG,GAAG,QAAQ,IAAI,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;QACjH,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;IACjC,CAAC;IAED,QAAQ,CAAC,QAAkB,EAAE,OAAgB;QAC3C,MAAM,GAAG,GAAG,GAAG,QAAQ,IAAI,OAAO,EAAE,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACpF,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,QAAkB,EAAE,OAAgB;QACrD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;QACpC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC9C,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC;YAC9C,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;YACpE,wBAAwB,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC;SAC1D,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,CAAC,CAAC;QAClE,OAAO;YACL,QAAQ,EAAE,OAAO;YACjB,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC;YACtB,aAAa,EAAE,KAAK,EAAE,UAAU,IAAI,IAAI;YACxC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;YAC3G,IAAI;YACJ,SAAS;YACT,QAAQ,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,QAAQ,IAAI,IAAI;YAC7H,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAC9C,UAAU,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;SACvH,CAAC;IACJ,CAAC;IAED,8CAA8C;IAC9C,KAAK,CAAC,QAAQ,CAAC,EAAU,EAAE,QAAgB;QACzC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,CAAC;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,QAAQ,0BAA0B,EAAE,CAAC;QACpF,IAAI,CAAC,CAAC,CAAC,aAAa;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,QAAQ,8BAA8B,EAAE,CAAC;QACtG,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC,aAAa,EAAE,CAAC;IACxC,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,EAAU,EAAE,KAAoB;QACrE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;QACpC,IAAI,CAAC,cAAc,KAAK,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;QAChF,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;QACtE,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC;QACrF,MAAM,SAAS,GAAG;YAChB,OAAO,EAAE,IAAI,QAAQ,6CAA6C,IAAI,QAAQ,KAAK,IAAI,gBAAgB,IAAI;gBACzG,qEAAqE;SACxE,CAAC;QACF,MAAM,WAAW,GAAG,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,QAAQ,KAAK,QAAQ,CAAC,CAAC;QACnI,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAAE,OAAO,SAAS,CAAC;QAC9D,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3E,IAAI,UAAU;YAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;QACjD,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,IAAI,EAAE;gBAC9E,MAAM,GAAG,GAA0B,EAAE,CAAC;gBACtC,SAAS,CAAC;oBACR,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,EACjE,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,iBAAiB,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC1G,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClB,IAAI,IAAI,CAAC,MAAM,GAAG,iBAAiB;wBAAE,OAAO,GAAG,CAAC;gBAClD,CAAC;YACH,CAAC,CAAC,EAAE,CAAC,CAAC;QACN,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QACtD,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAAE,OAAO,SAAS,CAAC;QAC1D,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,OAAO,EAAE,IAAI,QAAQ,4BAA4B,EAAE,mDAAmD,EAAE,CAAC;QAC7H,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,EAAY,EAAE,EAAU,EAAE,EAAU,EAAE,KAAoB;QACtE,MAAM,GAAG,GAAG,YAAY,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACjC,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,OAAO,oDAAoD,EAAE,CAAC;QACnH,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;IAC5E,CAAC;IAED,oGAAoG;IACpG,KAAK,CAAC,eAAe,CAAC,EAAY,EAAE,EAAU,EAAE,EAAU,EAAE,KAAK,GAAkB,EAAE;QACnF,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,kBAAkB,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC;YAC5D,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ;gBAAE,OAAO,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,EAAE,+BAA+B,EAAE,CAAC;YACpG,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,QAAQ;YAAE,OAAO,EAAE,OAAO,EAAE,iDAAiD,EAAE,CAAC;QACxF,OAAO,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAED,6FAA6F;IAC7F,KAAK,CAAC,YAAY,CAAC,EAAY,EAAE,GAAkB;QACjD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,GAAG,CAAC,MAAM,KAAK,oBAAoB,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;iBACrE,MAAM,CAAC,CAAC,GAAG,EAAiB,EAAE,CAAC,GAAG,KAAK,IAAI,CAAC;iBAC5C,GAAG,CAAC,CAAC,SAAS,EAAa,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;YAClD,2FAA2F;YAC3F,OAAO,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACxG,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,kBAAkB,EAAE,CAAC;YACtC,wFAAwF;YACxF,IAAI,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC;gBAAE,OAAO,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;YAC3D,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YAC7C,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ;gBAAE,OAAO,CAAC,EAAE,OAAO,EAAE,2CAA2C,EAAE,CAAC,CAAC;YAC/F,MAAM,GAAG,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;YAC5F,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;QACnE,CAAC;QACD,wEAAwE;QACxE,IAAI,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,SAAS,EAAE,EAAE;YAAE,OAAO,EAAE,CAAC;QAC3C,IAAI,KAAK,EAAE,IAAI,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC,CAAC,mBAAmB;QAC7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAC3D,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QACzB,+FAA+F;QAC/F,qFAAqF;QACrF,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC/C,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAqB,CAAC,CAAC;YACvE,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM;gBAAE,OAAO,CAAC,EAAE,OAAO,EAAE,qBAAqB,UAAU,qCAAqC,EAAE,CAAC,CAAC;YACtI,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QACzE,CAAC;QACD,IAAI,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO,EAAE,CAAC,CAAC,oDAAoD;QAC7F,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,KAAuB,EACvB,IAA0D;IAE1D,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAC1E,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACrI,MAAM,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;IAClC,MAAM,OAAO,GAAwB,EAAE,CAAC;IACxC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAoB,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAA2B,CAAC;QACjD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAyB,CAAC;QAC9C,MAAM,IAAI,GAAG,EAAE,SAAS,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QACtD,MAAM,OAAO,GAAG,CAAC,SAAwB,EAAE,MAAc,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/H,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,EAAE,CAAC;YAAC,OAAO,CAAC,IAAI,EAAE,kCAAkC,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAC3F,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpD,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC;YAAC,OAAO,CAAC,IAAI,EAAE,qEAAqE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QACnH,oGAAoG;QACpG,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,OAAO,EAAE,CAAC,EAAE,CAAC;YAC3D,MAAM,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC;YACrC,OAAO,CAAC,IAAI,EAAE,kFAAkF,CAAC,CAAC;QACpG,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC;YACnD,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;gBAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;YAC9B,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC;YAClC,IAAI,EAAE,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC;YACpF,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC;YACxF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACxE,IAAI,CAAC,MAAM,EAAE,CAAC;gBAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC;YACxF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC;YAChF,IAAI,CAAC;gBACH,IAAI,MAAM,uBAAuB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;oBACnD,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,8CAA8C;oBACvE,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBAC1C,SAAS;gBACX,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,SAAS,EAAE,gCAAgC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAClG,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,uBAAuB,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/G,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBAAC,OAAO,CAAC,SAAS,EAAE,8DAA8D,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC;YAC3H,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBAAC,OAAO,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC;YAC7F,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACxB,IAAI,MAAM,KAAK,UAAU;gBAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC;SAC9F,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;SAC7H,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC;gBAClF,KAAK,EAAE,SAAS,EAAE,qBAAqB,EAAE,KAAK,EAAE,YAAY;gBAC5D,QAAQ,EAAE,CAAC,CAAC,QAA2B,EAAE,OAAO,EAAE,CAAC,CAAC,OAAyB;gBAC7E,OAAO,EAAE,iBAAiB,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,KAAK;oBACvG,6FAA6F;aAChG,CAAC,CAAC;YACH,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC;gBAChD,KAAK,EAAE,SAAS,EAAE,qBAAqB,EAAE,KAAK,EAAE,eAAe;gBAC/D,QAAQ,EAAE,CAAC,CAAC,QAAoB,EAAE,OAAO,EAAE,CAAC,CAAC,OAAkB;gBAC/D,OAAO,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,+BAA+B,CAAC,CAAC,OAAO,6CAA6C;oBAChH,+EAA+E;aAClF,CAAC,CAAC;SACJ,CAAC,CAAC;IACL,CAAC;IACD,MAAM,MAAM,GAAG,EAAE,cAAc,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC/F,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7C,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI;QAClD,IAAI,EAAE,IAAI,CAAC,MAAM,GAAG,KAAK;QACzB,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,OAAO;QACP,MAAM;QACN,YAAY;KACb,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,oBAAoB,CAAC,KAAgB,EAAE,KAAsB,EAAE,EAAY;IACxF,MAAM,MAAM,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC;IAC9D,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;QAC5D,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;KAClE,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,CAAC,CAAgB,EAAiB,EAAE;QAClD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC;QAClC,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1C,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;QACvD,IAAI,OAAO;YAAE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,kGAAkG;IAClG,6FAA6F;IAC7F,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,MAAM,MAAM,GAAG,CAAC,CAAgB,EAAW,EAAE;QAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,cAAc;YAAE,OAAO,IAAI,CAAC;QACxF,OAAO,IAAI,GAAG,MAAM,CAAC;IACvB,CAAC,CAAC;IACF,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AACnF,CAAC"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { type PlatformActorId, type ScopeId, type TenantId } from '@substrat-run/contracts';
|
|
2
|
+
import { type HostAdmin, type ScopeScriptCopy } from '@substrat-run/kernel';
|
|
3
|
+
import type { VerticalClient } from './vertical-client.js';
|
|
4
|
+
export interface ScopeCopyCleanup {
|
|
5
|
+
admin: HostAdmin;
|
|
6
|
+
actor: PlatformActorId;
|
|
7
|
+
resolveRef: (scriptRef: string) => Promise<VerticalClient | undefined>;
|
|
8
|
+
}
|
|
9
|
+
/** Do not discard the directory's last pointer to a copy when dispatch is unavailable. */
|
|
10
|
+
export declare function assertNoUnreachableScopeCopies(admin: HostAdmin, actor: PlatformActorId, tenantId: TenantId, scopeId: ScopeId): Promise<void>;
|
|
11
|
+
/** Stable keyset walk for a scope. Reap holds a claim; erasure checks the final
|
|
12
|
+
* row count atomically, so a move recorded during this walk forces a retry. */
|
|
13
|
+
export declare function listAllScopeScriptCopies(admin: HostAdmin, actor: PlatformActorId, tenantId: TenantId, scopeId: ScopeId): Promise<ScopeScriptCopy[]>;
|
|
14
|
+
/** `routeOf` for a scope record already read; `versionOf` lets a caller share its version reads. */
|
|
15
|
+
export declare function routeOfScope(input: ScopeCopyCleanup, scope: {
|
|
16
|
+
servingRef?: string | null;
|
|
17
|
+
vertical?: string | null;
|
|
18
|
+
verticalVersionId?: string | null;
|
|
19
|
+
}, versionOf?: (id: string, vertical: string) => Promise<{
|
|
20
|
+
deploymentRef: string | null;
|
|
21
|
+
} | undefined>): Promise<string | null>;
|
|
22
|
+
/** Whether the scope's store in this script holds the tombstone a wipe leaves: proof it was wiped. */
|
|
23
|
+
export declare function hasCarriedAwayTombstone(holder: VerticalClient, scopeId: ScopeId): Promise<boolean>;
|
|
24
|
+
/** One eligible source. A changed store is kept, never discarded to satisfy the ledger. */
|
|
25
|
+
export declare function retryScopeScriptCopy(input: ScopeCopyCleanup, copy: ScopeScriptCopy): Promise<'done' | 'kept' | 'skipped'>;
|
|
26
|
+
/** Bounded scheduled pass. Failures stay eligible and are retried on the next pass. */
|
|
27
|
+
export declare function retryScopeScriptCopies(input: ScopeCopyCleanup, limit?: number): Promise<{
|
|
28
|
+
tried: number;
|
|
29
|
+
done: number;
|
|
30
|
+
failed: number;
|
|
31
|
+
}>;
|
|
32
|
+
/**
|
|
33
|
+
* Crash recovery (#1722): settle the pending entries of moves whose lease ran out, a carry, adopt
|
|
34
|
+
* or rebind that died between recording its copies and confirming its bind. Bounded, idempotent,
|
|
35
|
+
* and never racing a live move: an entry is claimed only once its lease has run out, and a move
|
|
36
|
+
* past its lease can no longer confirm a bind, so whichever claims it acts alone.
|
|
37
|
+
*
|
|
38
|
+
* An expired entry has no confirmed bind (the confirmation settles every entry of the move in
|
|
39
|
+
* the bind's own write). So for each one:
|
|
40
|
+
* - the scope routes to its script: that store is live, and the route reaches it (`done`);
|
|
41
|
+
* - a destination the scope does not route to: its restore, if it ever landed, is wiped, fenced
|
|
42
|
+
* on the load stamp the move recorded before restoring. The wipe loads the tombstone over the
|
|
43
|
+
* whole store, so nothing of the scope, an erased subject's rows included, is left in it, and
|
|
44
|
+
* its new load stamp refuses a late restore that still expects the store it read (`done`). A
|
|
45
|
+
* refused fence, or a deployment with no fenced wipe, keeps the entry (`retained`, still
|
|
46
|
+
* reached by reap and erasure): the restore may not have landed yet;
|
|
47
|
+
* - a source the scope no longer routes to (another move took the route since): `retained`.
|
|
48
|
+
*
|
|
49
|
+
* A failure leaves the entry claimed until the sweep's lease runs out, and a later pass takes it
|
|
50
|
+
* again. Until then erasure and reap answer retry-later, as they do for any pending move.
|
|
51
|
+
*/
|
|
52
|
+
export declare function settleExpiredScopeScriptCopies(input: ScopeCopyCleanup, opts?: {
|
|
53
|
+
now?: Date;
|
|
54
|
+
limit?: number;
|
|
55
|
+
}): Promise<{
|
|
56
|
+
claimed: number;
|
|
57
|
+
settled: number;
|
|
58
|
+
failed: number;
|
|
59
|
+
}>;
|
|
60
|
+
/** The scheduled pass over the ledger: crashed moves first, then the eligible sources' wipes. */
|
|
61
|
+
export declare function sweepScopeScriptCopies(input: ScopeCopyCleanup): Promise<{
|
|
62
|
+
expired: number;
|
|
63
|
+
retried: number;
|
|
64
|
+
failed: number;
|
|
65
|
+
}>;
|
|
66
|
+
/**
|
|
67
|
+
* The ops record of storage a reap left in place (#1722): a script that predates
|
|
68
|
+
* `/internal/delete-scope` answered 501. Every reap path that strands writes it, before the
|
|
69
|
+
* directory row can go, so the bytes left behind stay visible once the row is gone. `where`
|
|
70
|
+
* names the deployment (a script ref, quoted, or how it was resolved).
|
|
71
|
+
*/
|
|
72
|
+
export declare function recordStrandedStorage(admin: HostAdmin, actor: PlatformActorId, tenantId: TenantId, scopeId: ScopeId, where: string): Promise<void>;
|
|
73
|
+
/**
|
|
74
|
+
* A destructive reap reaches every named script before the directory forgets the scope.
|
|
75
|
+
* `storageStranded` when a script answered 501 to its delete: it predates the verb, so its bytes
|
|
76
|
+
* are unreachable through every platform verb and die with the script at orphan cleanup (#248).
|
|
77
|
+
* The reap goes on (a 501 must never pin a directory row forever), but never silently: each
|
|
78
|
+
* stranded script is recorded (`recordStrandedStorage`) first, for every caller (the reap
|
|
79
|
+
* routes, the preview reap and the GC sweep's reaps), so the bytes left behind stay visible after
|
|
80
|
+
* the row is gone. A record that cannot be written fails the reap, which is then retried. The
|
|
81
|
+
* reap routes' slug fallback (`reapScopeStorage`) records what it strands the same way.
|
|
82
|
+
*/
|
|
83
|
+
export declare function reapScopeScriptCopies(input: ScopeCopyCleanup, tenantId: TenantId, scopeId: ScopeId): Promise<{
|
|
84
|
+
storageStranded: boolean;
|
|
85
|
+
}>;
|
|
86
|
+
//# sourceMappingURL=scope-copy-cleanup.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scope-copy-cleanup.d.ts","sourceRoot":"","sources":["../src/scope-copy-cleanup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,KAAK,eAAe,EAAE,KAAK,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAEzG,OAAO,EAA+D,KAAK,SAAS,EAAE,KAAK,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACzI,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,eAAe,CAAC;IACvB,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;CACxE;AAED,0FAA0F;AAC1F,wBAAsB,8BAA8B,CAClD,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAC7E,OAAO,CAAC,IAAI,CAAC,CAMf;AAED;+EAC+E;AAC/E,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAC7E,OAAO,CAAC,eAAe,EAAE,CAAC,CAU5B;AAQD,oGAAoG;AACpG,wBAAsB,YAAY,CAChC,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,EAClG,SAAS,GAAE,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,SAAS,CAC5B,GACpE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED,sGAAsG;AACtG,wBAAsB,uBAAuB,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAIxG;AAED,2FAA2F;AAC3F,wBAAsB,oBAAoB,CAAC,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CA6B/H;AAED,uFAAuF;AACvF,wBAAsB,sBAAsB,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,SAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAa3I;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,8BAA8B,CAClD,KAAK,EAAE,gBAAgB,EAAE,IAAI,GAAE;IAAE,GAAG,CAAC,EAAE,IAAI,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GACjE,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAqB/D;AAuBD,iGAAiG;AACjG,wBAAsB,sBAAsB,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAInI;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAC5F,OAAO,CAAC,IAAI,CAAC,CAWf;AAED;;;;;;;;;GASG;AACH,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAC5D,OAAO,CAAC;IAAE,eAAe,EAAE,OAAO,CAAA;CAAE,CAAC,CAgCvC"}
|