@lunora/runtime 0.0.0 → 1.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +105 -0
- package/README.md +121 -9
- package/__assets__/package-og.svg +14 -0
- package/dist/index.d.mts +2259 -0
- package/dist/index.d.ts +2259 -0
- package/dist/index.mjs +14 -0
- package/dist/packem_shared/DEFAULT_REGISTRY_CACHE_TTL_MS-BpCwo_mo.mjs +75 -0
- package/dist/packem_shared/LunoraError-CL0aOtpo.mjs +46 -0
- package/dist/packem_shared/analyticsEngineSink-DqEvrQs0.mjs +141 -0
- package/dist/packem_shared/composeWorker-BmfKl1ei.mjs +2501 -0
- package/dist/packem_shared/createCrossShardRelationCapabilities-C0KOf7er.mjs +61 -0
- package/dist/packem_shared/createQueryCoordinator-DbxC7iUz.mjs +838 -0
- package/dist/packem_shared/decorateResponse-DbISh_Wi.mjs +233 -0
- package/dist/packem_shared/emitLogEvent-pEdtqAK8.mjs +20 -0
- package/dist/packem_shared/resolveShard-DDkzWtrU.mjs +9 -0
- package/dist/packem_shared/toAirbyteMessages-DrHdplb4.mjs +39 -0
- package/package.json +37 -17
|
@@ -0,0 +1,838 @@
|
|
|
1
|
+
import { LunoraError } from './LunoraError-CL0aOtpo.mjs';
|
|
2
|
+
import { resolveShard } from './resolveShard-DDkzWtrU.mjs';
|
|
3
|
+
|
|
4
|
+
const createStaticShardRegistry = (table_to_keys) => {
|
|
5
|
+
return {
|
|
6
|
+
listShardKeys(table) {
|
|
7
|
+
return table_to_keys[table] ?? [];
|
|
8
|
+
}
|
|
9
|
+
};
|
|
10
|
+
};
|
|
11
|
+
const mergeStrategyForAggregate = (input) => {
|
|
12
|
+
if (input.kind === "count") {
|
|
13
|
+
return { kind: "sum" };
|
|
14
|
+
}
|
|
15
|
+
if (input.kind === "scalar") {
|
|
16
|
+
if (input.op === "count" || input.op === "sum") {
|
|
17
|
+
return { kind: "sum" };
|
|
18
|
+
}
|
|
19
|
+
if (input.op === "max") {
|
|
20
|
+
return { kind: "max" };
|
|
21
|
+
}
|
|
22
|
+
if (input.op === "min") {
|
|
23
|
+
return { kind: "min" };
|
|
24
|
+
}
|
|
25
|
+
throw new LunoraError('aggregate({ op: "avg" }) is not supported across shards in v1 — fan out sum + count separately', {
|
|
26
|
+
code: "BAD_REQUEST",
|
|
27
|
+
status: 400
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
const op = input.agg?.op ?? "count";
|
|
31
|
+
if (op === "count" || op === "sum") {
|
|
32
|
+
return { kind: "groupBy", op: "sum" };
|
|
33
|
+
}
|
|
34
|
+
if (op === "max") {
|
|
35
|
+
return { kind: "groupBy", op: "max" };
|
|
36
|
+
}
|
|
37
|
+
if (op === "min") {
|
|
38
|
+
return { kind: "groupBy", op: "min" };
|
|
39
|
+
}
|
|
40
|
+
throw new LunoraError('groupBy({ agg: { op: "avg" } }) is not supported across shards in v1 — fan out sum + count separately', {
|
|
41
|
+
code: "BAD_REQUEST",
|
|
42
|
+
status: 400
|
|
43
|
+
});
|
|
44
|
+
};
|
|
45
|
+
const DEFAULT_CONCURRENCY = 16;
|
|
46
|
+
const DEFAULT_TIMEOUT_MS = 5e3;
|
|
47
|
+
const unwrapResult = (value) => value !== null && typeof value === "object" && "result" in value ? value.result : value;
|
|
48
|
+
const readRunCounts = (payload) => {
|
|
49
|
+
const run = payload ?? {};
|
|
50
|
+
return {
|
|
51
|
+
changed: typeof run.changed === "number" ? run.changed : 0,
|
|
52
|
+
processed: typeof run.processed === "number" ? run.processed : 0,
|
|
53
|
+
status: typeof run.status === "string" ? run.status : void 0
|
|
54
|
+
};
|
|
55
|
+
};
|
|
56
|
+
const rollUpStatus = (anyFailed, incomplete) => {
|
|
57
|
+
if (anyFailed) {
|
|
58
|
+
return "failed";
|
|
59
|
+
}
|
|
60
|
+
if (incomplete) {
|
|
61
|
+
return "in_progress";
|
|
62
|
+
}
|
|
63
|
+
return "completed";
|
|
64
|
+
};
|
|
65
|
+
const rollUpMigration = (results) => {
|
|
66
|
+
const shards = [];
|
|
67
|
+
let ok = 0;
|
|
68
|
+
let failed = 0;
|
|
69
|
+
let changed = 0;
|
|
70
|
+
let processed = 0;
|
|
71
|
+
let anyInProgress = false;
|
|
72
|
+
let anyFailed = false;
|
|
73
|
+
for (const result of results) {
|
|
74
|
+
if (result.kind === "err") {
|
|
75
|
+
failed += 1;
|
|
76
|
+
shards.push({ error: { message: result.message, timedOut: result.timedOut }, shardKey: result.shardKey });
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
ok += 1;
|
|
80
|
+
const payload = unwrapResult(result.value);
|
|
81
|
+
const counts = readRunCounts(payload);
|
|
82
|
+
changed += counts.changed;
|
|
83
|
+
processed += counts.processed;
|
|
84
|
+
anyInProgress ||= counts.status === "in_progress";
|
|
85
|
+
anyFailed ||= counts.status === "failed";
|
|
86
|
+
shards.push({ result: payload, shardKey: result.shardKey });
|
|
87
|
+
}
|
|
88
|
+
return { changed, failed, ok, processed, shards, status: rollUpStatus(anyFailed, anyInProgress || failed > 0) };
|
|
89
|
+
};
|
|
90
|
+
const readRankCounts = (payload) => {
|
|
91
|
+
const run = payload ?? {};
|
|
92
|
+
return {
|
|
93
|
+
before: typeof run.before === "number" && Number.isFinite(run.before) ? run.before : 0,
|
|
94
|
+
total: typeof run.total === "number" && Number.isFinite(run.total) ? run.total : 0
|
|
95
|
+
};
|
|
96
|
+
};
|
|
97
|
+
const rollUpRank = (results) => {
|
|
98
|
+
const shards = [];
|
|
99
|
+
let ok = 0;
|
|
100
|
+
let failed = 0;
|
|
101
|
+
let before = 0;
|
|
102
|
+
let total = 0;
|
|
103
|
+
for (const result of results) {
|
|
104
|
+
if (result.kind === "err") {
|
|
105
|
+
failed += 1;
|
|
106
|
+
shards.push({ error: { message: result.message, timedOut: result.timedOut }, shardKey: result.shardKey });
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
ok += 1;
|
|
110
|
+
const counts = readRankCounts(unwrapResult(result.value));
|
|
111
|
+
before += counts.before;
|
|
112
|
+
total += counts.total;
|
|
113
|
+
shards.push({ result: counts, shardKey: result.shardKey });
|
|
114
|
+
}
|
|
115
|
+
return { failed, ok, partial: failed > 0, position: before + 1, shards, total };
|
|
116
|
+
};
|
|
117
|
+
const RANK_CLASS_NULL = 0;
|
|
118
|
+
const RANK_CLASS_NUMBER = 1;
|
|
119
|
+
const RANK_CLASS_TEXT = 2;
|
|
120
|
+
const rankValueClass = (value) => {
|
|
121
|
+
if (value === null || value === void 0) {
|
|
122
|
+
return RANK_CLASS_NULL;
|
|
123
|
+
}
|
|
124
|
+
if (typeof value === "number") {
|
|
125
|
+
return RANK_CLASS_NUMBER;
|
|
126
|
+
}
|
|
127
|
+
return RANK_CLASS_TEXT;
|
|
128
|
+
};
|
|
129
|
+
const compareRankValueAsc = (a, b) => {
|
|
130
|
+
const classA = rankValueClass(a);
|
|
131
|
+
const classB = rankValueClass(b);
|
|
132
|
+
if (classA !== classB) {
|
|
133
|
+
return classA < classB ? -1 : 1;
|
|
134
|
+
}
|
|
135
|
+
if (classA === RANK_CLASS_NULL) {
|
|
136
|
+
return 0;
|
|
137
|
+
}
|
|
138
|
+
if (classA === RANK_CLASS_NUMBER) {
|
|
139
|
+
const numberA = a;
|
|
140
|
+
const numberB = b;
|
|
141
|
+
if (numberA < numberB) {
|
|
142
|
+
return -1;
|
|
143
|
+
}
|
|
144
|
+
return numberA > numberB ? 1 : 0;
|
|
145
|
+
}
|
|
146
|
+
const textA = String(a);
|
|
147
|
+
const textB = String(b);
|
|
148
|
+
if (textA < textB) {
|
|
149
|
+
return -1;
|
|
150
|
+
}
|
|
151
|
+
return textA > textB ? 1 : 0;
|
|
152
|
+
};
|
|
153
|
+
const compareRankKeys = (a, b, directions) => {
|
|
154
|
+
const partitionCmp = compareRankValueAsc(a.partitionKey, b.partitionKey);
|
|
155
|
+
if (partitionCmp !== 0) {
|
|
156
|
+
return partitionCmp;
|
|
157
|
+
}
|
|
158
|
+
const length = Math.max(a.sortValues.length, b.sortValues.length);
|
|
159
|
+
for (let i = 0; i < length; i += 1) {
|
|
160
|
+
const valueCmp = compareRankValueAsc(a.sortValues[i], b.sortValues[i]);
|
|
161
|
+
if (valueCmp !== 0) {
|
|
162
|
+
return directions[i] === "desc" ? -valueCmp : valueCmp;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return compareRankValueAsc(a.rowId, b.rowId);
|
|
166
|
+
};
|
|
167
|
+
const encodeRankPageCursor = (state) => {
|
|
168
|
+
const json = JSON.stringify(state);
|
|
169
|
+
const bytes = new TextEncoder().encode(json);
|
|
170
|
+
let binary = "";
|
|
171
|
+
for (const byte of bytes) {
|
|
172
|
+
binary += String.fromCodePoint(byte);
|
|
173
|
+
}
|
|
174
|
+
return btoa(binary);
|
|
175
|
+
};
|
|
176
|
+
const decodeRankPageCursor = (cursor) => {
|
|
177
|
+
let json;
|
|
178
|
+
try {
|
|
179
|
+
const binary = atob(cursor);
|
|
180
|
+
const bytes = Uint8Array.from(binary, (character) => character.codePointAt(0) ?? 0);
|
|
181
|
+
json = new TextDecoder().decode(bytes);
|
|
182
|
+
} catch {
|
|
183
|
+
return { perShard: {} };
|
|
184
|
+
}
|
|
185
|
+
try {
|
|
186
|
+
const parsed = JSON.parse(json);
|
|
187
|
+
if (parsed !== null && typeof parsed === "object" && "perShard" in parsed) {
|
|
188
|
+
const { perShard } = parsed;
|
|
189
|
+
if (perShard !== null && typeof perShard === "object") {
|
|
190
|
+
return { perShard };
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
} catch {
|
|
194
|
+
}
|
|
195
|
+
return { perShard: {} };
|
|
196
|
+
};
|
|
197
|
+
const readRankPageResult = (payload) => {
|
|
198
|
+
const value = payload ?? {};
|
|
199
|
+
const rows = Array.isArray(value.rows) ? value.rows : [];
|
|
200
|
+
const directions = Array.isArray(value.directions) ? value.directions : [];
|
|
201
|
+
return { directions, hasMore: value.hasMore === true, rows };
|
|
202
|
+
};
|
|
203
|
+
const pickSmallestHead = (slices, directions) => {
|
|
204
|
+
let best;
|
|
205
|
+
let bestKey;
|
|
206
|
+
for (const slice of slices) {
|
|
207
|
+
const candidate = slice.rows[slice.head];
|
|
208
|
+
if (candidate === void 0) {
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
if (bestKey === void 0 || compareRankKeys(candidate.key, bestKey, directions) < 0) {
|
|
212
|
+
best = slice;
|
|
213
|
+
bestKey = candidate.key;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return best;
|
|
217
|
+
};
|
|
218
|
+
const buildNextRankPageCursor = (slices, lastConsumedKey) => {
|
|
219
|
+
let anyRemaining = false;
|
|
220
|
+
const sliceKeys = /* @__PURE__ */ new Set();
|
|
221
|
+
for (const slice of slices) {
|
|
222
|
+
sliceKeys.add(slice.shardKey);
|
|
223
|
+
if (slice.head < slice.rows.length || slice.hasMore) {
|
|
224
|
+
anyRemaining = true;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
const perShard = { ...lastConsumedKey };
|
|
228
|
+
for (const shardKey of Object.keys(lastConsumedKey)) {
|
|
229
|
+
if (!sliceKeys.has(shardKey)) {
|
|
230
|
+
anyRemaining = true;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
if (!anyRemaining) {
|
|
234
|
+
return null;
|
|
235
|
+
}
|
|
236
|
+
return encodeRankPageCursor({ perShard });
|
|
237
|
+
};
|
|
238
|
+
const kWayMergeRankPages = (slices, take, directions, priorPerShard) => {
|
|
239
|
+
const page = [];
|
|
240
|
+
const lastConsumedKey = { ...priorPerShard };
|
|
241
|
+
while (page.length < take) {
|
|
242
|
+
const best = pickSmallestHead(slices, directions);
|
|
243
|
+
if (best === void 0) {
|
|
244
|
+
break;
|
|
245
|
+
}
|
|
246
|
+
const row = best.rows[best.head];
|
|
247
|
+
if (row !== void 0) {
|
|
248
|
+
page.push(row.doc);
|
|
249
|
+
lastConsumedKey[best.shardKey] = row.key;
|
|
250
|
+
}
|
|
251
|
+
best.head += 1;
|
|
252
|
+
}
|
|
253
|
+
const nextCursor = buildNextRankPageCursor(slices, lastConsumedKey);
|
|
254
|
+
return { isDone: nextCursor === null, nextCursor, page };
|
|
255
|
+
};
|
|
256
|
+
const rollUpExport = (results) => {
|
|
257
|
+
const shards = [];
|
|
258
|
+
let ok = 0;
|
|
259
|
+
let failed = 0;
|
|
260
|
+
for (const result of results) {
|
|
261
|
+
if (result.kind === "err") {
|
|
262
|
+
failed += 1;
|
|
263
|
+
shards.push({ error: { message: result.message, timedOut: result.timedOut }, shardKey: result.shardKey });
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
ok += 1;
|
|
267
|
+
const payload = unwrapResult(result.value);
|
|
268
|
+
const rows = Array.isArray(payload?.rows) ? payload.rows : [];
|
|
269
|
+
shards.push({ rows, shardKey: result.shardKey });
|
|
270
|
+
}
|
|
271
|
+
return { failed, ok, shards };
|
|
272
|
+
};
|
|
273
|
+
const rollUpCdcSync = (results) => {
|
|
274
|
+
const shards = [];
|
|
275
|
+
let ok = 0;
|
|
276
|
+
let failed = 0;
|
|
277
|
+
for (const { outcome, sinceSeq } of results) {
|
|
278
|
+
if (outcome.kind === "err") {
|
|
279
|
+
failed += 1;
|
|
280
|
+
shards.push({ cursor: sinceSeq, error: { message: outcome.message, timedOut: outcome.timedOut }, shardKey: outcome.shardKey });
|
|
281
|
+
continue;
|
|
282
|
+
}
|
|
283
|
+
ok += 1;
|
|
284
|
+
const payload = unwrapResult(outcome.value);
|
|
285
|
+
const changes = Array.isArray(payload?.changes) ? payload.changes : [];
|
|
286
|
+
const cursor = typeof payload?.cursor === "number" ? payload.cursor : sinceSeq;
|
|
287
|
+
shards.push({ changes, cursor, shardKey: outcome.shardKey });
|
|
288
|
+
}
|
|
289
|
+
return { failed, ok, shards };
|
|
290
|
+
};
|
|
291
|
+
const rollUpApplyCdc = (results) => {
|
|
292
|
+
let ok = 0;
|
|
293
|
+
let failed = 0;
|
|
294
|
+
let applied = 0;
|
|
295
|
+
for (const outcome of results) {
|
|
296
|
+
if (outcome.kind === "err") {
|
|
297
|
+
failed += 1;
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
ok += 1;
|
|
301
|
+
const payload = unwrapResult(outcome.value);
|
|
302
|
+
applied += typeof payload?.applied === "number" ? payload.applied : 0;
|
|
303
|
+
}
|
|
304
|
+
return { applied, failed, ok };
|
|
305
|
+
};
|
|
306
|
+
const readShardRequests = (payload) => {
|
|
307
|
+
const snapshot = payload ?? {};
|
|
308
|
+
return typeof snapshot.requests === "number" && Number.isFinite(snapshot.requests) && snapshot.requests >= 0 ? snapshot.requests : 0;
|
|
309
|
+
};
|
|
310
|
+
const rollUpShardTraffic = (results) => {
|
|
311
|
+
const shards = [];
|
|
312
|
+
let ok = 0;
|
|
313
|
+
let failed = 0;
|
|
314
|
+
for (const result of results) {
|
|
315
|
+
if (result.kind === "err") {
|
|
316
|
+
failed += 1;
|
|
317
|
+
shards.push({ requests: 0, shardKey: result.shardKey });
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
ok += 1;
|
|
321
|
+
shards.push({ requests: readShardRequests(unwrapResult(result.value)), shardKey: result.shardKey });
|
|
322
|
+
}
|
|
323
|
+
return { failed, ok, shards };
|
|
324
|
+
};
|
|
325
|
+
const rollUpImport = (results) => {
|
|
326
|
+
const shards = [];
|
|
327
|
+
const inserted = {};
|
|
328
|
+
const errors = [];
|
|
329
|
+
let conflicts = 0;
|
|
330
|
+
let ok = 0;
|
|
331
|
+
let failed = 0;
|
|
332
|
+
for (const result of results) {
|
|
333
|
+
if (result.kind === "err") {
|
|
334
|
+
failed += 1;
|
|
335
|
+
shards.push({ error: { message: result.message, timedOut: result.timedOut }, shardKey: result.shardKey });
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
ok += 1;
|
|
339
|
+
const payload = unwrapResult(result.value);
|
|
340
|
+
const shardInserted = payload?.inserted ?? {};
|
|
341
|
+
for (const [table, count] of Object.entries(shardInserted)) {
|
|
342
|
+
inserted[table] = (inserted[table] ?? 0) + count;
|
|
343
|
+
}
|
|
344
|
+
const payloadErrors = payload?.errors;
|
|
345
|
+
if (Array.isArray(payloadErrors)) {
|
|
346
|
+
errors.push(...payloadErrors);
|
|
347
|
+
}
|
|
348
|
+
conflicts += payload?.conflicts ?? 0;
|
|
349
|
+
shards.push({
|
|
350
|
+
result: {
|
|
351
|
+
conflicts: payload?.conflicts ?? 0,
|
|
352
|
+
errors: payload?.errors ?? [],
|
|
353
|
+
inserted: shardInserted
|
|
354
|
+
},
|
|
355
|
+
shardKey: result.shardKey
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
return { conflicts, errors, failed, inserted, ok, shards };
|
|
359
|
+
};
|
|
360
|
+
const prepareShardRpc = (request) => {
|
|
361
|
+
return {
|
|
362
|
+
body: JSON.stringify({ args: request.args ?? {}, functionPath: request.functionPath }),
|
|
363
|
+
headers: { "content-type": "application/json", ...request.headers }
|
|
364
|
+
};
|
|
365
|
+
};
|
|
366
|
+
const callOneShard = async (namespace, shardKey, prepared, timeoutMs) => {
|
|
367
|
+
const stub = resolveShard(namespace, shardKey);
|
|
368
|
+
const controller = new AbortController();
|
|
369
|
+
const forwarded = new Request("https://shard.internal/rpc", {
|
|
370
|
+
body: prepared.body,
|
|
371
|
+
headers: prepared.headers,
|
|
372
|
+
method: "POST",
|
|
373
|
+
signal: controller.signal
|
|
374
|
+
});
|
|
375
|
+
let timeoutId;
|
|
376
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
377
|
+
timeoutId = setTimeout(() => {
|
|
378
|
+
try {
|
|
379
|
+
controller.abort();
|
|
380
|
+
} catch {
|
|
381
|
+
}
|
|
382
|
+
resolve({ kind: "err", message: `shard "${shardKey}" timed out after ${String(timeoutMs)}ms`, shardKey, timedOut: true });
|
|
383
|
+
}, timeoutMs);
|
|
384
|
+
});
|
|
385
|
+
const fetchPromise = (async () => {
|
|
386
|
+
try {
|
|
387
|
+
const response = await stub.fetch(forwarded);
|
|
388
|
+
if (!response.ok) {
|
|
389
|
+
return { kind: "err", message: `shard "${shardKey}" returned ${String(response.status)}`, shardKey, timedOut: false };
|
|
390
|
+
}
|
|
391
|
+
const value = await response.json();
|
|
392
|
+
return { kind: "ok", shardKey, value };
|
|
393
|
+
} catch (error) {
|
|
394
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
395
|
+
return { kind: "err", message: `shard "${shardKey}" threw: ${message}`, shardKey, timedOut: false };
|
|
396
|
+
}
|
|
397
|
+
})();
|
|
398
|
+
try {
|
|
399
|
+
return await Promise.race([fetchPromise, timeoutPromise]);
|
|
400
|
+
} finally {
|
|
401
|
+
if (timeoutId !== void 0) {
|
|
402
|
+
clearTimeout(timeoutId);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
};
|
|
406
|
+
const runBoundedFanOut = async (namespace, keys, request, maxConcurrency, timeoutMs) => {
|
|
407
|
+
if (keys.length === 0) {
|
|
408
|
+
return [];
|
|
409
|
+
}
|
|
410
|
+
const prepared = prepareShardRpc(request);
|
|
411
|
+
const outcomes = Array.from({ length: keys.length });
|
|
412
|
+
let cursor = 0;
|
|
413
|
+
const worker = async () => {
|
|
414
|
+
while (true) {
|
|
415
|
+
const index = cursor;
|
|
416
|
+
cursor += 1;
|
|
417
|
+
const shardKey = keys[index];
|
|
418
|
+
if (index >= keys.length || shardKey === void 0) {
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
outcomes[index] = await callOneShard(namespace, shardKey, prepared, timeoutMs);
|
|
422
|
+
}
|
|
423
|
+
};
|
|
424
|
+
const concurrency = Math.min(maxConcurrency, keys.length);
|
|
425
|
+
await Promise.all(Array.from({ length: concurrency }, () => worker()));
|
|
426
|
+
return outcomes;
|
|
427
|
+
};
|
|
428
|
+
const compareCodeUnits = (a, b) => {
|
|
429
|
+
if (a < b) {
|
|
430
|
+
return -1;
|
|
431
|
+
}
|
|
432
|
+
return a > b ? 1 : 0;
|
|
433
|
+
};
|
|
434
|
+
const canonicalJson = (record) => {
|
|
435
|
+
const ordered = {};
|
|
436
|
+
for (const key of Object.keys(record).toSorted(compareCodeUnits)) {
|
|
437
|
+
ordered[key] = record[key] ?? null;
|
|
438
|
+
}
|
|
439
|
+
return JSON.stringify(ordered);
|
|
440
|
+
};
|
|
441
|
+
const mergeConcat = (values) => {
|
|
442
|
+
const out = [];
|
|
443
|
+
for (const v of values) {
|
|
444
|
+
if (Array.isArray(v)) {
|
|
445
|
+
out.push(...v);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
return out;
|
|
449
|
+
};
|
|
450
|
+
const combineGroupByValue = (current, incoming, op) => {
|
|
451
|
+
switch (op) {
|
|
452
|
+
case "max": {
|
|
453
|
+
return Math.max(current, incoming);
|
|
454
|
+
}
|
|
455
|
+
case "min": {
|
|
456
|
+
return Math.min(current, incoming);
|
|
457
|
+
}
|
|
458
|
+
case "sum": {
|
|
459
|
+
return current + incoming;
|
|
460
|
+
}
|
|
461
|
+
default: {
|
|
462
|
+
return current;
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
};
|
|
466
|
+
const accumulateGroupByEntry = (merged, entry, op) => {
|
|
467
|
+
if (entry === null || typeof entry !== "object") {
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
470
|
+
const entryKey = entry.key ?? {};
|
|
471
|
+
const entryValue = entry.value ?? null;
|
|
472
|
+
const stableKey = canonicalJson(entryKey);
|
|
473
|
+
const existing = merged.get(stableKey);
|
|
474
|
+
if (!existing) {
|
|
475
|
+
merged.set(stableKey, { key: entryKey, value: entryValue });
|
|
476
|
+
return;
|
|
477
|
+
}
|
|
478
|
+
if (existing.value === null) {
|
|
479
|
+
existing.value = entryValue;
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
if (entryValue === null) {
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
existing.value = combineGroupByValue(existing.value, entryValue, op);
|
|
486
|
+
};
|
|
487
|
+
const mergeGroupBy = (values, op) => {
|
|
488
|
+
const merged = /* @__PURE__ */ new Map();
|
|
489
|
+
for (const v of values) {
|
|
490
|
+
if (!Array.isArray(v)) {
|
|
491
|
+
continue;
|
|
492
|
+
}
|
|
493
|
+
for (const entry of v) {
|
|
494
|
+
accumulateGroupByEntry(merged, entry, op);
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
return [...merged.values()];
|
|
498
|
+
};
|
|
499
|
+
const mergeNumeric = (values, pick) => {
|
|
500
|
+
let best = null;
|
|
501
|
+
for (const v of values) {
|
|
502
|
+
if (typeof v === "number" && Number.isFinite(v)) {
|
|
503
|
+
best = best === null ? v : pick(best, v);
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
return best;
|
|
507
|
+
};
|
|
508
|
+
const mergeSum = (values) => {
|
|
509
|
+
let total = 0;
|
|
510
|
+
for (const v of values) {
|
|
511
|
+
if (typeof v === "number" && Number.isFinite(v)) {
|
|
512
|
+
total += v;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
return total;
|
|
516
|
+
};
|
|
517
|
+
const mergeRank = (values) => {
|
|
518
|
+
let before = 0;
|
|
519
|
+
let total = 0;
|
|
520
|
+
for (const v of values) {
|
|
521
|
+
if (v === null || typeof v !== "object") {
|
|
522
|
+
continue;
|
|
523
|
+
}
|
|
524
|
+
const payload = v;
|
|
525
|
+
if (typeof payload.before === "number" && Number.isFinite(payload.before)) {
|
|
526
|
+
before += payload.before;
|
|
527
|
+
}
|
|
528
|
+
if (typeof payload.total === "number" && Number.isFinite(payload.total)) {
|
|
529
|
+
total += payload.total;
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
return { position: before + 1, total };
|
|
533
|
+
};
|
|
534
|
+
const mergeTopK = (values, strategy) => {
|
|
535
|
+
const collected = [];
|
|
536
|
+
for (const v of values) {
|
|
537
|
+
if (!Array.isArray(v)) {
|
|
538
|
+
continue;
|
|
539
|
+
}
|
|
540
|
+
for (const row of v) {
|
|
541
|
+
if (row === null || typeof row !== "object") {
|
|
542
|
+
continue;
|
|
543
|
+
}
|
|
544
|
+
const raw = row[strategy.by];
|
|
545
|
+
const score = typeof raw === "number" && Number.isFinite(raw) ? raw : Number.NEGATIVE_INFINITY;
|
|
546
|
+
collected.push({ row, score });
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
const direction = strategy.direction ?? "desc";
|
|
550
|
+
collected.sort((a, b) => direction === "asc" ? a.score - b.score : b.score - a.score);
|
|
551
|
+
return collected.slice(0, strategy.k).map((entry) => entry.row);
|
|
552
|
+
};
|
|
553
|
+
const mergeShardResults = (values, strategy) => {
|
|
554
|
+
switch (strategy.kind) {
|
|
555
|
+
case "concat": {
|
|
556
|
+
return mergeConcat(values);
|
|
557
|
+
}
|
|
558
|
+
case "first": {
|
|
559
|
+
return values[0];
|
|
560
|
+
}
|
|
561
|
+
case "groupBy": {
|
|
562
|
+
return mergeGroupBy(values, strategy.op ?? "sum");
|
|
563
|
+
}
|
|
564
|
+
case "max": {
|
|
565
|
+
return mergeNumeric(values, (best, candidate) => Math.max(best, candidate));
|
|
566
|
+
}
|
|
567
|
+
case "min": {
|
|
568
|
+
return mergeNumeric(values, (best, candidate) => Math.min(best, candidate));
|
|
569
|
+
}
|
|
570
|
+
case "rank": {
|
|
571
|
+
return mergeRank(values);
|
|
572
|
+
}
|
|
573
|
+
case "sum": {
|
|
574
|
+
return mergeSum(values);
|
|
575
|
+
}
|
|
576
|
+
case "topK": {
|
|
577
|
+
return mergeTopK(values, strategy);
|
|
578
|
+
}
|
|
579
|
+
default: {
|
|
580
|
+
return values;
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
};
|
|
584
|
+
const createQueryCoordinator = (options) => {
|
|
585
|
+
const maxConcurrency = options.maxConcurrency ?? DEFAULT_CONCURRENCY;
|
|
586
|
+
const perShardTimeoutMs = options.perShardTimeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
587
|
+
if (maxConcurrency < 1) {
|
|
588
|
+
throw new LunoraError("maxConcurrency must be >= 1", { code: "BAD_REQUEST", status: 400 });
|
|
589
|
+
}
|
|
590
|
+
return {
|
|
591
|
+
async fanOut(namespace, request) {
|
|
592
|
+
const keys = await options.registry.listShardKeys(request.fanOut.table);
|
|
593
|
+
const results = await runBoundedFanOut(namespace, keys, request, maxConcurrency, perShardTimeoutMs);
|
|
594
|
+
const okValues = [];
|
|
595
|
+
const okShards = [];
|
|
596
|
+
const errors = [];
|
|
597
|
+
for (const result of results) {
|
|
598
|
+
if (result.kind === "ok") {
|
|
599
|
+
okValues.push(result.value);
|
|
600
|
+
okShards.push(result.shardKey);
|
|
601
|
+
} else {
|
|
602
|
+
errors.push({ message: result.message, shardKey: result.shardKey, timedOut: result.timedOut });
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
const merged = mergeShardResults(okValues, request.fanOut.merge);
|
|
606
|
+
return {
|
|
607
|
+
data: merged,
|
|
608
|
+
errors,
|
|
609
|
+
failed: errors.length,
|
|
610
|
+
ok: okShards.length
|
|
611
|
+
};
|
|
612
|
+
},
|
|
613
|
+
async orchestrateExport(namespace, request) {
|
|
614
|
+
const union = /* @__PURE__ */ new Set();
|
|
615
|
+
const perTableKeys = await Promise.all(request.tables.map(async (table) => options.registry.listShardKeys(table)));
|
|
616
|
+
for (const keys of perTableKeys) {
|
|
617
|
+
for (const key of keys) {
|
|
618
|
+
union.add(key);
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
const shardKeys = [...union];
|
|
622
|
+
const exportRequest = {
|
|
623
|
+
// Spread the caller's `args` (`batchSize`, future export knobs)
|
|
624
|
+
// before the `tables` field so they reach the shard RPC. The
|
|
625
|
+
// earlier `{ tables }` literal silently dropped them.
|
|
626
|
+
args: { ...request.args, tables: [...request.tables] },
|
|
627
|
+
functionPath: "__lunora_admin__:exportShard",
|
|
628
|
+
headers: request.headers
|
|
629
|
+
};
|
|
630
|
+
const results = await runBoundedFanOut(namespace, shardKeys, exportRequest, maxConcurrency, perShardTimeoutMs);
|
|
631
|
+
return rollUpExport(results);
|
|
632
|
+
},
|
|
633
|
+
async orchestrateCdcSync(namespace, request) {
|
|
634
|
+
const union = /* @__PURE__ */ new Set();
|
|
635
|
+
const perTableKeys = await Promise.all(request.tables.map(async (table) => options.registry.listShardKeys(table)));
|
|
636
|
+
for (const keys of perTableKeys) {
|
|
637
|
+
for (const key of keys) {
|
|
638
|
+
union.add(key);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
const shardKeys = [...union];
|
|
642
|
+
const cursors = request.cursors ?? {};
|
|
643
|
+
const results = Array.from({ length: shardKeys.length });
|
|
644
|
+
let cursor = 0;
|
|
645
|
+
const concurrency = Math.min(maxConcurrency, shardKeys.length);
|
|
646
|
+
const worker = async () => {
|
|
647
|
+
while (true) {
|
|
648
|
+
const index = cursor;
|
|
649
|
+
cursor += 1;
|
|
650
|
+
const shardKey = shardKeys[index];
|
|
651
|
+
if (index >= shardKeys.length || shardKey === void 0) {
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
const sinceSeq = cursors[shardKey] ?? 0;
|
|
655
|
+
const outcome = await callOneShard(
|
|
656
|
+
namespace,
|
|
657
|
+
shardKey,
|
|
658
|
+
prepareShardRpc({
|
|
659
|
+
args: { limit: request.limit, sinceSeq },
|
|
660
|
+
functionPath: "__lunora_admin__:cdcSync",
|
|
661
|
+
headers: request.headers
|
|
662
|
+
}),
|
|
663
|
+
perShardTimeoutMs
|
|
664
|
+
);
|
|
665
|
+
results[index] = { outcome, sinceSeq };
|
|
666
|
+
}
|
|
667
|
+
};
|
|
668
|
+
if (concurrency > 0) {
|
|
669
|
+
await Promise.all(Array.from({ length: concurrency }, () => worker()));
|
|
670
|
+
}
|
|
671
|
+
return rollUpCdcSync(results);
|
|
672
|
+
},
|
|
673
|
+
async orchestrateImport(namespace, request) {
|
|
674
|
+
const { batches } = request;
|
|
675
|
+
const outcomes = Array.from({ length: batches.length });
|
|
676
|
+
let cursor = 0;
|
|
677
|
+
const concurrency = Math.min(maxConcurrency, batches.length);
|
|
678
|
+
const worker = async () => {
|
|
679
|
+
while (true) {
|
|
680
|
+
const index = cursor;
|
|
681
|
+
cursor += 1;
|
|
682
|
+
const batch = batches[index];
|
|
683
|
+
if (index >= batches.length || batch === void 0) {
|
|
684
|
+
return;
|
|
685
|
+
}
|
|
686
|
+
outcomes[index] = await callOneShard(
|
|
687
|
+
namespace,
|
|
688
|
+
batch.shardKey,
|
|
689
|
+
prepareShardRpc({
|
|
690
|
+
args: { rows: [...batch.rows], startLine: batch.startLine ?? 1 },
|
|
691
|
+
functionPath: "__lunora_admin__:importShard",
|
|
692
|
+
headers: request.headers
|
|
693
|
+
}),
|
|
694
|
+
perShardTimeoutMs
|
|
695
|
+
);
|
|
696
|
+
}
|
|
697
|
+
};
|
|
698
|
+
if (concurrency > 0) {
|
|
699
|
+
await Promise.all(Array.from({ length: concurrency }, () => worker()));
|
|
700
|
+
}
|
|
701
|
+
return rollUpImport(outcomes);
|
|
702
|
+
},
|
|
703
|
+
async orchestrateApplyCdc(namespace, request) {
|
|
704
|
+
const { batches } = request;
|
|
705
|
+
const outcomes = Array.from({ length: batches.length });
|
|
706
|
+
let cursor = 0;
|
|
707
|
+
const concurrency = Math.min(maxConcurrency, batches.length);
|
|
708
|
+
const worker = async () => {
|
|
709
|
+
while (true) {
|
|
710
|
+
const index = cursor;
|
|
711
|
+
cursor += 1;
|
|
712
|
+
const batch = batches[index];
|
|
713
|
+
if (index >= batches.length || batch === void 0) {
|
|
714
|
+
return;
|
|
715
|
+
}
|
|
716
|
+
outcomes[index] = await callOneShard(
|
|
717
|
+
namespace,
|
|
718
|
+
batch.shardKey,
|
|
719
|
+
prepareShardRpc({
|
|
720
|
+
args: { changes: [...batch.changes] },
|
|
721
|
+
functionPath: "__lunora_admin__:applyCdc",
|
|
722
|
+
headers: request.headers
|
|
723
|
+
}),
|
|
724
|
+
perShardTimeoutMs
|
|
725
|
+
);
|
|
726
|
+
}
|
|
727
|
+
};
|
|
728
|
+
if (concurrency > 0) {
|
|
729
|
+
await Promise.all(Array.from({ length: concurrency }, () => worker()));
|
|
730
|
+
}
|
|
731
|
+
return rollUpApplyCdc(outcomes);
|
|
732
|
+
},
|
|
733
|
+
async orchestrateMigration(namespace, request) {
|
|
734
|
+
const keys = await options.registry.listShardKeys(request.table);
|
|
735
|
+
const results = await runBoundedFanOut(namespace, keys, request, maxConcurrency, perShardTimeoutMs);
|
|
736
|
+
return rollUpMigration(results);
|
|
737
|
+
},
|
|
738
|
+
async orchestrateRank(namespace, request) {
|
|
739
|
+
const keys = await options.registry.listShardKeys(request.table);
|
|
740
|
+
const rankRequest = {
|
|
741
|
+
args: {
|
|
742
|
+
index: request.index,
|
|
743
|
+
partitionKey: request.partitionKey,
|
|
744
|
+
rowId: request.rowId,
|
|
745
|
+
sortValues: [...request.sortValues],
|
|
746
|
+
table: request.table
|
|
747
|
+
},
|
|
748
|
+
functionPath: "__lunora_admin__:rankBefore",
|
|
749
|
+
headers: request.headers
|
|
750
|
+
};
|
|
751
|
+
const results = await runBoundedFanOut(namespace, keys, rankRequest, maxConcurrency, perShardTimeoutMs);
|
|
752
|
+
return rollUpRank(results);
|
|
753
|
+
},
|
|
754
|
+
async orchestrateRankPage(namespace, request) {
|
|
755
|
+
const keys = await options.registry.listShardKeys(request.table);
|
|
756
|
+
const take = Math.max(1, Math.min(1e3, Math.floor(request.take ?? 100)));
|
|
757
|
+
const requestedDirections = request.directions ?? [];
|
|
758
|
+
const cursorState = request.cursor ? decodeRankPageCursor(request.cursor) : { perShard: {} };
|
|
759
|
+
const outcomes = Array.from({ length: keys.length });
|
|
760
|
+
let cursor = 0;
|
|
761
|
+
const concurrency = Math.min(maxConcurrency, keys.length);
|
|
762
|
+
const worker = async () => {
|
|
763
|
+
while (true) {
|
|
764
|
+
const index = cursor;
|
|
765
|
+
cursor += 1;
|
|
766
|
+
const shardKey = keys[index];
|
|
767
|
+
if (index >= keys.length || shardKey === void 0) {
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
const shardCursorKey = cursorState.perShard[shardKey];
|
|
771
|
+
const args = {
|
|
772
|
+
index: request.index,
|
|
773
|
+
table: request.table,
|
|
774
|
+
take
|
|
775
|
+
};
|
|
776
|
+
if (request.partitionKey !== void 0) {
|
|
777
|
+
args["partitionKey"] = request.partitionKey;
|
|
778
|
+
}
|
|
779
|
+
if (shardCursorKey !== void 0) {
|
|
780
|
+
args["after"] = shardCursorKey;
|
|
781
|
+
}
|
|
782
|
+
const outcome = await callOneShard(
|
|
783
|
+
namespace,
|
|
784
|
+
shardKey,
|
|
785
|
+
prepareShardRpc({ args, functionPath: "__lunora_admin__:rankPage", headers: request.headers }),
|
|
786
|
+
perShardTimeoutMs
|
|
787
|
+
);
|
|
788
|
+
if (outcome.kind === "err") {
|
|
789
|
+
outcomes[index] = { error: { message: outcome.message, timedOut: outcome.timedOut }, shardKey };
|
|
790
|
+
continue;
|
|
791
|
+
}
|
|
792
|
+
const payload = readRankPageResult(unwrapResult(outcome.value));
|
|
793
|
+
outcomes[index] = { directions: payload.directions, hasMore: payload.hasMore, rows: payload.rows, shardKey };
|
|
794
|
+
}
|
|
795
|
+
};
|
|
796
|
+
if (concurrency > 0) {
|
|
797
|
+
await Promise.all(Array.from({ length: concurrency }, () => worker()));
|
|
798
|
+
}
|
|
799
|
+
const slices = [];
|
|
800
|
+
let ok = 0;
|
|
801
|
+
let failed = 0;
|
|
802
|
+
let reportedDirections;
|
|
803
|
+
for (const outcome of outcomes) {
|
|
804
|
+
if (outcome.error) {
|
|
805
|
+
failed += 1;
|
|
806
|
+
continue;
|
|
807
|
+
}
|
|
808
|
+
ok += 1;
|
|
809
|
+
if (reportedDirections === void 0 && outcome.directions && outcome.directions.length > 0) {
|
|
810
|
+
reportedDirections = outcome.directions;
|
|
811
|
+
}
|
|
812
|
+
slices.push({ hasMore: outcome.hasMore ?? false, head: 0, rows: outcome.rows ?? [], shardKey: outcome.shardKey });
|
|
813
|
+
}
|
|
814
|
+
const merged = kWayMergeRankPages(slices, take, reportedDirections ?? requestedDirections, cursorState.perShard);
|
|
815
|
+
return {
|
|
816
|
+
continueCursor: merged.nextCursor,
|
|
817
|
+
failed,
|
|
818
|
+
isDone: merged.isDone,
|
|
819
|
+
ok,
|
|
820
|
+
page: merged.page,
|
|
821
|
+
partial: failed > 0,
|
|
822
|
+
shards: outcomes
|
|
823
|
+
};
|
|
824
|
+
},
|
|
825
|
+
async orchestrateShardTraffic(namespace, request) {
|
|
826
|
+
const keys = await options.registry.listShardKeys(request.table);
|
|
827
|
+
const trafficRequest = {
|
|
828
|
+
functionPath: "__lunora_admin__:getMetrics",
|
|
829
|
+
headers: request.headers
|
|
830
|
+
};
|
|
831
|
+
const results = await runBoundedFanOut(namespace, keys, trafficRequest, maxConcurrency, perShardTimeoutMs);
|
|
832
|
+
return rollUpShardTraffic(results);
|
|
833
|
+
},
|
|
834
|
+
registry: options.registry
|
|
835
|
+
};
|
|
836
|
+
};
|
|
837
|
+
|
|
838
|
+
export { createQueryCoordinator, createStaticShardRegistry, mergeStrategyForAggregate };
|