@lunora/runtime 1.0.0-alpha.37 → 1.0.0-alpha.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +359 -8
- package/dist/index.d.ts +359 -8
- package/dist/index.mjs +1 -22
- package/dist/packem_shared/DEFAULT_LOG_COLUMNS-CpZ9kd-q.mjs +1 -0
- package/dist/packem_shared/DEFAULT_REGISTRY_CACHE_TTL_MS-BBf3mNrD.mjs +1 -0
- package/dist/packem_shared/GET_AUTH_AUDIT_LOG_OP-DmxHQbZT.mjs +1 -0
- package/dist/packem_shared/HEALTH_PATH-DiZqjEGp.mjs +1 -0
- package/dist/packem_shared/LOG_ARCHIVE_NOT_CONFIGURED-CDpi4yFD.mjs +1 -0
- package/dist/packem_shared/LOG_ARCHIVE_PATH-e8U3ftfM.mjs +1 -0
- package/dist/packem_shared/LunoraError-C08OP5Uq.mjs +1 -0
- package/dist/packem_shared/NOOP_EXECUTION_CONTEXT-YmXqH-jH.mjs +1 -0
- package/dist/packem_shared/analyticsEngineSink-DOoyBx4o.mjs +1 -0
- package/dist/packem_shared/applyJurisdiction-uRQLx282.mjs +1 -0
- package/dist/packem_shared/argsFromQuery-0KrWTkNx.mjs +1 -0
- package/dist/packem_shared/composeIdentityResolvers-DlBbYmBJ.mjs +1 -0
- package/dist/packem_shared/composeWorker-D7Zi6kZR.mjs +6 -0
- package/dist/packem_shared/createCrossShardRelationCapabilities-B2EKbSEs.mjs +1 -0
- package/dist/packem_shared/createKvCursorStore-C24tEuYk.mjs +3 -0
- package/dist/packem_shared/createQueryCoordinator-DLsrcOkB.mjs +1 -0
- package/dist/packem_shared/createShardClient-62qcYKGl.mjs +1 -0
- package/dist/packem_shared/decorateResponse-C6TZSzID.mjs +1 -0
- package/dist/packem_shared/emitLogEvent-CEbCh6FQ.mjs +1 -0
- package/dist/packem_shared/memoizeIdentity-DRYIK1yT.mjs +1 -0
- package/dist/packem_shared/method-guard-BbuR0VfS.mjs +1 -0
- package/dist/packem_shared/observability-B0MYwNDB.mjs +1 -0
- package/dist/packem_shared/otlp-resource-cAjGEywx.mjs +1 -0
- package/dist/packem_shared/pipeline-log-reader-tDuvc_Ty.mjs +1 -0
- package/dist/packem_shared/toAirbyteMessages-DBTuFjb5.mjs +1 -0
- package/package.json +1 -1
- package/dist/packem_shared/DEFAULT_LOG_COLUMNS-B7H3YdJ3.mjs +0 -2
- package/dist/packem_shared/DEFAULT_REGISTRY_CACHE_TTL_MS-B3pA7aXp.mjs +0 -79
- package/dist/packem_shared/HEALTH_PATH-e5J_NHBx.mjs +0 -150
- package/dist/packem_shared/LOG_ARCHIVE_NOT_CONFIGURED-acNcguqc.mjs +0 -3
- package/dist/packem_shared/LOG_ARCHIVE_PATH-CNs0bznX.mjs +0 -125
- package/dist/packem_shared/LunoraError-Bpb9EFJ3.mjs +0 -22
- package/dist/packem_shared/NOOP_EXECUTION_CONTEXT-CCTu0Bf1.mjs +0 -8
- package/dist/packem_shared/analyticsEngineSink-yLFNjHDt.mjs +0 -429
- package/dist/packem_shared/applyJurisdiction-BkZtTkct.mjs +0 -20
- package/dist/packem_shared/argsFromQuery-c-U1WRy-.mjs +0 -121
- package/dist/packem_shared/composeIdentityResolvers-XGjO7V1J.mjs +0 -55
- package/dist/packem_shared/composeWorker-DiWwOXXt.mjs +0 -3738
- package/dist/packem_shared/createCrossShardRelationCapabilities-CbcWjkAn.mjs +0 -63
- package/dist/packem_shared/createKvCursorStore-g8aA6B4L.mjs +0 -165
- package/dist/packem_shared/createQueryCoordinator-DNCJzOZE.mjs +0 -847
- package/dist/packem_shared/decorateResponse-DRWQFNhF.mjs +0 -273
- package/dist/packem_shared/emitLogEvent-BlGMnKZK.mjs +0 -1
- package/dist/packem_shared/method-guard-Qzw99aCj.mjs +0 -3
- package/dist/packem_shared/observability--NOFYBFc.mjs +0 -47
- package/dist/packem_shared/otlp-resource-Dow6-F_u.mjs +0 -163
- package/dist/packem_shared/pipeline-log-reader-BXULGNC3.mjs +0 -135
- package/dist/packem_shared/toAirbyteMessages-DrHdplb4.mjs +0 -39
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
import { LunoraError } from './LunoraError-Bpb9EFJ3.mjs';
|
|
2
|
-
|
|
3
|
-
const RPC_ENDPOINT = "/_lunora/rpc";
|
|
4
|
-
const buildIdentityHeaders = (options) => {
|
|
5
|
-
const headers = { "content-type": "application/json" };
|
|
6
|
-
if (options.userId !== void 0 && options.userId.length > 0) {
|
|
7
|
-
headers["x-lunora-userid"] = options.userId;
|
|
8
|
-
}
|
|
9
|
-
if (options.identity !== void 0) {
|
|
10
|
-
headers["x-lunora-identity"] = JSON.stringify(options.identity);
|
|
11
|
-
}
|
|
12
|
-
return headers;
|
|
13
|
-
};
|
|
14
|
-
const fanOutRelation = async (options, body, label) => {
|
|
15
|
-
const doFetch = options.fetch ?? globalThis.fetch;
|
|
16
|
-
const response = await doFetch(
|
|
17
|
-
new Request(`${options.origin}${RPC_ENDPOINT}`, {
|
|
18
|
-
body: JSON.stringify(body),
|
|
19
|
-
headers: buildIdentityHeaders(options),
|
|
20
|
-
method: "POST"
|
|
21
|
-
})
|
|
22
|
-
);
|
|
23
|
-
if (!response.ok) {
|
|
24
|
-
throw new LunoraError(`cross-shard relation ${label} failed: worker returned ${String(response.status)}`);
|
|
25
|
-
}
|
|
26
|
-
const result = await response.json();
|
|
27
|
-
if (typeof result.failed === "number" && result.failed > 0) {
|
|
28
|
-
const reached = (typeof result.ok === "number" ? result.ok : 0) + result.failed;
|
|
29
|
-
throw new LunoraError(
|
|
30
|
-
`cross-shard relation ${label} failed on ${String(result.failed)} of ${String(reached)} shard(s) — refusing to return a partial result`
|
|
31
|
-
);
|
|
32
|
-
}
|
|
33
|
-
return result.data;
|
|
34
|
-
};
|
|
35
|
-
const createCrossShardRelationCapabilities = (options) => {
|
|
36
|
-
const crossShardReader = async (table, args) => {
|
|
37
|
-
const data = await fanOutRelation(
|
|
38
|
-
options,
|
|
39
|
-
{
|
|
40
|
-
args: { orderBy: args?.orderBy, table, where: args?.where, with: args?.with },
|
|
41
|
-
fanOut: { merge: { kind: "concat" }, table },
|
|
42
|
-
functionPath: "__lunora_relation__:read"
|
|
43
|
-
},
|
|
44
|
-
"read"
|
|
45
|
-
);
|
|
46
|
-
return { continueCursor: null, isDone: true, page: Array.isArray(data) ? data : [] };
|
|
47
|
-
};
|
|
48
|
-
const crossShardCounter = async (table, where) => {
|
|
49
|
-
const data = await fanOutRelation(
|
|
50
|
-
options,
|
|
51
|
-
{
|
|
52
|
-
args: { table, where },
|
|
53
|
-
fanOut: { merge: { kind: "sum" }, table },
|
|
54
|
-
functionPath: "__lunora_relation__:count"
|
|
55
|
-
},
|
|
56
|
-
"count"
|
|
57
|
-
);
|
|
58
|
-
return typeof data === "number" ? data : 0;
|
|
59
|
-
};
|
|
60
|
-
return { crossShardCounter, crossShardReader };
|
|
61
|
-
};
|
|
62
|
-
|
|
63
|
-
export { createCrossShardRelationCapabilities };
|
|
@@ -1,165 +0,0 @@
|
|
|
1
|
-
const defaultSleep = (ms) => new Promise((resolve) => {
|
|
2
|
-
setTimeout(resolve, ms);
|
|
3
|
-
});
|
|
4
|
-
const sanitizeChange = (raw) => {
|
|
5
|
-
const table = typeof raw["table"] === "string" ? raw["table"] : "";
|
|
6
|
-
const rawOp = typeof raw["op"] === "string" ? raw["op"] : "";
|
|
7
|
-
const op = rawOp === "delete" || rawOp === "insert" || rawOp === "update" ? rawOp : "upsert";
|
|
8
|
-
const id = typeof raw["id"] === "string" ? raw["id"] : void 0;
|
|
9
|
-
const documentRow = raw["doc"] && typeof raw["doc"] === "object" ? raw["doc"] : void 0;
|
|
10
|
-
const seq = typeof raw["seq"] === "number" && Number.isFinite(raw["seq"]) ? raw["seq"] : void 0;
|
|
11
|
-
const ts = typeof raw["ts"] === "number" && Number.isFinite(raw["ts"]) ? raw["ts"] : void 0;
|
|
12
|
-
return {
|
|
13
|
-
op,
|
|
14
|
-
table,
|
|
15
|
-
...documentRow === void 0 ? {} : { doc: documentRow },
|
|
16
|
-
...id === void 0 ? {} : { id },
|
|
17
|
-
...seq === void 0 ? {} : { seq },
|
|
18
|
-
...ts === void 0 ? {} : { ts }
|
|
19
|
-
};
|
|
20
|
-
};
|
|
21
|
-
const deliverWithRetry = async (sink, batch, maxRetries, initialBackoffMs, maxBackoffMs, sleep) => {
|
|
22
|
-
let attempt = 0;
|
|
23
|
-
while (true) {
|
|
24
|
-
try {
|
|
25
|
-
await sink.deliver(batch);
|
|
26
|
-
return;
|
|
27
|
-
} catch (error) {
|
|
28
|
-
if (attempt >= maxRetries) {
|
|
29
|
-
throw error instanceof Error ? error : new Error(String(error));
|
|
30
|
-
}
|
|
31
|
-
const delay = Math.min(initialBackoffMs * 2 ** attempt, maxBackoffMs);
|
|
32
|
-
await sleep(delay);
|
|
33
|
-
attempt += 1;
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
};
|
|
37
|
-
const runExportTap = async (options) => {
|
|
38
|
-
const {
|
|
39
|
-
coordinator,
|
|
40
|
-
cursorStore,
|
|
41
|
-
headers,
|
|
42
|
-
initialBackoffMs = 100,
|
|
43
|
-
limit,
|
|
44
|
-
maxBackoffMs = 5e3,
|
|
45
|
-
maxRetries = 3,
|
|
46
|
-
shardDO,
|
|
47
|
-
sink,
|
|
48
|
-
sleep = defaultSleep,
|
|
49
|
-
tables
|
|
50
|
-
} = options;
|
|
51
|
-
const priorCursors = await cursorStore.read(sink.name);
|
|
52
|
-
const feed = await coordinator.orchestrateCdcSync(shardDO, { cursors: priorCursors, headers, limit, tables });
|
|
53
|
-
const nextCursors = { ...priorCursors };
|
|
54
|
-
const failures = [];
|
|
55
|
-
let delivered = 0;
|
|
56
|
-
let hasMore = false;
|
|
57
|
-
for (const shard of feed.shards) {
|
|
58
|
-
if (shard.error) {
|
|
59
|
-
failures.push({ error: shard.error.message, shardKey: shard.shardKey });
|
|
60
|
-
hasMore = true;
|
|
61
|
-
continue;
|
|
62
|
-
}
|
|
63
|
-
const rawChanges = shard.changes ?? [];
|
|
64
|
-
if (rawChanges.length === 0) {
|
|
65
|
-
nextCursors[shard.shardKey] = shard.cursor;
|
|
66
|
-
continue;
|
|
67
|
-
}
|
|
68
|
-
const changes = rawChanges.map((change) => sanitizeChange(change));
|
|
69
|
-
const batch = { changes, cursor: shard.cursor, shardKey: shard.shardKey, sink: sink.name };
|
|
70
|
-
try {
|
|
71
|
-
await deliverWithRetry(sink, batch, maxRetries, initialBackoffMs, maxBackoffMs, sleep);
|
|
72
|
-
nextCursors[shard.shardKey] = shard.cursor;
|
|
73
|
-
delivered += changes.length;
|
|
74
|
-
if (limit !== void 0 && rawChanges.length >= limit) {
|
|
75
|
-
hasMore = true;
|
|
76
|
-
}
|
|
77
|
-
} catch (error) {
|
|
78
|
-
failures.push({ error: error instanceof Error ? error.message : String(error), shardKey: shard.shardKey });
|
|
79
|
-
hasMore = true;
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
await cursorStore.write(sink.name, nextCursors);
|
|
83
|
-
return { cursors: nextCursors, delivered, failures, hasMore, shards: feed.shards.length };
|
|
84
|
-
};
|
|
85
|
-
const defineExportSink = (config) => {
|
|
86
|
-
if (typeof config.name !== "string" || config.name.length === 0) {
|
|
87
|
-
throw new Error("defineExportSink: `name` must be a non-empty string");
|
|
88
|
-
}
|
|
89
|
-
if (typeof config.deliver !== "function") {
|
|
90
|
-
throw new TypeError("defineExportSink: `deliver` must be a function");
|
|
91
|
-
}
|
|
92
|
-
return { deliver: config.deliver, name: config.name };
|
|
93
|
-
};
|
|
94
|
-
const encodeNdjson = (changes) => `${changes.map((change) => JSON.stringify(change)).join("\n")}
|
|
95
|
-
`;
|
|
96
|
-
const webhookExportSink = (config) => {
|
|
97
|
-
const fetchImpl = config.fetchImpl ?? ((input, init) => fetch(input, init));
|
|
98
|
-
return defineExportSink({
|
|
99
|
-
deliver: async (batch) => {
|
|
100
|
-
const response = await fetchImpl(config.url, {
|
|
101
|
-
body: encodeNdjson(batch.changes),
|
|
102
|
-
headers: {
|
|
103
|
-
"content-type": "application/x-ndjson",
|
|
104
|
-
"x-lunora-cursor": String(batch.cursor),
|
|
105
|
-
"x-lunora-shard": batch.shardKey,
|
|
106
|
-
"x-lunora-sink": batch.sink,
|
|
107
|
-
...config.headers
|
|
108
|
-
},
|
|
109
|
-
method: "POST"
|
|
110
|
-
});
|
|
111
|
-
if (!response.ok) {
|
|
112
|
-
throw new Error(`webhook export sink "${config.name}" returned ${String(response.status)}`);
|
|
113
|
-
}
|
|
114
|
-
},
|
|
115
|
-
name: config.name
|
|
116
|
-
});
|
|
117
|
-
};
|
|
118
|
-
const r2Sink = (config) => {
|
|
119
|
-
let prefix = config.prefix ?? "cdc";
|
|
120
|
-
while (prefix.endsWith("/")) {
|
|
121
|
-
prefix = prefix.slice(0, -1);
|
|
122
|
-
}
|
|
123
|
-
return defineExportSink({
|
|
124
|
-
deliver: async (batch) => {
|
|
125
|
-
const key = `${prefix}/${batch.shardKey}/${String(batch.cursor)}.ndjson`;
|
|
126
|
-
await config.bucket.put(key, encodeNdjson(batch.changes), { httpMetadata: { contentType: "application/x-ndjson" } });
|
|
127
|
-
},
|
|
128
|
-
name: config.name
|
|
129
|
-
});
|
|
130
|
-
};
|
|
131
|
-
const createMemoryCursorStore = () => {
|
|
132
|
-
const state = {};
|
|
133
|
-
return {
|
|
134
|
-
read: (sink) => Promise.resolve({ ...state[sink] }),
|
|
135
|
-
snapshot: () => structuredClone(state),
|
|
136
|
-
write: (sink, cursors) => {
|
|
137
|
-
state[sink] = { ...cursors };
|
|
138
|
-
return Promise.resolve();
|
|
139
|
-
}
|
|
140
|
-
};
|
|
141
|
-
};
|
|
142
|
-
const createKvCursorStore = (kv, options) => {
|
|
143
|
-
const keyPrefix = options?.keyPrefix ?? "__lunora_source_cursor:export";
|
|
144
|
-
const keyFor = (sink) => `${keyPrefix}:${sink}`;
|
|
145
|
-
return {
|
|
146
|
-
read: async (sink) => {
|
|
147
|
-
const raw = await kv.get(keyFor(sink), "json");
|
|
148
|
-
if (raw === null || typeof raw !== "object") {
|
|
149
|
-
return {};
|
|
150
|
-
}
|
|
151
|
-
const cursors = {};
|
|
152
|
-
for (const [shardKey, value] of Object.entries(raw)) {
|
|
153
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
154
|
-
cursors[shardKey] = value;
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
return cursors;
|
|
158
|
-
},
|
|
159
|
-
write: async (sink, cursors) => {
|
|
160
|
-
await kv.put(keyFor(sink), JSON.stringify(cursors));
|
|
161
|
-
}
|
|
162
|
-
};
|
|
163
|
-
};
|
|
164
|
-
|
|
165
|
-
export { createKvCursorStore, createMemoryCursorStore, defineExportSink, r2Sink, runExportTap, sanitizeChange, webhookExportSink };
|