@lunora/do 0.0.0 → 1.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +105 -0
- package/README.md +115 -9
- package/__assets__/package-og.svg +14 -0
- package/dist/index.d.mts +5599 -0
- package/dist/index.d.ts +5599 -0
- package/dist/index.mjs +35 -0
- package/dist/packem_shared/ADMIN_FUNCTION_PREFIX-Dzdqq5J2.mjs +313 -0
- package/dist/packem_shared/AUTH_METRICS_BUCKET_MS-CiHHYeJi.mjs +84 -0
- package/dist/packem_shared/ConflictError-C0STs6bU.mjs +13 -0
- package/dist/packem_shared/CountRlsUnsupportedError-28ZvvwKS.mjs +133 -0
- package/dist/packem_shared/DATA_MIGRATION_STATE_TABLE-PTtTiQ7U.mjs +237 -0
- package/dist/packem_shared/LogBuffer-B_Ezju_N.mjs +37 -0
- package/dist/packem_shared/NotFoundError-CMuMZt81.mjs +10 -0
- package/dist/packem_shared/ROOT_DO_SIZE_WARN_BYTES-DQkmGiCS.mjs +4009 -0
- package/dist/packem_shared/ReactiveCache-ByVzgH3d.mjs +259 -0
- package/dist/packem_shared/SCAN_DEP-DLJF8dsj.mjs +19 -0
- package/dist/packem_shared/SESSION_DO_TTL_DEFAULT-ilPZsVwu.mjs +180 -0
- package/dist/packem_shared/SHARD_REGISTRY_DO_NAME-BsAbi5Mn.mjs +146 -0
- package/dist/packem_shared/aggregateTableName-CxNqY1Sl.mjs +64 -0
- package/dist/packem_shared/applyCdcChanges-Ctdmxmrv.mjs +103 -0
- package/dist/packem_shared/applyOnDelete-CMif2RKw.mjs +165 -0
- package/dist/packem_shared/armRestore-BJk53Ro8.mjs +55 -0
- package/dist/packem_shared/assertFlatPredicate-DyVYReuT.mjs +160 -0
- package/dist/packem_shared/assertReadonly-dDcFE1YZ.mjs +29 -0
- package/dist/packem_shared/assertValidClientId-CBZ1zC96.mjs +1745 -0
- package/dist/packem_shared/backfillAggregateIndexes-BF5eL7kW.mjs +80 -0
- package/dist/packem_shared/buildSecurityAudit-CCAvoFlr.mjs +1 -0
- package/dist/packem_shared/buildSeekWhere-lVsNXSLy.mjs +84 -0
- package/dist/packem_shared/clearCapturedMail-CPpgl-dX.mjs +104 -0
- package/dist/packem_shared/compileWhereSql-CXrhFA3G.mjs +127 -0
- package/dist/packem_shared/createSystemReader-8CzSZP9V.mjs +80 -0
- package/dist/packem_shared/ctx-db-idempotency-DkC9rP91.mjs +35 -0
- package/dist/packem_shared/do-exec-5eQy5cEi.mjs +12 -0
- package/dist/packem_shared/do-sql-BCHCWtrD.mjs +87 -0
- package/dist/packem_shared/encodePartitionKey-C6blLR5K.mjs +1 -0
- package/dist/packem_shared/ensureFunctionMetricsTables-UDNVD7FS.mjs +248 -0
- package/dist/packem_shared/exportShardRows-DZEhUeyI.mjs +156 -0
- package/dist/packem_shared/ftsTableName-BLEMawrp.mjs +38 -0
- package/dist/packem_shared/guardWriter-u3UlnCH5.mjs +128 -0
- package/dist/packem_shared/matchesStaticWhere-CFk6adSu.mjs +54 -0
- package/dist/packem_shared/rank-CrkEIpF4.mjs +102 -0
- package/dist/packem_shared/renderSql-D6eUcn2N.mjs +16 -0
- package/dist/packem_shared/runShardMigrations-C3bn5r93.mjs +103 -0
- package/dist/packem_shared/runTriggers-5N6_Fx0A.mjs +20 -0
- package/dist/packem_shared/security-audit-CucgBice.mjs +158 -0
- package/dist/packem_shared/serveRelationFanout-Clr1a05L.mjs +24 -0
- package/package.json +41 -17
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
const DATA_MIGRATION_STATE_TABLE = "__lunora_migrations";
|
|
2
|
+
const DEFAULT_BATCH_SIZE = 100;
|
|
3
|
+
const STALE_CLAIM_TIMEOUT_MS = 3e4;
|
|
4
|
+
const CLAIM_HEARTBEAT_INTERVAL_MS = 1e4;
|
|
5
|
+
const runSql = (sql, query, ...params) => {
|
|
6
|
+
const runner = sql.exec;
|
|
7
|
+
return runner.call(sql, query, ...params);
|
|
8
|
+
};
|
|
9
|
+
const ensureStateTable = (sql) => {
|
|
10
|
+
runSql(
|
|
11
|
+
sql,
|
|
12
|
+
`CREATE TABLE IF NOT EXISTS "${DATA_MIGRATION_STATE_TABLE}" (
|
|
13
|
+
id TEXT PRIMARY KEY,
|
|
14
|
+
direction TEXT NOT NULL,
|
|
15
|
+
status TEXT NOT NULL,
|
|
16
|
+
cursor TEXT,
|
|
17
|
+
processed INTEGER NOT NULL DEFAULT 0,
|
|
18
|
+
changed INTEGER NOT NULL DEFAULT 0,
|
|
19
|
+
started_at REAL,
|
|
20
|
+
updated_at REAL,
|
|
21
|
+
error TEXT
|
|
22
|
+
)`
|
|
23
|
+
);
|
|
24
|
+
};
|
|
25
|
+
const readState = (sql, id) => {
|
|
26
|
+
const rows = runSql(sql, `SELECT * FROM "${DATA_MIGRATION_STATE_TABLE}" WHERE id = ?`, id).toArray();
|
|
27
|
+
const row = rows[0];
|
|
28
|
+
if (!row) {
|
|
29
|
+
return void 0;
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
changed: row.changed,
|
|
33
|
+
// eslint-disable-next-line unicorn/no-null -- mirrors the SQLite `cursor` column: a missing cursor is NULL, not undefined
|
|
34
|
+
cursor: typeof row.cursor === "string" ? row.cursor : null,
|
|
35
|
+
direction: row.direction === "down" ? "down" : "up",
|
|
36
|
+
processed: row.processed,
|
|
37
|
+
startedAt: typeof row.started_at === "number" ? row.started_at : void 0,
|
|
38
|
+
status: row.status === "completed" || row.status === "failed" ? row.status : "in_progress"
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
const deleteState = (sql, id) => {
|
|
42
|
+
runSql(sql, `DELETE FROM "${DATA_MIGRATION_STATE_TABLE}" WHERE id = ?`, id);
|
|
43
|
+
};
|
|
44
|
+
const persistState = (sql, state) => {
|
|
45
|
+
runSql(
|
|
46
|
+
sql,
|
|
47
|
+
`INSERT INTO "${DATA_MIGRATION_STATE_TABLE}"
|
|
48
|
+
(id, direction, status, cursor, processed, changed, started_at, updated_at, error)
|
|
49
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
50
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
51
|
+
direction = excluded.direction,
|
|
52
|
+
status = excluded.status,
|
|
53
|
+
cursor = excluded.cursor,
|
|
54
|
+
processed = excluded.processed,
|
|
55
|
+
changed = excluded.changed,
|
|
56
|
+
updated_at = excluded.updated_at,
|
|
57
|
+
error = excluded.error`,
|
|
58
|
+
state.id,
|
|
59
|
+
state.direction,
|
|
60
|
+
state.status,
|
|
61
|
+
state.cursor,
|
|
62
|
+
state.processed,
|
|
63
|
+
state.changed,
|
|
64
|
+
state.startedAt,
|
|
65
|
+
state.updatedAt,
|
|
66
|
+
state.error
|
|
67
|
+
);
|
|
68
|
+
};
|
|
69
|
+
const claimMigration = (sql, id, direction, now) => {
|
|
70
|
+
runSql(
|
|
71
|
+
sql,
|
|
72
|
+
`INSERT INTO "${DATA_MIGRATION_STATE_TABLE}"
|
|
73
|
+
(id, direction, status, cursor, processed, changed, started_at, updated_at, error)
|
|
74
|
+
VALUES (?, ?, 'in_progress', NULL, 0, 0, ?, ?, NULL)
|
|
75
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
76
|
+
status = 'in_progress',
|
|
77
|
+
updated_at = excluded.updated_at
|
|
78
|
+
WHERE
|
|
79
|
+
"${DATA_MIGRATION_STATE_TABLE}".direction <> excluded.direction
|
|
80
|
+
OR "${DATA_MIGRATION_STATE_TABLE}".status <> 'in_progress'
|
|
81
|
+
OR "${DATA_MIGRATION_STATE_TABLE}".updated_at IS NULL
|
|
82
|
+
OR "${DATA_MIGRATION_STATE_TABLE}".updated_at <= excluded.updated_at - ${String(STALE_CLAIM_TIMEOUT_MS)}`,
|
|
83
|
+
id,
|
|
84
|
+
direction,
|
|
85
|
+
now,
|
|
86
|
+
now
|
|
87
|
+
);
|
|
88
|
+
return runSql(sql, `SELECT changes() AS changed`).one().changed > 0;
|
|
89
|
+
};
|
|
90
|
+
const releaseClaim = (sql, id) => {
|
|
91
|
+
runSql(sql, `UPDATE "${DATA_MIGRATION_STATE_TABLE}" SET updated_at = 0 WHERE id = ? AND status = 'in_progress'`, id);
|
|
92
|
+
};
|
|
93
|
+
const touchClaim = (sql, id, now) => {
|
|
94
|
+
runSql(sql, `UPDATE "${DATA_MIGRATION_STATE_TABLE}" SET updated_at = ? WHERE id = ? AND status = 'in_progress'`, now, id);
|
|
95
|
+
};
|
|
96
|
+
const readMigrationStatus = (sql, id) => {
|
|
97
|
+
const exists = runSql(
|
|
98
|
+
sql,
|
|
99
|
+
`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1`,
|
|
100
|
+
DATA_MIGRATION_STATE_TABLE
|
|
101
|
+
).toArray();
|
|
102
|
+
if (exists.length === 0) {
|
|
103
|
+
return [];
|
|
104
|
+
}
|
|
105
|
+
const filter = id === void 0 ? " ORDER BY id" : " WHERE id = ?";
|
|
106
|
+
const params = id === void 0 ? [] : [id];
|
|
107
|
+
const rows = runSql(sql, `SELECT * FROM "${DATA_MIGRATION_STATE_TABLE}"${filter}`, ...params).toArray();
|
|
108
|
+
return rows.map((row) => {
|
|
109
|
+
return {
|
|
110
|
+
changed: row.changed,
|
|
111
|
+
cursor: typeof row.cursor === "string" ? row.cursor : null,
|
|
112
|
+
direction: row.direction === "down" ? "down" : "up",
|
|
113
|
+
error: typeof row.error === "string" ? row.error : null,
|
|
114
|
+
id: row.id,
|
|
115
|
+
processed: row.processed,
|
|
116
|
+
startedAt: typeof row.started_at === "number" ? row.started_at : null,
|
|
117
|
+
status: row.status === "completed" || row.status === "failed" ? row.status : "in_progress",
|
|
118
|
+
updatedAt: typeof row.updated_at === "number" ? row.updated_at : null
|
|
119
|
+
};
|
|
120
|
+
});
|
|
121
|
+
};
|
|
122
|
+
const runDataMigration = async (options) => {
|
|
123
|
+
const { migration, sql, writer } = options;
|
|
124
|
+
const direction = options.direction ?? "up";
|
|
125
|
+
const dryRun = options.dryRun ?? false;
|
|
126
|
+
const clock = options.clock ?? (() => Date.now());
|
|
127
|
+
const maxBatches = options.maxBatches ?? Number.POSITIVE_INFINITY;
|
|
128
|
+
const batchSize = options.batchSize ?? migration.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
129
|
+
const transform = direction === "up" ? migration.up : migration.down;
|
|
130
|
+
if (!transform) {
|
|
131
|
+
throw new Error(`data migration "${migration.id}" has no \`${direction}\` transform`);
|
|
132
|
+
}
|
|
133
|
+
let cursor = null;
|
|
134
|
+
let processed = 0;
|
|
135
|
+
let changed = 0;
|
|
136
|
+
let startedAt = clock();
|
|
137
|
+
if (!dryRun) {
|
|
138
|
+
ensureStateTable(sql);
|
|
139
|
+
const existing = readState(sql, migration.id);
|
|
140
|
+
if (existing?.direction === direction && existing.status === "completed") {
|
|
141
|
+
return { changed: existing.changed, cursor: null, direction, dryRun, id: migration.id, processed: existing.processed, status: "completed" };
|
|
142
|
+
}
|
|
143
|
+
if (existing && existing.direction !== direction) {
|
|
144
|
+
deleteState(sql, migration.id);
|
|
145
|
+
}
|
|
146
|
+
const claimed = claimMigration(sql, migration.id, direction, clock());
|
|
147
|
+
if (!claimed) {
|
|
148
|
+
const active = readState(sql, migration.id);
|
|
149
|
+
return {
|
|
150
|
+
changed: active?.changed ?? 0,
|
|
151
|
+
cursor: active?.cursor ?? cursor,
|
|
152
|
+
direction,
|
|
153
|
+
dryRun,
|
|
154
|
+
id: migration.id,
|
|
155
|
+
processed: active?.processed ?? 0,
|
|
156
|
+
status: active?.status ?? "in_progress"
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
const resume = existing?.direction === direction ? existing : void 0;
|
|
160
|
+
if (resume) {
|
|
161
|
+
cursor = resume.cursor;
|
|
162
|
+
processed = resume.processed;
|
|
163
|
+
changed = resume.changed;
|
|
164
|
+
startedAt = resume.startedAt ?? startedAt;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
let isDone = false;
|
|
168
|
+
let batches = 0;
|
|
169
|
+
let lastHeartbeatAt = startedAt;
|
|
170
|
+
try {
|
|
171
|
+
while (!isDone && batches < maxBatches) {
|
|
172
|
+
const batch = await writer.findMany(migration.table, { cursor, limit: batchSize });
|
|
173
|
+
for (const document of batch.page) {
|
|
174
|
+
processed += 1;
|
|
175
|
+
const next = transform(document);
|
|
176
|
+
if (next !== void 0) {
|
|
177
|
+
changed += 1;
|
|
178
|
+
if (!dryRun) {
|
|
179
|
+
await writer.replace(String(document["_id"]), { ...next, _creationTime: document["_creationTime"], _id: document["_id"] });
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
if (!dryRun) {
|
|
183
|
+
const now = clock();
|
|
184
|
+
if (now - lastHeartbeatAt >= CLAIM_HEARTBEAT_INTERVAL_MS) {
|
|
185
|
+
touchClaim(sql, migration.id, now);
|
|
186
|
+
lastHeartbeatAt = now;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
cursor = batch.continueCursor;
|
|
191
|
+
isDone = batch.isDone;
|
|
192
|
+
batches += 1;
|
|
193
|
+
if (!dryRun) {
|
|
194
|
+
const batchEndedAt = clock();
|
|
195
|
+
persistState(sql, {
|
|
196
|
+
changed,
|
|
197
|
+
// eslint-disable-next-line unicorn/no-null -- bound to the SQLite cursor column: a finished run stores NULL
|
|
198
|
+
cursor: isDone ? null : cursor,
|
|
199
|
+
direction,
|
|
200
|
+
// eslint-disable-next-line unicorn/no-null -- bound to the SQLite error column: a clean batch stores NULL
|
|
201
|
+
error: null,
|
|
202
|
+
id: migration.id,
|
|
203
|
+
processed,
|
|
204
|
+
startedAt,
|
|
205
|
+
status: isDone ? "completed" : "in_progress",
|
|
206
|
+
updatedAt: batchEndedAt
|
|
207
|
+
});
|
|
208
|
+
lastHeartbeatAt = batchEndedAt;
|
|
209
|
+
try {
|
|
210
|
+
await options.onBatch?.({ batches, changed, processed });
|
|
211
|
+
} catch {
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
} catch (error) {
|
|
216
|
+
if (!dryRun) {
|
|
217
|
+
persistState(sql, {
|
|
218
|
+
changed,
|
|
219
|
+
cursor,
|
|
220
|
+
direction,
|
|
221
|
+
error: error instanceof Error ? error.message : String(error),
|
|
222
|
+
id: migration.id,
|
|
223
|
+
processed,
|
|
224
|
+
startedAt,
|
|
225
|
+
status: "failed",
|
|
226
|
+
updatedAt: clock()
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
throw error;
|
|
230
|
+
}
|
|
231
|
+
if (!dryRun && !isDone) {
|
|
232
|
+
releaseClaim(sql, migration.id);
|
|
233
|
+
}
|
|
234
|
+
return { changed, cursor: isDone ? null : cursor, direction, dryRun, id: migration.id, processed, status: isDone ? "completed" : "in_progress" };
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
export { DATA_MIGRATION_STATE_TABLE, readMigrationStatus, runDataMigration };
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
const DEFAULT_CAPACITY = 500;
|
|
2
|
+
class LogBuffer {
|
|
3
|
+
/** Backing store, kept in insertion order (oldest first). */
|
|
4
|
+
buffer = [];
|
|
5
|
+
capacity;
|
|
6
|
+
constructor(capacity = DEFAULT_CAPACITY) {
|
|
7
|
+
this.capacity = capacity > 0 ? Math.trunc(capacity) : DEFAULT_CAPACITY;
|
|
8
|
+
}
|
|
9
|
+
/** Number of entries currently buffered. */
|
|
10
|
+
get size() {
|
|
11
|
+
return this.buffer.length;
|
|
12
|
+
}
|
|
13
|
+
/** Drop every buffered entry. */
|
|
14
|
+
clear() {
|
|
15
|
+
this.buffer.length = 0;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Snapshot of the buffered entries, **newest first** so the panel renders
|
|
19
|
+
* the most recent activity at the top without re-sorting. Returns a fresh
|
|
20
|
+
* array each call; the caller may mutate it freely.
|
|
21
|
+
*/
|
|
22
|
+
entries() {
|
|
23
|
+
return this.buffer.toReversed();
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Append an entry, evicting the oldest when at capacity so the buffer never
|
|
27
|
+
* grows past its bound.
|
|
28
|
+
*/
|
|
29
|
+
push(entry) {
|
|
30
|
+
this.buffer.push(entry);
|
|
31
|
+
if (this.buffer.length > this.capacity) {
|
|
32
|
+
this.buffer.shift();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export { LogBuffer };
|