@eltjs/elt-sqlite 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +248 -2
  3. package/dist/computed-dataset.d.ts +22 -0
  4. package/dist/computed-dataset.js +230 -0
  5. package/dist/index.d.ts +13 -0
  6. package/dist/index.js +13 -0
  7. package/dist/open-destination.d.ts +3 -0
  8. package/dist/open-destination.js +14 -0
  9. package/dist/reserved.d.ts +7 -0
  10. package/dist/reserved.js +9 -0
  11. package/dist/sqlite-append-writer.d.ts +8 -0
  12. package/dist/sqlite-append-writer.js +10 -0
  13. package/dist/sqlite-catalog.d.ts +3 -0
  14. package/dist/sqlite-catalog.js +33 -0
  15. package/dist/sqlite-checkpoint-store.d.ts +9 -0
  16. package/dist/sqlite-checkpoint-store.js +84 -0
  17. package/dist/sqlite-column.d.ts +53 -0
  18. package/dist/sqlite-column.js +306 -0
  19. package/dist/sqlite-columns.d.ts +14 -0
  20. package/dist/sqlite-columns.js +118 -0
  21. package/dist/sqlite-deduplicating-writer.d.ts +21 -0
  22. package/dist/sqlite-deduplicating-writer.js +124 -0
  23. package/dist/sqlite-descriptions.d.ts +8 -0
  24. package/dist/sqlite-descriptions.js +37 -0
  25. package/dist/sqlite-destination.d.ts +19 -0
  26. package/dist/sqlite-destination.js +67 -0
  27. package/dist/sqlite-file-store.d.ts +17 -0
  28. package/dist/sqlite-file-store.js +82 -0
  29. package/dist/sqlite-identifiers.d.ts +8 -0
  30. package/dist/sqlite-identifiers.js +11 -0
  31. package/dist/sqlite-overwrite-writer.d.ts +9 -0
  32. package/dist/sqlite-overwrite-writer.js +13 -0
  33. package/dist/sqlite-passes.d.ts +39 -0
  34. package/dist/sqlite-passes.js +174 -0
  35. package/dist/sqlite-sync-history-schema.d.ts +8 -0
  36. package/dist/sqlite-sync-history-schema.js +94 -0
  37. package/dist/sqlite-sync-history.d.ts +12 -0
  38. package/dist/sqlite-sync-history.js +119 -0
  39. package/dist/sqlite-table.d.ts +15 -0
  40. package/dist/sqlite-table.js +83 -0
  41. package/dist/sqlite-views.d.ts +16 -0
  42. package/dist/sqlite-views.js +62 -0
  43. package/dist/sqlite-writer.d.ts +38 -0
  44. package/dist/sqlite-writer.js +483 -0
  45. package/dist/view.d.ts +20 -0
  46. package/dist/view.js +59 -0
  47. package/package.json +52 -4
@@ -0,0 +1,483 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { Declaration } from '@zukhruf/sqlite-drift';
3
+ import { transaction } from '@zukhruf/sqlite-transaction';
4
+ import { FileContent, TargetOwnedError, Writer, describeTarget, errorMessage, reloadMode, } from '@eltjs/elt';
5
+ import { openDestination } from './open-destination.js';
6
+ import { describe } from './sqlite-descriptions.js';
7
+ import { SQLiteFileStore } from './sqlite-file-store.js';
8
+ import { identifiers } from './sqlite-identifiers.js';
9
+ const quote = (name) => `"${name.replaceAll('"', '""')}"`;
10
+ const writersTable = new Declaration([
11
+ 'CREATE TABLE "_elt_writers" ("target" TEXT PRIMARY KEY, "writer" TEXT NOT NULL) STRICT',
12
+ ], { rows: 'replace' });
13
+ export const seq = '"_elt_seq"';
14
+ export const op = '"_elt_op"';
15
+ // One stream's load. Operations, and the chunks of their files, wait in
16
+ // connection-private TEMP tables, so another stream's commit never publishes
17
+ // them and a crash leaves nothing behind. As in Postgres, no transaction stays
18
+ // open between commits: prepare only refuses and stages, and each commit is
19
+ // one transaction that merges the stage with the result of applying its
20
+ // operations one at a time, the first also creating the target.
21
+ export class SQLiteWriter extends Writer {
22
+ configuration;
23
+ path;
24
+ table;
25
+ #description;
26
+ constructor(configuration, path, table) {
27
+ super(configuration.stream);
28
+ this.configuration = configuration;
29
+ this.path = path;
30
+ this.table = table;
31
+ this.#description = describeTarget(configuration, table.columns, (column) => `Integer reference to the source file's original bytes. Join ${quote(SQLiteFileStore.tableName(table, column))} on file = this value and concatenate bytes in order of n. NULL when the source file is unavailable.`);
32
+ if (table.readerView === undefined)
33
+ return;
34
+ const { missing } = this.#description;
35
+ if (missing.length > 0)
36
+ throw new TypeError(`Reader view ${table.readerView} needs JSON Schema descriptions for ${missing.join(', ')} of stream ${this.stream.name}`);
37
+ }
38
+ // What the target needs at its first commit, before its first merge.
39
+ // replacing: that commit replaces the target, so its rows are moot. A mode
40
+ // without a key releases the index a deduplicating mode made.
41
+ initialize(database, _replacing) {
42
+ this.dropDedupIndexes(database, null);
43
+ }
44
+ // Drops the deduplication indexes this library made on the target, except
45
+ // kept.
46
+ dropDedupIndexes(database, kept) {
47
+ for (const { name } of database
48
+ .prepare(String.raw `SELECT "name" FROM sqlite_schema WHERE "type" = 'index' AND "tbl_name" = ? AND "name" LIKE '\_elt\_dedup\_%' ESCAPE '\'`)
49
+ .all(this.table.name))
50
+ if (name !== kept)
51
+ database.exec(`DROP INDEX ${quote(String(name))}`);
52
+ }
53
+ // Refuses, before anything is read, stored rows the load cannot keep.
54
+ inspect(_database) { }
55
+ // What a reload's hidden target needs once it is created, before the first
56
+ // merge fills it, and once it takes the target's name.
57
+ build(_database, _into) { }
58
+ adopt(_database) { }
59
+ // Inserts each staged record, in order.
60
+ append(database, stage, loadedAt, into) {
61
+ database
62
+ .prepare(`INSERT INTO ${into} (${this.fields.join(', ')}) SELECT ${this.table.columns.map((column) => column.quotedName).join(', ')}, ? FROM ${stage} WHERE ${op} = 'R' ORDER BY ${seq}`)
63
+ .run(loadedAt);
64
+ }
65
+ // An overwrite replaces the target at its first commit.
66
+ get replaces() {
67
+ return false;
68
+ }
69
+ // Empties the target a replacing commit is about to fill.
70
+ replace(database) {
71
+ database.exec(`DELETE FROM ${this.table.quotedName}`);
72
+ }
73
+ get hash() {
74
+ return createHash('sha256').update(this.table.location).digest('hex');
75
+ }
76
+ get fields() {
77
+ return [
78
+ ...this.table.columns.map((column) => column.quotedName),
79
+ '"loaded_at"',
80
+ ];
81
+ }
82
+ encode(record) {
83
+ return this.table.columns.map((column) => column.encode(record));
84
+ }
85
+ // Only a load that identifies rows by key can remove one.
86
+ deletionKeys(_key) {
87
+ throw new TypeError('Only deduplicating loads can apply deletions');
88
+ }
89
+ // The owner lives beside the table it guards. A dropped table releases it,
90
+ // since nothing it held remains, and so does a table of owners another
91
+ // version declared otherwise, which is replaced empty.
92
+ writers(database) {
93
+ writersTable.apply(database);
94
+ }
95
+ // Refuses, before anything is read, a target another writer owns.
96
+ #refuse(database, writer) {
97
+ if (!this.exists(database, '_elt_writers'))
98
+ return;
99
+ const owner = database
100
+ .prepare(`SELECT "writer" FROM "_elt_writers" WHERE "target" = ? AND "target" IN (SELECT lower("name") FROM sqlite_schema WHERE "type" = 'table')`)
101
+ .get(this.table.location)?.writer;
102
+ if (owner !== undefined && owner !== writer)
103
+ throw new TargetOwnedError(this.table.name, String(owner), writer);
104
+ }
105
+ // Records the owner with the target's first commit; prepare refused any
106
+ // other.
107
+ own(database, writer) {
108
+ this.writers(database);
109
+ database.exec(`DELETE FROM "_elt_writers" WHERE "target" NOT IN (SELECT lower("name") FROM sqlite_schema WHERE "type" = 'table')`);
110
+ database
111
+ .prepare('INSERT INTO "_elt_writers" ("target", "writer") VALUES (?, ?) ON CONFLICT ("target") DO NOTHING')
112
+ .run(this.table.location, writer);
113
+ }
114
+ exists(database, location) {
115
+ return (database
116
+ .prepare(`SELECT 1 FROM sqlite_schema WHERE "type" = 'table' AND lower("name") = ?`)
117
+ .get(location) !== undefined);
118
+ }
119
+ // The field's values in each of the tables, which while a reload is open
120
+ // are the target and its hidden target: both rows still refer to files.
121
+ values(database, tables) {
122
+ const { table } = this;
123
+ return async function* (field) {
124
+ const column = table.columns.find((column) => column.field === field);
125
+ if (column === undefined)
126
+ throw new TypeError(`Unknown target field: ${field}`);
127
+ for (const name of tables()) {
128
+ // clear also accepts a target or a declared column that was never loaded.
129
+ if (!database
130
+ .prepare('SELECT 1 FROM pragma_table_info(?) WHERE name = ? COLLATE NOCASE')
131
+ .get(name, column.name))
132
+ continue;
133
+ for (const row of database
134
+ .prepare(`SELECT ${column.quotedName} AS value FROM ${quote(name)}`)
135
+ .iterate())
136
+ yield row.value;
137
+ }
138
+ };
139
+ }
140
+ async clear(writer, committed) {
141
+ using database = openDestination(this.path);
142
+ database.exec('BEGIN IMMEDIATE');
143
+ try {
144
+ this.writers(database);
145
+ const owner = database
146
+ .prepare('SELECT "writer" FROM "_elt_writers" WHERE "target" = ?')
147
+ .get(this.table.location)?.writer;
148
+ if (owner !== undefined && owner !== writer)
149
+ throw new TargetOwnedError(this.table.name, String(owner), writer);
150
+ // Emptied, not dropped: views built on the table keep working. With no
151
+ // rows left, no stored chunk is referenced, including those a failed
152
+ // stream left behind that the file triggers never see.
153
+ if (this.exists(database, this.table.location)) {
154
+ database.exec(`DELETE FROM ${this.table.quotedName}`);
155
+ // Clearing a copy also abandons the reload it left open.
156
+ database.exec(`DROP TABLE IF EXISTS ${quote(this.#hiddenName)}`);
157
+ // A file column declared after the table's last load has no chunks.
158
+ for (const column of this.table.columns) {
159
+ const chunks = SQLiteFileStore.tableName(this.table, column);
160
+ if (column.storesFile && this.exists(database, chunks))
161
+ database.exec(`DELETE FROM ${quote(chunks)}`);
162
+ }
163
+ }
164
+ database
165
+ .prepare('DELETE FROM "_elt_writers" WHERE "target" = ?')
166
+ .run(this.table.location);
167
+ database.exec('COMMIT');
168
+ database.exec('BEGIN IMMEDIATE');
169
+ await committed?.(this.values(database, () => [this.table.name]));
170
+ database.exec('ROLLBACK');
171
+ }
172
+ catch (error) {
173
+ if (database.isTransaction)
174
+ database.exec('ROLLBACK');
175
+ throw error;
176
+ }
177
+ }
178
+ // What the table, its reader view and their columns say to readers,
179
+ // rewritten with every load so they follow the stream's schema.
180
+ describe(database) {
181
+ const dataTypes = new Map([
182
+ ...this.table.columns.map(({ name, dataType }) => [name, dataType]),
183
+ ['loaded_at', 'timestamp'],
184
+ ]);
185
+ const columns = Object.fromEntries(Object.entries(this.#description.columns).map(([name, description]) => [
186
+ name,
187
+ { description, dataType: dataTypes.get(name) },
188
+ ]));
189
+ describe(database, this.table.name, this.#description.table, columns);
190
+ if (this.table.readerView !== undefined)
191
+ describe(database, this.table.readerView, this.#description.table, columns);
192
+ }
193
+ #viewDefinition(view) {
194
+ return `CREATE VIEW ${quote(view)} AS SELECT ${this.fields.join(', ')} FROM ${this.table.quotedName}`;
195
+ }
196
+ // Refuses, before anything is read, a view of other columns or another
197
+ // table, rather than adopting it. A stale table's view still selects the
198
+ // stored columns, which evolving the table replaces.
199
+ #refuseReaderView(database, view, stale) {
200
+ const existing = database
201
+ .prepare('SELECT "type", "sql" FROM sqlite_schema WHERE lower("name") = lower(?)')
202
+ .get(view);
203
+ if (existing === undefined)
204
+ return;
205
+ const sql = String(existing.sql);
206
+ const ours = stale
207
+ ? sql.startsWith(`CREATE VIEW ${quote(view)} AS SELECT `) &&
208
+ sql.endsWith(` FROM ${this.table.quotedName}`)
209
+ : sql === this.#viewDefinition(view);
210
+ if (existing.type !== 'view' || !ours)
211
+ throw new TypeError(`${quote(view)} is not a view of exactly ${this.table.quotedName}; drop it or delete the database`);
212
+ }
213
+ // Created only when absent and never replaced by a commit: replacing a view
214
+ // readers can see would lock them out. prepare refused any view not this
215
+ // load's own.
216
+ installReaderView(database, view) {
217
+ if (database
218
+ .prepare('SELECT 1 FROM sqlite_schema WHERE lower("name") = lower(?)')
219
+ .get(view) === undefined)
220
+ database.exec(this.#viewDefinition(view));
221
+ }
222
+ // The target as its first commit makes it: its owner, the table and the
223
+ // index its mode owns, its reader view and its descriptions.
224
+ #create(database, writer, replacing) {
225
+ this.own(database, writer);
226
+ database.exec(this.table.createTableSQL);
227
+ this.initialize(database, replacing);
228
+ if (this.table.readerView !== undefined)
229
+ this.installReaderView(database, this.table.readerView);
230
+ this.describe(database);
231
+ }
232
+ // The rows of one partition, or every row when the stream is not
233
+ // partitioned, as a WHERE clause over the target's or the stage's columns.
234
+ #scope(partition) {
235
+ if (partition === null)
236
+ return ['', []];
237
+ const columns = Object.keys(partition).map((field) => {
238
+ const column = this.table.columns.find((column) => column.field === field);
239
+ if (column === undefined)
240
+ throw new TypeError(`Resetting a partition requires destination column ${field}`);
241
+ return column;
242
+ });
243
+ return [
244
+ ` WHERE ${columns.map((column) => `${column.quotedName} = ?`).join(' AND ')}`,
245
+ columns.map((column) => column.encode(partition)),
246
+ ];
247
+ }
248
+ // A reload's hidden target, beside the target and invisible to readers.
249
+ get #hiddenName() {
250
+ return `_elt_next_${this.hash}`;
251
+ }
252
+ // The table the stream needs under the name, by the text SQLite keeps of
253
+ // its CREATE TABLE, which a CHECK alone can change. Evolving it keeps its
254
+ // rows.
255
+ #declaration(name) {
256
+ return new Declaration([`CREATE TABLE ${this.table.definition(quote(name))}`], { rows: 'keep' });
257
+ }
258
+ // Drops the chunks of every stored column that is no longer one of the
259
+ // target's file columns, and the triggers that delete them, before the table
260
+ // is replaced: no row of the table that takes its place refers to them.
261
+ #dropRetiredChunks(database) {
262
+ const files = new Set(this.table.columns
263
+ .filter(({ storesFile }) => storesFile)
264
+ .map(({ name }) => identifiers.key(name)));
265
+ for (const { name } of database
266
+ .prepare('SELECT "name" FROM pragma_table_info(?)')
267
+ .all(this.table.name))
268
+ if (!files.has(identifiers.key(String(name)))) {
269
+ const chunks = SQLiteFileStore.tableName(this.table, {
270
+ name: String(name),
271
+ });
272
+ database.exec(`DROP TABLE IF EXISTS ${quote(chunks)}`);
273
+ for (const trigger of ['delete', 'update'])
274
+ database.exec(`DROP TRIGGER IF EXISTS ${quote(`${chunks}_${trigger}`)}`);
275
+ }
276
+ }
277
+ // Brings a stored table the stream no longer fits to its shape, keeping
278
+ // every row: drift rebuilds it from the new definition and copies the
279
+ // columns both have, by name, a column the stream added taking its default
280
+ // (NULL) and one it dropped left behind. A value the new definition refuses
281
+ // fails the load, naming the column, and the transaction leaves the table as
282
+ // it was. The reader view selects the stored columns, and the mode index
283
+ // can cover a dropped key, so both are made again for the new shape.
284
+ #evolve(database) {
285
+ if (this.table.readerView !== undefined)
286
+ database.exec(`DROP VIEW IF EXISTS ${quote(this.table.readerView)}`);
287
+ this.dropDedupIndexes(database, null);
288
+ this.#dropRetiredChunks(database);
289
+ try {
290
+ this.#declaration(this.table.name).apply(database);
291
+ }
292
+ catch (cause) {
293
+ throw new TypeError(`The rows stored in ${this.table.quotedName} do not fit the new shape of stream ${this.stream.name}: ${errorMessage(cause)}. Clear the copy to load it again, or change the stream so they fit.`, { cause });
294
+ }
295
+ this.initialize(database, false);
296
+ if (this.table.readerView !== undefined)
297
+ this.installReaderView(database, this.table.readerView);
298
+ this.describe(database);
299
+ }
300
+ // Refuses, in one short transaction, a target another writer owns, a
301
+ // reader view not its own and stored rows it cannot keep, evolves a stored
302
+ // table the stream no longer fits unless the load overwrites it, and stages
303
+ // in TEMP. Nothing else reaches the database until a commit. A load into the
304
+ // target creates it, or adopts the stored one, at its first commit; a reload
305
+ // leaves it to readers as it is, merges into a hidden target from its first
306
+ // commit, and swaps that in at complete().
307
+ prepare(database, { writer, reloading }, loadedAt) {
308
+ const name = quote(`_elt_stage_${this.hash}`);
309
+ const stage = `temp.${name}`;
310
+ const hidden = quote(this.#hiddenName);
311
+ const stores = this.table.columns
312
+ .filter((column) => column.storesFile)
313
+ .map((column) => ({
314
+ column,
315
+ store: new SQLiteFileStore(database, this.table, column),
316
+ }));
317
+ let mode;
318
+ const open = (at) => at === 'reload' || at === 'continue';
319
+ const into = () => (open(mode) ? hidden : this.table.quotedName);
320
+ const tables = () => [this.table.name, ...(open(mode) ? [this.#hiddenName] : [])].filter((table) => this.exists(database, identifiers.key(table)));
321
+ // Every stored file a row of the tables refers to.
322
+ // With no table yet, no row refers to a file: a query of no rows.
323
+ const referenced = (column) => {
324
+ const selects = tables().map((table) => `SELECT ${column.quotedName} FROM ${quote(table)} WHERE ${column.quotedName} IS NOT NULL`);
325
+ return selects.length === 0
326
+ ? 'SELECT NULL WHERE 0'
327
+ : selects.join(' UNION ');
328
+ };
329
+ // Drops the chunks of files no row of the tables refers to.
330
+ const prune = () => {
331
+ for (const { column, store } of stores)
332
+ if (store.published)
333
+ database.exec(`DELETE FROM ${quote(store.name)} WHERE "file" NOT IN (${referenced(column)})`);
334
+ };
335
+ mode = transaction(database, () => {
336
+ this.#refuse(database, writer);
337
+ const target = this.#declaration(this.table.name).drift(database);
338
+ let next = reloadMode({
339
+ reloading,
340
+ destinationSyncMode: this.configuration.destinationSyncMode,
341
+ target,
342
+ hidden: this.#declaration(this.#hiddenName).drift(database),
343
+ });
344
+ // A hidden target no reload continues is a leftover readers never saw.
345
+ if (next !== 'continue')
346
+ database.exec(`DROP TABLE IF EXISTS ${hidden}`);
347
+ if (!open(next) && this.table.readerView !== undefined)
348
+ this.#refuseReaderView(database, this.table.readerView, target === 'stale');
349
+ if (next === 'evolve') {
350
+ this.#evolve(database);
351
+ next = 'load';
352
+ }
353
+ if (next === 'load' && !this.replaces)
354
+ this.inspect(database);
355
+ database.exec(`DROP TABLE IF EXISTS ${stage}`);
356
+ database.exec(`CREATE TEMP TABLE ${name} (${seq} INTEGER PRIMARY KEY, ${op} TEXT NOT NULL, ${this.table.columns.map((column) => `${column.quotedName} ${column.storageType}`).join(', ')})`);
357
+ for (const { store } of stores)
358
+ store.stage();
359
+ return next;
360
+ }, { mode: 'immediate' });
361
+ const fresh = mode === 'create' || mode === 'reload';
362
+ const columns = this.table.columns.map((column) => column.quotedName);
363
+ const record = database.prepare(`INSERT INTO ${stage} (${op}, ${columns.join(', ')}) VALUES ('R', ${columns.map(() => '?').join(', ')})`);
364
+ const staged = (column) => `SELECT ${column.quotedName} FROM ${stage} WHERE ${column.quotedName} IS NOT NULL`;
365
+ // Partitions the stage dropped, which the next commit empties in the
366
+ // table it merges into.
367
+ let resets = [];
368
+ // Files staged but never merged, and files a merge did not keep.
369
+ const drop = () => {
370
+ for (const { column, store } of stores) {
371
+ if (store.published)
372
+ database.exec(`DELETE FROM ${quote(store.name)} WHERE "file" IN (${staged(column)}) AND "file" NOT IN (${referenced(column)})`);
373
+ store.discard();
374
+ }
375
+ database.exec(`DELETE FROM ${stage}`);
376
+ resets = [];
377
+ };
378
+ // The first commit claims the target: it owns the hidden target of a
379
+ // reload, or creates or adopts the table. Later commits find it claimed.
380
+ const claimTarget = () => {
381
+ if (open(mode)) {
382
+ this.own(database, writer);
383
+ return;
384
+ }
385
+ this.#create(database, writer, this.replaces);
386
+ if (this.replaces)
387
+ this.replace(database);
388
+ };
389
+ const claimed = () => { };
390
+ let claim = claimTarget;
391
+ return {
392
+ fresh,
393
+ get reloading() {
394
+ return open(mode);
395
+ },
396
+ values: this.values(database, tables),
397
+ apply: async (operation) => {
398
+ if (operation.type === 'RESET') {
399
+ const { partition } = operation;
400
+ if (partition === null) {
401
+ // The whole stream starts over in a new hidden target.
402
+ drop();
403
+ mode = 'reload';
404
+ return;
405
+ }
406
+ const [rows, values] = this.#scope(partition);
407
+ database.prepare(`DELETE FROM ${stage}${rows}`).run(...values);
408
+ resets.push(partition);
409
+ return;
410
+ }
411
+ if (operation.type === 'DELETE') {
412
+ const [keys, values] = this.deletionKeys(operation.key);
413
+ database
414
+ .prepare(`INSERT INTO ${stage} (${op}, ${keys.map((column) => column.quotedName).join(', ')}) VALUES ('D', ${keys.map(() => '?').join(', ')})`)
415
+ .run(...values);
416
+ return;
417
+ }
418
+ // Each file is read here, before the source advances past it.
419
+ let data = operation.data;
420
+ for (const { column, store } of stores) {
421
+ const content = Reflect.get(Object(data), column.field);
422
+ if (content instanceof FileContent)
423
+ data = {
424
+ ...Object(data),
425
+ [column.field]: await store.save(content),
426
+ };
427
+ }
428
+ record.run(...this.encode(data));
429
+ },
430
+ // Each commit is one transaction; a failed one leaves the database as
431
+ // the last commit left it.
432
+ commit: async () => {
433
+ transaction(database, () => {
434
+ if (mode === 'reload') {
435
+ database.exec(`DROP TABLE IF EXISTS ${hidden}`);
436
+ database.exec(`CREATE TABLE ${this.table.definition(hidden)}`);
437
+ this.build(database, hidden);
438
+ }
439
+ claim();
440
+ for (const partition of resets) {
441
+ const [rows, values] = this.#scope(partition);
442
+ database.prepare(`DELETE FROM ${into()}${rows}`).run(...values);
443
+ }
444
+ for (const { store } of stores)
445
+ store.publish();
446
+ this.merge(database, stage, loadedAt, into());
447
+ // Files of the rows a reset emptied, and of staged rows it dropped.
448
+ if (resets.length > 0)
449
+ prune();
450
+ drop();
451
+ }, { mode: 'immediate' });
452
+ claim = claimed;
453
+ // Its first commit made the hidden target, so the reload continues.
454
+ if (mode === 'reload')
455
+ mode = 'continue';
456
+ },
457
+ complete: async () => {
458
+ if (!open(mode))
459
+ return;
460
+ transaction(database, () => {
461
+ if (this.table.readerView !== undefined)
462
+ database.exec(`DROP VIEW IF EXISTS ${quote(this.table.readerView)}`);
463
+ this.#dropRetiredChunks(database);
464
+ database.exec(`DROP TABLE IF EXISTS ${this.table.quotedName}`);
465
+ database.exec(`ALTER TABLE ${hidden} RENAME TO ${this.table.quotedName}`);
466
+ this.adopt(database);
467
+ if (this.table.readerView !== undefined)
468
+ this.installReaderView(database, this.table.readerView);
469
+ this.describe(database);
470
+ mode = 'load';
471
+ prune();
472
+ }, { mode: 'immediate' });
473
+ },
474
+ discard: async () => drop(),
475
+ [Symbol.asyncDispose]: async () => {
476
+ drop();
477
+ database.exec(`DROP TABLE ${stage}`);
478
+ for (const { store } of stores)
479
+ store.unstage();
480
+ },
481
+ };
482
+ }
483
+ }
package/dist/view.d.ts ADDED
@@ -0,0 +1,20 @@
1
+ import { type Connection, DerivedDataset } from '@eltjs/elt';
2
+ import { SQLiteDestination } from './sqlite-destination.ts';
3
+ import type { SQLiteTable } from './sqlite-table.ts';
4
+ export declare class View extends DerivedDataset<SQLiteTable> {
5
+ #private;
6
+ readonly name: string;
7
+ readonly into: SQLiteDestination;
8
+ constructor({ into, name, description, columns, sql, }: {
9
+ into: SQLiteDestination;
10
+ name: string;
11
+ description: string;
12
+ columns: Readonly<Record<string, string>>;
13
+ sql: string;
14
+ });
15
+ validate(connections: readonly Connection<SQLiteTable>[]): void;
16
+ refresh({ connections, earlier, }: {
17
+ readonly connections: readonly Connection<SQLiteTable>[];
18
+ readonly earlier: readonly DerivedDataset<SQLiteTable>[];
19
+ }): Promise<void>;
20
+ }
package/dist/view.js ADDED
@@ -0,0 +1,59 @@
1
+ import { DerivedDataset } from '@eltjs/elt';
2
+ import { openDestination } from './open-destination.js';
3
+ import { SQLiteDestination } from './sqlite-destination.js';
4
+ import { publishSQLiteViews } from './sqlite-views.js';
5
+ // A view over what the pipeline loads into one SQLite file: derived data
6
+ // that SQLite computes on each read, as a dbt model materialized as a view
7
+ // is. Its inputs are the relations its SQL reads, so it names none of them.
8
+ // It appears once its query compiles, so a reader never finds a view that
9
+ // fails; until a copy of the pipeline has made each table it reads, and each
10
+ // derived dataset before it that it reads has appeared, it waits, as dbt
11
+ // skips a model whose upstream model was skipped. Any other error of its
12
+ // query is the view's own failure.
13
+ export class View extends DerivedDataset {
14
+ name;
15
+ // The SQLite file the view lives in, where a computed dataset after it
16
+ // reads it.
17
+ into;
18
+ #view;
19
+ constructor({ into, name, description, columns, sql, }) {
20
+ super();
21
+ this.name = name;
22
+ this.into = into;
23
+ this.#view = { name, description, columns, query: sql };
24
+ Object.freeze(this);
25
+ }
26
+ validate(connections) {
27
+ if (this.#made(connections).has(this.name.toLowerCase()))
28
+ throw new TypeError(`View ${this.name} has the name of a table or reader view a copy makes`);
29
+ }
30
+ async refresh({ connections, earlier, }) {
31
+ using database = openDestination(this.into.path);
32
+ try {
33
+ database.prepare(this.#view.query);
34
+ }
35
+ catch (error) {
36
+ const missing = error instanceof Error
37
+ ? /^no such table: (?:main\.)?(.+)$/.exec(error.message)?.[1]
38
+ : undefined;
39
+ const missingName = missing?.toLowerCase();
40
+ if (missingName !== undefined &&
41
+ (this.#made(connections).has(missingName) ||
42
+ earlier.some(({ name }) => name.toLowerCase() === missingName)))
43
+ return;
44
+ throw error;
45
+ }
46
+ publishSQLiteViews(database, { views: [this.#view] });
47
+ }
48
+ // The tables and reader views the pipeline's copies make in this view's
49
+ // file, in lower case, as SQLite compares names.
50
+ #made(connections) {
51
+ return new Set(connections
52
+ .filter(({ destination }) => destination instanceof SQLiteDestination &&
53
+ destination.path === this.into.path)
54
+ .flatMap(({ steps }) => steps)
55
+ .flatMap(({ to }) => [to.name, to.readerView])
56
+ .filter((name) => name !== undefined)
57
+ .map((name) => name.toLowerCase()));
58
+ }
59
+ }
package/package.json CHANGED
@@ -1,6 +1,54 @@
1
1
  {
2
2
  "name": "@eltjs/elt-sqlite",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.1.1",
4
+ "description": "The SQLite destination of elt: each stream is a raw table in one SQLite file, with a reader view over it and a catalog of each table, view and column.",
5
+ "license": "Apache-2.0",
6
+ "repository": {
7
+ "type": "git",
8
+ "url": "https://github.com/ezzabuzaid/elt",
9
+ "directory": "packages/destinations/sqlite"
10
+ },
11
+ "bugs": {
12
+ "url": "https://github.com/ezzabuzaid/elt/issues"
13
+ },
14
+ "homepage": "https://github.com/ezzabuzaid/elt/tree/main/packages/destinations/sqlite#readme",
15
+ "keywords": [
16
+ "elt",
17
+ "sqlite",
18
+ "destination",
19
+ "pipeline"
20
+ ],
21
+ "type": "module",
22
+ "exports": {
23
+ "./package.json": "./package.json",
24
+ ".": {
25
+ "types": "./dist/index.d.ts",
26
+ "default": "./dist/index.js"
27
+ }
28
+ },
29
+ "files": [
30
+ "dist",
31
+ "!dist/spec/**",
32
+ "!**/*.tsbuildinfo",
33
+ "!**/*.test.*"
34
+ ],
35
+ "publishConfig": {
36
+ "access": "public"
37
+ },
38
+ "engines": {
39
+ "node": ">=24"
40
+ },
41
+ "dependencies": {
42
+ "@eltjs/elt": "^0.1.1",
43
+ "@zukhruf/mutex": "^0.8.10",
44
+ "@zukhruf/sqlite-drift": "^0.8.10",
45
+ "@zukhruf/sqlite-open": "^0.8.10",
46
+ "@zukhruf/sqlite-transaction": "^0.8.10"
47
+ },
48
+ "devDependencies": {
49
+ "@sinonjs/fake-timers": "15.4.0",
50
+ "@zukhruf/async": "^0.8.10",
51
+ "@zukhruf/snapshot-differential": "^0.8.10",
52
+ "@zukhruf/testing": "^0.8.10"
53
+ }
54
+ }