@eltjs/elt-sqlite 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +248 -2
  3. package/dist/computed-dataset.d.ts +22 -0
  4. package/dist/computed-dataset.js +230 -0
  5. package/dist/index.d.ts +13 -0
  6. package/dist/index.js +13 -0
  7. package/dist/open-destination.d.ts +3 -0
  8. package/dist/open-destination.js +14 -0
  9. package/dist/reserved.d.ts +7 -0
  10. package/dist/reserved.js +9 -0
  11. package/dist/sqlite-append-writer.d.ts +8 -0
  12. package/dist/sqlite-append-writer.js +10 -0
  13. package/dist/sqlite-catalog.d.ts +3 -0
  14. package/dist/sqlite-catalog.js +33 -0
  15. package/dist/sqlite-checkpoint-store.d.ts +9 -0
  16. package/dist/sqlite-checkpoint-store.js +84 -0
  17. package/dist/sqlite-column.d.ts +53 -0
  18. package/dist/sqlite-column.js +306 -0
  19. package/dist/sqlite-columns.d.ts +14 -0
  20. package/dist/sqlite-columns.js +118 -0
  21. package/dist/sqlite-deduplicating-writer.d.ts +21 -0
  22. package/dist/sqlite-deduplicating-writer.js +124 -0
  23. package/dist/sqlite-descriptions.d.ts +8 -0
  24. package/dist/sqlite-descriptions.js +37 -0
  25. package/dist/sqlite-destination.d.ts +19 -0
  26. package/dist/sqlite-destination.js +67 -0
  27. package/dist/sqlite-file-store.d.ts +17 -0
  28. package/dist/sqlite-file-store.js +82 -0
  29. package/dist/sqlite-identifiers.d.ts +8 -0
  30. package/dist/sqlite-identifiers.js +11 -0
  31. package/dist/sqlite-overwrite-writer.d.ts +9 -0
  32. package/dist/sqlite-overwrite-writer.js +13 -0
  33. package/dist/sqlite-passes.d.ts +39 -0
  34. package/dist/sqlite-passes.js +174 -0
  35. package/dist/sqlite-sync-history-schema.d.ts +8 -0
  36. package/dist/sqlite-sync-history-schema.js +94 -0
  37. package/dist/sqlite-sync-history.d.ts +12 -0
  38. package/dist/sqlite-sync-history.js +119 -0
  39. package/dist/sqlite-table.d.ts +15 -0
  40. package/dist/sqlite-table.js +83 -0
  41. package/dist/sqlite-views.d.ts +16 -0
  42. package/dist/sqlite-views.js +62 -0
  43. package/dist/sqlite-writer.d.ts +38 -0
  44. package/dist/sqlite-writer.js +483 -0
  45. package/dist/view.d.ts +20 -0
  46. package/dist/view.js +59 -0
  47. package/package.json +52 -4
@@ -0,0 +1,67 @@
1
+ import { resolve } from 'node:path';
2
+ import { Destination } from '@eltjs/elt';
3
+ import { openDestination } from './open-destination.js';
4
+ import { SQLiteAppendWriter } from './sqlite-append-writer.js';
5
+ import { SQLiteColumns } from './sqlite-columns.js';
6
+ import { SQLiteDeduplicatingWriter } from './sqlite-deduplicating-writer.js';
7
+ import { SQLiteOverwriteWriter } from './sqlite-overwrite-writer.js';
8
+ import { SQLiteTable } from './sqlite-table.js';
9
+ export class SQLiteDestination extends Destination {
10
+ supportedDestinationSyncModes = Object.freeze([
11
+ 'overwrite',
12
+ 'append',
13
+ 'append_dedup',
14
+ 'overwrite_dedup',
15
+ ]);
16
+ path;
17
+ constructor({ path }) {
18
+ super();
19
+ if (!path)
20
+ throw new TypeError('SQLite requires a database path');
21
+ this.path = path === ':memory:' ? path : resolve(path);
22
+ Object.freeze(this);
23
+ }
24
+ location(target) {
25
+ return target.location;
26
+ }
27
+ // Each stream's commit is its own short transaction, so no transaction
28
+ // stays open while the sources read.
29
+ async load({ clock, }) {
30
+ const database = openDestination(this.path);
31
+ const loadedAt = new Date(clock.Date.now()).toISOString();
32
+ return {
33
+ prepare: async (configuration, target, binding) => this.createWriter(configuration, target).prepare(database, binding, loadedAt),
34
+ [Symbol.asyncDispose]: async () => {
35
+ try {
36
+ if (database.isTransaction)
37
+ database.exec('ROLLBACK');
38
+ }
39
+ finally {
40
+ database.close();
41
+ }
42
+ },
43
+ };
44
+ }
45
+ table(name, configure) {
46
+ return new SQLiteTable(name, configure?.(new SQLiteColumns()));
47
+ }
48
+ createWriter(configuration, target) {
49
+ this.validateConfiguration(configuration, target);
50
+ if (!(target instanceof SQLiteTable))
51
+ throw new TypeError('SQLite requires SQLite table targets');
52
+ if (configuration.syncMode === 'incremental' && this.path === ':memory:')
53
+ throw new TypeError('Incremental SQLite requires a persistent destination file');
54
+ const table = target.resolve(configuration.stream);
55
+ switch (configuration.destinationSyncMode) {
56
+ case 'append_dedup':
57
+ case 'overwrite_dedup':
58
+ return new SQLiteDeduplicatingWriter(configuration, this.path, table);
59
+ case 'append':
60
+ return new SQLiteAppendWriter(configuration, this.path, table);
61
+ case 'overwrite':
62
+ return new SQLiteOverwriteWriter(configuration, this.path, table);
63
+ default:
64
+ throw new TypeError(`Destination does not support ${configuration.destinationSyncMode}`);
65
+ }
66
+ }
67
+ }
@@ -0,0 +1,17 @@
1
+ import type { DatabaseSync } from 'node:sqlite';
2
+ import type { FileContent } from '@eltjs/elt';
3
+ import type { SQLiteColumn } from './sqlite-column.ts';
4
+ import type { SQLiteTable } from './sqlite-table.ts';
5
+ export declare class SQLiteFileStore {
6
+ #private;
7
+ static readonly chunkSize: number;
8
+ readonly name: string;
9
+ constructor(database: DatabaseSync, table: SQLiteTable, column: SQLiteColumn);
10
+ static tableName(table: SQLiteTable, column: Pick<SQLiteColumn, 'name'>): string;
11
+ get published(): boolean;
12
+ stage(): void;
13
+ save(content: FileContent): Promise<number>;
14
+ publish(): void;
15
+ discard(): void;
16
+ unstage(): void;
17
+ }
@@ -0,0 +1,82 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { identifiers } from './sqlite-identifiers.js';
3
+ const quote = (name) => `"${name.replaceAll('"', '""')}"`;
4
+ const tablePrefix = '_elt_files_';
5
+ const chunks = '("file" INTEGER NOT NULL, "n" INTEGER NOT NULL, "bytes" BLOB NOT NULL, PRIMARY KEY ("file", "n")) STRICT';
6
+ // A file column's bytes, in ordered chunks: SQLite caps one BLOB at 1e9 bytes
7
+ // and a whole-file BLOB would hold the file in memory. The row keeps the
8
+ // file's id; triggers drop a row's chunks in the same transaction as the row.
9
+ // A load stages chunks in a connection-private TEMP table and publishes them
10
+ // inside a commit, so a load holds no transaction while it reads files.
11
+ export class SQLiteFileStore {
12
+ static chunkSize = 4 * 1024 * 1024;
13
+ name;
14
+ #database;
15
+ #table;
16
+ #column;
17
+ #staged;
18
+ constructor(database, table, column) {
19
+ this.#database = database;
20
+ this.#table = table;
21
+ this.#column = column;
22
+ this.name = SQLiteFileStore.tableName(table, column);
23
+ this.#staged = quote(`${tablePrefix}stage_${this.name.slice(tablePrefix.length)}`);
24
+ }
25
+ // A hash of the table and column, as SQLite compares them: joined as text,
26
+ // table a_b with column c and table a with column b_c would share chunks
27
+ // and prune each other's.
28
+ static tableName(table, column) {
29
+ const key = createHash('sha256')
30
+ .update(JSON.stringify([table.location, identifiers.key(column.name)]))
31
+ .digest('hex')
32
+ .slice(0, 40);
33
+ return `${tablePrefix}${key}`;
34
+ }
35
+ #exists(location) {
36
+ return (this.#database
37
+ .prepare(`SELECT 1 FROM sqlite_schema WHERE "type" = 'table' AND lower("name") = ?`)
38
+ .get(location) !== undefined);
39
+ }
40
+ get published() {
41
+ return this.#exists(this.name);
42
+ }
43
+ stage() {
44
+ this.#database.exec(`DROP TABLE IF EXISTS temp.${this.#staged}`);
45
+ this.#database.exec(`CREATE TEMP TABLE ${this.#staged} ${chunks}`);
46
+ }
47
+ // An empty file still stores one empty chunk, so its id stays reserved. Ids
48
+ // continue after both the published and the staged chunks.
49
+ async save(content) {
50
+ const highest = (table) => Number(this.#database
51
+ .prepare(`SELECT coalesce(max("file"), 0) AS "file" FROM ${table}`)
52
+ .get()?.file);
53
+ const file = Math.max(this.published ? highest(quote(this.name)) : 0, highest(`temp.${this.#staged}`)) + 1;
54
+ const insert = this.#database.prepare(`INSERT INTO temp.${this.#staged} ("file", "n", "bytes") VALUES (?, ?, ?)`);
55
+ let n = 0;
56
+ for await (const chunk of content.chunks(SQLiteFileStore.chunkSize))
57
+ insert.run(file, n++, chunk);
58
+ if (n === 0)
59
+ insert.run(file, 0, new Uint8Array());
60
+ return file;
61
+ }
62
+ // Moves the staged chunks into the store, inside a commit, and attaches the
63
+ // triggers to the target once it exists. A reload swaps in a table without
64
+ // them, which its next commit attaches again.
65
+ publish() {
66
+ const table = quote(this.name);
67
+ const column = this.#column.quotedName;
68
+ this.#database.exec(`CREATE TABLE IF NOT EXISTS ${table} ${chunks}`);
69
+ if (this.#exists(this.#table.location)) {
70
+ this.#database.exec(`CREATE TRIGGER IF NOT EXISTS ${quote(`${this.name}_delete`)} AFTER DELETE ON ${this.#table.quotedName} BEGIN DELETE FROM ${table} WHERE "file" = old.${column}; END`);
71
+ this.#database.exec(`CREATE TRIGGER IF NOT EXISTS ${quote(`${this.name}_update`)} AFTER UPDATE OF ${column} ON ${this.#table.quotedName} WHEN old.${column} IS NOT new.${column} BEGIN DELETE FROM ${table} WHERE "file" = old.${column}; END`);
72
+ }
73
+ this.#database.exec(`INSERT INTO ${table} SELECT * FROM temp.${this.#staged}`);
74
+ this.discard();
75
+ }
76
+ discard() {
77
+ this.#database.exec(`DELETE FROM temp.${this.#staged}`);
78
+ }
79
+ unstage() {
80
+ this.#database.exec(`DROP TABLE IF EXISTS temp.${this.#staged}`);
81
+ }
82
+ }
@@ -0,0 +1,8 @@
1
+ import { Identifiers } from '@eltjs/elt';
2
+ declare class SQLiteIdentifiers extends Identifiers {
3
+ protected readonly maxBytes: number;
4
+ protected readonly ownColumns: string[];
5
+ key(name: string): string;
6
+ }
7
+ export declare const identifiers: Readonly<SQLiteIdentifiers>;
8
+ export {};
@@ -0,0 +1,11 @@
1
+ import { Identifiers } from '@eltjs/elt';
2
+ // SQLite holds a name of any length and compares two without ASCII case: it
3
+ // folds A-Z only, so `É` and `é` stay two names.
4
+ class SQLiteIdentifiers extends Identifiers {
5
+ maxBytes = Infinity;
6
+ ownColumns = ['loaded_at'];
7
+ key(name) {
8
+ return name.replaceAll(/[A-Z]/g, (letter) => letter.toLowerCase());
9
+ }
10
+ }
11
+ export const identifiers = Object.freeze(new SQLiteIdentifiers());
@@ -0,0 +1,9 @@
1
+ import type { DatabaseSync } from 'node:sqlite';
2
+ import type { CopyConfiguration } from '@eltjs/elt';
3
+ import type { SQLiteTable } from './sqlite-table.ts';
4
+ import { SQLiteWriter } from './sqlite-writer.ts';
5
+ export declare class SQLiteOverwriteWriter extends SQLiteWriter {
6
+ constructor(configuration: CopyConfiguration, path: string, table: SQLiteTable);
7
+ protected get replaces(): boolean;
8
+ protected merge(database: DatabaseSync, stage: string, loadedAt: string, into: string): void;
9
+ }
@@ -0,0 +1,13 @@
1
+ import { SQLiteWriter } from './sqlite-writer.js';
2
+ export class SQLiteOverwriteWriter extends SQLiteWriter {
3
+ constructor(configuration, path, table) {
4
+ super(configuration, path, table);
5
+ Object.freeze(this);
6
+ }
7
+ get replaces() {
8
+ return true;
9
+ }
10
+ merge(database, stage, loadedAt, into) {
11
+ this.append(database, stage, loadedAt, into);
12
+ }
13
+ }
@@ -0,0 +1,39 @@
1
+ import type { FailureType, SyncStatus } from '@eltjs/elt';
2
+ export type PassState = SyncStatus | 'running';
3
+ type Started = {
4
+ readonly startedAt: string;
5
+ readonly lastSucceededAt: string | null;
6
+ readonly currentAsOf: string | null;
7
+ };
8
+ export type PassStatus = (Started & {
9
+ readonly state: 'running';
10
+ readonly completedAt: null;
11
+ readonly error: null;
12
+ readonly failureType: null;
13
+ }) | (Started & {
14
+ readonly state: 'succeeded';
15
+ readonly completedAt: string;
16
+ readonly error: null;
17
+ readonly failureType: null;
18
+ }) | (Started & {
19
+ readonly state: 'cancelled';
20
+ readonly completedAt: string;
21
+ readonly error: string;
22
+ readonly failureType: null;
23
+ }) | (Started & {
24
+ readonly state: 'partial' | 'failed';
25
+ readonly completedAt: string;
26
+ readonly error: string;
27
+ readonly failureType: FailureType;
28
+ });
29
+ export type StreamPassStatus = {
30
+ readonly stream: string;
31
+ readonly state: PassState;
32
+ readonly lastSucceededAt: string | null;
33
+ readonly currentAsOf: string | null;
34
+ };
35
+ export declare function readPassStatus(path: string): {
36
+ readonly pass: PassStatus | null;
37
+ readonly streams: readonly StreamPassStatus[];
38
+ };
39
+ export {};
@@ -0,0 +1,174 @@
1
+ import { UnavailableError, open } from '@zukhruf/sqlite-open';
2
+ import { busyTimeout } from './open-destination.js';
3
+ import { syncHistory } from './sqlite-sync-history-schema.js';
4
+ // The latest pass of a SQLite destination file and each stream's latest
5
+ // outcome, as its sync history records them; null and empty until the history
6
+ // recorded a pass, and while the history in the file is one another version
7
+ // installed, and while no file is at the path. A file that several
8
+ // connections load reads as one pass, made of the latest pass of each
9
+ // connection.
10
+ export function readPassStatus(path) {
11
+ using data = reader(path);
12
+ if (data === null || syncHistory.drift(data) !== 'fits')
13
+ return { pass: null, streams: [] };
14
+ const [first, ...others] = data
15
+ .prepare('SELECT status, started_at, completed_at, error, failure_type, last_successful_sync_at, current_as_of FROM sync_status')
16
+ .all()
17
+ .map((row) => recorded(row, passState(row.status)));
18
+ return {
19
+ pass: first === undefined ? null : combined([first, ...others]),
20
+ streams: data
21
+ .prepare('SELECT stream, status, last_successful_sync_at, current_as_of FROM stream_status ORDER BY stream')
22
+ .all()
23
+ .map((row) => ({
24
+ stream: String(row.stream),
25
+ state: passState(row.status),
26
+ lastSucceededAt: text(row.last_successful_sync_at),
27
+ currentAsOf: text(row.current_as_of),
28
+ })),
29
+ };
30
+ }
31
+ // A reader of the file, or null while no file is at the path.
32
+ function reader(path) {
33
+ try {
34
+ return open(path, { readOnly: true, timeout: busyTimeout });
35
+ }
36
+ catch (error) {
37
+ if (error instanceof UnavailableError)
38
+ return null;
39
+ throw error;
40
+ }
41
+ }
42
+ // A sync_status row as the pass it records: the history completes every pass
43
+ // that ended, and gives an error to each that did not load completely.
44
+ function recorded(row, state) {
45
+ const started = {
46
+ startedAt: String(row.started_at),
47
+ lastSucceededAt: text(row.last_successful_sync_at),
48
+ currentAsOf: text(row.current_as_of),
49
+ };
50
+ switch (state) {
51
+ case 'running':
52
+ return {
53
+ ...started,
54
+ state,
55
+ completedAt: null,
56
+ error: null,
57
+ failureType: null,
58
+ };
59
+ case 'succeeded':
60
+ return {
61
+ ...started,
62
+ state,
63
+ completedAt: String(row.completed_at),
64
+ error: null,
65
+ failureType: null,
66
+ };
67
+ case 'cancelled':
68
+ return {
69
+ ...started,
70
+ state,
71
+ completedAt: String(row.completed_at),
72
+ error: String(row.error),
73
+ failureType: null,
74
+ };
75
+ default:
76
+ return {
77
+ ...started,
78
+ state,
79
+ completedAt: String(row.completed_at),
80
+ error: String(row.error),
81
+ failureType: failureType(row.failure_type),
82
+ };
83
+ }
84
+ }
85
+ // The latest passes of the connections of one file as one pass, which is only
86
+ // as good as its worst connection: it runs while any of them runs, began when
87
+ // the first began, ended when the last ended, and is current only as of the
88
+ // least current. A failure of one connection makes the pass partial, not
89
+ // succeeded, and its error keeps the error of each connection that did not
90
+ // succeed, as Go's errors.Join does, one on each line.
91
+ function combined(passes) {
92
+ const started = {
93
+ startedAt: earliest(passes.map(({ startedAt }) => startedAt)),
94
+ lastSucceededAt: earliestOfAll(passes.map(({ lastSucceededAt }) => lastSucceededAt)),
95
+ currentAsOf: earliestOfAll(passes.map(({ currentAsOf }) => currentAsOf)),
96
+ };
97
+ const ended = passes.filter((pass) => pass.state !== 'running');
98
+ if (ended.length < passes.length)
99
+ return {
100
+ ...started,
101
+ state: 'running',
102
+ completedAt: null,
103
+ error: null,
104
+ failureType: null,
105
+ };
106
+ const completedAt = latest(ended.map(({ completedAt }) => completedAt));
107
+ const error = ended
108
+ .flatMap((pass) => (pass.error === null ? [] : [pass.error]))
109
+ .join('\n');
110
+ if (ended.some(({ state }) => state === 'cancelled'))
111
+ return {
112
+ ...started,
113
+ state: 'cancelled',
114
+ completedAt,
115
+ error,
116
+ failureType: null,
117
+ };
118
+ if (ended.every(({ state }) => state === 'succeeded'))
119
+ return {
120
+ ...started,
121
+ state: 'succeeded',
122
+ completedAt,
123
+ error: null,
124
+ failureType: null,
125
+ };
126
+ return {
127
+ ...started,
128
+ state: ended.every(({ state }) => state === 'failed')
129
+ ? 'failed'
130
+ : 'partial',
131
+ completedAt,
132
+ error,
133
+ failureType: ended.some(({ failureType }) => failureType === 'config')
134
+ ? 'config'
135
+ : 'system',
136
+ };
137
+ }
138
+ function earliest(instants) {
139
+ return instants.reduce((first, next) => Date.parse(next) < Date.parse(first) ? next : first);
140
+ }
141
+ function latest(instants) {
142
+ return instants.reduce((last, next) => Date.parse(next) > Date.parse(last) ? next : last);
143
+ }
144
+ // The earliest instant, or null when any of them is null.
145
+ function earliestOfAll(instants) {
146
+ const known = instants.filter((instant) => instant !== null);
147
+ return known.length < instants.length ? null : earliest(known);
148
+ }
149
+ function failureType(value) {
150
+ switch (value) {
151
+ case 'config':
152
+ case 'system':
153
+ return value;
154
+ default:
155
+ throw new TypeError(`Unknown failure type ${String(value)}`);
156
+ }
157
+ }
158
+ function passState(status) {
159
+ switch (status) {
160
+ case 'running':
161
+ case 'succeeded':
162
+ case 'partial':
163
+ case 'failed':
164
+ case 'cancelled':
165
+ return status;
166
+ default:
167
+ throw new TypeError(`Unknown pass status ${String(status)}`);
168
+ }
169
+ }
170
+ function text(value) {
171
+ if (value === undefined)
172
+ throw new TypeError('A sync history view lacks a column it declares');
173
+ return value === null ? null : String(value);
174
+ }
@@ -0,0 +1,8 @@
1
+ import { Declaration } from '@zukhruf/sqlite-drift';
2
+ import { type SQLiteView } from './sqlite-views.ts';
3
+ export declare const attempts = "\"_elt_sync_attempts\"";
4
+ export declare const coverage = "\"_elt_extraction_coverage\"";
5
+ export declare const checks = "\"_elt_stream_checks\"";
6
+ export declare const syncHistoryTables: Declaration;
7
+ export declare const syncHistoryViews: readonly SQLiteView[];
8
+ export declare const syncHistory: Declaration;
@@ -0,0 +1,94 @@
1
+ import { Declaration } from '@zukhruf/sqlite-drift';
2
+ import { syncHistoryRelations } from '@eltjs/elt';
3
+ import { viewDefinition } from './sqlite-views.js';
4
+ export const attempts = '"_elt_sync_attempts"';
5
+ export const coverage = '"_elt_extraction_coverage"';
6
+ export const checks = '"_elt_stream_checks"';
7
+ const status = `"status" TEXT NOT NULL DEFAULT 'running' CHECK ("status" IN ('running', 'succeeded', 'partial', 'failed', 'cancelled'))`;
8
+ const tables = [
9
+ `CREATE TABLE ${attempts} (
10
+ "id" INTEGER PRIMARY KEY,
11
+ "connector" TEXT NOT NULL CHECK (trim("connector") <> ''), "source" TEXT NOT NULL,
12
+ "started_at" TEXT NOT NULL, "completed_at" TEXT, ${status}, "error" TEXT,
13
+ "failure_type" TEXT CHECK ("failure_type" IN ('config', 'system')),
14
+ CHECK (("status" = 'running') = ("completed_at" IS NULL)),
15
+ CHECK (("status" IN ('partial', 'failed')) = ("failure_type" IS NOT NULL))
16
+ ) STRICT`,
17
+ `CREATE INDEX "_elt_sync_attempts_connector" ON ${attempts} ("connector", "id" DESC)`,
18
+ `CREATE TABLE ${coverage} (
19
+ "attempt_id" INTEGER NOT NULL REFERENCES ${attempts}("id"),
20
+ "stream" TEXT NOT NULL, "target_schema" TEXT NOT NULL, "target_table" TEXT NOT NULL,
21
+ "sync_mode" TEXT NOT NULL, "destination_sync_mode" TEXT NOT NULL,
22
+ "description" TEXT NOT NULL CHECK (trim("description") <> ''),
23
+ "selection" TEXT NOT NULL CHECK (json_valid("selection")), ${status},
24
+ "written_count" INTEGER CHECK ("written_count" >= 0),
25
+ "deleted_count" INTEGER CHECK ("deleted_count" >= 0),
26
+ "failures" TEXT NOT NULL DEFAULT '[]' CHECK (json_valid("failures")),
27
+ "upstream_current_as_of" TEXT,
28
+ PRIMARY KEY ("attempt_id", "stream")
29
+ ) STRICT`,
30
+ // Each stream's latest check by a watch that found it unchanged.
31
+ `CREATE TABLE ${checks} (
32
+ "connector" TEXT NOT NULL, "stream" TEXT NOT NULL, "checked_at" TEXT NOT NULL,
33
+ PRIMARY KEY ("connector", "stream")
34
+ ) STRICT`,
35
+ ];
36
+ // The history's tables, which install() brings to this declaration.
37
+ export const syncHistoryTables = new Declaration(tables, { rows: 'replace' });
38
+ // When each declared stream last matched its upstream: the start of its last
39
+ // successful pass, or a later check that found it unchanged, each no later
40
+ // than when the source said the upstream itself was last current.
41
+ const currency = `"stream_success" AS (
42
+ SELECT a."connector", c."stream", min(a."started_at", coalesce(c."upstream_current_as_of", a."started_at")) AS "current_as_of",
43
+ row_number() OVER (PARTITION BY a."connector", c."stream" ORDER BY a."completed_at" DESC, a."id" DESC) AS "rank"
44
+ FROM ${coverage} c JOIN ${attempts} a ON a."id" = c."attempt_id" WHERE c."status" = 'succeeded'
45
+ ), "currency" AS (
46
+ SELECT d."connector", d."stream",
47
+ CASE WHEN s."current_as_of" IS NULL THEN NULL ELSE max(s."current_as_of", coalesce(k."checked_at", s."current_as_of")) END AS "current_as_of"
48
+ FROM (SELECT DISTINCT a."connector", c."stream" FROM ${coverage} c JOIN ${attempts} a ON a."id" = c."attempt_id") d
49
+ LEFT JOIN "stream_success" s ON s."connector" = d."connector" AND s."stream" = d."stream" AND s."rank" = 1
50
+ LEFT JOIN ${checks} k ON k."connector" = d."connector" AND k."stream" = d."stream"
51
+ )`;
52
+ // The same relations Postgres publishes; SQLite ranks with window functions
53
+ // where Postgres uses DISTINCT ON and LATERAL.
54
+ const queries = {
55
+ sync_attempts: `SELECT "id" AS "attempt_id", "connector", "source", "started_at", "completed_at", "status", "error", "failure_type" FROM ${attempts}`,
56
+ extraction_coverage: `SELECT c."attempt_id", a."connector", a."source", a."started_at", a."completed_at",
57
+ c."stream", c."target_schema", c."target_table",
58
+ EXISTS (SELECT 1 FROM sqlite_schema t WHERE t."type" = 'table' AND lower(t."name") = lower(c."target_table")) AS "target_exists",
59
+ c."sync_mode", c."destination_sync_mode", c."description", c."selection",
60
+ c."status", c."written_count", c."deleted_count", c."failures"
61
+ FROM ${coverage} c JOIN ${attempts} a ON a."id" = c."attempt_id"`,
62
+ sync_status: `WITH ${currency}, "success" AS (
63
+ SELECT "connector", "id", "completed_at", row_number() OVER (PARTITION BY "connector" ORDER BY "completed_at" DESC, "id" DESC) AS "rank"
64
+ FROM ${attempts} WHERE "status" = 'succeeded'
65
+ ), "latest" AS (
66
+ SELECT *, row_number() OVER (PARTITION BY "connector" ORDER BY "id" DESC) AS "rank" FROM ${attempts}
67
+ )
68
+ SELECT a."connector", a."id" AS "latest_attempt_id", a."started_at", a."completed_at", a."status", a."error", a."failure_type",
69
+ s."id" AS "last_successful_attempt_id", s."completed_at" AS "last_successful_sync_at",
70
+ (SELECT CASE WHEN count(*) = count(c."current_as_of") THEN min(c."current_as_of") END FROM "currency" c WHERE c."connector" = a."connector") AS "current_as_of"
71
+ FROM "latest" a LEFT JOIN "success" s ON s."connector" = a."connector" AND s."rank" = 1
72
+ WHERE a."rank" = 1 ORDER BY a."connector"`,
73
+ stream_status: `WITH ${currency}, "declared" AS (
74
+ SELECT a."connector", c."stream", c."target_schema", c."target_table", a."id", a."started_at", a."completed_at", c."status",
75
+ row_number() OVER (PARTITION BY a."connector", c."stream" ORDER BY a."id" DESC) AS "rank"
76
+ FROM ${coverage} c JOIN ${attempts} a ON a."id" = c."attempt_id"
77
+ ), "success" AS (
78
+ SELECT a."connector", c."stream", a."id", a."completed_at",
79
+ row_number() OVER (PARTITION BY a."connector", c."stream" ORDER BY a."completed_at" DESC, a."id" DESC) AS "rank"
80
+ FROM ${coverage} c JOIN ${attempts} a ON a."id" = c."attempt_id" WHERE c."status" = 'succeeded'
81
+ )
82
+ SELECT d."connector", d."stream", d."target_schema", d."target_table",
83
+ d."id" AS "latest_attempt_id", d."started_at", d."completed_at", d."status",
84
+ s."id" AS "last_successful_attempt_id", s."completed_at" AS "last_successful_sync_at", c."current_as_of"
85
+ FROM "declared" d LEFT JOIN "success" s ON s."connector" = d."connector" AND s."stream" = d."stream" AND s."rank" = 1
86
+ LEFT JOIN "currency" c ON c."connector" = d."connector" AND c."stream" = d."stream"
87
+ WHERE d."rank" = 1 ORDER BY d."connector", d."stream"`,
88
+ };
89
+ // How readers see the sync history: what each pass declared and loaded.
90
+ export const syncHistoryViews = Object.values(syncHistoryRelations).map((relation) => ({ ...relation, query: queries[relation.name] }));
91
+ // The whole history as this version reads it: a file whose tables or views
92
+ // another version installed has no history this version can read until its
93
+ // next pass installs it again.
94
+ export const syncHistory = new Declaration([...tables, ...syncHistoryViews.map(viewDefinition)], { rows: 'replace' });
@@ -0,0 +1,12 @@
1
+ import { type Connection, type DeclaredCopy, type RecordedPass, type StreamCheck, SyncHistory } from '@eltjs/elt';
2
+ import type { SQLiteTable } from './sqlite-table.ts';
3
+ export declare class SQLiteSyncHistory extends SyncHistory<SQLiteTable> {
4
+ #private;
5
+ constructor(options?: {
6
+ readonly soleWriter?: boolean;
7
+ });
8
+ validate(connection: Connection<SQLiteTable>): void;
9
+ protected recordChecks(connection: Connection<SQLiteTable>, streamChecks: readonly StreamCheck[]): Promise<void>;
10
+ begin(connection: Connection<SQLiteTable>, copies: readonly DeclaredCopy<SQLiteTable>[], at: Date): Promise<RecordedPass<SQLiteTable>>;
11
+ install(connections: readonly Connection<SQLiteTable>[], at: Date): Promise<void>;
12
+ }