@eltjs/elt-sqlserver 0.0.0-stage → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +487 -2
  3. package/dist/computed-dataset.d.ts +31 -0
  4. package/dist/computed-dataset.js +377 -0
  5. package/dist/index.d.ts +13 -0
  6. package/dist/index.js +10 -0
  7. package/dist/sql-server-append-writer.d.ts +8 -0
  8. package/dist/sql-server-append-writer.js +13 -0
  9. package/dist/sql-server-as-of.d.ts +14 -0
  10. package/dist/sql-server-as-of.js +49 -0
  11. package/dist/sql-server-checkpoint-store.d.ts +10 -0
  12. package/dist/sql-server-checkpoint-store.js +101 -0
  13. package/dist/sql-server-column.d.ts +53 -0
  14. package/dist/sql-server-column.js +295 -0
  15. package/dist/sql-server-columns.d.ts +15 -0
  16. package/dist/sql-server-columns.js +49 -0
  17. package/dist/sql-server-connection.d.ts +6 -0
  18. package/dist/sql-server-connection.js +56 -0
  19. package/dist/sql-server-deduplicating-writer.d.ts +18 -0
  20. package/dist/sql-server-deduplicating-writer.js +102 -0
  21. package/dist/sql-server-dependents.d.ts +9 -0
  22. package/dist/sql-server-dependents.js +54 -0
  23. package/dist/sql-server-derived-status.d.ts +9 -0
  24. package/dist/sql-server-derived-status.js +60 -0
  25. package/dist/sql-server-descriptions.d.ts +6 -0
  26. package/dist/sql-server-descriptions.js +53 -0
  27. package/dist/sql-server-destination.d.ts +32 -0
  28. package/dist/sql-server-destination.js +95 -0
  29. package/dist/sql-server-failure-type.d.ts +2 -0
  30. package/dist/sql-server-failure-type.js +25 -0
  31. package/dist/sql-server-file-store.d.ts +21 -0
  32. package/dist/sql-server-file-store.js +91 -0
  33. package/dist/sql-server-fit.d.ts +20 -0
  34. package/dist/sql-server-fit.js +52 -0
  35. package/dist/sql-server-history.d.ts +15 -0
  36. package/dist/sql-server-history.js +148 -0
  37. package/dist/sql-server-identifiers.d.ts +10 -0
  38. package/dist/sql-server-identifiers.js +27 -0
  39. package/dist/sql-server-indexes.d.ts +20 -0
  40. package/dist/sql-server-indexes.js +68 -0
  41. package/dist/sql-server-load.d.ts +19 -0
  42. package/dist/sql-server-load.js +58 -0
  43. package/dist/sql-server-numbering.d.ts +32 -0
  44. package/dist/sql-server-numbering.js +255 -0
  45. package/dist/sql-server-overwrite-deduplicating-writer.d.ts +4 -0
  46. package/dist/sql-server-overwrite-deduplicating-writer.js +8 -0
  47. package/dist/sql-server-overwrite-writer.d.ts +4 -0
  48. package/dist/sql-server-overwrite-writer.js +9 -0
  49. package/dist/sql-server-parts-table.d.ts +24 -0
  50. package/dist/sql-server-parts-table.js +165 -0
  51. package/dist/sql-server-passes.d.ts +6 -0
  52. package/dist/sql-server-passes.js +71 -0
  53. package/dist/sql-server-reader-view.d.ts +12 -0
  54. package/dist/sql-server-reader-view.js +71 -0
  55. package/dist/sql-server-rule-columns.d.ts +12 -0
  56. package/dist/sql-server-rule-columns.js +69 -0
  57. package/dist/sql-server-schemas.d.ts +4 -0
  58. package/dist/sql-server-schemas.js +8 -0
  59. package/dist/sql-server-setup-error.d.ts +3 -0
  60. package/dist/sql-server-setup-error.js +6 -0
  61. package/dist/sql-server-sync-history-schema.d.ts +7 -0
  62. package/dist/sql-server-sync-history-schema.js +120 -0
  63. package/dist/sql-server-sync-history.d.ts +14 -0
  64. package/dist/sql-server-sync-history.js +165 -0
  65. package/dist/sql-server-table.d.ts +10 -0
  66. package/dist/sql-server-table.js +40 -0
  67. package/dist/sql-server-views.d.ts +7 -0
  68. package/dist/sql-server-views.js +70 -0
  69. package/dist/sql-server-writer.d.ts +40 -0
  70. package/dist/sql-server-writer.js +914 -0
  71. package/dist/view.d.ts +23 -0
  72. package/dist/view.js +82 -0
  73. package/package.json +48 -4
@@ -0,0 +1,165 @@
1
+ import { hash } from 'node:crypto';
2
+ import { partsView, } from '@eltjs/elt';
3
+ import { TargetDescriptions } from './sql-server-descriptions.js';
4
+ import { TargetHistory, rowHash } from './sql-server-history.js';
5
+ import { quote, rawTable } from './sql-server-identifiers.js';
6
+ import { TableIndexes } from './sql-server-indexes.js';
7
+ import { TargetReaderView } from './sql-server-reader-view.js';
8
+ import { batchValue, keyHash, ruleBatchType, ruleColumnType, ruleStored, storedValue, } from './sql-server-rule-columns.js';
9
+ import { inWarehouse, marts, warehouse } from './sql-server-schemas.js';
10
+ // The parts of a computed dataset whose rule is a table rule, in the table
11
+ // raw_<name>_parts of _warehouse beside the output rows of raw_<name>, as in
12
+ // elt-postgresql: one row for each part, with the content columns and the
13
+ // place columns of the rule. Readers query the view <name>_parts of marts.
14
+ // The table numbers its rows with _elt_row, its clustered key (ADR 0003),
15
+ // and is unique on the key hash and seq (ADR 0004). Each row names the
16
+ // commit that last wrote it, and the history keeps each row that a commit
17
+ // replaced or removed, as for the output rows.
18
+ export class SqlServerPartsTable {
19
+ table;
20
+ declared;
21
+ statement;
22
+ history;
23
+ view;
24
+ indexes;
25
+ // The relations that a copy of the pipeline must not make.
26
+ names;
27
+ #key;
28
+ #rule;
29
+ // Every column of a part that readers see and the row hash counts, in
30
+ // table order.
31
+ #columns;
32
+ constructor({ dataset, description, rule, indexes, }) {
33
+ const view = partsView({ dataset, description, rule });
34
+ const name = rawTable(view.name);
35
+ this.names = Object.freeze([
36
+ `${marts}.${view.name}`,
37
+ `${warehouse}.${name}`,
38
+ ]);
39
+ this.table = inWarehouse(name);
40
+ this.#key = rule.key.name;
41
+ this.#rule = rule;
42
+ this.#columns = Object.freeze(view.columns.map(({ name }) => name));
43
+ this.declared = Object.freeze([
44
+ { name: '_elt_row', type: 'bigint', required: true },
45
+ { name: rule.key.name, type: 'nvarchar(max)', required: true },
46
+ { name: 'seq', type: 'int', required: true },
47
+ { name: 'part_hash', type: 'nvarchar(max)', required: true },
48
+ ...Object.entries({ ...rule.output, ...rule.place }).map(([column, ruleColumn]) => ({
49
+ name: column,
50
+ type: ruleColumnType(column, ruleColumn),
51
+ required: false,
52
+ })),
53
+ { name: 'committed_in', type: 'bigint', required: true },
54
+ { name: '_elt_row_hash', type: 'binary(32)', required: true },
55
+ { name: '_elt_key_hash', type: 'binary(32)', required: true },
56
+ ]);
57
+ const columns = this.declared
58
+ .slice(1)
59
+ .map(({ name, type, required }) => `${quote(name)} ${type} ${required ? 'NOT NULL' : 'NULL'}`);
60
+ this.statement = `CREATE TABLE ${this.table} ([_elt_row] bigint IDENTITY(1, 1) NOT NULL PRIMARY KEY CLUSTERED, ${columns.join(', ')});
61
+ CREATE UNIQUE INDEX [key_hash_seq] ON ${this.table} ([_elt_key_hash], [seq]);`;
62
+ this.indexes = new TableIndexes({
63
+ dataset,
64
+ table: this.table,
65
+ columns: this.declared,
66
+ indexes,
67
+ });
68
+ this.history = new TargetHistory(hash('sha256', this.table).slice(0, 40));
69
+ this.view = new TargetReaderView(view.name, this.table, this.history.qualifiedName, this.#columns, new TargetDescriptions(view.description, Object.fromEntries(view.columns.map((column) => [column.name, column.description]))));
70
+ Object.freeze(this);
71
+ }
72
+ // Writes the parts of each output row in place of its stored parts, by
73
+ // seq, in the commit that commit numbers: a part with the row hash of the
74
+ // stored part keeps its commit, the other parts are written in one MERGE,
75
+ // and each stored part beyond the new parts of its row is deleted. The
76
+ // history keeps each part that the commit replaces or deletes. A row with
77
+ // no parts, empty or failed, keeps none.
78
+ async write(connection, commit, outputs) {
79
+ const ruleColumns = Object.entries({
80
+ ...this.#rule.output,
81
+ ...this.#rule.place,
82
+ });
83
+ const hashed = this.#columns.map((name) => ({ name }));
84
+ const rows = outputs.map((output) => {
85
+ const key = keyHash(output.key);
86
+ const parts = 'parts' in output ? output.parts : [];
87
+ return {
88
+ key,
89
+ count: parts.length,
90
+ parts: parts.map((part, seq) => {
91
+ const values = [
92
+ output.key,
93
+ seq,
94
+ part.partHash,
95
+ ...ruleColumns.map(([column]) => storedValue(part.values[column])),
96
+ ];
97
+ return { key, seq, version: rowHash(hashed, values), values };
98
+ }),
99
+ };
100
+ });
101
+ const stored = Map.groupBy((await connection.query(`SELECT LOWER(CONVERT(char(64), [_elt_key_hash], 2)) AS [key], [seq], LOWER(CONVERT(char(64), [_elt_row_hash], 2)) AS [version]
102
+ FROM ${this.table} WHERE [_elt_key_hash] IN (SELECT CONVERT(binary(32), [value], 2) FROM OPENJSON(@keys))`, { keys: JSON.stringify(rows.map(({ key }) => key)) })).map(({ key, seq, version }) => ({
103
+ key: String(key),
104
+ seq: Number(seq),
105
+ version: String(version),
106
+ })), ({ key }) => key);
107
+ const changed = rows
108
+ .flatMap(({ parts }) => parts)
109
+ .filter((part) => stored.get(part.key)?.find(({ seq }) => seq === part.seq)?.version !==
110
+ part.version);
111
+ if (changed.length > 0)
112
+ await this.#merge(connection, commit, changed);
113
+ if (rows.some(({ key, count }) => stored.get(key)?.some(({ seq }) => seq >= count)))
114
+ await this.history.remove(connection, await commit(), this.table, `EXISTS (SELECT 1 FROM OPENJSON(@counts) WITH ([k] char(64) '$[0]', [n] int '$[1]') AS c WHERE [_elt_key_hash] = CONVERT(binary(32), c.[k], 2) AND [seq] >= c.[n])`, { counts: JSON.stringify(rows.map(({ key, count }) => [key, count])) });
115
+ }
116
+ // Deletes every part of the keys whose key hashes are given, in the commit
117
+ // replacedIn, and the history keeps them.
118
+ async remove(connection, replacedIn, keyHashes) {
119
+ await this.history.remove(connection, replacedIn, this.table, '[_elt_key_hash] IN (SELECT CONVERT(binary(32), [value], 2) FROM OPENJSON(@gone))', { gone: JSON.stringify(keyHashes) });
120
+ }
121
+ // One MERGE on the key hash and seq of the changed parts, from OPENJSON of
122
+ // their values, whose output the history keeps (ADR 0005, 0011).
123
+ async #merge(connection, commit, changed) {
124
+ // Each column as OPENJSON reads it and as the table stores it.
125
+ const same = (at) => at;
126
+ const fields = [
127
+ { name: this.#key, type: 'nvarchar(max)', stored: same },
128
+ { name: 'seq', type: 'int', stored: same },
129
+ { name: 'part_hash', type: 'nvarchar(max)', stored: same },
130
+ ...Object.entries({ ...this.#rule.output, ...this.#rule.place }).map(([name, column]) => ({
131
+ name,
132
+ type: ruleBatchType(column),
133
+ stored: (at) => ruleStored(name, column, at),
134
+ })),
135
+ ];
136
+ const written = [
137
+ ...fields.map(({ name }) => name),
138
+ 'committed_in',
139
+ '_elt_row_hash',
140
+ '_elt_key_hash',
141
+ ].map(quote);
142
+ const source = `SELECT ${[
143
+ ...fields.map(({ name, stored }, index) => `${stored(`j.[v${index}]`)} AS ${quote(name)}`),
144
+ 'CAST(@committed AS bigint) AS [committed_in]',
145
+ 'CONVERT(binary(32), j.[h], 2) AS [_elt_row_hash]',
146
+ 'CONVERT(binary(32), j.[k], 2) AS [_elt_key_hash]',
147
+ ].join(', ')} FROM OPENJSON(@batch) WITH (${[
148
+ `[k] char(64) '$[0]'`,
149
+ `[h] char(64) '$[1]'`,
150
+ ...fields.map(({ type }, index) => `[v${index}] ${type} '$[${index + 2}]'`),
151
+ ].join(', ')}) AS j`;
152
+ const committed = await commit();
153
+ await this.history.keep(connection, committed, this.table, `MERGE ${this.table} AS t USING (${source}) AS s ON t.[_elt_key_hash] = s.[_elt_key_hash] AND t.[seq] = s.[seq]
154
+ WHEN MATCHED THEN UPDATE SET ${written.map((column) => `t.${column} = s.${column}`).join(', ')}
155
+ WHEN NOT MATCHED THEN INSERT (${written.join(', ')}) VALUES (${written.map((column) => `s.${column}`).join(', ')})
156
+ OUTPUT deleted.*`, {
157
+ committed: String(committed),
158
+ batch: JSON.stringify(changed.map((part) => [
159
+ part.key,
160
+ part.version,
161
+ ...part.values.map(batchValue),
162
+ ])),
163
+ });
164
+ }
165
+ }
@@ -0,0 +1,6 @@
1
+ import { type PassStatus, type StreamPassStatus } from '@eltjs/elt';
2
+ import { type SqlServerLogin } from './sql-server-connection.ts';
3
+ export declare function readPassStatus(login: SqlServerLogin): Promise<{
4
+ readonly pass: PassStatus | null;
5
+ readonly streams: readonly StreamPassStatus[];
6
+ }>;
@@ -0,0 +1,71 @@
1
+ import { combinedPass, recordedPassStatus, recordedStreamStatus, } from '@eltjs/elt';
2
+ import { SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
3
+ import { connect } from './sql-server-connection.js';
4
+ import { inMarts } from './sql-server-schemas.js';
5
+ // The latest pass of each connection and each stream's latest outcome, as
6
+ // one result set: one statement reads one snapshot of the history when the
7
+ // database keeps row versions, and needs no SNAPSHOT transaction, which a
8
+ // database that keeps none refuses, though its history records why its
9
+ // loads failed.
10
+ const status = `SELECT * FROM (SELECT N'pass' AS [kind], CAST(NULL AS nvarchar(max)) AS [stream], [status], [started_at], [completed_at], [error], [failure_type], [last_successful_sync_at], [current_as_of]
11
+ FROM ${inMarts('sync_status')}
12
+ UNION ALL
13
+ SELECT N'stream', s.[stream], s.[status], NULL, NULL,
14
+ (SELECT STRING_AGG(e.[error], NCHAR(10)) WITHIN GROUP (ORDER BY CAST(f.[key] AS int))
15
+ FROM ${inMarts('extraction_coverage')} AS c
16
+ CROSS APPLY OPENJSON(c.[failures]) AS f
17
+ CROSS APPLY OPENJSON(f.[value]) WITH ([error] nvarchar(max) '$.error') AS e
18
+ WHERE c.[attempt_id] = s.[latest_attempt_id] AND c.[stream] = s.[stream]),
19
+ NULL, s.[last_successful_sync_at], s.[current_as_of]
20
+ FROM ${inMarts('stream_status')} AS s) AS [status]
21
+ ORDER BY [kind], [stream] COLLATE Latin1_General_100_BIN2`;
22
+ // The latest pass of the connections that the sync history of a database
23
+ // records, and each stream's latest outcome, with the errors of its failures
24
+ // in that attempt, one on each line; null and empty until the history
25
+ // recorded a pass, also while no history is installed (error 208, an
26
+ // invalid object name). The latest passes of several connections read as
27
+ // one pass, as in elt-postgresql.
28
+ export async function readPassStatus(login) {
29
+ await using connection = await connect(new SqlServerDatabase(login));
30
+ let rows;
31
+ try {
32
+ rows = await connection.query(status);
33
+ }
34
+ catch (error) {
35
+ if (Object(error).number === 208)
36
+ return { pass: null, streams: [] };
37
+ throw error;
38
+ }
39
+ const [first, ...others] = rows
40
+ .filter(({ kind }) => kind === 'pass')
41
+ .map((row) => recordedPassStatus({
42
+ status: row.status,
43
+ startedAt: instant(row.started_at),
44
+ completedAt: optionalInstant(row.completed_at),
45
+ error: row.error,
46
+ failureType: row.failure_type,
47
+ lastSucceededAt: optionalInstant(row.last_successful_sync_at),
48
+ currentAsOf: optionalInstant(row.current_as_of),
49
+ }));
50
+ return {
51
+ pass: first === undefined ? null : combinedPass([first, ...others]),
52
+ streams: rows
53
+ .filter(({ kind }) => kind === 'stream')
54
+ .map((row) => recordedStreamStatus({
55
+ stream: row.stream,
56
+ status: row.status,
57
+ error: row.error,
58
+ lastSucceededAt: optionalInstant(row.last_successful_sync_at),
59
+ currentAsOf: optionalInstant(row.current_as_of),
60
+ })),
61
+ };
62
+ }
63
+ // A datetimeoffset as the ISO text that elt-sqlite stores and gives.
64
+ function instant(value) {
65
+ if (value instanceof Date)
66
+ return value.toISOString();
67
+ throw new TypeError(`A sync history view gave ${String(value)} for a time`);
68
+ }
69
+ function optionalInstant(value) {
70
+ return value === null ? null : instant(value);
71
+ }
@@ -0,0 +1,12 @@
1
+ import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
2
+ import type { TargetDescriptions } from './sql-server-descriptions.ts';
3
+ export declare const now = "9223372036854775807";
4
+ export declare const position = "ISNULL(TRY_CAST(SESSION_CONTEXT(N'elt.position') AS bigint), 9223372036854775807)";
5
+ export declare class TargetReaderView {
6
+ #private;
7
+ constructor(name: string, target: string, history: string, columns: readonly string[], descriptions: TargetDescriptions);
8
+ get qualifiedName(): string;
9
+ refuse(connection: SqlServerConnection, stale: boolean): Promise<void>;
10
+ describe(connection: SqlServerConnection): Promise<void>;
11
+ replace(connection: SqlServerConnection): Promise<void>;
12
+ }
@@ -0,0 +1,71 @@
1
+ import { quote } from './sql-server-identifiers.js';
2
+ import { inMarts, marts } from './sql-server-schemas.js';
3
+ // The position past every commit, which reads each target as it is now and
4
+ // no row of its history.
5
+ export const now = '9223372036854775807';
6
+ // The position a reader reads at: the one a reading set in its session
7
+ // context, or now for a session that set none (ADR 0010).
8
+ export const position = `ISNULL(TRY_CAST(SESSION_CONTEXT(N'elt.position') AS bigint), ${now})`;
9
+ // The view in marts that readers see a target through: exactly the stream's
10
+ // columns and loaded_at, none of the internal ones, made with the target and
11
+ // described by the descriptions of those columns. It reads the target as of
12
+ // the position its reader set: the rows that the target had then, and that
13
+ // it still has or its history kept.
14
+ export class TargetReaderView {
15
+ #name;
16
+ #target;
17
+ #history;
18
+ #columns;
19
+ #descriptions;
20
+ constructor(name, target, history, columns, descriptions) {
21
+ this.#name = name;
22
+ this.#target = target;
23
+ this.#history = history;
24
+ this.#columns = columns;
25
+ this.#descriptions = descriptions;
26
+ Object.freeze(this);
27
+ }
28
+ get qualifiedName() {
29
+ return inMarts(this.#name);
30
+ }
31
+ // Refuses, before anything is read, a reader view this load did not make:
32
+ // not a view, or a view of other columns or another table. A stale
33
+ // target's view still shows the stored columns, which evolving it replaces.
34
+ async refuse(connection, stale) {
35
+ const view = this.qualifiedName;
36
+ const [existing] = await connection.query(`SELECT o.[type],
37
+ (SELECT c.[name] FROM sys.columns AS c WHERE c.[object_id] = o.[object_id] ORDER BY c.[column_id] FOR JSON PATH) AS [columns],
38
+ IIF(EXISTS (SELECT 1 FROM sys.sql_expression_dependencies AS d WHERE d.[referencing_id] = o.[object_id] AND d.[referenced_id] = OBJECT_ID(@target)), 1, 0) AS [reads]
39
+ FROM sys.objects AS o WHERE o.[object_id] = OBJECT_ID(@view)`, { view, target: this.#target });
40
+ if (existing === undefined)
41
+ return;
42
+ const columns = JSON.parse(String(existing.columns ?? '[]'));
43
+ const named = Array.isArray(columns)
44
+ ? columns.map((column) => String(Object(column).name))
45
+ : [];
46
+ if (String(existing.type).trim() !== 'V' ||
47
+ existing.reads !== 1 ||
48
+ (!stale && named.join('\0') !== this.#columns.join('\0')))
49
+ throw new TypeError(`${view} is not a view of exactly ${this.#target}; drop it or reset the warehouse`);
50
+ }
51
+ // Made only when absent and never replaced by a commit: prepare refused any
52
+ // view not this load's own.
53
+ async describe(connection) {
54
+ const [existing] = await connection.query(`SELECT IIF(OBJECT_ID(@view, N'V') IS NULL, 0, 1) AS [made]`, { view: this.qualifiedName });
55
+ if (existing?.made !== 1)
56
+ await this.#define(connection, 'CREATE');
57
+ await this.#descriptions.write(connection, 'VIEW', marts, this.#name);
58
+ }
59
+ // Defined again over a target that took the place of another, whose
60
+ // columns it then shows, with its grants and the views over it kept.
61
+ async replace(connection) {
62
+ await this.#define(connection, 'CREATE OR ALTER');
63
+ await this.#descriptions.write(connection, 'VIEW', marts, this.#name);
64
+ }
65
+ // CREATE VIEW must be the only statement of its batch.
66
+ async #define(connection, verb) {
67
+ const columns = this.#columns.map(quote).join(', ');
68
+ const kept = this.#columns.map((column) => `o.${quote(column)}`);
69
+ await connection.query(`${verb} VIEW ${this.qualifiedName} AS SELECT ${columns} FROM ${this.#target} WHERE [committed_in] <= ${position} UNION ALL SELECT ${kept.join(', ')} FROM ${this.#history} AS o WHERE o.[committed_in] <= ${position} AND o.[replaced_in] > ${position}`);
70
+ }
71
+ }
@@ -0,0 +1,12 @@
1
+ import type { RuleColumn, RuleColumns, RuleValues } from '@eltjs/elt';
2
+ export declare function ruleColumnType(name: string, column: RuleColumn): string;
3
+ export declare function ruleBatchType(column: RuleColumn): string;
4
+ export declare function ruleStored(name: string, column: RuleColumn, alias: string): string;
5
+ export declare function storedValue(value: RuleValues<RuleColumns>[string]): string | number | Uint8Array | null;
6
+ export declare function batchValue(value: string | number | Uint8Array | null): string | number | null;
7
+ export type Declared = {
8
+ readonly name: string;
9
+ readonly type: string;
10
+ readonly required: boolean;
11
+ };
12
+ export declare function keyHash(key: string): string;
@@ -0,0 +1,69 @@
1
+ import { hash } from 'node:crypto';
2
+ import { Vector } from '@zukhruf/vectors';
3
+ import { sqlType, toSql } from '@zukhruf/vectors/sqlserver';
4
+ // The most dimensions of a float32 VECTOR that SQL Server holds (ADR 0007).
5
+ const mostDimensions = 1998;
6
+ // The type of a rule column in the raw table, as sys.columns spells it:
7
+ // text NVARCHAR(MAX), bytes VARBINARY(MAX), an integer BIGINT, and a vector
8
+ // VECTOR(n) of float32 values. A vector column of more dimensions than SQL
9
+ // Server holds is a TypeError.
10
+ export function ruleColumnType(name, column) {
11
+ switch (column.type) {
12
+ case 'text':
13
+ return 'nvarchar(max)';
14
+ case 'blob':
15
+ return 'varbinary(max)';
16
+ case 'integer':
17
+ return 'bigint';
18
+ case 'vector':
19
+ if (column.dimensions > mostDimensions)
20
+ throw new TypeError(`The vector column ${name} has ${column.dimensions} dimensions; a float32 VECTOR of SQL Server holds at most ${mostDimensions}`);
21
+ return sqlType(Vector, column.dimensions).toLowerCase();
22
+ }
23
+ }
24
+ // The type that OPENJSON reads a value of the column as: bytes travel as
25
+ // base64 text, which BASE64_DECODE takes only as varchar (ADR 0011).
26
+ export function ruleBatchType(column) {
27
+ switch (column.type) {
28
+ case 'blob':
29
+ return 'varchar(max)';
30
+ case 'integer':
31
+ return 'bigint';
32
+ case 'text':
33
+ case 'vector':
34
+ return 'nvarchar(max)';
35
+ }
36
+ }
37
+ // The stored value of what OPENJSON read as alias: base64 text decoded, and
38
+ // the JSON text of a vector cast to the type of its column.
39
+ export function ruleStored(name, column, alias) {
40
+ switch (column.type) {
41
+ case 'blob':
42
+ return `BASE64_DECODE(${alias})`;
43
+ case 'vector':
44
+ return `CAST(${alias} AS ${ruleColumnType(name, column)})`;
45
+ case 'text':
46
+ case 'integer':
47
+ return alias;
48
+ }
49
+ }
50
+ // The value of a rule column in the form the table stores it, which the row
51
+ // hash counts: a vector as the JSON text of @zukhruf/vectors/sqlserver, and
52
+ // a column with no value as null.
53
+ export function storedValue(value) {
54
+ if (value instanceof Vector)
55
+ return toSql(value);
56
+ return value ?? null;
57
+ }
58
+ // A stored value as JSON carries it: bytes as base64 text.
59
+ export function batchValue(value) {
60
+ return value instanceof Uint8Array
61
+ ? Buffer.from(value).toString('base64')
62
+ : value;
63
+ }
64
+ // The key hash of an output row and of each of its parts: the SHA-256, in
65
+ // lower-case hex, of the JSON array of its key, as a load hashes the key of a
66
+ // staged row (ADR 0004).
67
+ export function keyHash(key) {
68
+ return hash('sha256', JSON.stringify([key]));
69
+ }
@@ -0,0 +1,4 @@
1
+ export declare const marts = "marts";
2
+ export declare const warehouse = "_warehouse";
3
+ export declare const inMarts: (name: string) => string;
4
+ export declare const inWarehouse: (name: string) => string;
@@ -0,0 +1,8 @@
1
+ import { quote } from './sql-server-identifiers.js';
2
+ // The two schemas of a warehouse, as in elt-postgresql: readers query the
3
+ // views of marts, and elt keeps each table of its own in _warehouse.
4
+ export const marts = 'marts';
5
+ export const warehouse = '_warehouse';
6
+ // A relation of marts or of _warehouse, as T-SQL names it.
7
+ export const inMarts = (name) => `${quote(marts)}.${quote(name)}`;
8
+ export const inWarehouse = (name) => `${quote(warehouse)}.${quote(name)}`;
@@ -0,0 +1,3 @@
1
+ export declare class SqlServerSetupError extends Error {
2
+ name: string;
3
+ }
@@ -0,0 +1,6 @@
1
+ // The database is not set up as the destination needs it: an engine older
2
+ // than SQL Server 2025, or no row versions for readers. The message gives
3
+ // what to change, such as the ALTER DATABASE statement to run.
4
+ export class SqlServerSetupError extends Error {
5
+ name = 'SqlServerSetupError';
6
+ }
@@ -0,0 +1,7 @@
1
+ import type { SqlServerView } from './sql-server-views.ts';
2
+ export declare const attempts: string;
3
+ export declare const coverage: string;
4
+ export declare const checks: string;
5
+ export declare const syncHistoryTables: string;
6
+ export declare const syncHistoryViews: readonly SqlServerView[];
7
+ export declare const catalog: SqlServerView;
@@ -0,0 +1,120 @@
1
+ import { readerCatalog, syncHistoryRelations } from '@eltjs/elt';
2
+ import { derivedStatusTable } from './sql-server-derived-status.js';
3
+ import { columnType } from './sql-server-fit.js';
4
+ import { now, position } from './sql-server-reader-view.js';
5
+ import { inWarehouse, marts, warehouse } from './sql-server-schemas.js';
6
+ // The tables of the history in _warehouse.
7
+ export const attempts = inWarehouse('sync_attempts');
8
+ export const coverage = inWarehouse('extraction_coverage');
9
+ export const checks = inWarehouse('stream_checks');
10
+ const derivedStatus = inWarehouse('derived_status');
11
+ // A name of a connection or a stream compares by its code points, as in
12
+ // Postgres. A key of text of any length is unique by its SHA-256 (ADR 0004).
13
+ const name = 'nvarchar(max) COLLATE Latin1_General_100_BIN2';
14
+ const keyOf = (column) => `CAST(HASHBYTES('SHA2_256', [${column}]) AS binary(32)) PERSISTED NOT NULL`;
15
+ const statuses = `(N'running', N'succeeded', N'partial', N'failed', N'cancelled')`;
16
+ export const syncHistoryTables = `
17
+ IF SCHEMA_ID(N'${warehouse}') IS NULL EXEC (N'CREATE SCHEMA [${warehouse}]');
18
+ IF OBJECT_ID(N'${attempts}', N'U') IS NULL
19
+ CREATE TABLE ${attempts} (
20
+ [id] bigint IDENTITY(1, 1) NOT NULL PRIMARY KEY,
21
+ [connector] ${name} NOT NULL CHECK (TRIM([connector]) <> N''), [source] nvarchar(max) NOT NULL,
22
+ [started_at] datetimeoffset(3) NOT NULL, [completed_at] datetimeoffset(3) NULL,
23
+ [status] nvarchar(20) NOT NULL DEFAULT N'running' CHECK ([status] IN ${statuses}),
24
+ [error] nvarchar(max) NULL,
25
+ [failure_type] nvarchar(20) NULL CHECK ([failure_type] IN (N'config', N'system')),
26
+ CHECK (([status] = N'running' AND [completed_at] IS NULL) OR ([status] <> N'running' AND [completed_at] IS NOT NULL)),
27
+ CHECK (([status] IN (N'partial', N'failed') AND [failure_type] IS NOT NULL) OR ([status] NOT IN (N'partial', N'failed') AND [failure_type] IS NULL)));
28
+ IF OBJECT_ID(N'${coverage}', N'U') IS NULL
29
+ CREATE TABLE ${coverage} (
30
+ [attempt_id] bigint NOT NULL REFERENCES ${attempts} ([id]),
31
+ [stream] ${name} NOT NULL, [stream_key] AS ${keyOf('stream')}, [target_table] nvarchar(128) NOT NULL,
32
+ [sync_mode] nvarchar(20) NOT NULL, [destination_sync_mode] nvarchar(20) NOT NULL,
33
+ [description] nvarchar(max) NOT NULL CHECK (TRIM([description]) <> N''), [selection] json NOT NULL,
34
+ [status] nvarchar(20) NOT NULL DEFAULT N'running' CHECK ([status] IN ${statuses}),
35
+ [written_count] bigint NULL CHECK ([written_count] >= 0), [deleted_count] bigint NULL CHECK ([deleted_count] >= 0),
36
+ [failures] json NOT NULL DEFAULT N'[]', [upstream_current_as_of] datetimeoffset(3) NULL,
37
+ PRIMARY KEY ([attempt_id], [stream_key]));
38
+ IF OBJECT_ID(N'${checks}', N'U') IS NULL
39
+ CREATE TABLE ${checks} (
40
+ [connector] ${name} NOT NULL, [connector_key] AS ${keyOf('connector')},
41
+ [stream] ${name} NOT NULL, [stream_key] AS ${keyOf('stream')},
42
+ [checked_at] datetimeoffset(3) NOT NULL,
43
+ PRIMARY KEY ([connector_key], [stream_key]));
44
+ ${derivedStatusTable}`;
45
+ // When connector's stream last matched its upstream: the start of its last
46
+ // successful pass, or a later check that found it unchanged, each no later
47
+ // than when the source said the upstream itself was last current. NULL if it
48
+ // never succeeded. LEAST and GREATEST skip NULLs.
49
+ const currentAsOf = (connector, stream) => `(
50
+ SELECT CASE WHEN success.[id] IS NULL THEN NULL ELSE GREATEST(LEAST(success.[started_at], success.[upstream_current_as_of]), k.[checked_at]) END
51
+ FROM (SELECT 1 AS [one]) AS one
52
+ OUTER APPLY (
53
+ SELECT TOP (1) s.[id], s.[started_at], sc.[upstream_current_as_of] FROM ${coverage} AS sc
54
+ JOIN ${attempts} AS s ON s.[id] = sc.[attempt_id]
55
+ WHERE s.[connector] = ${connector} AND sc.[stream] = ${stream} AND sc.[status] = N'succeeded'
56
+ ORDER BY s.[completed_at] DESC, s.[id] DESC
57
+ ) AS success
58
+ LEFT JOIN ${checks} AS k ON k.[connector] = ${connector} AND k.[stream] = ${stream})`;
59
+ const lastSuccess = (connector) => `OUTER APPLY (
60
+ SELECT TOP (1) [id], [completed_at] FROM ${attempts}
61
+ WHERE [connector] = ${connector} AND [status] = N'succeeded'
62
+ ORDER BY [completed_at] DESC, [id] DESC) AS success`;
63
+ const queries = {
64
+ sync_attempts: `SELECT [id] AS [attempt_id], [connector], [source], [started_at], [completed_at], [status], [error], [failure_type] FROM ${attempts}`,
65
+ extraction_coverage: `SELECT c.[attempt_id], a.[connector], a.[source], a.[started_at], a.[completed_at],
66
+ c.[stream], c.[target_table],
67
+ CAST(IIF(OBJECT_ID(QUOTENAME(N'${marts}') + N'.' + QUOTENAME(c.[target_table]), N'V') IS NULL, 0, 1) AS bit) AS [target_exists],
68
+ c.[sync_mode], c.[destination_sync_mode], c.[description], c.[selection],
69
+ c.[status], c.[written_count], c.[deleted_count], c.[failures]
70
+ FROM ${coverage} AS c JOIN ${attempts} AS a ON a.[id] = c.[attempt_id]`,
71
+ sync_status: `SELECT a.[connector],
72
+ a.[id] AS [latest_attempt_id], a.[started_at], a.[completed_at], a.[status], a.[error], a.[failure_type],
73
+ success.[id] AS [last_successful_attempt_id], success.[completed_at] AS [last_successful_sync_at],
74
+ (SELECT CASE WHEN COUNT(*) = COUNT(streams.[current_as_of]) THEN MIN(streams.[current_as_of]) END
75
+ FROM (SELECT ${currentAsOf('a.[connector]', 'd.[stream]')} AS [current_as_of]
76
+ FROM (SELECT DISTINCT dc.[stream] FROM ${coverage} AS dc
77
+ JOIN ${attempts} AS da ON da.[id] = dc.[attempt_id]
78
+ WHERE da.[connector] = a.[connector]) AS d) AS streams) AS [current_as_of]
79
+ FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY [connector] ORDER BY [id] DESC) AS [latest] FROM ${attempts}) AS a
80
+ ${lastSuccess('a.[connector]')}
81
+ WHERE a.[latest] = 1`,
82
+ stream_status: `SELECT a.[connector], c.[stream], c.[target_table],
83
+ a.[id] AS [latest_attempt_id], a.[started_at], a.[completed_at], c.[status],
84
+ success.[id] AS [last_successful_attempt_id], success.[completed_at] AS [last_successful_sync_at],
85
+ ${currentAsOf('a.[connector]', 'c.[stream]')} AS [current_as_of]
86
+ FROM (SELECT cc.*, ROW_NUMBER() OVER (PARTITION BY ca.[connector], cc.[stream] ORDER BY ca.[id] DESC) AS [latest]
87
+ FROM ${coverage} AS cc JOIN ${attempts} AS ca ON ca.[id] = cc.[attempt_id]) AS c
88
+ JOIN ${attempts} AS a ON a.[id] = c.[attempt_id]
89
+ OUTER APPLY (
90
+ SELECT TOP (1) s.[id], s.[completed_at] FROM ${coverage} AS sc
91
+ JOIN ${attempts} AS s ON s.[id] = sc.[attempt_id]
92
+ WHERE s.[connector] = a.[connector] AND sc.[stream] = c.[stream] AND sc.[status] = N'succeeded'
93
+ ORDER BY s.[completed_at] DESC, s.[id] DESC) AS success
94
+ WHERE c.[latest] = 1`,
95
+ derived_status: `SELECT [dataset], [kind], [state], [as_of], [refreshed_at], [refreshing_since], [error], [inputs], [computed], [empty], [failed], [pending] FROM ${derivedStatus}`,
96
+ };
97
+ // How readers see the sync history: what each pass declared and loaded.
98
+ // The sync history keeps no row versions, so a session that reads as of a
99
+ // position before now fails each of its views. A view cannot raise an
100
+ // error, so the condition converts the reason to int, which fails with
101
+ // error 245 and the reason in its message, also over no rows.
102
+ export const syncHistoryViews = Object.values(syncHistoryRelations).map((relation) => ({
103
+ ...relation,
104
+ query: `SELECT * FROM (${queries[relation.name]}) AS [history] WHERE 1 = IIF(${position} = ${now}, 1, CAST(N'The sync history cannot be read as of a position: it keeps no row versions' AS int))`,
105
+ }));
106
+ // Lists what readers of marts query, its views, from the descriptions every
107
+ // load and publication writes there.
108
+ export const catalog = {
109
+ ...readerCatalog,
110
+ query: `SELECT N'view' AS [kind], v.[name] AS [name], CAST(NULL AS nvarchar(max)) AS [data_type], CAST(ep.[value] AS nvarchar(max)) AS [description]
111
+ FROM sys.views AS v
112
+ LEFT JOIN sys.extended_properties AS ep ON ep.[class] = 1 AND ep.[major_id] = v.[object_id] AND ep.[minor_id] = 0 AND ep.[name] = N'MS_Description'
113
+ WHERE v.[schema_id] = SCHEMA_ID(N'${marts}')
114
+ UNION ALL
115
+ SELECT N'column', v.[name] + N'.' + c.[name], ${columnType}, CAST(ep.[value] AS nvarchar(max))
116
+ FROM sys.views AS v
117
+ JOIN sys.columns AS c ON c.[object_id] = v.[object_id]
118
+ LEFT JOIN sys.extended_properties AS ep ON ep.[class] = 1 AND ep.[major_id] = v.[object_id] AND ep.[minor_id] = c.[column_id] AND ep.[name] = N'MS_Description'
119
+ WHERE v.[schema_id] = SCHEMA_ID(N'${marts}')`,
120
+ };
@@ -0,0 +1,14 @@
1
+ import { type Connection, type DeclaredCopy, type RecordedPass, type StreamCheck, SyncHistory } from '@eltjs/elt';
2
+ import { type SqlServerLogin } from './sql-server-connection.ts';
3
+ import type { SqlServerTable } from './sql-server-table.ts';
4
+ export declare class SqlServerSyncHistory extends SyncHistory<SqlServerTable> {
5
+ #private;
6
+ constructor({ login, readers, }: {
7
+ login: SqlServerLogin;
8
+ readers?: readonly string[];
9
+ });
10
+ validate({ name, destination }: Connection<SqlServerTable>): void;
11
+ protected recordChecks(connection: Connection<SqlServerTable>, streamChecks: readonly StreamCheck[]): Promise<void>;
12
+ begin(connection: Connection<SqlServerTable>, copies: readonly DeclaredCopy<SqlServerTable>[], at: Date): Promise<RecordedPass<SqlServerTable>>;
13
+ install(): Promise<void>;
14
+ }