@eltjs/elt-sqlserver 0.0.0-stage → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +487 -2
- package/dist/computed-dataset.d.ts +31 -0
- package/dist/computed-dataset.js +377 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +10 -0
- package/dist/sql-server-append-writer.d.ts +8 -0
- package/dist/sql-server-append-writer.js +13 -0
- package/dist/sql-server-as-of.d.ts +14 -0
- package/dist/sql-server-as-of.js +49 -0
- package/dist/sql-server-checkpoint-store.d.ts +10 -0
- package/dist/sql-server-checkpoint-store.js +101 -0
- package/dist/sql-server-column.d.ts +53 -0
- package/dist/sql-server-column.js +295 -0
- package/dist/sql-server-columns.d.ts +15 -0
- package/dist/sql-server-columns.js +49 -0
- package/dist/sql-server-connection.d.ts +6 -0
- package/dist/sql-server-connection.js +56 -0
- package/dist/sql-server-deduplicating-writer.d.ts +18 -0
- package/dist/sql-server-deduplicating-writer.js +102 -0
- package/dist/sql-server-dependents.d.ts +9 -0
- package/dist/sql-server-dependents.js +54 -0
- package/dist/sql-server-derived-status.d.ts +9 -0
- package/dist/sql-server-derived-status.js +60 -0
- package/dist/sql-server-descriptions.d.ts +6 -0
- package/dist/sql-server-descriptions.js +53 -0
- package/dist/sql-server-destination.d.ts +32 -0
- package/dist/sql-server-destination.js +95 -0
- package/dist/sql-server-failure-type.d.ts +2 -0
- package/dist/sql-server-failure-type.js +25 -0
- package/dist/sql-server-file-store.d.ts +21 -0
- package/dist/sql-server-file-store.js +91 -0
- package/dist/sql-server-fit.d.ts +20 -0
- package/dist/sql-server-fit.js +52 -0
- package/dist/sql-server-history.d.ts +15 -0
- package/dist/sql-server-history.js +148 -0
- package/dist/sql-server-identifiers.d.ts +10 -0
- package/dist/sql-server-identifiers.js +27 -0
- package/dist/sql-server-indexes.d.ts +20 -0
- package/dist/sql-server-indexes.js +68 -0
- package/dist/sql-server-load.d.ts +19 -0
- package/dist/sql-server-load.js +58 -0
- package/dist/sql-server-numbering.d.ts +32 -0
- package/dist/sql-server-numbering.js +255 -0
- package/dist/sql-server-overwrite-deduplicating-writer.d.ts +4 -0
- package/dist/sql-server-overwrite-deduplicating-writer.js +8 -0
- package/dist/sql-server-overwrite-writer.d.ts +4 -0
- package/dist/sql-server-overwrite-writer.js +9 -0
- package/dist/sql-server-parts-table.d.ts +24 -0
- package/dist/sql-server-parts-table.js +165 -0
- package/dist/sql-server-passes.d.ts +6 -0
- package/dist/sql-server-passes.js +71 -0
- package/dist/sql-server-reader-view.d.ts +12 -0
- package/dist/sql-server-reader-view.js +71 -0
- package/dist/sql-server-rule-columns.d.ts +12 -0
- package/dist/sql-server-rule-columns.js +69 -0
- package/dist/sql-server-schemas.d.ts +4 -0
- package/dist/sql-server-schemas.js +8 -0
- package/dist/sql-server-setup-error.d.ts +3 -0
- package/dist/sql-server-setup-error.js +6 -0
- package/dist/sql-server-sync-history-schema.d.ts +7 -0
- package/dist/sql-server-sync-history-schema.js +120 -0
- package/dist/sql-server-sync-history.d.ts +14 -0
- package/dist/sql-server-sync-history.js +165 -0
- package/dist/sql-server-table.d.ts +10 -0
- package/dist/sql-server-table.js +40 -0
- package/dist/sql-server-views.d.ts +7 -0
- package/dist/sql-server-views.js +70 -0
- package/dist/sql-server-writer.d.ts +40 -0
- package/dist/sql-server-writer.js +914 -0
- package/dist/view.d.ts +23 -0
- package/dist/view.js +82 -0
- package/package.json +48 -4
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { hash } from 'node:crypto';
|
|
2
|
+
import { partsView, } from '@eltjs/elt';
|
|
3
|
+
import { TargetDescriptions } from './sql-server-descriptions.js';
|
|
4
|
+
import { TargetHistory, rowHash } from './sql-server-history.js';
|
|
5
|
+
import { quote, rawTable } from './sql-server-identifiers.js';
|
|
6
|
+
import { TableIndexes } from './sql-server-indexes.js';
|
|
7
|
+
import { TargetReaderView } from './sql-server-reader-view.js';
|
|
8
|
+
import { batchValue, keyHash, ruleBatchType, ruleColumnType, ruleStored, storedValue, } from './sql-server-rule-columns.js';
|
|
9
|
+
import { inWarehouse, marts, warehouse } from './sql-server-schemas.js';
|
|
10
|
+
// The parts of a computed dataset whose rule is a table rule, in the table
|
|
11
|
+
// raw_<name>_parts of _warehouse beside the output rows of raw_<name>, as in
|
|
12
|
+
// elt-postgresql: one row for each part, with the content columns and the
|
|
13
|
+
// place columns of the rule. Readers query the view <name>_parts of marts.
|
|
14
|
+
// The table numbers its rows with _elt_row, its clustered key (ADR 0003),
|
|
15
|
+
// and is unique on the key hash and seq (ADR 0004). Each row names the
|
|
16
|
+
// commit that last wrote it, and the history keeps each row that a commit
|
|
17
|
+
// replaced or removed, as for the output rows.
|
|
18
|
+
export class SqlServerPartsTable {
|
|
19
|
+
table;
|
|
20
|
+
declared;
|
|
21
|
+
statement;
|
|
22
|
+
history;
|
|
23
|
+
view;
|
|
24
|
+
indexes;
|
|
25
|
+
// The relations that a copy of the pipeline must not make.
|
|
26
|
+
names;
|
|
27
|
+
#key;
|
|
28
|
+
#rule;
|
|
29
|
+
// Every column of a part that readers see and the row hash counts, in
|
|
30
|
+
// table order.
|
|
31
|
+
#columns;
|
|
32
|
+
constructor({ dataset, description, rule, indexes, }) {
|
|
33
|
+
const view = partsView({ dataset, description, rule });
|
|
34
|
+
const name = rawTable(view.name);
|
|
35
|
+
this.names = Object.freeze([
|
|
36
|
+
`${marts}.${view.name}`,
|
|
37
|
+
`${warehouse}.${name}`,
|
|
38
|
+
]);
|
|
39
|
+
this.table = inWarehouse(name);
|
|
40
|
+
this.#key = rule.key.name;
|
|
41
|
+
this.#rule = rule;
|
|
42
|
+
this.#columns = Object.freeze(view.columns.map(({ name }) => name));
|
|
43
|
+
this.declared = Object.freeze([
|
|
44
|
+
{ name: '_elt_row', type: 'bigint', required: true },
|
|
45
|
+
{ name: rule.key.name, type: 'nvarchar(max)', required: true },
|
|
46
|
+
{ name: 'seq', type: 'int', required: true },
|
|
47
|
+
{ name: 'part_hash', type: 'nvarchar(max)', required: true },
|
|
48
|
+
...Object.entries({ ...rule.output, ...rule.place }).map(([column, ruleColumn]) => ({
|
|
49
|
+
name: column,
|
|
50
|
+
type: ruleColumnType(column, ruleColumn),
|
|
51
|
+
required: false,
|
|
52
|
+
})),
|
|
53
|
+
{ name: 'committed_in', type: 'bigint', required: true },
|
|
54
|
+
{ name: '_elt_row_hash', type: 'binary(32)', required: true },
|
|
55
|
+
{ name: '_elt_key_hash', type: 'binary(32)', required: true },
|
|
56
|
+
]);
|
|
57
|
+
const columns = this.declared
|
|
58
|
+
.slice(1)
|
|
59
|
+
.map(({ name, type, required }) => `${quote(name)} ${type} ${required ? 'NOT NULL' : 'NULL'}`);
|
|
60
|
+
this.statement = `CREATE TABLE ${this.table} ([_elt_row] bigint IDENTITY(1, 1) NOT NULL PRIMARY KEY CLUSTERED, ${columns.join(', ')});
|
|
61
|
+
CREATE UNIQUE INDEX [key_hash_seq] ON ${this.table} ([_elt_key_hash], [seq]);`;
|
|
62
|
+
this.indexes = new TableIndexes({
|
|
63
|
+
dataset,
|
|
64
|
+
table: this.table,
|
|
65
|
+
columns: this.declared,
|
|
66
|
+
indexes,
|
|
67
|
+
});
|
|
68
|
+
this.history = new TargetHistory(hash('sha256', this.table).slice(0, 40));
|
|
69
|
+
this.view = new TargetReaderView(view.name, this.table, this.history.qualifiedName, this.#columns, new TargetDescriptions(view.description, Object.fromEntries(view.columns.map((column) => [column.name, column.description]))));
|
|
70
|
+
Object.freeze(this);
|
|
71
|
+
}
|
|
72
|
+
// Writes the parts of each output row in place of its stored parts, by
|
|
73
|
+
// seq, in the commit that commit numbers: a part with the row hash of the
|
|
74
|
+
// stored part keeps its commit, the other parts are written in one MERGE,
|
|
75
|
+
// and each stored part beyond the new parts of its row is deleted. The
|
|
76
|
+
// history keeps each part that the commit replaces or deletes. A row with
|
|
77
|
+
// no parts, empty or failed, keeps none.
|
|
78
|
+
async write(connection, commit, outputs) {
|
|
79
|
+
const ruleColumns = Object.entries({
|
|
80
|
+
...this.#rule.output,
|
|
81
|
+
...this.#rule.place,
|
|
82
|
+
});
|
|
83
|
+
const hashed = this.#columns.map((name) => ({ name }));
|
|
84
|
+
const rows = outputs.map((output) => {
|
|
85
|
+
const key = keyHash(output.key);
|
|
86
|
+
const parts = 'parts' in output ? output.parts : [];
|
|
87
|
+
return {
|
|
88
|
+
key,
|
|
89
|
+
count: parts.length,
|
|
90
|
+
parts: parts.map((part, seq) => {
|
|
91
|
+
const values = [
|
|
92
|
+
output.key,
|
|
93
|
+
seq,
|
|
94
|
+
part.partHash,
|
|
95
|
+
...ruleColumns.map(([column]) => storedValue(part.values[column])),
|
|
96
|
+
];
|
|
97
|
+
return { key, seq, version: rowHash(hashed, values), values };
|
|
98
|
+
}),
|
|
99
|
+
};
|
|
100
|
+
});
|
|
101
|
+
const stored = Map.groupBy((await connection.query(`SELECT LOWER(CONVERT(char(64), [_elt_key_hash], 2)) AS [key], [seq], LOWER(CONVERT(char(64), [_elt_row_hash], 2)) AS [version]
|
|
102
|
+
FROM ${this.table} WHERE [_elt_key_hash] IN (SELECT CONVERT(binary(32), [value], 2) FROM OPENJSON(@keys))`, { keys: JSON.stringify(rows.map(({ key }) => key)) })).map(({ key, seq, version }) => ({
|
|
103
|
+
key: String(key),
|
|
104
|
+
seq: Number(seq),
|
|
105
|
+
version: String(version),
|
|
106
|
+
})), ({ key }) => key);
|
|
107
|
+
const changed = rows
|
|
108
|
+
.flatMap(({ parts }) => parts)
|
|
109
|
+
.filter((part) => stored.get(part.key)?.find(({ seq }) => seq === part.seq)?.version !==
|
|
110
|
+
part.version);
|
|
111
|
+
if (changed.length > 0)
|
|
112
|
+
await this.#merge(connection, commit, changed);
|
|
113
|
+
if (rows.some(({ key, count }) => stored.get(key)?.some(({ seq }) => seq >= count)))
|
|
114
|
+
await this.history.remove(connection, await commit(), this.table, `EXISTS (SELECT 1 FROM OPENJSON(@counts) WITH ([k] char(64) '$[0]', [n] int '$[1]') AS c WHERE [_elt_key_hash] = CONVERT(binary(32), c.[k], 2) AND [seq] >= c.[n])`, { counts: JSON.stringify(rows.map(({ key, count }) => [key, count])) });
|
|
115
|
+
}
|
|
116
|
+
// Deletes every part of the keys whose key hashes are given, in the commit
|
|
117
|
+
// replacedIn, and the history keeps them.
|
|
118
|
+
async remove(connection, replacedIn, keyHashes) {
|
|
119
|
+
await this.history.remove(connection, replacedIn, this.table, '[_elt_key_hash] IN (SELECT CONVERT(binary(32), [value], 2) FROM OPENJSON(@gone))', { gone: JSON.stringify(keyHashes) });
|
|
120
|
+
}
|
|
121
|
+
// One MERGE on the key hash and seq of the changed parts, from OPENJSON of
|
|
122
|
+
// their values, whose output the history keeps (ADR 0005, 0011).
|
|
123
|
+
async #merge(connection, commit, changed) {
|
|
124
|
+
// Each column as OPENJSON reads it and as the table stores it.
|
|
125
|
+
const same = (at) => at;
|
|
126
|
+
const fields = [
|
|
127
|
+
{ name: this.#key, type: 'nvarchar(max)', stored: same },
|
|
128
|
+
{ name: 'seq', type: 'int', stored: same },
|
|
129
|
+
{ name: 'part_hash', type: 'nvarchar(max)', stored: same },
|
|
130
|
+
...Object.entries({ ...this.#rule.output, ...this.#rule.place }).map(([name, column]) => ({
|
|
131
|
+
name,
|
|
132
|
+
type: ruleBatchType(column),
|
|
133
|
+
stored: (at) => ruleStored(name, column, at),
|
|
134
|
+
})),
|
|
135
|
+
];
|
|
136
|
+
const written = [
|
|
137
|
+
...fields.map(({ name }) => name),
|
|
138
|
+
'committed_in',
|
|
139
|
+
'_elt_row_hash',
|
|
140
|
+
'_elt_key_hash',
|
|
141
|
+
].map(quote);
|
|
142
|
+
const source = `SELECT ${[
|
|
143
|
+
...fields.map(({ name, stored }, index) => `${stored(`j.[v${index}]`)} AS ${quote(name)}`),
|
|
144
|
+
'CAST(@committed AS bigint) AS [committed_in]',
|
|
145
|
+
'CONVERT(binary(32), j.[h], 2) AS [_elt_row_hash]',
|
|
146
|
+
'CONVERT(binary(32), j.[k], 2) AS [_elt_key_hash]',
|
|
147
|
+
].join(', ')} FROM OPENJSON(@batch) WITH (${[
|
|
148
|
+
`[k] char(64) '$[0]'`,
|
|
149
|
+
`[h] char(64) '$[1]'`,
|
|
150
|
+
...fields.map(({ type }, index) => `[v${index}] ${type} '$[${index + 2}]'`),
|
|
151
|
+
].join(', ')}) AS j`;
|
|
152
|
+
const committed = await commit();
|
|
153
|
+
await this.history.keep(connection, committed, this.table, `MERGE ${this.table} AS t USING (${source}) AS s ON t.[_elt_key_hash] = s.[_elt_key_hash] AND t.[seq] = s.[seq]
|
|
154
|
+
WHEN MATCHED THEN UPDATE SET ${written.map((column) => `t.${column} = s.${column}`).join(', ')}
|
|
155
|
+
WHEN NOT MATCHED THEN INSERT (${written.join(', ')}) VALUES (${written.map((column) => `s.${column}`).join(', ')})
|
|
156
|
+
OUTPUT deleted.*`, {
|
|
157
|
+
committed: String(committed),
|
|
158
|
+
batch: JSON.stringify(changed.map((part) => [
|
|
159
|
+
part.key,
|
|
160
|
+
part.version,
|
|
161
|
+
...part.values.map(batchValue),
|
|
162
|
+
])),
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { type PassStatus, type StreamPassStatus } from '@eltjs/elt';
|
|
2
|
+
import { type SqlServerLogin } from './sql-server-connection.ts';
|
|
3
|
+
export declare function readPassStatus(login: SqlServerLogin): Promise<{
|
|
4
|
+
readonly pass: PassStatus | null;
|
|
5
|
+
readonly streams: readonly StreamPassStatus[];
|
|
6
|
+
}>;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { combinedPass, recordedPassStatus, recordedStreamStatus, } from '@eltjs/elt';
|
|
2
|
+
import { SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
|
|
3
|
+
import { connect } from './sql-server-connection.js';
|
|
4
|
+
import { inMarts } from './sql-server-schemas.js';
|
|
5
|
+
// The latest pass of each connection and each stream's latest outcome, as
|
|
6
|
+
// one result set: one statement reads one snapshot of the history when the
|
|
7
|
+
// database keeps row versions, and needs no SNAPSHOT transaction, which a
|
|
8
|
+
// database that keeps none refuses, though its history records why its
|
|
9
|
+
// loads failed.
|
|
10
|
+
const status = `SELECT * FROM (SELECT N'pass' AS [kind], CAST(NULL AS nvarchar(max)) AS [stream], [status], [started_at], [completed_at], [error], [failure_type], [last_successful_sync_at], [current_as_of]
|
|
11
|
+
FROM ${inMarts('sync_status')}
|
|
12
|
+
UNION ALL
|
|
13
|
+
SELECT N'stream', s.[stream], s.[status], NULL, NULL,
|
|
14
|
+
(SELECT STRING_AGG(e.[error], NCHAR(10)) WITHIN GROUP (ORDER BY CAST(f.[key] AS int))
|
|
15
|
+
FROM ${inMarts('extraction_coverage')} AS c
|
|
16
|
+
CROSS APPLY OPENJSON(c.[failures]) AS f
|
|
17
|
+
CROSS APPLY OPENJSON(f.[value]) WITH ([error] nvarchar(max) '$.error') AS e
|
|
18
|
+
WHERE c.[attempt_id] = s.[latest_attempt_id] AND c.[stream] = s.[stream]),
|
|
19
|
+
NULL, s.[last_successful_sync_at], s.[current_as_of]
|
|
20
|
+
FROM ${inMarts('stream_status')} AS s) AS [status]
|
|
21
|
+
ORDER BY [kind], [stream] COLLATE Latin1_General_100_BIN2`;
|
|
22
|
+
// The latest pass of the connections that the sync history of a database
|
|
23
|
+
// records, and each stream's latest outcome, with the errors of its failures
|
|
24
|
+
// in that attempt, one on each line; null and empty until the history
|
|
25
|
+
// recorded a pass, also while no history is installed (error 208, an
|
|
26
|
+
// invalid object name). The latest passes of several connections read as
|
|
27
|
+
// one pass, as in elt-postgresql.
|
|
28
|
+
export async function readPassStatus(login) {
|
|
29
|
+
await using connection = await connect(new SqlServerDatabase(login));
|
|
30
|
+
let rows;
|
|
31
|
+
try {
|
|
32
|
+
rows = await connection.query(status);
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
if (Object(error).number === 208)
|
|
36
|
+
return { pass: null, streams: [] };
|
|
37
|
+
throw error;
|
|
38
|
+
}
|
|
39
|
+
const [first, ...others] = rows
|
|
40
|
+
.filter(({ kind }) => kind === 'pass')
|
|
41
|
+
.map((row) => recordedPassStatus({
|
|
42
|
+
status: row.status,
|
|
43
|
+
startedAt: instant(row.started_at),
|
|
44
|
+
completedAt: optionalInstant(row.completed_at),
|
|
45
|
+
error: row.error,
|
|
46
|
+
failureType: row.failure_type,
|
|
47
|
+
lastSucceededAt: optionalInstant(row.last_successful_sync_at),
|
|
48
|
+
currentAsOf: optionalInstant(row.current_as_of),
|
|
49
|
+
}));
|
|
50
|
+
return {
|
|
51
|
+
pass: first === undefined ? null : combinedPass([first, ...others]),
|
|
52
|
+
streams: rows
|
|
53
|
+
.filter(({ kind }) => kind === 'stream')
|
|
54
|
+
.map((row) => recordedStreamStatus({
|
|
55
|
+
stream: row.stream,
|
|
56
|
+
status: row.status,
|
|
57
|
+
error: row.error,
|
|
58
|
+
lastSucceededAt: optionalInstant(row.last_successful_sync_at),
|
|
59
|
+
currentAsOf: optionalInstant(row.current_as_of),
|
|
60
|
+
})),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
// A datetimeoffset as the ISO text that elt-sqlite stores and gives.
|
|
64
|
+
function instant(value) {
|
|
65
|
+
if (value instanceof Date)
|
|
66
|
+
return value.toISOString();
|
|
67
|
+
throw new TypeError(`A sync history view gave ${String(value)} for a time`);
|
|
68
|
+
}
|
|
69
|
+
function optionalInstant(value) {
|
|
70
|
+
return value === null ? null : instant(value);
|
|
71
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
|
|
2
|
+
import type { TargetDescriptions } from './sql-server-descriptions.ts';
|
|
3
|
+
export declare const now = "9223372036854775807";
|
|
4
|
+
export declare const position = "ISNULL(TRY_CAST(SESSION_CONTEXT(N'elt.position') AS bigint), 9223372036854775807)";
|
|
5
|
+
export declare class TargetReaderView {
|
|
6
|
+
#private;
|
|
7
|
+
constructor(name: string, target: string, history: string, columns: readonly string[], descriptions: TargetDescriptions);
|
|
8
|
+
get qualifiedName(): string;
|
|
9
|
+
refuse(connection: SqlServerConnection, stale: boolean): Promise<void>;
|
|
10
|
+
describe(connection: SqlServerConnection): Promise<void>;
|
|
11
|
+
replace(connection: SqlServerConnection): Promise<void>;
|
|
12
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { quote } from './sql-server-identifiers.js';
|
|
2
|
+
import { inMarts, marts } from './sql-server-schemas.js';
|
|
3
|
+
// The position past every commit, which reads each target as it is now and
|
|
4
|
+
// no row of its history.
|
|
5
|
+
export const now = '9223372036854775807';
|
|
6
|
+
// The position a reader reads at: the one a reading set in its session
|
|
7
|
+
// context, or now for a session that set none (ADR 0010).
|
|
8
|
+
export const position = `ISNULL(TRY_CAST(SESSION_CONTEXT(N'elt.position') AS bigint), ${now})`;
|
|
9
|
+
// The view in marts that readers see a target through: exactly the stream's
|
|
10
|
+
// columns and loaded_at, none of the internal ones, made with the target and
|
|
11
|
+
// described by the descriptions of those columns. It reads the target as of
|
|
12
|
+
// the position its reader set: the rows that the target had then, and that
|
|
13
|
+
// it still has or its history kept.
|
|
14
|
+
export class TargetReaderView {
|
|
15
|
+
#name;
|
|
16
|
+
#target;
|
|
17
|
+
#history;
|
|
18
|
+
#columns;
|
|
19
|
+
#descriptions;
|
|
20
|
+
constructor(name, target, history, columns, descriptions) {
|
|
21
|
+
this.#name = name;
|
|
22
|
+
this.#target = target;
|
|
23
|
+
this.#history = history;
|
|
24
|
+
this.#columns = columns;
|
|
25
|
+
this.#descriptions = descriptions;
|
|
26
|
+
Object.freeze(this);
|
|
27
|
+
}
|
|
28
|
+
get qualifiedName() {
|
|
29
|
+
return inMarts(this.#name);
|
|
30
|
+
}
|
|
31
|
+
// Refuses, before anything is read, a reader view this load did not make:
|
|
32
|
+
// not a view, or a view of other columns or another table. A stale
|
|
33
|
+
// target's view still shows the stored columns, which evolving it replaces.
|
|
34
|
+
async refuse(connection, stale) {
|
|
35
|
+
const view = this.qualifiedName;
|
|
36
|
+
const [existing] = await connection.query(`SELECT o.[type],
|
|
37
|
+
(SELECT c.[name] FROM sys.columns AS c WHERE c.[object_id] = o.[object_id] ORDER BY c.[column_id] FOR JSON PATH) AS [columns],
|
|
38
|
+
IIF(EXISTS (SELECT 1 FROM sys.sql_expression_dependencies AS d WHERE d.[referencing_id] = o.[object_id] AND d.[referenced_id] = OBJECT_ID(@target)), 1, 0) AS [reads]
|
|
39
|
+
FROM sys.objects AS o WHERE o.[object_id] = OBJECT_ID(@view)`, { view, target: this.#target });
|
|
40
|
+
if (existing === undefined)
|
|
41
|
+
return;
|
|
42
|
+
const columns = JSON.parse(String(existing.columns ?? '[]'));
|
|
43
|
+
const named = Array.isArray(columns)
|
|
44
|
+
? columns.map((column) => String(Object(column).name))
|
|
45
|
+
: [];
|
|
46
|
+
if (String(existing.type).trim() !== 'V' ||
|
|
47
|
+
existing.reads !== 1 ||
|
|
48
|
+
(!stale && named.join('\0') !== this.#columns.join('\0')))
|
|
49
|
+
throw new TypeError(`${view} is not a view of exactly ${this.#target}; drop it or reset the warehouse`);
|
|
50
|
+
}
|
|
51
|
+
// Made only when absent and never replaced by a commit: prepare refused any
|
|
52
|
+
// view not this load's own.
|
|
53
|
+
async describe(connection) {
|
|
54
|
+
const [existing] = await connection.query(`SELECT IIF(OBJECT_ID(@view, N'V') IS NULL, 0, 1) AS [made]`, { view: this.qualifiedName });
|
|
55
|
+
if (existing?.made !== 1)
|
|
56
|
+
await this.#define(connection, 'CREATE');
|
|
57
|
+
await this.#descriptions.write(connection, 'VIEW', marts, this.#name);
|
|
58
|
+
}
|
|
59
|
+
// Defined again over a target that took the place of another, whose
|
|
60
|
+
// columns it then shows, with its grants and the views over it kept.
|
|
61
|
+
async replace(connection) {
|
|
62
|
+
await this.#define(connection, 'CREATE OR ALTER');
|
|
63
|
+
await this.#descriptions.write(connection, 'VIEW', marts, this.#name);
|
|
64
|
+
}
|
|
65
|
+
// CREATE VIEW must be the only statement of its batch.
|
|
66
|
+
async #define(connection, verb) {
|
|
67
|
+
const columns = this.#columns.map(quote).join(', ');
|
|
68
|
+
const kept = this.#columns.map((column) => `o.${quote(column)}`);
|
|
69
|
+
await connection.query(`${verb} VIEW ${this.qualifiedName} AS SELECT ${columns} FROM ${this.#target} WHERE [committed_in] <= ${position} UNION ALL SELECT ${kept.join(', ')} FROM ${this.#history} AS o WHERE o.[committed_in] <= ${position} AND o.[replaced_in] > ${position}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { RuleColumn, RuleColumns, RuleValues } from '@eltjs/elt';
|
|
2
|
+
export declare function ruleColumnType(name: string, column: RuleColumn): string;
|
|
3
|
+
export declare function ruleBatchType(column: RuleColumn): string;
|
|
4
|
+
export declare function ruleStored(name: string, column: RuleColumn, alias: string): string;
|
|
5
|
+
export declare function storedValue(value: RuleValues<RuleColumns>[string]): string | number | Uint8Array | null;
|
|
6
|
+
export declare function batchValue(value: string | number | Uint8Array | null): string | number | null;
|
|
7
|
+
export type Declared = {
|
|
8
|
+
readonly name: string;
|
|
9
|
+
readonly type: string;
|
|
10
|
+
readonly required: boolean;
|
|
11
|
+
};
|
|
12
|
+
export declare function keyHash(key: string): string;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { hash } from 'node:crypto';
|
|
2
|
+
import { Vector } from '@zukhruf/vectors';
|
|
3
|
+
import { sqlType, toSql } from '@zukhruf/vectors/sqlserver';
|
|
4
|
+
// The most dimensions of a float32 VECTOR that SQL Server holds (ADR 0007).
|
|
5
|
+
const mostDimensions = 1998;
|
|
6
|
+
// The type of a rule column in the raw table, as sys.columns spells it:
|
|
7
|
+
// text NVARCHAR(MAX), bytes VARBINARY(MAX), an integer BIGINT, and a vector
|
|
8
|
+
// VECTOR(n) of float32 values. A vector column of more dimensions than SQL
|
|
9
|
+
// Server holds is a TypeError.
|
|
10
|
+
export function ruleColumnType(name, column) {
|
|
11
|
+
switch (column.type) {
|
|
12
|
+
case 'text':
|
|
13
|
+
return 'nvarchar(max)';
|
|
14
|
+
case 'blob':
|
|
15
|
+
return 'varbinary(max)';
|
|
16
|
+
case 'integer':
|
|
17
|
+
return 'bigint';
|
|
18
|
+
case 'vector':
|
|
19
|
+
if (column.dimensions > mostDimensions)
|
|
20
|
+
throw new TypeError(`The vector column ${name} has ${column.dimensions} dimensions; a float32 VECTOR of SQL Server holds at most ${mostDimensions}`);
|
|
21
|
+
return sqlType(Vector, column.dimensions).toLowerCase();
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
// The type that OPENJSON reads a value of the column as: bytes travel as
|
|
25
|
+
// base64 text, which BASE64_DECODE takes only as varchar (ADR 0011).
|
|
26
|
+
export function ruleBatchType(column) {
|
|
27
|
+
switch (column.type) {
|
|
28
|
+
case 'blob':
|
|
29
|
+
return 'varchar(max)';
|
|
30
|
+
case 'integer':
|
|
31
|
+
return 'bigint';
|
|
32
|
+
case 'text':
|
|
33
|
+
case 'vector':
|
|
34
|
+
return 'nvarchar(max)';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
// The stored value of what OPENJSON read as alias: base64 text decoded, and
|
|
38
|
+
// the JSON text of a vector cast to the type of its column.
|
|
39
|
+
export function ruleStored(name, column, alias) {
|
|
40
|
+
switch (column.type) {
|
|
41
|
+
case 'blob':
|
|
42
|
+
return `BASE64_DECODE(${alias})`;
|
|
43
|
+
case 'vector':
|
|
44
|
+
return `CAST(${alias} AS ${ruleColumnType(name, column)})`;
|
|
45
|
+
case 'text':
|
|
46
|
+
case 'integer':
|
|
47
|
+
return alias;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
// The value of a rule column in the form the table stores it, which the row
|
|
51
|
+
// hash counts: a vector as the JSON text of @zukhruf/vectors/sqlserver, and
|
|
52
|
+
// a column with no value as null.
|
|
53
|
+
export function storedValue(value) {
|
|
54
|
+
if (value instanceof Vector)
|
|
55
|
+
return toSql(value);
|
|
56
|
+
return value ?? null;
|
|
57
|
+
}
|
|
58
|
+
// A stored value as JSON carries it: bytes as base64 text.
|
|
59
|
+
export function batchValue(value) {
|
|
60
|
+
return value instanceof Uint8Array
|
|
61
|
+
? Buffer.from(value).toString('base64')
|
|
62
|
+
: value;
|
|
63
|
+
}
|
|
64
|
+
// The key hash of an output row and of each of its parts: the SHA-256, in
|
|
65
|
+
// lower-case hex, of the JSON array of its key, as a load hashes the key of a
|
|
66
|
+
// staged row (ADR 0004).
|
|
67
|
+
export function keyHash(key) {
|
|
68
|
+
return hash('sha256', JSON.stringify([key]));
|
|
69
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { quote } from './sql-server-identifiers.js';
|
|
2
|
+
// The two schemas of a warehouse, as in elt-postgresql: readers query the
|
|
3
|
+
// views of marts, and elt keeps each table of its own in _warehouse.
|
|
4
|
+
export const marts = 'marts';
|
|
5
|
+
export const warehouse = '_warehouse';
|
|
6
|
+
// A relation of marts or of _warehouse, as T-SQL names it.
|
|
7
|
+
export const inMarts = (name) => `${quote(marts)}.${quote(name)}`;
|
|
8
|
+
export const inWarehouse = (name) => `${quote(warehouse)}.${quote(name)}`;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
// The database is not set up as the destination needs it: an engine older
|
|
2
|
+
// than SQL Server 2025, or no row versions for readers. The message gives
|
|
3
|
+
// what to change, such as the ALTER DATABASE statement to run.
|
|
4
|
+
export class SqlServerSetupError extends Error {
|
|
5
|
+
name = 'SqlServerSetupError';
|
|
6
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { SqlServerView } from './sql-server-views.ts';
|
|
2
|
+
export declare const attempts: string;
|
|
3
|
+
export declare const coverage: string;
|
|
4
|
+
export declare const checks: string;
|
|
5
|
+
export declare const syncHistoryTables: string;
|
|
6
|
+
export declare const syncHistoryViews: readonly SqlServerView[];
|
|
7
|
+
export declare const catalog: SqlServerView;
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { readerCatalog, syncHistoryRelations } from '@eltjs/elt';
|
|
2
|
+
import { derivedStatusTable } from './sql-server-derived-status.js';
|
|
3
|
+
import { columnType } from './sql-server-fit.js';
|
|
4
|
+
import { now, position } from './sql-server-reader-view.js';
|
|
5
|
+
import { inWarehouse, marts, warehouse } from './sql-server-schemas.js';
|
|
6
|
+
// The tables of the history in _warehouse.
|
|
7
|
+
export const attempts = inWarehouse('sync_attempts');
|
|
8
|
+
export const coverage = inWarehouse('extraction_coverage');
|
|
9
|
+
export const checks = inWarehouse('stream_checks');
|
|
10
|
+
const derivedStatus = inWarehouse('derived_status');
|
|
11
|
+
// A name of a connection or a stream compares by its code points, as in
|
|
12
|
+
// Postgres. A key of text of any length is unique by its SHA-256 (ADR 0004).
|
|
13
|
+
const name = 'nvarchar(max) COLLATE Latin1_General_100_BIN2';
|
|
14
|
+
const keyOf = (column) => `CAST(HASHBYTES('SHA2_256', [${column}]) AS binary(32)) PERSISTED NOT NULL`;
|
|
15
|
+
const statuses = `(N'running', N'succeeded', N'partial', N'failed', N'cancelled')`;
|
|
16
|
+
export const syncHistoryTables = `
|
|
17
|
+
IF SCHEMA_ID(N'${warehouse}') IS NULL EXEC (N'CREATE SCHEMA [${warehouse}]');
|
|
18
|
+
IF OBJECT_ID(N'${attempts}', N'U') IS NULL
|
|
19
|
+
CREATE TABLE ${attempts} (
|
|
20
|
+
[id] bigint IDENTITY(1, 1) NOT NULL PRIMARY KEY,
|
|
21
|
+
[connector] ${name} NOT NULL CHECK (TRIM([connector]) <> N''), [source] nvarchar(max) NOT NULL,
|
|
22
|
+
[started_at] datetimeoffset(3) NOT NULL, [completed_at] datetimeoffset(3) NULL,
|
|
23
|
+
[status] nvarchar(20) NOT NULL DEFAULT N'running' CHECK ([status] IN ${statuses}),
|
|
24
|
+
[error] nvarchar(max) NULL,
|
|
25
|
+
[failure_type] nvarchar(20) NULL CHECK ([failure_type] IN (N'config', N'system')),
|
|
26
|
+
CHECK (([status] = N'running' AND [completed_at] IS NULL) OR ([status] <> N'running' AND [completed_at] IS NOT NULL)),
|
|
27
|
+
CHECK (([status] IN (N'partial', N'failed') AND [failure_type] IS NOT NULL) OR ([status] NOT IN (N'partial', N'failed') AND [failure_type] IS NULL)));
|
|
28
|
+
IF OBJECT_ID(N'${coverage}', N'U') IS NULL
|
|
29
|
+
CREATE TABLE ${coverage} (
|
|
30
|
+
[attempt_id] bigint NOT NULL REFERENCES ${attempts} ([id]),
|
|
31
|
+
[stream] ${name} NOT NULL, [stream_key] AS ${keyOf('stream')}, [target_table] nvarchar(128) NOT NULL,
|
|
32
|
+
[sync_mode] nvarchar(20) NOT NULL, [destination_sync_mode] nvarchar(20) NOT NULL,
|
|
33
|
+
[description] nvarchar(max) NOT NULL CHECK (TRIM([description]) <> N''), [selection] json NOT NULL,
|
|
34
|
+
[status] nvarchar(20) NOT NULL DEFAULT N'running' CHECK ([status] IN ${statuses}),
|
|
35
|
+
[written_count] bigint NULL CHECK ([written_count] >= 0), [deleted_count] bigint NULL CHECK ([deleted_count] >= 0),
|
|
36
|
+
[failures] json NOT NULL DEFAULT N'[]', [upstream_current_as_of] datetimeoffset(3) NULL,
|
|
37
|
+
PRIMARY KEY ([attempt_id], [stream_key]));
|
|
38
|
+
IF OBJECT_ID(N'${checks}', N'U') IS NULL
|
|
39
|
+
CREATE TABLE ${checks} (
|
|
40
|
+
[connector] ${name} NOT NULL, [connector_key] AS ${keyOf('connector')},
|
|
41
|
+
[stream] ${name} NOT NULL, [stream_key] AS ${keyOf('stream')},
|
|
42
|
+
[checked_at] datetimeoffset(3) NOT NULL,
|
|
43
|
+
PRIMARY KEY ([connector_key], [stream_key]));
|
|
44
|
+
${derivedStatusTable}`;
|
|
45
|
+
// When connector's stream last matched its upstream: the start of its last
|
|
46
|
+
// successful pass, or a later check that found it unchanged, each no later
|
|
47
|
+
// than when the source said the upstream itself was last current. NULL if it
|
|
48
|
+
// never succeeded. LEAST and GREATEST skip NULLs.
|
|
49
|
+
const currentAsOf = (connector, stream) => `(
|
|
50
|
+
SELECT CASE WHEN success.[id] IS NULL THEN NULL ELSE GREATEST(LEAST(success.[started_at], success.[upstream_current_as_of]), k.[checked_at]) END
|
|
51
|
+
FROM (SELECT 1 AS [one]) AS one
|
|
52
|
+
OUTER APPLY (
|
|
53
|
+
SELECT TOP (1) s.[id], s.[started_at], sc.[upstream_current_as_of] FROM ${coverage} AS sc
|
|
54
|
+
JOIN ${attempts} AS s ON s.[id] = sc.[attempt_id]
|
|
55
|
+
WHERE s.[connector] = ${connector} AND sc.[stream] = ${stream} AND sc.[status] = N'succeeded'
|
|
56
|
+
ORDER BY s.[completed_at] DESC, s.[id] DESC
|
|
57
|
+
) AS success
|
|
58
|
+
LEFT JOIN ${checks} AS k ON k.[connector] = ${connector} AND k.[stream] = ${stream})`;
|
|
59
|
+
const lastSuccess = (connector) => `OUTER APPLY (
|
|
60
|
+
SELECT TOP (1) [id], [completed_at] FROM ${attempts}
|
|
61
|
+
WHERE [connector] = ${connector} AND [status] = N'succeeded'
|
|
62
|
+
ORDER BY [completed_at] DESC, [id] DESC) AS success`;
|
|
63
|
+
const queries = {
|
|
64
|
+
sync_attempts: `SELECT [id] AS [attempt_id], [connector], [source], [started_at], [completed_at], [status], [error], [failure_type] FROM ${attempts}`,
|
|
65
|
+
extraction_coverage: `SELECT c.[attempt_id], a.[connector], a.[source], a.[started_at], a.[completed_at],
|
|
66
|
+
c.[stream], c.[target_table],
|
|
67
|
+
CAST(IIF(OBJECT_ID(QUOTENAME(N'${marts}') + N'.' + QUOTENAME(c.[target_table]), N'V') IS NULL, 0, 1) AS bit) AS [target_exists],
|
|
68
|
+
c.[sync_mode], c.[destination_sync_mode], c.[description], c.[selection],
|
|
69
|
+
c.[status], c.[written_count], c.[deleted_count], c.[failures]
|
|
70
|
+
FROM ${coverage} AS c JOIN ${attempts} AS a ON a.[id] = c.[attempt_id]`,
|
|
71
|
+
sync_status: `SELECT a.[connector],
|
|
72
|
+
a.[id] AS [latest_attempt_id], a.[started_at], a.[completed_at], a.[status], a.[error], a.[failure_type],
|
|
73
|
+
success.[id] AS [last_successful_attempt_id], success.[completed_at] AS [last_successful_sync_at],
|
|
74
|
+
(SELECT CASE WHEN COUNT(*) = COUNT(streams.[current_as_of]) THEN MIN(streams.[current_as_of]) END
|
|
75
|
+
FROM (SELECT ${currentAsOf('a.[connector]', 'd.[stream]')} AS [current_as_of]
|
|
76
|
+
FROM (SELECT DISTINCT dc.[stream] FROM ${coverage} AS dc
|
|
77
|
+
JOIN ${attempts} AS da ON da.[id] = dc.[attempt_id]
|
|
78
|
+
WHERE da.[connector] = a.[connector]) AS d) AS streams) AS [current_as_of]
|
|
79
|
+
FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY [connector] ORDER BY [id] DESC) AS [latest] FROM ${attempts}) AS a
|
|
80
|
+
${lastSuccess('a.[connector]')}
|
|
81
|
+
WHERE a.[latest] = 1`,
|
|
82
|
+
stream_status: `SELECT a.[connector], c.[stream], c.[target_table],
|
|
83
|
+
a.[id] AS [latest_attempt_id], a.[started_at], a.[completed_at], c.[status],
|
|
84
|
+
success.[id] AS [last_successful_attempt_id], success.[completed_at] AS [last_successful_sync_at],
|
|
85
|
+
${currentAsOf('a.[connector]', 'c.[stream]')} AS [current_as_of]
|
|
86
|
+
FROM (SELECT cc.*, ROW_NUMBER() OVER (PARTITION BY ca.[connector], cc.[stream] ORDER BY ca.[id] DESC) AS [latest]
|
|
87
|
+
FROM ${coverage} AS cc JOIN ${attempts} AS ca ON ca.[id] = cc.[attempt_id]) AS c
|
|
88
|
+
JOIN ${attempts} AS a ON a.[id] = c.[attempt_id]
|
|
89
|
+
OUTER APPLY (
|
|
90
|
+
SELECT TOP (1) s.[id], s.[completed_at] FROM ${coverage} AS sc
|
|
91
|
+
JOIN ${attempts} AS s ON s.[id] = sc.[attempt_id]
|
|
92
|
+
WHERE s.[connector] = a.[connector] AND sc.[stream] = c.[stream] AND sc.[status] = N'succeeded'
|
|
93
|
+
ORDER BY s.[completed_at] DESC, s.[id] DESC) AS success
|
|
94
|
+
WHERE c.[latest] = 1`,
|
|
95
|
+
derived_status: `SELECT [dataset], [kind], [state], [as_of], [refreshed_at], [refreshing_since], [error], [inputs], [computed], [empty], [failed], [pending] FROM ${derivedStatus}`,
|
|
96
|
+
};
|
|
97
|
+
// How readers see the sync history: what each pass declared and loaded.
|
|
98
|
+
// The sync history keeps no row versions, so a session that reads as of a
|
|
99
|
+
// position before now fails each of its views. A view cannot raise an
|
|
100
|
+
// error, so the condition converts the reason to int, which fails with
|
|
101
|
+
// error 245 and the reason in its message, also over no rows.
|
|
102
|
+
export const syncHistoryViews = Object.values(syncHistoryRelations).map((relation) => ({
|
|
103
|
+
...relation,
|
|
104
|
+
query: `SELECT * FROM (${queries[relation.name]}) AS [history] WHERE 1 = IIF(${position} = ${now}, 1, CAST(N'The sync history cannot be read as of a position: it keeps no row versions' AS int))`,
|
|
105
|
+
}));
|
|
106
|
+
// Lists what readers of marts query, its views, from the descriptions every
|
|
107
|
+
// load and publication writes there.
|
|
108
|
+
export const catalog = {
|
|
109
|
+
...readerCatalog,
|
|
110
|
+
query: `SELECT N'view' AS [kind], v.[name] AS [name], CAST(NULL AS nvarchar(max)) AS [data_type], CAST(ep.[value] AS nvarchar(max)) AS [description]
|
|
111
|
+
FROM sys.views AS v
|
|
112
|
+
LEFT JOIN sys.extended_properties AS ep ON ep.[class] = 1 AND ep.[major_id] = v.[object_id] AND ep.[minor_id] = 0 AND ep.[name] = N'MS_Description'
|
|
113
|
+
WHERE v.[schema_id] = SCHEMA_ID(N'${marts}')
|
|
114
|
+
UNION ALL
|
|
115
|
+
SELECT N'column', v.[name] + N'.' + c.[name], ${columnType}, CAST(ep.[value] AS nvarchar(max))
|
|
116
|
+
FROM sys.views AS v
|
|
117
|
+
JOIN sys.columns AS c ON c.[object_id] = v.[object_id]
|
|
118
|
+
LEFT JOIN sys.extended_properties AS ep ON ep.[class] = 1 AND ep.[major_id] = v.[object_id] AND ep.[minor_id] = c.[column_id] AND ep.[name] = N'MS_Description'
|
|
119
|
+
WHERE v.[schema_id] = SCHEMA_ID(N'${marts}')`,
|
|
120
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type Connection, type DeclaredCopy, type RecordedPass, type StreamCheck, SyncHistory } from '@eltjs/elt';
|
|
2
|
+
import { type SqlServerLogin } from './sql-server-connection.ts';
|
|
3
|
+
import type { SqlServerTable } from './sql-server-table.ts';
|
|
4
|
+
export declare class SqlServerSyncHistory extends SyncHistory<SqlServerTable> {
|
|
5
|
+
#private;
|
|
6
|
+
constructor({ login, readers, }: {
|
|
7
|
+
login: SqlServerLogin;
|
|
8
|
+
readers?: readonly string[];
|
|
9
|
+
});
|
|
10
|
+
validate({ name, destination }: Connection<SqlServerTable>): void;
|
|
11
|
+
protected recordChecks(connection: Connection<SqlServerTable>, streamChecks: readonly StreamCheck[]): Promise<void>;
|
|
12
|
+
begin(connection: Connection<SqlServerTable>, copies: readonly DeclaredCopy<SqlServerTable>[], at: Date): Promise<RecordedPass<SqlServerTable>>;
|
|
13
|
+
install(): Promise<void>;
|
|
14
|
+
}
|