@eltjs/elt-sqlserver 0.0.0-stage → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +485 -2
- package/dist/computed-dataset.d.ts +31 -0
- package/dist/computed-dataset.js +380 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +10 -0
- package/dist/sql-server-append-writer.d.ts +8 -0
- package/dist/sql-server-append-writer.js +13 -0
- package/dist/sql-server-as-of.d.ts +14 -0
- package/dist/sql-server-as-of.js +44 -0
- package/dist/sql-server-checkpoint-store.d.ts +10 -0
- package/dist/sql-server-checkpoint-store.js +101 -0
- package/dist/sql-server-column.d.ts +53 -0
- package/dist/sql-server-column.js +295 -0
- package/dist/sql-server-columns.d.ts +15 -0
- package/dist/sql-server-columns.js +49 -0
- package/dist/sql-server-connection.d.ts +6 -0
- package/dist/sql-server-connection.js +56 -0
- package/dist/sql-server-deduplicating-writer.d.ts +18 -0
- package/dist/sql-server-deduplicating-writer.js +102 -0
- package/dist/sql-server-dependents.d.ts +9 -0
- package/dist/sql-server-dependents.js +54 -0
- package/dist/sql-server-derived-status.d.ts +9 -0
- package/dist/sql-server-derived-status.js +60 -0
- package/dist/sql-server-descriptions.d.ts +6 -0
- package/dist/sql-server-descriptions.js +53 -0
- package/dist/sql-server-destination.d.ts +32 -0
- package/dist/sql-server-destination.js +95 -0
- package/dist/sql-server-failure-type.d.ts +2 -0
- package/dist/sql-server-failure-type.js +25 -0
- package/dist/sql-server-file-store.d.ts +21 -0
- package/dist/sql-server-file-store.js +91 -0
- package/dist/sql-server-fit.d.ts +20 -0
- package/dist/sql-server-fit.js +52 -0
- package/dist/sql-server-history.d.ts +15 -0
- package/dist/sql-server-history.js +148 -0
- package/dist/sql-server-identifiers.d.ts +10 -0
- package/dist/sql-server-identifiers.js +27 -0
- package/dist/sql-server-indexes.d.ts +23 -0
- package/dist/sql-server-indexes.js +121 -0
- package/dist/sql-server-load.d.ts +19 -0
- package/dist/sql-server-load.js +58 -0
- package/dist/sql-server-numbering.d.ts +32 -0
- package/dist/sql-server-numbering.js +255 -0
- package/dist/sql-server-overwrite-deduplicating-writer.d.ts +4 -0
- package/dist/sql-server-overwrite-deduplicating-writer.js +8 -0
- package/dist/sql-server-overwrite-writer.d.ts +4 -0
- package/dist/sql-server-overwrite-writer.js +9 -0
- package/dist/sql-server-parts-table.d.ts +24 -0
- package/dist/sql-server-parts-table.js +167 -0
- package/dist/sql-server-passes.d.ts +6 -0
- package/dist/sql-server-passes.js +71 -0
- package/dist/sql-server-reader-view.d.ts +12 -0
- package/dist/sql-server-reader-view.js +71 -0
- package/dist/sql-server-rule-columns.d.ts +12 -0
- package/dist/sql-server-rule-columns.js +69 -0
- package/dist/sql-server-schemas.d.ts +4 -0
- package/dist/sql-server-schemas.js +8 -0
- package/dist/sql-server-setup-error.d.ts +3 -0
- package/dist/sql-server-setup-error.js +6 -0
- package/dist/sql-server-sync-history-schema.d.ts +7 -0
- package/dist/sql-server-sync-history-schema.js +120 -0
- package/dist/sql-server-sync-history.d.ts +14 -0
- package/dist/sql-server-sync-history.js +165 -0
- package/dist/sql-server-table.d.ts +10 -0
- package/dist/sql-server-table.js +40 -0
- package/dist/sql-server-views.d.ts +7 -0
- package/dist/sql-server-views.js +70 -0
- package/dist/sql-server-writer.d.ts +40 -0
- package/dist/sql-server-writer.js +914 -0
- package/dist/view.d.ts +23 -0
- package/dist/view.js +82 -0
- package/package.json +48 -4
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { Identifiers } from '@eltjs/elt';
|
|
2
|
+
// SQL Server keeps 128 characters of a name, and the default collation of a
|
|
3
|
+
// database compares two names without case. The destination's own columns
|
|
4
|
+
// are loaded_at, which readers see, and the columns that only its own SQL
|
|
5
|
+
// reads: _elt_row, committed_in and _elt_row_hash in each table, replaced_in
|
|
6
|
+
// in each history, and _elt_key_hash of a key.
|
|
7
|
+
class SqlServerIdentifiers extends Identifiers {
|
|
8
|
+
maxBytes = 128;
|
|
9
|
+
ownColumns = [
|
|
10
|
+
'loaded_at',
|
|
11
|
+
'committed_in',
|
|
12
|
+
'replaced_in',
|
|
13
|
+
'_elt_row',
|
|
14
|
+
'_elt_row_hash',
|
|
15
|
+
'_elt_key_hash',
|
|
16
|
+
];
|
|
17
|
+
key(name) {
|
|
18
|
+
return name.toLowerCase();
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export const identifiers = Object.freeze(new SqlServerIdentifiers());
|
|
22
|
+
// The table in _warehouse that keeps the rows of the reader relation name:
|
|
23
|
+
// raw_<name>, under the naming rule of every relation.
|
|
24
|
+
export const rawTable = (name) => identifiers.relation(`raw_${name}`);
|
|
25
|
+
export function quote(name) {
|
|
26
|
+
return `[${name.replaceAll(']', ']]')}]`;
|
|
27
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
|
|
2
|
+
import type { Declared } from './sql-server-rule-columns.ts';
|
|
3
|
+
export type SqlServerIndex = {
|
|
4
|
+
readonly type: 'nonclustered';
|
|
5
|
+
readonly columns: readonly [string, ...string[]];
|
|
6
|
+
} | {
|
|
7
|
+
readonly type: 'vector';
|
|
8
|
+
readonly column: string;
|
|
9
|
+
readonly metric: 'cosine' | 'euclidean' | 'dot';
|
|
10
|
+
};
|
|
11
|
+
export declare class TableIndexes {
|
|
12
|
+
#private;
|
|
13
|
+
readonly names: readonly string[];
|
|
14
|
+
constructor({ dataset, table, reader, keys, columns, indexes, }: {
|
|
15
|
+
dataset: string;
|
|
16
|
+
table: string;
|
|
17
|
+
reader: string;
|
|
18
|
+
keys: readonly string[];
|
|
19
|
+
columns: readonly Declared[];
|
|
20
|
+
indexes: readonly SqlServerIndex[];
|
|
21
|
+
});
|
|
22
|
+
reconcile(connection: SqlServerConnection): Promise<void>;
|
|
23
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { hash } from 'node:crypto';
|
|
2
|
+
import { identifiers, quote } from './sql-server-identifiers.js';
|
|
3
|
+
import { inMarts, marts } from './sql-server-schemas.js';
|
|
4
|
+
// The start of the name of each index that the destination manages. A
|
|
5
|
+
// person's own index has another name, so the destination never drops it.
|
|
6
|
+
const managed = 'ix_elt_';
|
|
7
|
+
// The declared indexes of one table of a computed dataset, each under a name
|
|
8
|
+
// that the destination derives from its declaration, so a changed
|
|
9
|
+
// declaration is another index. reconcile brings the table to them in place,
|
|
10
|
+
// with no make of the table again and no recompute.
|
|
11
|
+
// A vector index has its search, the inline table-valued function
|
|
12
|
+
// marts.<reader>_nearest(@query, @k), named after the reader view of the
|
|
13
|
+
// table: VECTOR_SEARCH uses the index only on a base table, and readers read
|
|
14
|
+
// only marts. It gives the keys and the distance of the k nearest rows of the
|
|
15
|
+
// table as it is now. It and the table have one owner, so a reader that may
|
|
16
|
+
// read the function needs nothing of _warehouse (ADR 0014).
|
|
17
|
+
export class TableIndexes {
|
|
18
|
+
// The relations of marts that the table's search takes, schema-qualified:
|
|
19
|
+
// none when the table has no vector index.
|
|
20
|
+
names;
|
|
21
|
+
#table;
|
|
22
|
+
// The statement that makes each declared index, by its name.
|
|
23
|
+
#declared;
|
|
24
|
+
// The function that searches the vector index, in marts.
|
|
25
|
+
#search;
|
|
26
|
+
// The definition of the search, from FUNCTION on, as SQL Server keeps it;
|
|
27
|
+
// undefined when the table has no vector index.
|
|
28
|
+
#nearest;
|
|
29
|
+
// Refuses an index that the table cannot have, before anything is made: a
|
|
30
|
+
// column that the table does not have, a vector index on a column that is
|
|
31
|
+
// not a vector, an index key that SQL Server refuses (error 1919 for a
|
|
32
|
+
// (max) type, 1978 for a vector), and a search that cannot have its name
|
|
33
|
+
// or its columns: a second vector index, and a key named distance. keys
|
|
34
|
+
// are the columns that name a row of the table, which the search gives.
|
|
35
|
+
constructor({ dataset, table, reader, keys, columns, indexes, }) {
|
|
36
|
+
const column = (name) => {
|
|
37
|
+
const found = columns.find((declared) => identifiers.key(declared.name) === identifiers.key(name));
|
|
38
|
+
if (found === undefined)
|
|
39
|
+
throw new TypeError(`Computed dataset ${dataset} declares an index on ${name}, which ${table} does not have`);
|
|
40
|
+
return found;
|
|
41
|
+
};
|
|
42
|
+
this.#table = table;
|
|
43
|
+
this.#declared = new Map(indexes.map((index) => {
|
|
44
|
+
if (index.type === 'vector') {
|
|
45
|
+
const { name, type } = column(index.column);
|
|
46
|
+
if (!type.startsWith('vector('))
|
|
47
|
+
throw new TypeError(`Computed dataset ${dataset} declares a vector index on ${name}, which is ${type}, not a vector`);
|
|
48
|
+
const indexName = derivedName(['vector', name, index.metric]);
|
|
49
|
+
return [
|
|
50
|
+
indexName,
|
|
51
|
+
`CREATE VECTOR INDEX ${quote(indexName)} ON ${table} (${quote(name)}) WITH (METRIC = '${index.metric}', TYPE = 'DiskANN')`,
|
|
52
|
+
];
|
|
53
|
+
}
|
|
54
|
+
const keys = index.columns.map((key) => {
|
|
55
|
+
const { name, type } = column(key);
|
|
56
|
+
if (type.endsWith('(max)') || type.startsWith('vector('))
|
|
57
|
+
throw new TypeError(`Computed dataset ${dataset} cannot make ${name} an index key: SQL Server refuses a column of type ${type} as one`);
|
|
58
|
+
return name;
|
|
59
|
+
});
|
|
60
|
+
const indexName = derivedName(['nonclustered', ...keys]);
|
|
61
|
+
return [
|
|
62
|
+
indexName,
|
|
63
|
+
`CREATE NONCLUSTERED INDEX ${quote(indexName)} ON ${table} (${keys.map(quote).join(', ')})`,
|
|
64
|
+
];
|
|
65
|
+
}));
|
|
66
|
+
const name = identifiers.relation(`${reader}_nearest`);
|
|
67
|
+
this.#search = inMarts(name);
|
|
68
|
+
const vectors = indexes.filter((index) => index.type === 'vector');
|
|
69
|
+
const [vector] = vectors;
|
|
70
|
+
if (vectors.length > 1)
|
|
71
|
+
throw new TypeError(`Computed dataset ${dataset} declares ${vectors.length} vector indexes on ${table}, and ${this.#search} searches one`);
|
|
72
|
+
const distance = keys.find((key) => identifiers.key(key) === 'distance');
|
|
73
|
+
if (vector !== undefined && distance !== undefined)
|
|
74
|
+
throw new TypeError(`Computed dataset ${dataset} cannot search ${table} through ${this.#search}: its column ${distance} has the name of the distance that the search gives`);
|
|
75
|
+
this.names = Object.freeze(vector === undefined ? [] : [`${marts}.${name}`]);
|
|
76
|
+
if (vector === undefined)
|
|
77
|
+
this.#nearest = undefined;
|
|
78
|
+
else {
|
|
79
|
+
const searched = column(vector.column);
|
|
80
|
+
this.#nearest = `FUNCTION ${this.#search}(@query ${searched.type}, @k int) RETURNS TABLE AS RETURN SELECT TOP (@k) WITH APPROXIMATE ${keys.map((key) => `t.${quote(key)}`).join(', ')}, r.[distance] FROM VECTOR_SEARCH(TABLE = ${table} AS t, COLUMN = ${quote(searched.name)}, SIMILAR_TO = @query, METRIC = '${vector.metric}') AS r ORDER BY r.[distance]`;
|
|
81
|
+
}
|
|
82
|
+
Object.freeze(this);
|
|
83
|
+
}
|
|
84
|
+
// Drops each managed index of the table that is no longer declared, then
|
|
85
|
+
// makes each declared index that the table does not have, then makes the
|
|
86
|
+
// search of the vector index, or makes it again when its stored
|
|
87
|
+
// definition differs, or drops it when the table has no vector index.
|
|
88
|
+
// Each statement runs outside a transaction, since SQL Server makes a
|
|
89
|
+
// vector index only there (error 574); a refresh that stops between two of
|
|
90
|
+
// them makes the rest the next time.
|
|
91
|
+
async reconcile(connection) {
|
|
92
|
+
const stored = new Set((await connection.query(`SELECT [name] FROM sys.indexes WHERE [object_id] = OBJECT_ID(@table) AND [name] LIKE N'${managed.replaceAll('_', '[_]')}%'`, { table: this.#table })).map(({ name }) => String(name)));
|
|
93
|
+
for (const name of stored)
|
|
94
|
+
if (!this.#declared.has(name))
|
|
95
|
+
await connection.query(`DROP INDEX ${quote(name)} ON ${this.#table}`);
|
|
96
|
+
for (const [name, statement] of this.#declared)
|
|
97
|
+
if (!stored.has(name))
|
|
98
|
+
await connection.query(statement);
|
|
99
|
+
const search = await searchDefinition(connection, this.#search);
|
|
100
|
+
if (this.#nearest === undefined) {
|
|
101
|
+
if (search !== undefined)
|
|
102
|
+
await connection.query(`DROP FUNCTION ${this.#search}`);
|
|
103
|
+
}
|
|
104
|
+
else if (search !== this.#nearest)
|
|
105
|
+
await connection.query(`CREATE OR ALTER ${this.#nearest}`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// The definition of the inline table-valued function name, from FUNCTION on,
|
|
109
|
+
// or undefined when the database has none of that name. SQL Server keeps the
|
|
110
|
+
// text of the statement that made the function, with CREATE OR ALTER kept
|
|
111
|
+
// as CREATE (Azure probe of ADR 0014), so the words before FUNCTION are
|
|
112
|
+
// skipped.
|
|
113
|
+
async function searchDefinition(connection, name) {
|
|
114
|
+
const [stored] = await connection.query("SELECT OBJECT_DEFINITION(OBJECT_ID(@name, N'IF')) AS [definition]", { name });
|
|
115
|
+
return typeof stored?.definition === 'string'
|
|
116
|
+
? stored.definition.replace(/^CREATE\s+(?:OR\s+ALTER\s+)?/i, '')
|
|
117
|
+
: undefined;
|
|
118
|
+
}
|
|
119
|
+
// The name of an index: the managed prefix and the first 16 hex digits of the
|
|
120
|
+
// SHA-256 of its declaration.
|
|
121
|
+
const derivedName = (declaration) => `${managed}${hash('sha256', JSON.stringify(declaration)).slice(0, 16)}`;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { PipelineClock } from '@eltjs/elt';
|
|
2
|
+
import type { SqlServerConnection, SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
|
|
3
|
+
import { type LoadOwner } from './sql-server-numbering.ts';
|
|
4
|
+
export declare class SqlServerLoad implements AsyncDisposable {
|
|
5
|
+
#private;
|
|
6
|
+
readonly connection: SqlServerConnection;
|
|
7
|
+
readonly loadedAt: string;
|
|
8
|
+
private constructor();
|
|
9
|
+
static open(database: SqlServerDatabase, owner: LoadOwner, retaining: Retaining): Promise<SqlServerLoad>;
|
|
10
|
+
transaction<T>(work: () => Promise<T>): Promise<T>;
|
|
11
|
+
commit<T>(work: (committedIn: number) => Promise<T>): Promise<T>;
|
|
12
|
+
number(): Promise<number>;
|
|
13
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
type Retaining = {
|
|
16
|
+
readonly clock: PipelineClock;
|
|
17
|
+
readonly retention: Temporal.Duration;
|
|
18
|
+
};
|
|
19
|
+
export {};
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { connect, transaction } from './sql-server-connection.js';
|
|
2
|
+
import { closeLoad, commitNumber, databaseCopy, installNumbering, openLoad, } from './sql-server-numbering.js';
|
|
3
|
+
// A run's one connection to the database, and its record in the numbering of
|
|
4
|
+
// the database: open from the start of the load until it is disposed, with a
|
|
5
|
+
// number for each commit. The stage of each stream is a #temp table of this
|
|
6
|
+
// connection. No transaction stays open between commits.
|
|
7
|
+
export class SqlServerLoad {
|
|
8
|
+
connection;
|
|
9
|
+
#clock;
|
|
10
|
+
#retention;
|
|
11
|
+
#owner;
|
|
12
|
+
#id;
|
|
13
|
+
// When the load began, which each row it writes keeps in loaded_at.
|
|
14
|
+
loadedAt;
|
|
15
|
+
constructor(connection, owner, { clock, retention }, id, loadedAt) {
|
|
16
|
+
this.connection = connection;
|
|
17
|
+
this.#clock = clock;
|
|
18
|
+
this.#retention = retention;
|
|
19
|
+
this.#owner = owner;
|
|
20
|
+
this.#id = id;
|
|
21
|
+
this.loadedAt = loadedAt.toISOString();
|
|
22
|
+
}
|
|
23
|
+
// Checks the engine and the row versioning of the database first, so a
|
|
24
|
+
// database that the destination cannot load fails before a row is read.
|
|
25
|
+
static async open(database, owner, retaining) {
|
|
26
|
+
const connection = await connect(database);
|
|
27
|
+
try {
|
|
28
|
+
const loadedAt = new Date(retaining.clock.Date.now());
|
|
29
|
+
await installNumbering(connection, await databaseCopy(connection));
|
|
30
|
+
const id = await openLoad(connection, owner, loadedAt);
|
|
31
|
+
return new SqlServerLoad(connection, owner, retaining, id, loadedAt);
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
await connection[Symbol.asyncDispose]();
|
|
35
|
+
throw error;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
transaction(work) {
|
|
39
|
+
return transaction(this.connection, work);
|
|
40
|
+
}
|
|
41
|
+
// A transaction that is a commit of this load, with its number.
|
|
42
|
+
commit(work) {
|
|
43
|
+
return this.transaction(async () => work(await this.number()));
|
|
44
|
+
}
|
|
45
|
+
// The number of a commit of this load, taken inside the commit's
|
|
46
|
+
// transaction, so a transaction that takes none is no commit.
|
|
47
|
+
number() {
|
|
48
|
+
return commitNumber(this.connection, { load: this.#id, owner: this.#owner }, new Date(this.#clock.Date.now()), this.#retention);
|
|
49
|
+
}
|
|
50
|
+
async [Symbol.asyncDispose]() {
|
|
51
|
+
try {
|
|
52
|
+
await closeLoad(this.connection, this.#id, new Date(this.#clock.Date.now()));
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
await this.connection[Symbol.asyncDispose]();
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { type NumberedStore, type Numbering } from '@eltjs/elt';
|
|
2
|
+
import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
|
|
3
|
+
export declare const sqlServerDatabase: NumberedStore;
|
|
4
|
+
export type LoadOwner = {
|
|
5
|
+
readonly connector: string;
|
|
6
|
+
} | {
|
|
7
|
+
readonly dataset: string;
|
|
8
|
+
};
|
|
9
|
+
export declare function requireRowVersions(connection: SqlServerConnection): Promise<{
|
|
10
|
+
readonly name: string;
|
|
11
|
+
readonly azure: boolean;
|
|
12
|
+
}>;
|
|
13
|
+
export declare function databaseCopy(connection: SqlServerConnection): Promise<string>;
|
|
14
|
+
export declare function installNumbering(connection: SqlServerConnection, mark: string): Promise<void>;
|
|
15
|
+
export declare function grantReading(connection: SqlServerConnection, reader: string): Promise<void>;
|
|
16
|
+
export declare function openLoad(connection: SqlServerConnection, owner: LoadOwner, openedAt: Date): Promise<string>;
|
|
17
|
+
export declare function closeLoad(connection: SqlServerConnection, load: string, closedAt: Date): Promise<void>;
|
|
18
|
+
export declare function commitNumber(connection: SqlServerConnection, { load, owner }: {
|
|
19
|
+
load: string;
|
|
20
|
+
owner: LoadOwner;
|
|
21
|
+
}, committedAt: Date, retention: Temporal.Duration): Promise<number>;
|
|
22
|
+
export declare function forgetCommit(connection: SqlServerConnection, number: number): Promise<void>;
|
|
23
|
+
export declare function minValid(connection: SqlServerConnection): Promise<number>;
|
|
24
|
+
export declare function commitOnce(connection: SqlServerConnection, connector: string, at: Date, retention: Temporal.Duration): Promise<number>;
|
|
25
|
+
export declare function expireBefore(connection: SqlServerConnection, changedIn: number, by: 'type' | 'clear'): Promise<void>;
|
|
26
|
+
export declare function numberingIn(connection: SqlServerConnection, settled: number): Promise<Numbering | null>;
|
|
27
|
+
export declare function settledMark(connection: SqlServerConnection): Promise<number | null>;
|
|
28
|
+
export declare function committedAt(connection: SqlServerConnection, number: number): Promise<{
|
|
29
|
+
readonly at: Date;
|
|
30
|
+
readonly newest: number;
|
|
31
|
+
} | undefined>;
|
|
32
|
+
export declare function readNumbering(connection: SqlServerConnection): Promise<Numbering | null>;
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { numbering } from '@eltjs/elt';
|
|
2
|
+
import { takeLock, transaction } from './sql-server-connection.js';
|
|
3
|
+
import { quote } from './sql-server-identifiers.js';
|
|
4
|
+
import { inWarehouse, warehouse } from './sql-server-schemas.js';
|
|
5
|
+
import { SqlServerSetupError } from './sql-server-setup-error.js';
|
|
6
|
+
// In _warehouse, beside the sync history, so one numbering orders the commits
|
|
7
|
+
// of every load of the database.
|
|
8
|
+
const epochs = inWarehouse('_elt_epoch');
|
|
9
|
+
const loads = inWarehouse('_elt_loads');
|
|
10
|
+
const commits = inWarehouse('_elt_commits');
|
|
11
|
+
// How the database numbers its commits (ADR 0002). The number of a commit is
|
|
12
|
+
// the rowversion of its row in _elt_commits, which the commit inserts before
|
|
13
|
+
// it changes a row; no other table of the destination has a rowversion
|
|
14
|
+
// column. The epoch names the numbering, and mark names the copy of the
|
|
15
|
+
// database that gave it: a restored copy gives numbers again that the
|
|
16
|
+
// original gave, so it starts a new epoch. A name of a connection or a
|
|
17
|
+
// dataset compares by its code points, as in Postgres.
|
|
18
|
+
const tables = `
|
|
19
|
+
IF SCHEMA_ID(N'${warehouse}') IS NULL EXEC (N'CREATE SCHEMA ${quote(warehouse)}');
|
|
20
|
+
IF OBJECT_ID(N'${epochs}', N'U') IS NULL
|
|
21
|
+
CREATE TABLE ${epochs} ([id] int NOT NULL PRIMARY KEY CHECK ([id] = 1), [epoch] nvarchar(36) NOT NULL, [mark] nvarchar(400) NOT NULL, [min_valid] bigint NOT NULL DEFAULT 0, [expired_by] nvarchar(20) NULL CHECK ([expired_by] IN (N'retention', N'type', N'clear')));
|
|
22
|
+
IF OBJECT_ID(N'${loads}', N'U') IS NULL
|
|
23
|
+
CREATE TABLE ${loads} ([id] bigint IDENTITY(1, 1) NOT NULL PRIMARY KEY, [connector] nvarchar(max) COLLATE Latin1_General_100_BIN2 NULL, [dataset] nvarchar(max) COLLATE Latin1_General_100_BIN2 NULL, [opened_at] datetimeoffset(3) NOT NULL, [closed_at] datetimeoffset(3) NULL, CHECK (([connector] IS NULL AND [dataset] IS NOT NULL) OR ([connector] IS NOT NULL AND [dataset] IS NULL)));
|
|
24
|
+
IF OBJECT_ID(N'${commits}', N'U') IS NULL
|
|
25
|
+
BEGIN
|
|
26
|
+
CREATE TABLE ${commits} ([number] rowversion NOT NULL PRIMARY KEY, [load] bigint NOT NULL REFERENCES ${loads} ([id]), [committed_at] datetimeoffset(3) NOT NULL);
|
|
27
|
+
CREATE INDEX [_elt_commits_load] ON ${commits} ([load], [number]);
|
|
28
|
+
END;
|
|
29
|
+
INSERT INTO ${epochs} ([id], [epoch], [mark])
|
|
30
|
+
SELECT 1, CONVERT(nvarchar(36), NEWID()), @mark WHERE NOT EXISTS (SELECT 1 FROM ${epochs});`;
|
|
31
|
+
// How a SQL Server database names itself in the reasons of its positions.
|
|
32
|
+
export const sqlServerDatabase = {
|
|
33
|
+
engine: 'SQL Server',
|
|
34
|
+
noun: 'database',
|
|
35
|
+
};
|
|
36
|
+
function owned(owner) {
|
|
37
|
+
return 'connector' in owner
|
|
38
|
+
? ['connector', owner.connector]
|
|
39
|
+
: ['dataset', owner.dataset];
|
|
40
|
+
}
|
|
41
|
+
// Refuses a database that the destination cannot load or read as ADR 0010
|
|
42
|
+
// says: an engine older than SQL Server 2025 has no VECTOR and no JSON type,
|
|
43
|
+
// and a database that keeps no row versions makes readers wait for a load
|
|
44
|
+
// and refuses a SNAPSHOT transaction. Any user of the database can read its
|
|
45
|
+
// row of sys.databases. Gives whether the database is an Azure SQL Database.
|
|
46
|
+
export async function requireRowVersions(connection) {
|
|
47
|
+
const [setup] = await connection.query(`SELECT CAST(SERVERPROPERTY('EngineEdition') AS int) AS [edition], CAST(SERVERPROPERTY('ProductMajorVersion') AS int) AS [major],
|
|
48
|
+
CAST([is_read_committed_snapshot_on] AS int) AS [committed_snapshot], CAST([snapshot_isolation_state] AS int) AS [snapshot], DB_NAME() AS [name]
|
|
49
|
+
FROM sys.databases WHERE [database_id] = DB_ID()`);
|
|
50
|
+
const name = String(setup?.name);
|
|
51
|
+
// Azure SQL Database (5) and Managed Instance (8) report version 12 but run
|
|
52
|
+
// the current engine.
|
|
53
|
+
const azure = setup?.edition === 5;
|
|
54
|
+
if (!azure && setup?.edition !== 8 && Number(setup?.major) < 17)
|
|
55
|
+
throw new SqlServerSetupError(`Database ${name} runs SQL Server ${String(setup?.major)}; the SQL Server destination needs SQL Server 2025 (17.x), Azure SQL Database or Azure SQL Managed Instance`);
|
|
56
|
+
const statements = [
|
|
57
|
+
...(setup?.snapshot === 1
|
|
58
|
+
? []
|
|
59
|
+
: [`ALTER DATABASE ${quote(name)} SET ALLOW_SNAPSHOT_ISOLATION ON`]),
|
|
60
|
+
...(setup?.committed_snapshot === 1
|
|
61
|
+
? []
|
|
62
|
+
: [
|
|
63
|
+
`ALTER DATABASE ${quote(name)} SET READ_COMMITTED_SNAPSHOT ON WITH ROLLBACK IMMEDIATE`,
|
|
64
|
+
]),
|
|
65
|
+
];
|
|
66
|
+
if (statements.length > 0)
|
|
67
|
+
throw new SqlServerSetupError(`Database ${name} keeps no row versions for readers, so they would wait for each load; run ${statements.join('; ')}`);
|
|
68
|
+
return { name, azure };
|
|
69
|
+
}
|
|
70
|
+
// Refuses a database that the destination cannot load, and gives the mark of
|
|
71
|
+
// the copy of the database: the recovery fork of SQL Server, which each
|
|
72
|
+
// restore changes and a restart does not; Azure SQL Database has no such
|
|
73
|
+
// view, so there each value of sys.databases that a restore changes.
|
|
74
|
+
export async function databaseCopy(connection) {
|
|
75
|
+
const { name, azure } = await requireRowVersions(connection);
|
|
76
|
+
const [copy] = await connection.query(azure
|
|
77
|
+
? `SELECT CONCAT([database_id], N' ', CONVERT(nvarchar(30), [create_date], 126), N' ', [physical_database_name]) AS [mark] FROM sys.databases WHERE [database_id] = DB_ID()`
|
|
78
|
+
: 'SELECT CONVERT(nvarchar(36), [recovery_fork_guid]) AS [mark] FROM sys.database_recovery_status WHERE [database_id] = DB_ID()');
|
|
79
|
+
if (typeof copy?.mark !== 'string')
|
|
80
|
+
throw new TypeError(`Database ${name} gave no mark of its copy`);
|
|
81
|
+
return copy.mark;
|
|
82
|
+
}
|
|
83
|
+
// Makes the numbering if the database has none, under the lock of
|
|
84
|
+
// _warehouse, and starts a new epoch when the database is another copy than
|
|
85
|
+
// the one that numbered it, such as one restored from a backup. Only one of
|
|
86
|
+
// two loads that find a new copy at once changes the epoch.
|
|
87
|
+
export async function installNumbering(connection, mark) {
|
|
88
|
+
const [made] = await connection.query(`SELECT CASE WHEN OBJECT_ID(N'${epochs}', N'U') IS NULL THEN 0 ELSE 1 END AS [made]`);
|
|
89
|
+
if (made?.made !== 1)
|
|
90
|
+
await transaction(connection, async () => {
|
|
91
|
+
await takeLock(connection, warehouse);
|
|
92
|
+
await connection.query(tables, { mark });
|
|
93
|
+
});
|
|
94
|
+
await connection.query(`UPDATE ${epochs} SET [epoch] = CONVERT(nvarchar(36), NEWID()), [mark] = @mark WHERE [id] = 1 AND [mark] <> @mark`, { mark });
|
|
95
|
+
}
|
|
96
|
+
// Lets a reader read the positions and the readings of the database with its
|
|
97
|
+
// own session: the numbering, and nothing else of _warehouse.
|
|
98
|
+
export async function grantReading(connection, reader) {
|
|
99
|
+
await connection.query(`GRANT SELECT ON ${epochs} TO ${quote(reader)}; GRANT SELECT ON ${commits} TO ${quote(reader)}; GRANT SELECT ON ${loads} TO ${quote(reader)}`);
|
|
100
|
+
}
|
|
101
|
+
// Records a load of owner that opens at openedAt, and gives its id. A load of
|
|
102
|
+
// the same owner that is still open is one whose process ended during it:
|
|
103
|
+
// the new load closes it. The UPDATE that closes it reads each row of
|
|
104
|
+
// _elt_loads with an update lock, as an nvarchar(max) owner has no index, so
|
|
105
|
+
// two opens that each held the row they inserted would wait for each other
|
|
106
|
+
// until SQL Server killed one (error 1205). The open takes the lock of
|
|
107
|
+
// _elt_loads before it inserts, so two opens run one at a time.
|
|
108
|
+
export async function openLoad(connection, owner, openedAt) {
|
|
109
|
+
const [column, name] = owned(owner);
|
|
110
|
+
const [load] = await transaction(connection, async () => {
|
|
111
|
+
await takeLock(connection, `${warehouse}._elt_loads`);
|
|
112
|
+
return connection.query(`DECLARE @load table ([id] bigint);
|
|
113
|
+
INSERT INTO ${loads} ([${column}], [opened_at]) OUTPUT inserted.[id] INTO @load VALUES (@name, CAST(@at AS datetimeoffset(3)));
|
|
114
|
+
UPDATE ${loads} SET [closed_at] = CAST(@at AS datetimeoffset(3))
|
|
115
|
+
WHERE [${column}] = @name AND [closed_at] IS NULL AND [id] <> (SELECT [id] FROM @load);
|
|
116
|
+
SELECT CONVERT(nvarchar(20), [id]) AS [id] FROM @load;`, { name, at: openedAt.toISOString() });
|
|
117
|
+
});
|
|
118
|
+
if (typeof load?.id !== 'string')
|
|
119
|
+
throw new Error('The load was not recorded');
|
|
120
|
+
return load.id;
|
|
121
|
+
}
|
|
122
|
+
export async function closeLoad(connection, load, closedAt) {
|
|
123
|
+
await connection.query(`UPDATE ${loads} SET [closed_at] = CAST(@at AS datetimeoffset(3)) WHERE [id] = CAST(@load AS bigint) AND [closed_at] IS NULL`, { load, at: closedAt.toISOString() });
|
|
124
|
+
}
|
|
125
|
+
// Inserts the commit row of a commit of load, as the first write of its
|
|
126
|
+
// transaction, and gives its rowversion, the number of the commit. A load
|
|
127
|
+
// that a later load of its owner closed has ended, and a commit after its end
|
|
128
|
+
// would move a position that readers were already given: the commit holds a
|
|
129
|
+
// shared lock on the row of its load, so a close that runs at the same time
|
|
130
|
+
// waits for the commit.
|
|
131
|
+
export async function commitNumber(connection, { load, owner }, committedAt, retention) {
|
|
132
|
+
const [open] = await connection.query(`SELECT 1 AS [open] FROM ${loads} WITH (HOLDLOCK, ROWLOCK) WHERE [id] = CAST(@load AS bigint) AND [closed_at] IS NULL`, { load });
|
|
133
|
+
if (open === undefined)
|
|
134
|
+
throw new Error('connector' in owner
|
|
135
|
+
? `A later load of connection ${owner.connector} closed this load, so it cannot commit; a connection runs one pass at a time`
|
|
136
|
+
: `A later refresh of dataset ${owner.dataset} closed this one, so it cannot commit; a dataset refreshes one at a time`);
|
|
137
|
+
const [taken] = await connection.query(`INSERT INTO ${commits} ([load], [committed_at]) OUTPUT CONVERT(nvarchar(20), CAST(inserted.[number] AS bigint)) AS [number] VALUES (CAST(@load AS bigint), CAST(@at AS datetimeoffset(3)))`, { load, at: committedAt.toISOString() });
|
|
138
|
+
if (typeof taken?.number !== 'string')
|
|
139
|
+
throw new Error('No commit number was taken');
|
|
140
|
+
await retain(connection, committedAt, retention);
|
|
141
|
+
return Number(taken.number);
|
|
142
|
+
}
|
|
143
|
+
// Takes back the commit row of a commit that changed no row, before it
|
|
144
|
+
// commits: its number is a gap, as the number of a commit that rolled back
|
|
145
|
+
// is, so no reader sees a position that changed nothing.
|
|
146
|
+
export async function forgetCommit(connection, number) {
|
|
147
|
+
await connection.query(`DELETE FROM ${commits} WHERE [number] = CAST(CAST(@number AS bigint) AS binary(8))`, { number: String(number) });
|
|
148
|
+
}
|
|
149
|
+
// Applies the retention inside a commit's transaction, as elt-postgresql
|
|
150
|
+
// does: the newest position whose commit is older than the window becomes
|
|
151
|
+
// min_valid, and the numbering then keeps no commit before it and no closed
|
|
152
|
+
// load left with none. min_valid only moves forward, as commits of other
|
|
153
|
+
// connections run at the same time.
|
|
154
|
+
async function retain(connection, at, retention) {
|
|
155
|
+
const cutoff = new Date(at.getTime() - retention.total('milliseconds'));
|
|
156
|
+
const [row] = await connection.query(`SELECT CONVERT(nvarchar(20), (SELECT ISNULL(MAX(CAST([number] AS bigint)), 0) FROM ${commits} WHERE [committed_at] < CAST(@cutoff AS datetimeoffset(3)))) AS [older],
|
|
157
|
+
CONVERT(nvarchar(20), [min_valid]) AS [min_valid]
|
|
158
|
+
FROM ${epochs}`, { cutoff: cutoff.toISOString() });
|
|
159
|
+
if (row === undefined)
|
|
160
|
+
return;
|
|
161
|
+
const older = Number(row.older);
|
|
162
|
+
const kept = Number(row.min_valid);
|
|
163
|
+
if (older <= kept)
|
|
164
|
+
return;
|
|
165
|
+
const expired = (await numberingIn(connection, older))?.ends.at(-1);
|
|
166
|
+
if (expired === undefined || expired <= kept)
|
|
167
|
+
return;
|
|
168
|
+
await connection.query(`UPDATE ${epochs} SET [min_valid] = CAST(@expired AS bigint), [expired_by] = N'retention' WHERE [min_valid] < CAST(@expired AS bigint);
|
|
169
|
+
IF @@ROWCOUNT > 0
|
|
170
|
+
BEGIN
|
|
171
|
+
DELETE FROM ${commits} WHERE CAST([number] AS bigint) < CAST(@expired AS bigint);
|
|
172
|
+
DELETE FROM ${loads} WHERE [closed_at] IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${commits} AS c WHERE c.[load] = ${loads}.[id]);
|
|
173
|
+
END;`, { expired: String(expired) });
|
|
174
|
+
}
|
|
175
|
+
// The oldest commit that a reading can read as of, inside the transaction
|
|
176
|
+
// connection runs in.
|
|
177
|
+
export async function minValid(connection) {
|
|
178
|
+
const [row] = await connection.query(`SELECT CONVERT(nvarchar(20), [min_valid]) AS [min_valid] FROM ${epochs}`);
|
|
179
|
+
return Number(row?.min_valid ?? 0);
|
|
180
|
+
}
|
|
181
|
+
// The number of a change that is a load of one commit, such as a clear,
|
|
182
|
+
// taken inside the caller's transaction: the load opens, commits and closes
|
|
183
|
+
// in it. It is not a pass, so it leaves the open loads of its connector open.
|
|
184
|
+
export async function commitOnce(connection, connector, at, retention) {
|
|
185
|
+
const [load] = await connection.query(`INSERT INTO ${loads} ([connector], [opened_at]) OUTPUT CONVERT(nvarchar(20), inserted.[id]) AS [id] VALUES (@connector, CAST(@at AS datetimeoffset(3)))`, { connector, at: at.toISOString() });
|
|
186
|
+
if (typeof load?.id !== 'string')
|
|
187
|
+
throw new Error('The load was not recorded');
|
|
188
|
+
const number = await commitNumber(connection, { load: load.id, owner: { connector } }, at, retention);
|
|
189
|
+
await closeLoad(connection, load.id, at);
|
|
190
|
+
return number;
|
|
191
|
+
}
|
|
192
|
+
// Makes each position before the commit changedIn expire, inside that
|
|
193
|
+
// commit, for by: a change of type that is not a widening, or a clear that
|
|
194
|
+
// forgets the rows it removes.
|
|
195
|
+
export async function expireBefore(connection, changedIn, by) {
|
|
196
|
+
await connection.query(`UPDATE ${epochs} SET [min_valid] = CAST(@changed AS bigint), [expired_by] = @by WHERE [min_valid] < CAST(@changed AS bigint)`, { changed: String(changedIn), by });
|
|
197
|
+
}
|
|
198
|
+
// The numbering as the transaction that connection runs in sees it, with
|
|
199
|
+
// settled taken before its snapshot.
|
|
200
|
+
export async function numberingIn(connection, settled) {
|
|
201
|
+
const [row] = await connection.query(`SELECT [epoch], CONVERT(nvarchar(20), [min_valid]) AS [min_valid], [expired_by] FROM ${epochs}`);
|
|
202
|
+
if (row === undefined)
|
|
203
|
+
return null;
|
|
204
|
+
const spans = await connection.query(`SELECT CONVERT(nvarchar(20), MIN(CAST(c.[number] AS bigint))) AS [first],
|
|
205
|
+
CASE WHEN l.[closed_at] IS NULL THEN NULL ELSE CONVERT(nvarchar(20), MAX(CAST(c.[number] AS bigint))) END AS [last]
|
|
206
|
+
FROM ${commits} AS c JOIN ${loads} AS l ON l.[id] = c.[load]
|
|
207
|
+
GROUP BY l.[id], l.[closed_at]`);
|
|
208
|
+
const { expired_by: expiredBy } = row;
|
|
209
|
+
return numbering({
|
|
210
|
+
epoch: String(row.epoch),
|
|
211
|
+
minValid: Number(row.min_valid),
|
|
212
|
+
expiredBy: expiredBy === 'retention' || expiredBy === 'type' || expiredBy === 'clear'
|
|
213
|
+
? expiredBy
|
|
214
|
+
: null,
|
|
215
|
+
settled,
|
|
216
|
+
spans: spans.map(({ first, last }) => ({
|
|
217
|
+
first: Number(first),
|
|
218
|
+
last: last === null ? null : Number(last),
|
|
219
|
+
})),
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
// The high-water mark: MIN_ACTIVE_ROWVERSION() is the lowest number that a
|
|
223
|
+
// transaction still holds, so each number below it is settled, and a
|
|
224
|
+
// snapshot that begins after the read sees each commit at or below the mark
|
|
225
|
+
// (ADR 0002). Null when the database has no numbering.
|
|
226
|
+
export async function settledMark(connection) {
|
|
227
|
+
const [mark] = await connection.query(`SELECT CONVERT(nvarchar(20), CAST(MIN_ACTIVE_ROWVERSION() AS bigint) - 1) AS [settled], CASE WHEN OBJECT_ID(N'${epochs}', N'U') IS NULL THEN 0 ELSE 1 END AS [made]`);
|
|
228
|
+
return mark?.made === 1 ? Number(mark.settled) : null;
|
|
229
|
+
}
|
|
230
|
+
// When the commit number was made, and the newest commit, as the transaction
|
|
231
|
+
// that connection runs in sees them; undefined for a number of no commit.
|
|
232
|
+
export async function committedAt(connection, number) {
|
|
233
|
+
const [row] = await connection.query(`SELECT [committed_at] AS [at], (SELECT CONVERT(nvarchar(20), MAX(CAST([number] AS bigint))) FROM ${commits}) AS [newest]
|
|
234
|
+
FROM ${commits} WHERE [number] = CAST(CAST(@number AS bigint) AS binary(8))`, { number: String(number) });
|
|
235
|
+
if (row === undefined)
|
|
236
|
+
return undefined;
|
|
237
|
+
if (!(row.at instanceof Date))
|
|
238
|
+
throw new TypeError(`Commit ${number} gave no time`);
|
|
239
|
+
return { at: row.at, newest: Number(row.newest) };
|
|
240
|
+
}
|
|
241
|
+
// The numbering in a SNAPSHOT transaction of its own, with the high-water
|
|
242
|
+
// mark read before it. connection is a session of its own, which keeps the
|
|
243
|
+
// SNAPSHOT isolation level after.
|
|
244
|
+
export async function readNumbering(connection) {
|
|
245
|
+
const settled = await settledMark(connection);
|
|
246
|
+
if (settled === null)
|
|
247
|
+
return null;
|
|
248
|
+
await connection.query('SET TRANSACTION ISOLATION LEVEL SNAPSHOT; BEGIN TRANSACTION');
|
|
249
|
+
try {
|
|
250
|
+
return await numberingIn(connection, settled);
|
|
251
|
+
}
|
|
252
|
+
finally {
|
|
253
|
+
await connection.query('COMMIT');
|
|
254
|
+
}
|
|
255
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { SqlServerDeduplicatingWriter } from './sql-server-deduplicating-writer.js';
|
|
2
|
+
// Replaces the rows of the target with one record for each key of a run, by
|
|
3
|
+
// difference.
|
|
4
|
+
export class SqlServerOverwriteDeduplicatingWriter extends SqlServerDeduplicatingWriter {
|
|
5
|
+
get replaces() {
|
|
6
|
+
return true;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { SqlServerAppendWriter } from './sql-server-append-writer.js';
|
|
2
|
+
// Replaces the rows of the target with the records of a run, by difference
|
|
3
|
+
// and with DELETE, not TRUNCATE: TRUNCATE takes a schema modification lock,
|
|
4
|
+
// which readers wait for.
|
|
5
|
+
export class SqlServerOverwriteWriter extends SqlServerAppendWriter {
|
|
6
|
+
get replaces() {
|
|
7
|
+
return true;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { type OutputRow, type RuleColumns, type TableOutputRow, type TableRule } from '@eltjs/elt';
|
|
2
|
+
import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
|
|
3
|
+
import { TargetHistory } from './sql-server-history.ts';
|
|
4
|
+
import { type SqlServerIndex, TableIndexes } from './sql-server-indexes.ts';
|
|
5
|
+
import { TargetReaderView } from './sql-server-reader-view.ts';
|
|
6
|
+
import { type Declared } from './sql-server-rule-columns.ts';
|
|
7
|
+
export declare class SqlServerPartsTable {
|
|
8
|
+
#private;
|
|
9
|
+
readonly table: string;
|
|
10
|
+
readonly declared: readonly Declared[];
|
|
11
|
+
readonly statement: string;
|
|
12
|
+
readonly history: TargetHistory;
|
|
13
|
+
readonly view: TargetReaderView;
|
|
14
|
+
readonly indexes: TableIndexes;
|
|
15
|
+
readonly names: readonly string[];
|
|
16
|
+
constructor({ dataset, description, rule, indexes, }: {
|
|
17
|
+
dataset: string;
|
|
18
|
+
description: string;
|
|
19
|
+
rule: Pick<TableRule<unknown, RuleColumns, RuleColumns>, 'key' | 'output' | 'place'>;
|
|
20
|
+
indexes: readonly SqlServerIndex[];
|
|
21
|
+
});
|
|
22
|
+
write(connection: SqlServerConnection, commit: () => Promise<number>, outputs: readonly (OutputRow | TableOutputRow)[]): Promise<void>;
|
|
23
|
+
remove(connection: SqlServerConnection, replacedIn: number, keyHashes: readonly string[]): Promise<void>;
|
|
24
|
+
}
|