@eltjs/elt-sqlserver 0.0.0-stage → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +485 -2
  3. package/dist/computed-dataset.d.ts +31 -0
  4. package/dist/computed-dataset.js +380 -0
  5. package/dist/index.d.ts +13 -0
  6. package/dist/index.js +10 -0
  7. package/dist/sql-server-append-writer.d.ts +8 -0
  8. package/dist/sql-server-append-writer.js +13 -0
  9. package/dist/sql-server-as-of.d.ts +14 -0
  10. package/dist/sql-server-as-of.js +44 -0
  11. package/dist/sql-server-checkpoint-store.d.ts +10 -0
  12. package/dist/sql-server-checkpoint-store.js +101 -0
  13. package/dist/sql-server-column.d.ts +53 -0
  14. package/dist/sql-server-column.js +295 -0
  15. package/dist/sql-server-columns.d.ts +15 -0
  16. package/dist/sql-server-columns.js +49 -0
  17. package/dist/sql-server-connection.d.ts +6 -0
  18. package/dist/sql-server-connection.js +56 -0
  19. package/dist/sql-server-deduplicating-writer.d.ts +18 -0
  20. package/dist/sql-server-deduplicating-writer.js +102 -0
  21. package/dist/sql-server-dependents.d.ts +9 -0
  22. package/dist/sql-server-dependents.js +54 -0
  23. package/dist/sql-server-derived-status.d.ts +9 -0
  24. package/dist/sql-server-derived-status.js +60 -0
  25. package/dist/sql-server-descriptions.d.ts +6 -0
  26. package/dist/sql-server-descriptions.js +53 -0
  27. package/dist/sql-server-destination.d.ts +32 -0
  28. package/dist/sql-server-destination.js +95 -0
  29. package/dist/sql-server-failure-type.d.ts +2 -0
  30. package/dist/sql-server-failure-type.js +25 -0
  31. package/dist/sql-server-file-store.d.ts +21 -0
  32. package/dist/sql-server-file-store.js +91 -0
  33. package/dist/sql-server-fit.d.ts +20 -0
  34. package/dist/sql-server-fit.js +52 -0
  35. package/dist/sql-server-history.d.ts +15 -0
  36. package/dist/sql-server-history.js +148 -0
  37. package/dist/sql-server-identifiers.d.ts +10 -0
  38. package/dist/sql-server-identifiers.js +27 -0
  39. package/dist/sql-server-indexes.d.ts +23 -0
  40. package/dist/sql-server-indexes.js +121 -0
  41. package/dist/sql-server-load.d.ts +19 -0
  42. package/dist/sql-server-load.js +58 -0
  43. package/dist/sql-server-numbering.d.ts +32 -0
  44. package/dist/sql-server-numbering.js +255 -0
  45. package/dist/sql-server-overwrite-deduplicating-writer.d.ts +4 -0
  46. package/dist/sql-server-overwrite-deduplicating-writer.js +8 -0
  47. package/dist/sql-server-overwrite-writer.d.ts +4 -0
  48. package/dist/sql-server-overwrite-writer.js +9 -0
  49. package/dist/sql-server-parts-table.d.ts +24 -0
  50. package/dist/sql-server-parts-table.js +167 -0
  51. package/dist/sql-server-passes.d.ts +6 -0
  52. package/dist/sql-server-passes.js +71 -0
  53. package/dist/sql-server-reader-view.d.ts +12 -0
  54. package/dist/sql-server-reader-view.js +71 -0
  55. package/dist/sql-server-rule-columns.d.ts +12 -0
  56. package/dist/sql-server-rule-columns.js +69 -0
  57. package/dist/sql-server-schemas.d.ts +4 -0
  58. package/dist/sql-server-schemas.js +8 -0
  59. package/dist/sql-server-setup-error.d.ts +3 -0
  60. package/dist/sql-server-setup-error.js +6 -0
  61. package/dist/sql-server-sync-history-schema.d.ts +7 -0
  62. package/dist/sql-server-sync-history-schema.js +120 -0
  63. package/dist/sql-server-sync-history.d.ts +14 -0
  64. package/dist/sql-server-sync-history.js +165 -0
  65. package/dist/sql-server-table.d.ts +10 -0
  66. package/dist/sql-server-table.js +40 -0
  67. package/dist/sql-server-views.d.ts +7 -0
  68. package/dist/sql-server-views.js +70 -0
  69. package/dist/sql-server-writer.d.ts +40 -0
  70. package/dist/sql-server-writer.js +914 -0
  71. package/dist/view.d.ts +23 -0
  72. package/dist/view.js +82 -0
  73. package/package.json +48 -4
@@ -0,0 +1,27 @@
1
+ import { Identifiers } from '@eltjs/elt';
2
+ // SQL Server keeps 128 characters of a name, and the default collation of a
3
+ // database compares two names without case. The destination's own columns
4
+ // are loaded_at, which readers see, and the columns that only its own SQL
5
+ // reads: _elt_row, committed_in and _elt_row_hash in each table, replaced_in
6
+ // in each history, and _elt_key_hash of a key.
7
+ class SqlServerIdentifiers extends Identifiers {
8
+ maxBytes = 128;
9
+ ownColumns = [
10
+ 'loaded_at',
11
+ 'committed_in',
12
+ 'replaced_in',
13
+ '_elt_row',
14
+ '_elt_row_hash',
15
+ '_elt_key_hash',
16
+ ];
17
+ key(name) {
18
+ return name.toLowerCase();
19
+ }
20
+ }
21
+ export const identifiers = Object.freeze(new SqlServerIdentifiers());
22
+ // The table in _warehouse that keeps the rows of the reader relation name:
23
+ // raw_<name>, under the naming rule of every relation.
24
+ export const rawTable = (name) => identifiers.relation(`raw_${name}`);
25
+ export function quote(name) {
26
+ return `[${name.replaceAll(']', ']]')}]`;
27
+ }
@@ -0,0 +1,23 @@
1
+ import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
2
+ import type { Declared } from './sql-server-rule-columns.ts';
3
+ export type SqlServerIndex = {
4
+ readonly type: 'nonclustered';
5
+ readonly columns: readonly [string, ...string[]];
6
+ } | {
7
+ readonly type: 'vector';
8
+ readonly column: string;
9
+ readonly metric: 'cosine' | 'euclidean' | 'dot';
10
+ };
11
+ export declare class TableIndexes {
12
+ #private;
13
+ readonly names: readonly string[];
14
+ constructor({ dataset, table, reader, keys, columns, indexes, }: {
15
+ dataset: string;
16
+ table: string;
17
+ reader: string;
18
+ keys: readonly string[];
19
+ columns: readonly Declared[];
20
+ indexes: readonly SqlServerIndex[];
21
+ });
22
+ reconcile(connection: SqlServerConnection): Promise<void>;
23
+ }
@@ -0,0 +1,121 @@
1
+ import { hash } from 'node:crypto';
2
+ import { identifiers, quote } from './sql-server-identifiers.js';
3
+ import { inMarts, marts } from './sql-server-schemas.js';
4
+ // The start of the name of each index that the destination manages. A
5
+ // person's own index has another name, so the destination never drops it.
6
+ const managed = 'ix_elt_';
7
+ // The declared indexes of one table of a computed dataset, each under a name
8
+ // that the destination derives from its declaration, so a changed
9
+ // declaration is another index. reconcile brings the table to them in place,
10
+ // with no make of the table again and no recompute.
11
+ // A vector index has its search, the inline table-valued function
12
+ // marts.<reader>_nearest(@query, @k), named after the reader view of the
13
+ // table: VECTOR_SEARCH uses the index only on a base table, and readers read
14
+ // only marts. It gives the keys and the distance of the k nearest rows of the
15
+ // table as it is now. It and the table have one owner, so a reader that may
16
+ // read the function needs nothing of _warehouse (ADR 0014).
17
+ export class TableIndexes {
18
+ // The relations of marts that the table's search takes, schema-qualified:
19
+ // none when the table has no vector index.
20
+ names;
21
+ #table;
22
+ // The statement that makes each declared index, by its name.
23
+ #declared;
24
+ // The function that searches the vector index, in marts.
25
+ #search;
26
+ // The definition of the search, from FUNCTION on, as SQL Server keeps it;
27
+ // undefined when the table has no vector index.
28
+ #nearest;
29
+ // Refuses an index that the table cannot have, before anything is made: a
30
+ // column that the table does not have, a vector index on a column that is
31
+ // not a vector, an index key that SQL Server refuses (error 1919 for a
32
+ // (max) type, 1978 for a vector), and a search that cannot have its name
33
+ // or its columns: a second vector index, and a key named distance. keys
34
+ // are the columns that name a row of the table, which the search gives.
35
+ constructor({ dataset, table, reader, keys, columns, indexes, }) {
36
+ const column = (name) => {
37
+ const found = columns.find((declared) => identifiers.key(declared.name) === identifiers.key(name));
38
+ if (found === undefined)
39
+ throw new TypeError(`Computed dataset ${dataset} declares an index on ${name}, which ${table} does not have`);
40
+ return found;
41
+ };
42
+ this.#table = table;
43
+ this.#declared = new Map(indexes.map((index) => {
44
+ if (index.type === 'vector') {
45
+ const { name, type } = column(index.column);
46
+ if (!type.startsWith('vector('))
47
+ throw new TypeError(`Computed dataset ${dataset} declares a vector index on ${name}, which is ${type}, not a vector`);
48
+ const indexName = derivedName(['vector', name, index.metric]);
49
+ return [
50
+ indexName,
51
+ `CREATE VECTOR INDEX ${quote(indexName)} ON ${table} (${quote(name)}) WITH (METRIC = '${index.metric}', TYPE = 'DiskANN')`,
52
+ ];
53
+ }
54
+ const keys = index.columns.map((key) => {
55
+ const { name, type } = column(key);
56
+ if (type.endsWith('(max)') || type.startsWith('vector('))
57
+ throw new TypeError(`Computed dataset ${dataset} cannot make ${name} an index key: SQL Server refuses a column of type ${type} as one`);
58
+ return name;
59
+ });
60
+ const indexName = derivedName(['nonclustered', ...keys]);
61
+ return [
62
+ indexName,
63
+ `CREATE NONCLUSTERED INDEX ${quote(indexName)} ON ${table} (${keys.map(quote).join(', ')})`,
64
+ ];
65
+ }));
66
+ const name = identifiers.relation(`${reader}_nearest`);
67
+ this.#search = inMarts(name);
68
+ const vectors = indexes.filter((index) => index.type === 'vector');
69
+ const [vector] = vectors;
70
+ if (vectors.length > 1)
71
+ throw new TypeError(`Computed dataset ${dataset} declares ${vectors.length} vector indexes on ${table}, and ${this.#search} searches one`);
72
+ const distance = keys.find((key) => identifiers.key(key) === 'distance');
73
+ if (vector !== undefined && distance !== undefined)
74
+ throw new TypeError(`Computed dataset ${dataset} cannot search ${table} through ${this.#search}: its column ${distance} has the name of the distance that the search gives`);
75
+ this.names = Object.freeze(vector === undefined ? [] : [`${marts}.${name}`]);
76
+ if (vector === undefined)
77
+ this.#nearest = undefined;
78
+ else {
79
+ const searched = column(vector.column);
80
+ this.#nearest = `FUNCTION ${this.#search}(@query ${searched.type}, @k int) RETURNS TABLE AS RETURN SELECT TOP (@k) WITH APPROXIMATE ${keys.map((key) => `t.${quote(key)}`).join(', ')}, r.[distance] FROM VECTOR_SEARCH(TABLE = ${table} AS t, COLUMN = ${quote(searched.name)}, SIMILAR_TO = @query, METRIC = '${vector.metric}') AS r ORDER BY r.[distance]`;
81
+ }
82
+ Object.freeze(this);
83
+ }
84
+ // Drops each managed index of the table that is no longer declared, then
85
+ // makes each declared index that the table does not have, then makes the
86
+ // search of the vector index, or makes it again when its stored
87
+ // definition differs, or drops it when the table has no vector index.
88
+ // Each statement runs outside a transaction, since SQL Server makes a
89
+ // vector index only there (error 574); a refresh that stops between two of
90
+ // them makes the rest the next time.
91
+ async reconcile(connection) {
92
+ const stored = new Set((await connection.query(`SELECT [name] FROM sys.indexes WHERE [object_id] = OBJECT_ID(@table) AND [name] LIKE N'${managed.replaceAll('_', '[_]')}%'`, { table: this.#table })).map(({ name }) => String(name)));
93
+ for (const name of stored)
94
+ if (!this.#declared.has(name))
95
+ await connection.query(`DROP INDEX ${quote(name)} ON ${this.#table}`);
96
+ for (const [name, statement] of this.#declared)
97
+ if (!stored.has(name))
98
+ await connection.query(statement);
99
+ const search = await searchDefinition(connection, this.#search);
100
+ if (this.#nearest === undefined) {
101
+ if (search !== undefined)
102
+ await connection.query(`DROP FUNCTION ${this.#search}`);
103
+ }
104
+ else if (search !== this.#nearest)
105
+ await connection.query(`CREATE OR ALTER ${this.#nearest}`);
106
+ }
107
+ }
108
+ // The definition of the inline table-valued function name, from FUNCTION on,
109
+ // or undefined when the database has none of that name. SQL Server keeps the
110
+ // text of the statement that made the function, with CREATE OR ALTER kept
111
+ // as CREATE (Azure probe of ADR 0014), so the words before FUNCTION are
112
+ // skipped.
113
+ async function searchDefinition(connection, name) {
114
+ const [stored] = await connection.query("SELECT OBJECT_DEFINITION(OBJECT_ID(@name, N'IF')) AS [definition]", { name });
115
+ return typeof stored?.definition === 'string'
116
+ ? stored.definition.replace(/^CREATE\s+(?:OR\s+ALTER\s+)?/i, '')
117
+ : undefined;
118
+ }
119
+ // The name of an index: the managed prefix and the first 16 hex digits of the
120
+ // SHA-256 of its declaration.
121
+ const derivedName = (declaration) => `${managed}${hash('sha256', JSON.stringify(declaration)).slice(0, 16)}`;
@@ -0,0 +1,19 @@
1
+ import type { PipelineClock } from '@eltjs/elt';
2
+ import type { SqlServerConnection, SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
3
+ import { type LoadOwner } from './sql-server-numbering.ts';
4
+ export declare class SqlServerLoad implements AsyncDisposable {
5
+ #private;
6
+ readonly connection: SqlServerConnection;
7
+ readonly loadedAt: string;
8
+ private constructor();
9
+ static open(database: SqlServerDatabase, owner: LoadOwner, retaining: Retaining): Promise<SqlServerLoad>;
10
+ transaction<T>(work: () => Promise<T>): Promise<T>;
11
+ commit<T>(work: (committedIn: number) => Promise<T>): Promise<T>;
12
+ number(): Promise<number>;
13
+ [Symbol.asyncDispose](): Promise<void>;
14
+ }
15
+ type Retaining = {
16
+ readonly clock: PipelineClock;
17
+ readonly retention: Temporal.Duration;
18
+ };
19
+ export {};
@@ -0,0 +1,58 @@
1
+ import { connect, transaction } from './sql-server-connection.js';
2
+ import { closeLoad, commitNumber, databaseCopy, installNumbering, openLoad, } from './sql-server-numbering.js';
3
+ // A run's one connection to the database, and its record in the numbering of
4
+ // the database: open from the start of the load until it is disposed, with a
5
+ // number for each commit. The stage of each stream is a #temp table of this
6
+ // connection. No transaction stays open between commits.
7
+ export class SqlServerLoad {
8
+ connection;
9
+ #clock;
10
+ #retention;
11
+ #owner;
12
+ #id;
13
+ // When the load began, which each row it writes keeps in loaded_at.
14
+ loadedAt;
15
+ constructor(connection, owner, { clock, retention }, id, loadedAt) {
16
+ this.connection = connection;
17
+ this.#clock = clock;
18
+ this.#retention = retention;
19
+ this.#owner = owner;
20
+ this.#id = id;
21
+ this.loadedAt = loadedAt.toISOString();
22
+ }
23
+ // Checks the engine and the row versioning of the database first, so a
24
+ // database that the destination cannot load fails before a row is read.
25
+ static async open(database, owner, retaining) {
26
+ const connection = await connect(database);
27
+ try {
28
+ const loadedAt = new Date(retaining.clock.Date.now());
29
+ await installNumbering(connection, await databaseCopy(connection));
30
+ const id = await openLoad(connection, owner, loadedAt);
31
+ return new SqlServerLoad(connection, owner, retaining, id, loadedAt);
32
+ }
33
+ catch (error) {
34
+ await connection[Symbol.asyncDispose]();
35
+ throw error;
36
+ }
37
+ }
38
+ transaction(work) {
39
+ return transaction(this.connection, work);
40
+ }
41
+ // A transaction that is a commit of this load, with its number.
42
+ commit(work) {
43
+ return this.transaction(async () => work(await this.number()));
44
+ }
45
+ // The number of a commit of this load, taken inside the commit's
46
+ // transaction, so a transaction that takes none is no commit.
47
+ number() {
48
+ return commitNumber(this.connection, { load: this.#id, owner: this.#owner }, new Date(this.#clock.Date.now()), this.#retention);
49
+ }
50
+ async [Symbol.asyncDispose]() {
51
+ try {
52
+ await closeLoad(this.connection, this.#id, new Date(this.#clock.Date.now()));
53
+ }
54
+ finally {
55
+ await this.connection[Symbol.asyncDispose]();
56
+ }
57
+ }
58
+ }
@@ -0,0 +1,32 @@
1
+ import { type NumberedStore, type Numbering } from '@eltjs/elt';
2
+ import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
3
+ export declare const sqlServerDatabase: NumberedStore;
4
+ export type LoadOwner = {
5
+ readonly connector: string;
6
+ } | {
7
+ readonly dataset: string;
8
+ };
9
+ export declare function requireRowVersions(connection: SqlServerConnection): Promise<{
10
+ readonly name: string;
11
+ readonly azure: boolean;
12
+ }>;
13
+ export declare function databaseCopy(connection: SqlServerConnection): Promise<string>;
14
+ export declare function installNumbering(connection: SqlServerConnection, mark: string): Promise<void>;
15
+ export declare function grantReading(connection: SqlServerConnection, reader: string): Promise<void>;
16
+ export declare function openLoad(connection: SqlServerConnection, owner: LoadOwner, openedAt: Date): Promise<string>;
17
+ export declare function closeLoad(connection: SqlServerConnection, load: string, closedAt: Date): Promise<void>;
18
+ export declare function commitNumber(connection: SqlServerConnection, { load, owner }: {
19
+ load: string;
20
+ owner: LoadOwner;
21
+ }, committedAt: Date, retention: Temporal.Duration): Promise<number>;
22
+ export declare function forgetCommit(connection: SqlServerConnection, number: number): Promise<void>;
23
+ export declare function minValid(connection: SqlServerConnection): Promise<number>;
24
+ export declare function commitOnce(connection: SqlServerConnection, connector: string, at: Date, retention: Temporal.Duration): Promise<number>;
25
+ export declare function expireBefore(connection: SqlServerConnection, changedIn: number, by: 'type' | 'clear'): Promise<void>;
26
+ export declare function numberingIn(connection: SqlServerConnection, settled: number): Promise<Numbering | null>;
27
+ export declare function settledMark(connection: SqlServerConnection): Promise<number | null>;
28
+ export declare function committedAt(connection: SqlServerConnection, number: number): Promise<{
29
+ readonly at: Date;
30
+ readonly newest: number;
31
+ } | undefined>;
32
+ export declare function readNumbering(connection: SqlServerConnection): Promise<Numbering | null>;
@@ -0,0 +1,255 @@
1
+ import { numbering } from '@eltjs/elt';
2
+ import { takeLock, transaction } from './sql-server-connection.js';
3
+ import { quote } from './sql-server-identifiers.js';
4
+ import { inWarehouse, warehouse } from './sql-server-schemas.js';
5
+ import { SqlServerSetupError } from './sql-server-setup-error.js';
6
+ // In _warehouse, beside the sync history, so one numbering orders the commits
7
+ // of every load of the database.
8
+ const epochs = inWarehouse('_elt_epoch');
9
+ const loads = inWarehouse('_elt_loads');
10
+ const commits = inWarehouse('_elt_commits');
11
+ // How the database numbers its commits (ADR 0002). The number of a commit is
12
+ // the rowversion of its row in _elt_commits, which the commit inserts before
13
+ // it changes a row; no other table of the destination has a rowversion
14
+ // column. The epoch names the numbering, and mark names the copy of the
15
+ // database that gave it: a restored copy gives numbers again that the
16
+ // original gave, so it starts a new epoch. A name of a connection or a
17
+ // dataset compares by its code points, as in Postgres.
18
+ const tables = `
19
+ IF SCHEMA_ID(N'${warehouse}') IS NULL EXEC (N'CREATE SCHEMA ${quote(warehouse)}');
20
+ IF OBJECT_ID(N'${epochs}', N'U') IS NULL
21
+ CREATE TABLE ${epochs} ([id] int NOT NULL PRIMARY KEY CHECK ([id] = 1), [epoch] nvarchar(36) NOT NULL, [mark] nvarchar(400) NOT NULL, [min_valid] bigint NOT NULL DEFAULT 0, [expired_by] nvarchar(20) NULL CHECK ([expired_by] IN (N'retention', N'type', N'clear')));
22
+ IF OBJECT_ID(N'${loads}', N'U') IS NULL
23
+ CREATE TABLE ${loads} ([id] bigint IDENTITY(1, 1) NOT NULL PRIMARY KEY, [connector] nvarchar(max) COLLATE Latin1_General_100_BIN2 NULL, [dataset] nvarchar(max) COLLATE Latin1_General_100_BIN2 NULL, [opened_at] datetimeoffset(3) NOT NULL, [closed_at] datetimeoffset(3) NULL, CHECK (([connector] IS NULL AND [dataset] IS NOT NULL) OR ([connector] IS NOT NULL AND [dataset] IS NULL)));
24
+ IF OBJECT_ID(N'${commits}', N'U') IS NULL
25
+ BEGIN
26
+ CREATE TABLE ${commits} ([number] rowversion NOT NULL PRIMARY KEY, [load] bigint NOT NULL REFERENCES ${loads} ([id]), [committed_at] datetimeoffset(3) NOT NULL);
27
+ CREATE INDEX [_elt_commits_load] ON ${commits} ([load], [number]);
28
+ END;
29
+ INSERT INTO ${epochs} ([id], [epoch], [mark])
30
+ SELECT 1, CONVERT(nvarchar(36), NEWID()), @mark WHERE NOT EXISTS (SELECT 1 FROM ${epochs});`;
31
+ // How a SQL Server database names itself in the reasons of its positions.
32
+ export const sqlServerDatabase = {
33
+ engine: 'SQL Server',
34
+ noun: 'database',
35
+ };
36
+ function owned(owner) {
37
+ return 'connector' in owner
38
+ ? ['connector', owner.connector]
39
+ : ['dataset', owner.dataset];
40
+ }
41
+ // Refuses a database that the destination cannot load or read as ADR 0010
42
+ // says: an engine older than SQL Server 2025 has no VECTOR and no JSON type,
43
+ // and a database that keeps no row versions makes readers wait for a load
44
+ // and refuses a SNAPSHOT transaction. Any user of the database can read its
45
+ // row of sys.databases. Gives whether the database is an Azure SQL Database.
46
+ export async function requireRowVersions(connection) {
47
+ const [setup] = await connection.query(`SELECT CAST(SERVERPROPERTY('EngineEdition') AS int) AS [edition], CAST(SERVERPROPERTY('ProductMajorVersion') AS int) AS [major],
48
+ CAST([is_read_committed_snapshot_on] AS int) AS [committed_snapshot], CAST([snapshot_isolation_state] AS int) AS [snapshot], DB_NAME() AS [name]
49
+ FROM sys.databases WHERE [database_id] = DB_ID()`);
50
+ const name = String(setup?.name);
51
+ // Azure SQL Database (5) and Managed Instance (8) report version 12 but run
52
+ // the current engine.
53
+ const azure = setup?.edition === 5;
54
+ if (!azure && setup?.edition !== 8 && Number(setup?.major) < 17)
55
+ throw new SqlServerSetupError(`Database ${name} runs SQL Server ${String(setup?.major)}; the SQL Server destination needs SQL Server 2025 (17.x), Azure SQL Database or Azure SQL Managed Instance`);
56
+ const statements = [
57
+ ...(setup?.snapshot === 1
58
+ ? []
59
+ : [`ALTER DATABASE ${quote(name)} SET ALLOW_SNAPSHOT_ISOLATION ON`]),
60
+ ...(setup?.committed_snapshot === 1
61
+ ? []
62
+ : [
63
+ `ALTER DATABASE ${quote(name)} SET READ_COMMITTED_SNAPSHOT ON WITH ROLLBACK IMMEDIATE`,
64
+ ]),
65
+ ];
66
+ if (statements.length > 0)
67
+ throw new SqlServerSetupError(`Database ${name} keeps no row versions for readers, so they would wait for each load; run ${statements.join('; ')}`);
68
+ return { name, azure };
69
+ }
70
+ // Refuses a database that the destination cannot load, and gives the mark of
71
+ // the copy of the database: the recovery fork of SQL Server, which each
72
+ // restore changes and a restart does not; Azure SQL Database has no such
73
+ // view, so there each value of sys.databases that a restore changes.
74
+ export async function databaseCopy(connection) {
75
+ const { name, azure } = await requireRowVersions(connection);
76
+ const [copy] = await connection.query(azure
77
+ ? `SELECT CONCAT([database_id], N' ', CONVERT(nvarchar(30), [create_date], 126), N' ', [physical_database_name]) AS [mark] FROM sys.databases WHERE [database_id] = DB_ID()`
78
+ : 'SELECT CONVERT(nvarchar(36), [recovery_fork_guid]) AS [mark] FROM sys.database_recovery_status WHERE [database_id] = DB_ID()');
79
+ if (typeof copy?.mark !== 'string')
80
+ throw new TypeError(`Database ${name} gave no mark of its copy`);
81
+ return copy.mark;
82
+ }
83
+ // Makes the numbering if the database has none, under the lock of
84
+ // _warehouse, and starts a new epoch when the database is another copy than
85
+ // the one that numbered it, such as one restored from a backup. Only one of
86
+ // two loads that find a new copy at once changes the epoch.
87
+ export async function installNumbering(connection, mark) {
88
+ const [made] = await connection.query(`SELECT CASE WHEN OBJECT_ID(N'${epochs}', N'U') IS NULL THEN 0 ELSE 1 END AS [made]`);
89
+ if (made?.made !== 1)
90
+ await transaction(connection, async () => {
91
+ await takeLock(connection, warehouse);
92
+ await connection.query(tables, { mark });
93
+ });
94
+ await connection.query(`UPDATE ${epochs} SET [epoch] = CONVERT(nvarchar(36), NEWID()), [mark] = @mark WHERE [id] = 1 AND [mark] <> @mark`, { mark });
95
+ }
96
+ // Lets a reader read the positions and the readings of the database with its
97
+ // own session: the numbering, and nothing else of _warehouse.
98
+ export async function grantReading(connection, reader) {
99
+ await connection.query(`GRANT SELECT ON ${epochs} TO ${quote(reader)}; GRANT SELECT ON ${commits} TO ${quote(reader)}; GRANT SELECT ON ${loads} TO ${quote(reader)}`);
100
+ }
101
+ // Records a load of owner that opens at openedAt, and gives its id. A load of
102
+ // the same owner that is still open is one whose process ended during it:
103
+ // the new load closes it. The UPDATE that closes it reads each row of
104
+ // _elt_loads with an update lock, as an nvarchar(max) owner has no index, so
105
+ // two opens that each held the row they inserted would wait for each other
106
+ // until SQL Server killed one (error 1205). The open takes the lock of
107
+ // _elt_loads before it inserts, so two opens run one at a time.
108
+ export async function openLoad(connection, owner, openedAt) {
109
+ const [column, name] = owned(owner);
110
+ const [load] = await transaction(connection, async () => {
111
+ await takeLock(connection, `${warehouse}._elt_loads`);
112
+ return connection.query(`DECLARE @load table ([id] bigint);
113
+ INSERT INTO ${loads} ([${column}], [opened_at]) OUTPUT inserted.[id] INTO @load VALUES (@name, CAST(@at AS datetimeoffset(3)));
114
+ UPDATE ${loads} SET [closed_at] = CAST(@at AS datetimeoffset(3))
115
+ WHERE [${column}] = @name AND [closed_at] IS NULL AND [id] <> (SELECT [id] FROM @load);
116
+ SELECT CONVERT(nvarchar(20), [id]) AS [id] FROM @load;`, { name, at: openedAt.toISOString() });
117
+ });
118
+ if (typeof load?.id !== 'string')
119
+ throw new Error('The load was not recorded');
120
+ return load.id;
121
+ }
122
+ export async function closeLoad(connection, load, closedAt) {
123
+ await connection.query(`UPDATE ${loads} SET [closed_at] = CAST(@at AS datetimeoffset(3)) WHERE [id] = CAST(@load AS bigint) AND [closed_at] IS NULL`, { load, at: closedAt.toISOString() });
124
+ }
125
+ // Inserts the commit row of a commit of load, as the first write of its
126
+ // transaction, and gives its rowversion, the number of the commit. A load
127
+ // that a later load of its owner closed has ended, and a commit after its end
128
+ // would move a position that readers were already given: the commit holds a
129
+ // shared lock on the row of its load, so a close that runs at the same time
130
+ // waits for the commit.
131
+ export async function commitNumber(connection, { load, owner }, committedAt, retention) {
132
+ const [open] = await connection.query(`SELECT 1 AS [open] FROM ${loads} WITH (HOLDLOCK, ROWLOCK) WHERE [id] = CAST(@load AS bigint) AND [closed_at] IS NULL`, { load });
133
+ if (open === undefined)
134
+ throw new Error('connector' in owner
135
+ ? `A later load of connection ${owner.connector} closed this load, so it cannot commit; a connection runs one pass at a time`
136
+ : `A later refresh of dataset ${owner.dataset} closed this one, so it cannot commit; a dataset refreshes one at a time`);
137
+ const [taken] = await connection.query(`INSERT INTO ${commits} ([load], [committed_at]) OUTPUT CONVERT(nvarchar(20), CAST(inserted.[number] AS bigint)) AS [number] VALUES (CAST(@load AS bigint), CAST(@at AS datetimeoffset(3)))`, { load, at: committedAt.toISOString() });
138
+ if (typeof taken?.number !== 'string')
139
+ throw new Error('No commit number was taken');
140
+ await retain(connection, committedAt, retention);
141
+ return Number(taken.number);
142
+ }
143
+ // Takes back the commit row of a commit that changed no row, before it
144
+ // commits: its number is a gap, as the number of a commit that rolled back
145
+ // is, so no reader sees a position that changed nothing.
146
+ export async function forgetCommit(connection, number) {
147
+ await connection.query(`DELETE FROM ${commits} WHERE [number] = CAST(CAST(@number AS bigint) AS binary(8))`, { number: String(number) });
148
+ }
149
+ // Applies the retention inside a commit's transaction, as elt-postgresql
150
+ // does: the newest position whose commit is older than the window becomes
151
+ // min_valid, and the numbering then keeps no commit before it and no closed
152
+ // load left with none. min_valid only moves forward, as commits of other
153
+ // connections run at the same time.
154
+ async function retain(connection, at, retention) {
155
+ const cutoff = new Date(at.getTime() - retention.total('milliseconds'));
156
+ const [row] = await connection.query(`SELECT CONVERT(nvarchar(20), (SELECT ISNULL(MAX(CAST([number] AS bigint)), 0) FROM ${commits} WHERE [committed_at] < CAST(@cutoff AS datetimeoffset(3)))) AS [older],
157
+ CONVERT(nvarchar(20), [min_valid]) AS [min_valid]
158
+ FROM ${epochs}`, { cutoff: cutoff.toISOString() });
159
+ if (row === undefined)
160
+ return;
161
+ const older = Number(row.older);
162
+ const kept = Number(row.min_valid);
163
+ if (older <= kept)
164
+ return;
165
+ const expired = (await numberingIn(connection, older))?.ends.at(-1);
166
+ if (expired === undefined || expired <= kept)
167
+ return;
168
+ await connection.query(`UPDATE ${epochs} SET [min_valid] = CAST(@expired AS bigint), [expired_by] = N'retention' WHERE [min_valid] < CAST(@expired AS bigint);
169
+ IF @@ROWCOUNT > 0
170
+ BEGIN
171
+ DELETE FROM ${commits} WHERE CAST([number] AS bigint) < CAST(@expired AS bigint);
172
+ DELETE FROM ${loads} WHERE [closed_at] IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${commits} AS c WHERE c.[load] = ${loads}.[id]);
173
+ END;`, { expired: String(expired) });
174
+ }
175
+ // The oldest commit that a reading can read as of, inside the transaction
176
+ // connection runs in.
177
+ export async function minValid(connection) {
178
+ const [row] = await connection.query(`SELECT CONVERT(nvarchar(20), [min_valid]) AS [min_valid] FROM ${epochs}`);
179
+ return Number(row?.min_valid ?? 0);
180
+ }
181
+ // The number of a change that is a load of one commit, such as a clear,
182
+ // taken inside the caller's transaction: the load opens, commits and closes
183
+ // in it. It is not a pass, so it leaves the open loads of its connector open.
184
+ export async function commitOnce(connection, connector, at, retention) {
185
+ const [load] = await connection.query(`INSERT INTO ${loads} ([connector], [opened_at]) OUTPUT CONVERT(nvarchar(20), inserted.[id]) AS [id] VALUES (@connector, CAST(@at AS datetimeoffset(3)))`, { connector, at: at.toISOString() });
186
+ if (typeof load?.id !== 'string')
187
+ throw new Error('The load was not recorded');
188
+ const number = await commitNumber(connection, { load: load.id, owner: { connector } }, at, retention);
189
+ await closeLoad(connection, load.id, at);
190
+ return number;
191
+ }
192
+ // Makes each position before the commit changedIn expire, inside that
193
+ // commit, for by: a change of type that is not a widening, or a clear that
194
+ // forgets the rows it removes.
195
+ export async function expireBefore(connection, changedIn, by) {
196
+ await connection.query(`UPDATE ${epochs} SET [min_valid] = CAST(@changed AS bigint), [expired_by] = @by WHERE [min_valid] < CAST(@changed AS bigint)`, { changed: String(changedIn), by });
197
+ }
198
+ // The numbering as the transaction that connection runs in sees it, with
199
+ // settled taken before its snapshot.
200
+ export async function numberingIn(connection, settled) {
201
+ const [row] = await connection.query(`SELECT [epoch], CONVERT(nvarchar(20), [min_valid]) AS [min_valid], [expired_by] FROM ${epochs}`);
202
+ if (row === undefined)
203
+ return null;
204
+ const spans = await connection.query(`SELECT CONVERT(nvarchar(20), MIN(CAST(c.[number] AS bigint))) AS [first],
205
+ CASE WHEN l.[closed_at] IS NULL THEN NULL ELSE CONVERT(nvarchar(20), MAX(CAST(c.[number] AS bigint))) END AS [last]
206
+ FROM ${commits} AS c JOIN ${loads} AS l ON l.[id] = c.[load]
207
+ GROUP BY l.[id], l.[closed_at]`);
208
+ const { expired_by: expiredBy } = row;
209
+ return numbering({
210
+ epoch: String(row.epoch),
211
+ minValid: Number(row.min_valid),
212
+ expiredBy: expiredBy === 'retention' || expiredBy === 'type' || expiredBy === 'clear'
213
+ ? expiredBy
214
+ : null,
215
+ settled,
216
+ spans: spans.map(({ first, last }) => ({
217
+ first: Number(first),
218
+ last: last === null ? null : Number(last),
219
+ })),
220
+ });
221
+ }
222
+ // The high-water mark: MIN_ACTIVE_ROWVERSION() is the lowest number that a
223
+ // transaction still holds, so each number below it is settled, and a
224
+ // snapshot that begins after the read sees each commit at or below the mark
225
+ // (ADR 0002). Null when the database has no numbering.
226
+ export async function settledMark(connection) {
227
+ const [mark] = await connection.query(`SELECT CONVERT(nvarchar(20), CAST(MIN_ACTIVE_ROWVERSION() AS bigint) - 1) AS [settled], CASE WHEN OBJECT_ID(N'${epochs}', N'U') IS NULL THEN 0 ELSE 1 END AS [made]`);
228
+ return mark?.made === 1 ? Number(mark.settled) : null;
229
+ }
230
+ // When the commit number was made, and the newest commit, as the transaction
231
+ // that connection runs in sees them; undefined for a number of no commit.
232
+ export async function committedAt(connection, number) {
233
+ const [row] = await connection.query(`SELECT [committed_at] AS [at], (SELECT CONVERT(nvarchar(20), MAX(CAST([number] AS bigint))) FROM ${commits}) AS [newest]
234
+ FROM ${commits} WHERE [number] = CAST(CAST(@number AS bigint) AS binary(8))`, { number: String(number) });
235
+ if (row === undefined)
236
+ return undefined;
237
+ if (!(row.at instanceof Date))
238
+ throw new TypeError(`Commit ${number} gave no time`);
239
+ return { at: row.at, newest: Number(row.newest) };
240
+ }
241
+ // The numbering in a SNAPSHOT transaction of its own, with the high-water
242
+ // mark read before it. connection is a session of its own, which keeps the
243
+ // SNAPSHOT isolation level after.
244
+ export async function readNumbering(connection) {
245
+ const settled = await settledMark(connection);
246
+ if (settled === null)
247
+ return null;
248
+ await connection.query('SET TRANSACTION ISOLATION LEVEL SNAPSHOT; BEGIN TRANSACTION');
249
+ try {
250
+ return await numberingIn(connection, settled);
251
+ }
252
+ finally {
253
+ await connection.query('COMMIT');
254
+ }
255
+ }
@@ -0,0 +1,4 @@
1
+ import { SqlServerDeduplicatingWriter } from './sql-server-deduplicating-writer.ts';
2
+ export declare class SqlServerOverwriteDeduplicatingWriter extends SqlServerDeduplicatingWriter {
3
+ protected get replaces(): boolean;
4
+ }
@@ -0,0 +1,8 @@
1
+ import { SqlServerDeduplicatingWriter } from './sql-server-deduplicating-writer.js';
2
+ // Replaces the rows of the target with one record for each key of a run, by
3
+ // difference.
4
+ export class SqlServerOverwriteDeduplicatingWriter extends SqlServerDeduplicatingWriter {
5
+ get replaces() {
6
+ return true;
7
+ }
8
+ }
@@ -0,0 +1,4 @@
1
+ import { SqlServerAppendWriter } from './sql-server-append-writer.ts';
2
+ export declare class SqlServerOverwriteWriter extends SqlServerAppendWriter {
3
+ protected get replaces(): boolean;
4
+ }
@@ -0,0 +1,9 @@
1
+ import { SqlServerAppendWriter } from './sql-server-append-writer.js';
2
+ // Replaces the rows of the target with the records of a run, by difference
3
+ // and with DELETE, not TRUNCATE: TRUNCATE takes a schema modification lock,
4
+ // which readers wait for.
5
+ export class SqlServerOverwriteWriter extends SqlServerAppendWriter {
6
+ get replaces() {
7
+ return true;
8
+ }
9
+ }
@@ -0,0 +1,24 @@
1
+ import { type OutputRow, type RuleColumns, type TableOutputRow, type TableRule } from '@eltjs/elt';
2
+ import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
3
+ import { TargetHistory } from './sql-server-history.ts';
4
+ import { type SqlServerIndex, TableIndexes } from './sql-server-indexes.ts';
5
+ import { TargetReaderView } from './sql-server-reader-view.ts';
6
+ import { type Declared } from './sql-server-rule-columns.ts';
7
+ export declare class SqlServerPartsTable {
8
+ #private;
9
+ readonly table: string;
10
+ readonly declared: readonly Declared[];
11
+ readonly statement: string;
12
+ readonly history: TargetHistory;
13
+ readonly view: TargetReaderView;
14
+ readonly indexes: TableIndexes;
15
+ readonly names: readonly string[];
16
+ constructor({ dataset, description, rule, indexes, }: {
17
+ dataset: string;
18
+ description: string;
19
+ rule: Pick<TableRule<unknown, RuleColumns, RuleColumns>, 'key' | 'output' | 'place'>;
20
+ indexes: readonly SqlServerIndex[];
21
+ });
22
+ write(connection: SqlServerConnection, commit: () => Promise<number>, outputs: readonly (OutputRow | TableOutputRow)[]): Promise<void>;
23
+ remove(connection: SqlServerConnection, replacedIn: number, keyHashes: readonly string[]): Promise<void>;
24
+ }