@eltjs/elt-sqlserver 0.0.0-stage → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +487 -2
  3. package/dist/computed-dataset.d.ts +31 -0
  4. package/dist/computed-dataset.js +377 -0
  5. package/dist/index.d.ts +13 -0
  6. package/dist/index.js +10 -0
  7. package/dist/sql-server-append-writer.d.ts +8 -0
  8. package/dist/sql-server-append-writer.js +13 -0
  9. package/dist/sql-server-as-of.d.ts +14 -0
  10. package/dist/sql-server-as-of.js +49 -0
  11. package/dist/sql-server-checkpoint-store.d.ts +10 -0
  12. package/dist/sql-server-checkpoint-store.js +101 -0
  13. package/dist/sql-server-column.d.ts +53 -0
  14. package/dist/sql-server-column.js +295 -0
  15. package/dist/sql-server-columns.d.ts +15 -0
  16. package/dist/sql-server-columns.js +49 -0
  17. package/dist/sql-server-connection.d.ts +6 -0
  18. package/dist/sql-server-connection.js +56 -0
  19. package/dist/sql-server-deduplicating-writer.d.ts +18 -0
  20. package/dist/sql-server-deduplicating-writer.js +102 -0
  21. package/dist/sql-server-dependents.d.ts +9 -0
  22. package/dist/sql-server-dependents.js +54 -0
  23. package/dist/sql-server-derived-status.d.ts +9 -0
  24. package/dist/sql-server-derived-status.js +60 -0
  25. package/dist/sql-server-descriptions.d.ts +6 -0
  26. package/dist/sql-server-descriptions.js +53 -0
  27. package/dist/sql-server-destination.d.ts +32 -0
  28. package/dist/sql-server-destination.js +95 -0
  29. package/dist/sql-server-failure-type.d.ts +2 -0
  30. package/dist/sql-server-failure-type.js +25 -0
  31. package/dist/sql-server-file-store.d.ts +21 -0
  32. package/dist/sql-server-file-store.js +91 -0
  33. package/dist/sql-server-fit.d.ts +20 -0
  34. package/dist/sql-server-fit.js +52 -0
  35. package/dist/sql-server-history.d.ts +15 -0
  36. package/dist/sql-server-history.js +148 -0
  37. package/dist/sql-server-identifiers.d.ts +10 -0
  38. package/dist/sql-server-identifiers.js +27 -0
  39. package/dist/sql-server-indexes.d.ts +20 -0
  40. package/dist/sql-server-indexes.js +68 -0
  41. package/dist/sql-server-load.d.ts +19 -0
  42. package/dist/sql-server-load.js +58 -0
  43. package/dist/sql-server-numbering.d.ts +32 -0
  44. package/dist/sql-server-numbering.js +255 -0
  45. package/dist/sql-server-overwrite-deduplicating-writer.d.ts +4 -0
  46. package/dist/sql-server-overwrite-deduplicating-writer.js +8 -0
  47. package/dist/sql-server-overwrite-writer.d.ts +4 -0
  48. package/dist/sql-server-overwrite-writer.js +9 -0
  49. package/dist/sql-server-parts-table.d.ts +24 -0
  50. package/dist/sql-server-parts-table.js +165 -0
  51. package/dist/sql-server-passes.d.ts +6 -0
  52. package/dist/sql-server-passes.js +71 -0
  53. package/dist/sql-server-reader-view.d.ts +12 -0
  54. package/dist/sql-server-reader-view.js +71 -0
  55. package/dist/sql-server-rule-columns.d.ts +12 -0
  56. package/dist/sql-server-rule-columns.js +69 -0
  57. package/dist/sql-server-schemas.d.ts +4 -0
  58. package/dist/sql-server-schemas.js +8 -0
  59. package/dist/sql-server-setup-error.d.ts +3 -0
  60. package/dist/sql-server-setup-error.js +6 -0
  61. package/dist/sql-server-sync-history-schema.d.ts +7 -0
  62. package/dist/sql-server-sync-history-schema.js +120 -0
  63. package/dist/sql-server-sync-history.d.ts +14 -0
  64. package/dist/sql-server-sync-history.js +165 -0
  65. package/dist/sql-server-table.d.ts +10 -0
  66. package/dist/sql-server-table.js +40 -0
  67. package/dist/sql-server-views.d.ts +7 -0
  68. package/dist/sql-server-views.js +70 -0
  69. package/dist/sql-server-writer.d.ts +40 -0
  70. package/dist/sql-server-writer.js +914 -0
  71. package/dist/view.d.ts +23 -0
  72. package/dist/view.js +82 -0
  73. package/package.json +48 -4
@@ -0,0 +1,60 @@
1
+ import { DerivedStatus } from '@eltjs/elt';
2
+ import { takeLock, transaction } from './sql-server-connection.js';
3
+ import { inWarehouse, warehouse } from './sql-server-schemas.js';
4
+ const table = inWarehouse('derived_status');
5
+ // One row for each derived dataset of the database: what its refreshes left,
6
+ // which readers see through marts.derived_status. The sync history makes it
7
+ // with its tables, and a dataset makes it too, so it never depends on the
8
+ // order of the two.
9
+ export const derivedStatusTable = `
10
+ IF OBJECT_ID(N'${table}', N'U') IS NULL
11
+ CREATE TABLE ${table} (
12
+ [dataset] nvarchar(300) NOT NULL PRIMARY KEY,
13
+ [kind] nvarchar(20) NOT NULL CHECK ([kind] IN (N'view', N'computed')),
14
+ [state] nvarchar(20) NOT NULL CHECK ([state] IN (N'waiting', N'refreshing', N'current', N'failed')),
15
+ [as_of] datetimeoffset(3) NULL, [refreshed_at] datetimeoffset(3) NULL, [refreshing_since] datetimeoffset(3) NULL, [error] nvarchar(max) NULL,
16
+ [inputs] bigint NULL, [computed] bigint NULL, [empty] bigint NULL, [failed] bigint NULL, [pending] bigint NULL);`;
17
+ // The derived_status of the database, where dataset is the view of the
18
+ // dataset in marts, such as marts.lengths.
19
+ export class SqlServerDerivedStatus extends DerivedStatus {
20
+ #connection;
21
+ #dataset;
22
+ #kind;
23
+ // connection is a session outside any transaction, between the dataset's
24
+ // own.
25
+ constructor(connection, dataset, kind, clock) {
26
+ super(clock);
27
+ this.#connection = connection;
28
+ this.#dataset = dataset;
29
+ this.#kind = kind;
30
+ }
31
+ // Inside the transaction of a commit that settled count inputs, on the
32
+ // session of the status.
33
+ async settled(count) {
34
+ await this.#connection.query(`UPDATE ${table} SET [pending] = GREATEST([pending] - CAST(@count AS bigint), 0) WHERE [dataset] = @dataset`, { count: String(count), dataset: this.#dataset });
35
+ }
36
+ // In a transaction of its own, which takes no commit number: the status is
37
+ // no data of a position. The lock of _warehouse serializes the make of the
38
+ // table, and the update and the insert of one row. Each value is text,
39
+ // which SQL Server converts to the type of its column: an ISO 8601 time to
40
+ // datetimeoffset, a count to bigint.
41
+ async write(values) {
42
+ const columns = ['kind', ...Object.keys(values)];
43
+ await transaction(this.#connection, async () => {
44
+ await takeLock(this.#connection, warehouse);
45
+ await this.#connection.query(`IF SCHEMA_ID(N'${warehouse}') IS NULL EXEC (N'CREATE SCHEMA [${warehouse}]');`);
46
+ await this.#connection.query(derivedStatusTable);
47
+ await this.#connection.query(`UPDATE ${table} SET ${columns.map((column) => `[${column}] = @${column}`).join(', ')} WHERE [dataset] = @dataset;
48
+ IF @@ROWCOUNT = 0
49
+ INSERT INTO ${table} ([dataset], ${columns.map((column) => `[${column}]`).join(', ')})
50
+ VALUES (@dataset, ${columns.map((column) => `@${column}`).join(', ')});`, {
51
+ dataset: this.#dataset,
52
+ kind: this.#kind,
53
+ ...Object.fromEntries(Object.entries(values).map(([column, value]) => [
54
+ column,
55
+ value === null ? null : String(value),
56
+ ])),
57
+ });
58
+ });
59
+ }
60
+ }
@@ -0,0 +1,6 @@
1
+ import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
2
+ export declare class TargetDescriptions {
3
+ #private;
4
+ constructor(table: string, columns: Readonly<Record<string, string | null>>);
5
+ write(connection: SqlServerConnection, kind: 'TABLE' | 'VIEW', schema: string, name: string): Promise<void>;
6
+ }
@@ -0,0 +1,53 @@
1
+ // SQL Server keeps at most 7,500 bytes in an extended property, and a value
2
+ // of nvarchar takes 2 bytes for each UTF-16 code unit.
3
+ const mostBytes = 7500;
4
+ // A target's descriptions as SQL Server keeps them: one MS_Description
5
+ // extended property on the table or its view, and one on each column, the
6
+ // property that SQL Server Management Studio and the SQL Server source of
7
+ // elt read. A null description removes the property.
8
+ export class TargetDescriptions {
9
+ #table;
10
+ #columns;
11
+ constructor(table, columns) {
12
+ for (const [name, description] of [
13
+ ['the table', table],
14
+ ...Object.entries(columns),
15
+ ])
16
+ if (description !== null && description.length * 2 > mostBytes)
17
+ throw new TypeError(`The description of ${name} takes ${description.length * 2} bytes; a SQL Server extended property holds at most ${mostBytes}`);
18
+ this.#table = table;
19
+ this.#columns = columns;
20
+ Object.freeze(this);
21
+ }
22
+ async write(connection, kind, schema, name) {
23
+ await connection.query(`DECLARE @object int = OBJECT_ID(QUOTENAME(@schema) + N'.' + QUOTENAME(@name));
24
+ DECLARE @column sysname, @description nvarchar(4000), @level2 varchar(10), @has bit;
25
+ DECLARE descriptions CURSOR LOCAL FAST_FORWARD FOR
26
+ SELECT [column], [description] FROM OPENJSON(@descriptions) WITH ([column] sysname '$[0]', [description] nvarchar(4000) '$[1]');
27
+ OPEN descriptions;
28
+ FETCH NEXT FROM descriptions INTO @column, @description;
29
+ WHILE @@FETCH_STATUS = 0
30
+ BEGIN
31
+ SET @level2 = IIF(@column IS NULL, NULL, 'COLUMN');
32
+ SET @has = IIF(EXISTS (SELECT 1 FROM sys.extended_properties WHERE [class] = 1 AND [major_id] = @object
33
+ AND [minor_id] = IIF(@column IS NULL, 0, COLUMNPROPERTY(@object, @column, 'ColumnId')) AND [name] = N'MS_Description'), 1, 0);
34
+ IF @has = 1 AND @description IS NULL
35
+ EXEC sp_dropextendedproperty N'MS_Description', 'SCHEMA', @schema, @kind, @name, @level2, @column;
36
+ ELSE IF @has = 1
37
+ EXEC sp_updateextendedproperty N'MS_Description', @description, 'SCHEMA', @schema, @kind, @name, @level2, @column;
38
+ ELSE IF @description IS NOT NULL
39
+ EXEC sp_addextendedproperty N'MS_Description', @description, 'SCHEMA', @schema, @kind, @name, @level2, @column;
40
+ FETCH NEXT FROM descriptions INTO @column, @description;
41
+ END;
42
+ CLOSE descriptions;
43
+ DEALLOCATE descriptions;`, {
44
+ schema,
45
+ name,
46
+ kind,
47
+ descriptions: JSON.stringify([
48
+ [null, this.#table],
49
+ ...Object.entries(this.#columns),
50
+ ]),
51
+ });
52
+ }
53
+ }
@@ -0,0 +1,32 @@
1
+ import { type CopyConfiguration, Destination, type FailureType, type Load, type PipelineClock } from '@eltjs/elt';
2
+ import { SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
3
+ import { type SqlServerReading } from './sql-server-as-of.ts';
4
+ import type { SqlServerColumn } from './sql-server-column.ts';
5
+ import { SqlServerColumns } from './sql-server-columns.ts';
6
+ import { type SqlServerLogin } from './sql-server-connection.ts';
7
+ import { SqlServerTable } from './sql-server-table.ts';
8
+ import type { SqlServerWriter } from './sql-server-writer.ts';
9
+ export declare const database: unique symbol;
10
+ export declare class SqlServerDestination extends Destination<SqlServerTable> {
11
+ readonly supportedDestinationSyncModes: readonly ["append", "overwrite", "append_dedup", "overwrite_dedup"];
12
+ readonly retention: Temporal.Duration;
13
+ readonly [database]: SqlServerDatabase;
14
+ constructor({ login, retention, }: {
15
+ login: SqlServerLogin;
16
+ retention?: Temporal.Duration;
17
+ });
18
+ location(target: SqlServerTable): string;
19
+ load(owner: {
20
+ clock: PipelineClock;
21
+ connection: string;
22
+ }): Promise<Load<SqlServerTable>>;
23
+ failureType(error: unknown): FailureType;
24
+ positions(after?: string): Promise<readonly string[] | {
25
+ readonly expired: string;
26
+ }>;
27
+ reading(position?: string): Promise<SqlServerReading | {
28
+ readonly expired: string;
29
+ }>;
30
+ table(name: string, configure?: (columns: SqlServerColumns) => readonly SqlServerColumn[]): SqlServerTable;
31
+ createWriter(configuration: CopyConfiguration, target: SqlServerTable): SqlServerWriter;
32
+ }
@@ -0,0 +1,95 @@
1
+ import { Destination, parsePosition, positionsAfter, } from '@eltjs/elt';
2
+ import { SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
3
+ import { SqlServerAppendWriter } from './sql-server-append-writer.js';
4
+ import { readAsOf } from './sql-server-as-of.js';
5
+ import { SqlServerColumns } from './sql-server-columns.js';
6
+ import { connect } from './sql-server-connection.js';
7
+ import { SqlServerDeduplicatingWriter } from './sql-server-deduplicating-writer.js';
8
+ import { failureType } from './sql-server-failure-type.js';
9
+ import { identifiers } from './sql-server-identifiers.js';
10
+ import { SqlServerLoad } from './sql-server-load.js';
11
+ import { readNumbering, sqlServerDatabase } from './sql-server-numbering.js';
12
+ import { SqlServerOverwriteDeduplicatingWriter } from './sql-server-overwrite-deduplicating-writer.js';
13
+ import { SqlServerOverwriteWriter } from './sql-server-overwrite-writer.js';
14
+ import { SqlServerTable } from './sql-server-table.js';
15
+ // The key of a destination's database, for the views and computed datasets
16
+ // of this package, which live in it. The index does not export it, so only
17
+ // this package connects through a destination.
18
+ export const database = Symbol('database');
19
+ // Loads into the database of a login: each target is a reader view in marts
20
+ // over a raw table in _warehouse, as in elt-postgresql (ADR 0001). The login
21
+ // carries credentials, so it stays private and out of location(), which is
22
+ // persisted with checkpoints.
23
+ export class SqlServerDestination extends Destination {
24
+ supportedDestinationSyncModes = Object.freeze([
25
+ 'append',
26
+ 'overwrite',
27
+ 'append_dedup',
28
+ 'overwrite_dedup',
29
+ ]);
30
+ // How long the history keeps a position readable: 30 days unless set. The
31
+ // destinations of one database share one numbering, so the shortest
32
+ // retention among them decides.
33
+ retention;
34
+ [database];
35
+ constructor({ login, retention = Temporal.Duration.from({ days: 30 }), }) {
36
+ super();
37
+ if (retention.sign < 0)
38
+ throw new RangeError(`Retention ${retention} must not be negative`);
39
+ this[database] = new SqlServerDatabase(login);
40
+ this.retention = retention;
41
+ Object.freeze(this);
42
+ }
43
+ // SQL Server compares names without case, so one raw table has one
44
+ // location.
45
+ location(target) {
46
+ return identifiers.key(target.raw);
47
+ }
48
+ // One connection for the whole run; each commit is its own transaction,
49
+ // numbered in the numbering that the destinations of the database share.
50
+ async load(owner) {
51
+ const load = await SqlServerLoad.open(this[database], { connector: owner.connection }, { clock: owner.clock, retention: this.retention });
52
+ return {
53
+ prepare: (configuration, target, binding) => this.createWriter(configuration, target).prepare(load, binding),
54
+ [Symbol.asyncDispose]: () => load[Symbol.asyncDispose](),
55
+ };
56
+ }
57
+ failureType(error) {
58
+ return failureType(error);
59
+ }
60
+ // The positions of the database after a position, ascending, or all of
61
+ // them; each one, `${epoch}:${number}`, names a commit where no pass of the
62
+ // database is half read. A position of another epoch, or past the newest
63
+ // commit, is expired.
64
+ async positions(after) {
65
+ const since = after === undefined ? undefined : parsePosition(after, sqlServerDatabase);
66
+ await using connection = await connect(this[database]);
67
+ return positionsAfter(await readNumbering(connection), since, sqlServerDatabase);
68
+ }
69
+ // The database as of a position, or as of its newest one, read through the
70
+ // reader views: { expired } for a position of another epoch, past the
71
+ // newest commit or before min_valid, and a TypeError for one inside a pass
72
+ // or not a position.
73
+ reading(position) {
74
+ return readAsOf(this[database], position);
75
+ }
76
+ table(name, configure) {
77
+ return new SqlServerTable(name, configure?.(new SqlServerColumns()));
78
+ }
79
+ createWriter(configuration, target) {
80
+ this.validateConfiguration(configuration, target);
81
+ if (!(target instanceof SqlServerTable))
82
+ throw new TypeError('SQL Server requires SQL Server table targets');
83
+ const table = target.resolve(configuration.stream);
84
+ switch (configuration.destinationSyncMode) {
85
+ case 'append':
86
+ return new SqlServerAppendWriter(configuration, this[database], table, this.retention);
87
+ case 'overwrite':
88
+ return new SqlServerOverwriteWriter(configuration, this[database], table, this.retention);
89
+ case 'append_dedup':
90
+ return new SqlServerDeduplicatingWriter(configuration, this[database], table, this.retention);
91
+ case 'overwrite_dedup':
92
+ return new SqlServerOverwriteDeduplicatingWriter(configuration, this[database], table, this.retention);
93
+ }
94
+ }
95
+ }
@@ -0,0 +1,2 @@
1
+ import type { FailureType } from '@eltjs/elt';
2
+ export declare function failureType(error: unknown): FailureType;
@@ -0,0 +1,25 @@
1
+ import { SqlServerLoginError, SqlServerPermissionError, } from '@eltjs/sdk-microsoft-sql-server';
2
+ import { SqlServerSetupError } from './sql-server-setup-error.js';
3
+ // The numbers of the errors of SQL Server for a login without a permission
4
+ // that a statement needs: on an object (229) or a column (230), or in the
5
+ // database, such as CREATE TABLE (262).
6
+ const permissionNumbers = new Set([229, 230, 262]);
7
+ // What kind of failure an error of the database that the destination or the
8
+ // checkpoint store writes is: config for a database that is not set up as the
9
+ // destination needs it (SqlServerSetupError), for a login that the server
10
+ // refuses (SqlServerLoginError: wrong credentials, a disabled login, a
11
+ // database it may not open) and for a login without a permission
12
+ // (SqlServerPermissionError, or an error with one of permissionNumbers), as
13
+ // elt-postgresql calls a wrong password config; system for every other error,
14
+ // such as a server that cannot be reached.
15
+ export function failureType(error) {
16
+ return error instanceof SqlServerSetupError ||
17
+ error instanceof SqlServerLoginError ||
18
+ error instanceof SqlServerPermissionError ||
19
+ (Error.isError(error) &&
20
+ 'number' in error &&
21
+ typeof error.number === 'number' &&
22
+ permissionNumbers.has(error.number))
23
+ ? 'config'
24
+ : 'system';
25
+ }
@@ -0,0 +1,21 @@
1
+ import { type FileContent } from '@eltjs/elt';
2
+ import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
3
+ import type { SqlServerColumn } from './sql-server-column.ts';
4
+ import type { SqlServerTable } from './sql-server-table.ts';
5
+ export declare class SqlServerFileStore {
6
+ #private;
7
+ readonly column: SqlServerColumn;
8
+ readonly qualifiedName: string;
9
+ readonly view: string;
10
+ constructor(table: SqlServerTable, column: SqlServerColumn);
11
+ static drop(connection: SqlServerConnection, table: SqlServerTable, column: string): Promise<void>;
12
+ stage(connection: SqlServerConnection): Promise<void>;
13
+ save(connection: SqlServerConnection, content: FileContent): Promise<{
14
+ readonly file: string;
15
+ readonly sha256: Uint8Array;
16
+ }>;
17
+ publish(connection: SqlServerConnection): Promise<void>;
18
+ discard(connection: SqlServerConnection): Promise<void>;
19
+ unstage(connection: SqlServerConnection): Promise<void>;
20
+ prune(connection: SqlServerConnection, tables: readonly string[]): Promise<void>;
21
+ }
@@ -0,0 +1,91 @@
1
+ import { hash, randomUUID } from 'node:crypto';
2
+ import { storedChunkSize, writeChunks } from '@eltjs/elt';
3
+ import { TargetDescriptions } from './sql-server-descriptions.js';
4
+ import { storedColumns } from './sql-server-fit.js';
5
+ import { inMarts, inWarehouse, marts } from './sql-server-schemas.js';
6
+ const chunks = '([file] uniqueidentifier NOT NULL, [n] bigint NOT NULL, [bytes] varbinary(max) NOT NULL, PRIMARY KEY ([file], [n]))';
7
+ // The original bytes of the files of one column, in chunks, as in
8
+ // elt-postgresql: the column holds the UNIQUEIDENTIFIER of its file, the
9
+ // chunks are a table in _warehouse, and readers read them through a view of
10
+ // the same name in marts. The chunks of a load wait in a #temp table of its
11
+ // connection until the commit of their rows moves them.
12
+ export class SqlServerFileStore {
13
+ column;
14
+ qualifiedName;
15
+ // The view in marts that readers join on file.
16
+ view;
17
+ #name;
18
+ #descriptions;
19
+ #staged;
20
+ constructor(table, column) {
21
+ this.column = column;
22
+ this.#name = SqlServerFileStore.#nameOf(table, column.name);
23
+ this.qualifiedName = inWarehouse(this.#name);
24
+ this.view = inMarts(this.#name);
25
+ this.#descriptions = new TargetDescriptions(`The original bytes of the files that column ${column.name} of ${marts}.${table.name} names, in chunks: join on file = that column and concatenate bytes in order of n.`, {
26
+ file: `The UNIQUEIDENTIFIER that column ${column.name} of ${marts}.${table.name} holds.`,
27
+ n: 'The place of the chunk in its file, from 0.',
28
+ bytes: `The bytes of the chunk, at most ${storedChunkSize} bytes.`,
29
+ });
30
+ this.#staged = `#elt_files_stage_${SqlServerFileStore.#key(table, column.name)}`;
31
+ }
32
+ // Drops the chunks of the column named column, which may no longer be one
33
+ // of the table's file columns, and their view.
34
+ static async drop(connection, table, column) {
35
+ const name = SqlServerFileStore.#nameOf(table, column);
36
+ await connection.query(`DROP VIEW IF EXISTS ${inMarts(name)}; DROP TABLE IF EXISTS ${inWarehouse(name)};`);
37
+ }
38
+ static #nameOf(table, column) {
39
+ return `_elt_files_${SqlServerFileStore.#key(table, column)}`;
40
+ }
41
+ static #key(table, column) {
42
+ return hash('sha256', JSON.stringify([table.raw, column])).slice(0, 40);
43
+ }
44
+ // A #temp table that a batch with parameters makes is dropped at the end
45
+ // of that batch, so the stage is made by a batch with none.
46
+ async stage(connection) {
47
+ await connection.query(`DROP TABLE IF EXISTS ${this.#staged}; CREATE TABLE ${this.#staged} ${chunks};`);
48
+ }
49
+ // Stages the file's bytes under a new id, and gives the id and the SHA-256
50
+ // of the bytes, which the row hash counts in place of the id. Each chunk
51
+ // travels as base64 in one JSON parameter, which OPENJSON reads as varchar:
52
+ // BASE64_DECODE takes no nvarchar (ADR 0011).
53
+ async save(connection, content) {
54
+ const file = randomUUID();
55
+ const sha256 = await writeChunks(content, async (n, bytes) => {
56
+ await connection.query(`INSERT INTO ${this.#staged} ([file], [n], [bytes]) SELECT CAST(@file AS uniqueidentifier), j.[n], BASE64_DECODE(j.[b]) FROM OPENJSON(@chunk) WITH ([n] bigint '$.n', [b] varchar(max) '$.b') AS j`, { file, chunk: JSON.stringify({ n, b: bytes.toBase64() }) });
57
+ });
58
+ return { file, sha256 };
59
+ }
60
+ // Moves the staged chunks into the store, inside a commit, and makes the
61
+ // store and its view when absent. CREATE VIEW must be the only statement
62
+ // of its batch.
63
+ async publish(connection) {
64
+ const [made] = await connection.query(`IF OBJECT_ID(@store, N'U') IS NULL CREATE TABLE ${this.qualifiedName} ${chunks};
65
+ SELECT IIF(OBJECT_ID(@view, N'V') IS NULL, 0, 1) AS [made];`, { store: this.qualifiedName, view: this.view });
66
+ if (made?.made !== 1) {
67
+ await connection.query(`CREATE VIEW ${this.view} AS SELECT [file], [n], [bytes] FROM ${this.qualifiedName}`);
68
+ await this.#descriptions.write(connection, 'VIEW', marts, this.#name);
69
+ }
70
+ await connection.query(`INSERT INTO ${this.qualifiedName} ([file], [n], [bytes]) SELECT [file], [n], [bytes] FROM ${this.#staged}; TRUNCATE TABLE ${this.#staged};`);
71
+ }
72
+ async discard(connection) {
73
+ await connection.query(`TRUNCATE TABLE ${this.#staged}`);
74
+ }
75
+ async unstage(connection) {
76
+ await connection.query(`DROP TABLE IF EXISTS ${this.#staged}`);
77
+ }
78
+ // Called inside each commit, after the merge: chunks that no row of the
79
+ // tables names, such as the chunks of a record that deduplication refused,
80
+ // go. The tables are the target, its history, whose rows keep their files
81
+ // for a reading as of a position before the commit that replaced them,
82
+ // and, while a reload is open, its hidden target. Only a table that keeps
83
+ // the column as a UNIQUEIDENTIFIER names a file.
84
+ async prune(connection, tables) {
85
+ const kept = [];
86
+ for (const table of tables)
87
+ if ((await storedColumns(connection, { table })).some(({ name, type }) => name === this.column.name && type === 'uniqueidentifier'))
88
+ kept.push(`NOT EXISTS (SELECT 1 FROM ${table} AS [kept] WHERE [kept].${this.column.quotedName} = ${this.qualifiedName}.[file])`);
89
+ await connection.query(`IF OBJECT_ID(@store, N'U') IS NOT NULL DELETE FROM ${this.qualifiedName}${kept.length === 0 ? '' : ` WHERE ${kept.join(' AND ')}`}`, { store: this.qualifiedName });
90
+ }
91
+ }
@@ -0,0 +1,20 @@
1
+ import type { StoredFit } from '@eltjs/elt';
2
+ import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
3
+ import type { SqlServerTable } from './sql-server-table.ts';
4
+ export declare const columnType = "LOWER(TYPE_NAME(c.[user_type_id])) + CASE\n WHEN TYPE_NAME(c.[user_type_id]) IN (N'nvarchar', N'nchar') THEN IIF(c.[max_length] = -1, N'(max)', CONCAT(N'(', c.[max_length] / 2, N')'))\n WHEN TYPE_NAME(c.[user_type_id]) IN (N'varchar', N'char', N'varbinary', N'binary') THEN IIF(c.[max_length] = -1, N'(max)', CONCAT(N'(', c.[max_length], N')'))\n WHEN TYPE_NAME(c.[user_type_id]) IN (N'decimal', N'numeric') THEN CONCAT(N'(', c.[precision], N',', c.[scale], N')')\n WHEN TYPE_NAME(c.[user_type_id]) IN (N'datetime2', N'datetimeoffset', N'time') THEN CONCAT(N'(', c.[scale], N')')\n WHEN TYPE_NAME(c.[user_type_id]) = N'vector' THEN CONCAT(N'(', c.[vector_dimensions], N')')\n ELSE N'' END";
5
+ type StoredColumn = {
6
+ readonly name: string;
7
+ readonly type: string;
8
+ readonly required: boolean;
9
+ };
10
+ export declare function storedColumns(connection: SqlServerConnection, relation: {
11
+ readonly table: string;
12
+ } | {
13
+ readonly stage: string;
14
+ }): Promise<readonly StoredColumn[]>;
15
+ export declare function storedFit(connection: SqlServerConnection, target: SqlServerTable, { table, stage, keyIndex, }: {
16
+ table: string;
17
+ stage: string;
18
+ keyIndex: string | null;
19
+ }): Promise<StoredFit>;
20
+ export {};
@@ -0,0 +1,52 @@
1
+ // A column's type as SqlServerColumn and sqlType of
2
+ // @zukhruf/vectors/sqlserver spell it, in lower case, from sys.columns.
3
+ export const columnType = `LOWER(TYPE_NAME(c.[user_type_id])) + CASE
4
+ WHEN TYPE_NAME(c.[user_type_id]) IN (N'nvarchar', N'nchar') THEN IIF(c.[max_length] = -1, N'(max)', CONCAT(N'(', c.[max_length] / 2, N')'))
5
+ WHEN TYPE_NAME(c.[user_type_id]) IN (N'varchar', N'char', N'varbinary', N'binary') THEN IIF(c.[max_length] = -1, N'(max)', CONCAT(N'(', c.[max_length], N')'))
6
+ WHEN TYPE_NAME(c.[user_type_id]) IN (N'decimal', N'numeric') THEN CONCAT(N'(', c.[precision], N',', c.[scale], N')')
7
+ WHEN TYPE_NAME(c.[user_type_id]) IN (N'datetime2', N'datetimeoffset', N'time') THEN CONCAT(N'(', c.[scale], N')')
8
+ WHEN TYPE_NAME(c.[user_type_id]) = N'vector' THEN CONCAT(N'(', c.[vector_dimensions], N')')
9
+ ELSE N'' END`;
10
+ // The columns a table stores, in order, with each one's type and whether it
11
+ // is NOT NULL; none when it does not exist. stage names a #temp table of the
12
+ // session, which tempdb keeps.
13
+ export async function storedColumns(connection, relation) {
14
+ const [catalog, object] = 'table' in relation
15
+ ? ['sys', relation.table]
16
+ : ['tempdb.sys', `tempdb..${relation.stage}`];
17
+ const rows = await connection.query(`SELECT c.[name], ${columnType} AS [type], CAST(c.[is_nullable] AS int) AS [nullable]
18
+ FROM ${catalog}.columns AS c WHERE c.[object_id] = OBJECT_ID(@object) ORDER BY c.[column_id]`, { object });
19
+ return rows.map(({ name, type, nullable }) => ({
20
+ name: String(name),
21
+ type: String(type),
22
+ required: nullable === 0,
23
+ }));
24
+ }
25
+ // Whether a stored table has the columns, types and NOT NULL the stream
26
+ // needs, and the index of its key hash, which names the key that the stored
27
+ // hashes are of. The stage is built from the same column declarations, so
28
+ // SQL Server spells both alike.
29
+ export async function storedFit(connection, target, { table, stage, keyIndex, }) {
30
+ const stored = await storedColumns(connection, { table });
31
+ if (stored.length === 0)
32
+ return 'missing';
33
+ const types = new Map((await storedColumns(connection, { stage })).map(({ name, type }) => [
34
+ name,
35
+ type,
36
+ ]));
37
+ const needed = [
38
+ ...target.columns.map((column) => `${column.name} ${types.get(column.name)} ${column.isPrimaryKey}`),
39
+ '_elt_row bigint true',
40
+ 'loaded_at datetimeoffset(3) true',
41
+ 'committed_in bigint true',
42
+ '_elt_row_hash binary(32) false',
43
+ ...(keyIndex === null ? [] : ['_elt_key_hash binary(32) true']),
44
+ ];
45
+ const has = stored.map(({ name, type, required }) => `${name} ${type} ${required}`);
46
+ if (needed.sort().join('\0') !== has.sort().join('\0'))
47
+ return 'stale';
48
+ if (keyIndex === null)
49
+ return 'fits';
50
+ const [index] = await connection.query('SELECT 1 AS [made] FROM sys.indexes WHERE [object_id] = OBJECT_ID(@table) AND [name] = @index', { table, index: keyIndex });
51
+ return index === undefined ? 'stale' : 'fits';
52
+ }
@@ -0,0 +1,15 @@
1
+ import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
2
+ import type { EncodedValue } from './sql-server-column.ts';
3
+ export declare class TargetHistory {
4
+ readonly name: string;
5
+ constructor(key: string);
6
+ get qualifiedName(): string;
7
+ create(connection: SqlServerConnection, target: string): Promise<void>;
8
+ follow(connection: SqlServerConnection, table: string): Promise<'follows' | 'retyped'>;
9
+ keep(connection: SqlServerConnection, replacedIn: number, target: string, changes: string, params: Readonly<Record<string, string>>): Promise<number>;
10
+ remove(connection: SqlServerConnection, replacedIn: number, target: string, where: string, params: Readonly<Record<string, string>>): Promise<number>;
11
+ expire(connection: SqlServerConnection): Promise<void>;
12
+ }
13
+ export declare function rowHash(columns: readonly {
14
+ readonly name: string;
15
+ }[], values: readonly (EncodedValue | Uint8Array)[]): string;
@@ -0,0 +1,148 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { storedColumns } from './sql-server-fit.js';
3
+ import { quote } from './sql-server-identifiers.js';
4
+ import { minValid } from './sql-server-numbering.js';
5
+ import { inWarehouse } from './sql-server-schemas.js';
6
+ // The rows of a target that commits replaced or removed, each with the
7
+ // number of the commit that did, as in elt-postgresql. Each column has the
8
+ // type of the target's column of the same name; the row number of the target
9
+ // is not kept (ADR 0003).
10
+ export class TargetHistory {
11
+ name;
12
+ constructor(key) {
13
+ this.name = `_elt_history_${key}`;
14
+ }
15
+ get qualifiedName() {
16
+ return inWarehouse(this.name);
17
+ }
18
+ // Made with the target, with the target's columns, none NOT NULL.
19
+ async create(connection, target) {
20
+ const columns = (await storedColumns(connection, { table: target })).filter(({ name }) => name !== '_elt_row');
21
+ await connection.query(`IF OBJECT_ID(@history, N'U') IS NULL
22
+ BEGIN
23
+ CREATE TABLE ${this.qualifiedName} (${[...columns.map(({ name, type }) => `${quote(name)} ${type} NULL`), '[replaced_in] bigint NOT NULL'].join(', ')});
24
+ CREATE INDEX ${quote(`${this.name}_replaced`)} ON ${this.qualifiedName} ([replaced_in]);
25
+ END`, { history: this.qualifiedName });
26
+ }
27
+ // Takes the shape of table, the target as a change of shape leaves it or
28
+ // the table that is to replace it: each column that table adds enters,
29
+ // nullable, a column that table drops stays, and a column that a widening
30
+ // changes takes the new type, which keeps each value. The history holds
31
+ // one type for each column, so when table changes the type of one it keeps
32
+ // in another way, it drops itself and gives 'retyped': the caller then
33
+ // expires each position before the change, which only its rows read.
34
+ async follow(connection, table) {
35
+ const kept = new Map((await storedColumns(connection, { table: this.qualifiedName })).map(({ name, type }) => [name, type]));
36
+ if (kept.size === 0)
37
+ return 'follows';
38
+ const next = (await storedColumns(connection, { table })).filter(({ name }) => name !== '_elt_row');
39
+ const changed = next.filter(({ name, type }) => {
40
+ const had = kept.get(name);
41
+ return had !== undefined && had !== type;
42
+ });
43
+ if (changed.some(({ name, type }) => !widens(String(kept.get(name)), type))) {
44
+ await connection.query(`DROP TABLE ${this.qualifiedName}`);
45
+ return 'retyped';
46
+ }
47
+ for (const { name, type } of changed)
48
+ await connection.query(`ALTER TABLE ${this.qualifiedName} ALTER COLUMN ${quote(name)} ${type} NULL`);
49
+ const added = next.filter(({ name }) => !kept.has(name));
50
+ if (added.length > 0)
51
+ await connection.query(`ALTER TABLE ${this.qualifiedName} ADD ${added.map(({ name, type }) => `${quote(name)} ${type} NULL`).join(', ')}`);
52
+ return 'follows';
53
+ }
54
+ // Runs changes, a statement that changes or removes rows of target and
55
+ // outputs deleted.*, and keeps each row it output that target stored, as
56
+ // replaced in commit replacedIn, in the same statement (ADR 0005). An
57
+ // insert outputs a deleted row of nulls, which it does not keep. Gives how
58
+ // many rows it kept.
59
+ async keep(connection, replacedIn, target, changes, params) {
60
+ const columns = (await storedColumns(connection, { table: target }))
61
+ .filter(({ name }) => name !== '_elt_row')
62
+ .map(({ name }) => quote(name));
63
+ const [kept] = await connection.query(`INSERT INTO ${this.qualifiedName} (${[...columns, '[replaced_in]'].join(', ')})
64
+ SELECT ${[...columns.map((column) => `[_elt_old].${column}`), 'CAST(@replacedIn AS bigint)'].join(', ')}
65
+ FROM (${changes}) AS [_elt_old] WHERE [_elt_old].[committed_in] IS NOT NULL;
66
+ SELECT @@ROWCOUNT AS [kept];`, { ...params, replacedIn: String(replacedIn) });
67
+ return Number(kept?.kept);
68
+ }
69
+ // Deletes the rows of target that where selects, in commit replacedIn, and
70
+ // keeps them in the same statement. where is a condition over the columns
71
+ // of target. Gives how many rows it kept.
72
+ remove(connection, replacedIn, target, where, params) {
73
+ return this.keep(connection, replacedIn, target, `DELETE FROM ${target} OUTPUT deleted.* WHERE ${where}`, params);
74
+ }
75
+ // Deletes the rows that only a reading before min_valid reads, which the
76
+ // retention left, once the history exists.
77
+ async expire(connection) {
78
+ await connection.query(`IF OBJECT_ID(@history, N'U') IS NOT NULL DELETE FROM ${this.qualifiedName} WHERE [replaced_in] <= CAST(@kept AS bigint)`, {
79
+ history: this.qualifiedName,
80
+ kept: String(await minValid(connection)),
81
+ });
82
+ }
83
+ }
84
+ // Whether ALTER COLUMN from type from to type to maps each old value to one
85
+ // value of the new type exactly, with no loss: a widening, as in
86
+ // elt-postgresql, with the types that SQL Server declares and converts.
87
+ // Text keeps the exact form of a number, a bit, a date, a time and a
88
+ // decimal, but not of a float, whose conversion keeps 6 digits, nor of
89
+ // bytes or JSON. A bigint fits a decimal with 19 digits before its point, a
90
+ // date a datetime2, a time more digits of its fraction, and a decimal more
91
+ // digits on each side of its point. bigint does not widen to float: past
92
+ // 2^53 it loses digits.
93
+ function widens(from, to) {
94
+ const [was, now] = [typeOf(from), typeOf(to)];
95
+ if (to === 'nvarchar(max)')
96
+ return [
97
+ 'bigint',
98
+ 'bit',
99
+ 'date',
100
+ 'decimal',
101
+ 'datetime2',
102
+ 'datetimeoffset',
103
+ 'time',
104
+ ].includes(was.name);
105
+ if (was.name === 'bigint' && now.name === 'decimal')
106
+ return now.precision - now.scale >= 19;
107
+ if (was.name === 'date' && now.name === 'datetime2')
108
+ return true;
109
+ if (was.name !== now.name)
110
+ return false;
111
+ if (was.name === 'decimal')
112
+ return (now.scale >= was.scale &&
113
+ now.precision - now.scale >= was.precision - was.scale);
114
+ return (['datetime2', 'datetimeoffset', 'time'].includes(was.name) &&
115
+ now.precision > was.precision);
116
+ }
117
+ // A type as SQL Server spells it in sys.columns: its name, and its
118
+ // precision and scale, or its fraction digits as its precision.
119
+ function typeOf(type) {
120
+ const [, name = type, precision = '0', scale = '0'] = /^(\w+)(?:\((\d+)(?:,(\d+))?\))?$/.exec(type) ?? [];
121
+ return { name, precision: Number(precision), scale: Number(scale) };
122
+ }
123
+ // The SHA-256 of the values of a staged row, which names its version, as in
124
+ // elt-postgresql: each value that is not null counts with its column's name,
125
+ // so a column the stream adds, null in each stored row, makes no new
126
+ // version. A stored file counts as the SHA-256 of its bytes, not as its id,
127
+ // a string as its text, and every other value as the JSON that the stage
128
+ // carries.
129
+ export function rowHash(columns, values) {
130
+ const digest = createHash('sha256');
131
+ const add = (tag, data) => {
132
+ digest.update(`${tag}${Buffer.byteLength(data)}:`);
133
+ digest.update(data);
134
+ };
135
+ columns.forEach((column, index) => {
136
+ const value = values[index];
137
+ if (value === null || value === undefined)
138
+ return;
139
+ add('c', column.name);
140
+ if (value instanceof Uint8Array)
141
+ add('b', value);
142
+ else if (typeof value === 'string')
143
+ add('t', value);
144
+ else
145
+ add('j', JSON.stringify(value));
146
+ });
147
+ return digest.digest('hex');
148
+ }
@@ -0,0 +1,10 @@
1
+ import { Identifiers } from '@eltjs/elt';
2
+ declare class SqlServerIdentifiers extends Identifiers {
3
+ protected readonly maxBytes = 128;
4
+ protected readonly ownColumns: string[];
5
+ key(name: string): string;
6
+ }
7
+ export declare const identifiers: Readonly<SqlServerIdentifiers>;
8
+ export declare const rawTable: (name: string) => string;
9
+ export declare function quote(name: string): string;
10
+ export {};