@eltjs/elt-sqlserver 0.0.0-stage → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +487 -2
  3. package/dist/computed-dataset.d.ts +31 -0
  4. package/dist/computed-dataset.js +377 -0
  5. package/dist/index.d.ts +13 -0
  6. package/dist/index.js +10 -0
  7. package/dist/sql-server-append-writer.d.ts +8 -0
  8. package/dist/sql-server-append-writer.js +13 -0
  9. package/dist/sql-server-as-of.d.ts +14 -0
  10. package/dist/sql-server-as-of.js +49 -0
  11. package/dist/sql-server-checkpoint-store.d.ts +10 -0
  12. package/dist/sql-server-checkpoint-store.js +101 -0
  13. package/dist/sql-server-column.d.ts +53 -0
  14. package/dist/sql-server-column.js +295 -0
  15. package/dist/sql-server-columns.d.ts +15 -0
  16. package/dist/sql-server-columns.js +49 -0
  17. package/dist/sql-server-connection.d.ts +6 -0
  18. package/dist/sql-server-connection.js +56 -0
  19. package/dist/sql-server-deduplicating-writer.d.ts +18 -0
  20. package/dist/sql-server-deduplicating-writer.js +102 -0
  21. package/dist/sql-server-dependents.d.ts +9 -0
  22. package/dist/sql-server-dependents.js +54 -0
  23. package/dist/sql-server-derived-status.d.ts +9 -0
  24. package/dist/sql-server-derived-status.js +60 -0
  25. package/dist/sql-server-descriptions.d.ts +6 -0
  26. package/dist/sql-server-descriptions.js +53 -0
  27. package/dist/sql-server-destination.d.ts +32 -0
  28. package/dist/sql-server-destination.js +95 -0
  29. package/dist/sql-server-failure-type.d.ts +2 -0
  30. package/dist/sql-server-failure-type.js +25 -0
  31. package/dist/sql-server-file-store.d.ts +21 -0
  32. package/dist/sql-server-file-store.js +91 -0
  33. package/dist/sql-server-fit.d.ts +20 -0
  34. package/dist/sql-server-fit.js +52 -0
  35. package/dist/sql-server-history.d.ts +15 -0
  36. package/dist/sql-server-history.js +148 -0
  37. package/dist/sql-server-identifiers.d.ts +10 -0
  38. package/dist/sql-server-identifiers.js +27 -0
  39. package/dist/sql-server-indexes.d.ts +20 -0
  40. package/dist/sql-server-indexes.js +68 -0
  41. package/dist/sql-server-load.d.ts +19 -0
  42. package/dist/sql-server-load.js +58 -0
  43. package/dist/sql-server-numbering.d.ts +32 -0
  44. package/dist/sql-server-numbering.js +255 -0
  45. package/dist/sql-server-overwrite-deduplicating-writer.d.ts +4 -0
  46. package/dist/sql-server-overwrite-deduplicating-writer.js +8 -0
  47. package/dist/sql-server-overwrite-writer.d.ts +4 -0
  48. package/dist/sql-server-overwrite-writer.js +9 -0
  49. package/dist/sql-server-parts-table.d.ts +24 -0
  50. package/dist/sql-server-parts-table.js +165 -0
  51. package/dist/sql-server-passes.d.ts +6 -0
  52. package/dist/sql-server-passes.js +71 -0
  53. package/dist/sql-server-reader-view.d.ts +12 -0
  54. package/dist/sql-server-reader-view.js +71 -0
  55. package/dist/sql-server-rule-columns.d.ts +12 -0
  56. package/dist/sql-server-rule-columns.js +69 -0
  57. package/dist/sql-server-schemas.d.ts +4 -0
  58. package/dist/sql-server-schemas.js +8 -0
  59. package/dist/sql-server-setup-error.d.ts +3 -0
  60. package/dist/sql-server-setup-error.js +6 -0
  61. package/dist/sql-server-sync-history-schema.d.ts +7 -0
  62. package/dist/sql-server-sync-history-schema.js +120 -0
  63. package/dist/sql-server-sync-history.d.ts +14 -0
  64. package/dist/sql-server-sync-history.js +165 -0
  65. package/dist/sql-server-table.d.ts +10 -0
  66. package/dist/sql-server-table.js +40 -0
  67. package/dist/sql-server-views.d.ts +7 -0
  68. package/dist/sql-server-views.js +70 -0
  69. package/dist/sql-server-writer.d.ts +40 -0
  70. package/dist/sql-server-writer.js +914 -0
  71. package/dist/view.d.ts +23 -0
  72. package/dist/view.js +82 -0
  73. package/package.json +48 -4
package/README.md CHANGED
@@ -1,3 +1,488 @@
1
- # Temporary Holding Version
1
+ # @eltjs/elt-sqlserver
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ The SQL Server destination of elt. It loads each stream of a connection into a SQL Server or Azure SQL database, with typed columns, and it does not block readers while it loads. Each target is a reader view in `marts`, which the host names, over a raw table in `_warehouse`. It also keeps the views and computed datasets that the pipeline derives. A computed dataset can keep a vector column as a native `VECTOR(n)`, so Azure SQL searches it with `VECTOR_DISTANCE` or a DiskANN index. The checkpoint store and the sync history keep their data in the same database.
4
+
5
+ The destination gives the contract of [elt-postgresql](../postgresql/README.md): the same declarations, the same columns for readers, the same positions, history, retention and sync history. A host changes its destination and nothing else. Where SQL Server is different, this README says how and why ([ADR 0001](./docs/adr/0001-the-sql-server-destination-takes-the-declarations-of-elt-postgresql.md)).
6
+
7
+ The words in these documents have one meaning each. See the glossary in [CONTEXT.md](./CONTEXT.md), and the glossary of [elt](../../elt/CONTEXT.md) for the words of all of elt.
8
+
9
+ ## The map
10
+
11
+ | Now (first release) | Later | Never |
12
+ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- |
13
+ | `SqlServerDestination`: the four destination sync modes, evolve, reload, reader views, extended properties, file chunks | Native `VECTOR` over TDS, when tedious has it ([tedious #1841](https://github.com/tediousjs/tedious/pull/1841)) | A search API in the destination: search is the host's SQL (`@zukhruf/vectors/sqlserver`) |
14
+ | Numbered commits, positions, history, readings, retention, a clear that forgets | `float16` vector columns (`HalfVector`), for a model of more than 1,998 dimensions, such as `text-embedding-3-large` (3,072) | System-versioned temporal tables as the history ([ADR 0001](./docs/adr/0001-the-sql-server-destination-takes-the-declarations-of-elt-postgresql.md)) |
15
+ | `View` and `ComputedDataset`, with `vector(n)` rule columns as `VECTOR(n)` and declared indexes: nonclustered, and DiskANN for `VECTOR_SEARCH` | A bulk load through Azure Blob Storage and `BULK INSERT`, as the bulk mode of Airbyte, when a load is too slow through JSON batches | An ORM, or a query builder over the reader's SQL |
16
+ | `SqlServerCheckpointStore`, `SqlServerSyncHistory` (with `derived_status`, `catalog` and named readers), `readPassStatus` | Microsoft Entra ID logins (managed identity), which `mssql` supports through `authentication` | An `mssql://` URL |
17
+ | The prerequisites in other packages: the `vector` rule column of elt, `dimensions` on `Embedding`, and the extraction of the rules that the destinations share ([ADR 0012](./docs/adr/0012-the-rules-that-the-destinations-share-move-into-elt-before-the-third-destination.md)) | SQL Server 2022 and older, which have no `VECTOR` and no `JSON` type | A migration of stored data between versions (elt AGENTS.md: stored data is disposable) |
18
+
19
+ ### What the first release does
20
+
21
+ - `SqlServerDestination` with the four destination sync modes: the raw table with its `_elt_row` identity key (ADR 0003), the unique index on `_elt_key_hash` when the copy deduplicates or the columns declare a key (ADR 0004), the `MERGE` of a deduplicating commit (ADR 0005, see "Deduplication" below), the replacement by difference of an overwrite (see "Locks and commits" below), the history table, the `MS_Description` extended properties, the reader view, and `positions()`.
22
+ - The JSON batches through `OPENJSON` (ADR 0011), the check of the engine and the row versioning at each load (ADR 0010), the numbering with the rowversion of the commit row and the mark of the database copy (ADR 0002), and the clear that forgets.
23
+ - The reset and the reload: a `RESET` of the stream or of a partition, and an overwrite whose stored table no longer has the shape of its stream, load into a hidden table, which the end of the stream gives to the table by difference or swaps in (see "Shape changes" below).
24
+ - `SqlServerCheckpointStore` and `SqlServerSyncHistory`, with the application locks of ADR 0009.
25
+ - The evolve: a load that is not an overwrite, whose stored table no longer has the shape of its stream, changes the table in place with `ALTER TABLE`, in one commit of its own, and the history follows each widening (see "Shape changes" below).
26
+ - The file columns: parsed text, references to the files of a storage, and the original bytes of a file in a file store (see "File bytes in the database" below).
27
+ - `reading(position?)`: a session of its own that reads the database as of a position (see "Commit numbers and positions" below), and `readPassStatus(login)`.
28
+ - `View`, and its rows of `derived_status` (see "Declare a view" and "Derived status" below).
29
+ - `ComputedDataset` of a `Rule` or a `TableRule`, with `text`, `blob`, `integer` and `vector(n)` rule columns, the parts table of a `TableRule`, declared indexes, and its rows of `derived_status` (see "Declare a computed dataset" below).
30
+ - A failure of the setup of the database is a `SqlServerSetupError`. Its [failure type](../../elt/README.md#failure-types) is `config`. A login without a permission that a statement of the destination or of `SqlServerCheckpointStore` needs is also `config`: a `SqlServerPermissionError` of sdk-microsoft-sql-server, and the SQL Server errors 229 and 230 (a permission on an object or a column) and 262 (a permission in the database, such as `CREATE TABLE`). A login that the server refuses (`SqlServerLoginError`: wrong credentials, a disabled login, a database that it may not open) is `config` too, as a wrong password is in elt-postgresql. Each other error, for example a server that cannot be reached, is `system`.
31
+
32
+ ## Load a stream into SQL Server
33
+
34
+ ```ts
35
+ import { Connection, Copy, Pipeline } from '@eltjs/elt';
36
+ import {
37
+ SqlServerCheckpointStore,
38
+ SqlServerDestination,
39
+ SqlServerSyncHistory,
40
+ } from '@eltjs/elt-sqlserver';
41
+
42
+ const login = {
43
+ server: 'datahub.database.windows.net',
44
+ database: 'semantic',
45
+ user: 'elt_writer',
46
+ password: '…',
47
+ };
48
+ const warehouse = new SqlServerDestination({ login });
49
+
50
+ await new Pipeline({
51
+ history: new SqlServerSyncHistory({ login }),
52
+ connections: [
53
+ new Connection({
54
+ name: 'sharepoint',
55
+ source: sharepoint,
56
+ destination: warehouse,
57
+ checkpoints: new SqlServerCheckpointStore({ login }),
58
+ steps: [
59
+ new Copy(sharepoint.driveItems, warehouse.table('drive_items'), {
60
+ id: 'sharepoint:drive-items',
61
+ syncMode: 'incremental',
62
+ destinationSyncMode: 'append_dedup',
63
+ }),
64
+ ],
65
+ }),
66
+ ],
67
+ }).run();
68
+ ```
69
+
70
+ - `SqlServerDestination({ login })` loads into the database of the login. Each relation that readers query is a view in `marts`, and each table of elt is in `_warehouse`, as in Postgres ([elt-postgresql ADR 0010](../postgresql/docs/adr/0010-readers-look-in-marts-and-elt-keeps-its-tables-in-warehouse.md)). The first load makes the two schemas.
71
+ - `warehouse.table('drive_items')` names the reader view `marts.drive_items`, the reader name. The destination keeps the rows of the target in the raw table `_warehouse.raw_drive_items`.
72
+ - `login` is the parts of a login (`server`, `port`, `database`, `user`, `password`, `trustServerCertificate`) or an ADO.NET connection string, the two forms of `@eltjs/sdk-microsoft-sql-server`. The destination opens its sessions through the `SqlServerDatabase` of the SDK, which gets write sessions for it.
73
+ - The login holds credentials, so `location()`, which checkpoints keep, names only the raw table. A getter gives the login to the views and computed datasets of the destination, so JSON and `util.inspect` do not show it.
74
+ - The destination checks the declarations with no connection to the database.
75
+ - A reader name follows the [naming rule](../../elt/README.md#names), with the 128 characters of a SQL Server name. `table(name)` refuses a reader name that the sync history publishes in `marts`, such as `catalog`.
76
+ - The database must be SQL Server 2025 (17.x), Azure SQL Database or Azure SQL Managed Instance with the always-up-to-date policy, because the destination uses the `VECTOR` and `JSON` types. It must keep row versions for readers: `READ_COMMITTED_SNAPSHOT` and `ALLOW_SNAPSHOT_ISOLATION` on ([ADR 0010](./docs/adr/0010-a-reading-sets-its-position-in-the-session-context-and-reads-row-versions.md)). The first load checks both, and fails with the failure type `config` and the `ALTER DATABASE` statement that turns them on.
77
+ - The destination holds one connection of its own for each load, because a `#stage` table belongs to one connection. It does not take connections from a shared pool.
78
+
79
+ ## The API
80
+
81
+ ```ts
82
+ // @eltjs/elt-sqlserver
83
+ type SqlServerLogin =
84
+ | string // an ADO.NET connection string
85
+ | {
86
+ server: string;
87
+ port?: number;
88
+ database: string;
89
+ user: string;
90
+ password: string;
91
+ trustServerCertificate?: boolean;
92
+ };
93
+
94
+ class SqlServerDestination extends Destination<SqlServerTable> {
95
+ constructor(options: {
96
+ login: SqlServerLogin;
97
+ retention?: Temporal.Duration; // 30 days unless set
98
+ });
99
+ readonly retention: Temporal.Duration;
100
+ table(
101
+ name: string, // the reader name: marts.<name> over _warehouse.raw_<name>
102
+ columns?: (c: SqlServerColumns) => SqlServerColumn[],
103
+ ): SqlServerTable;
104
+ location(target: SqlServerTable): string; // 'raw_<name>'
105
+ positions(after?: string): Promise<readonly string[] | { expired: string }>;
106
+ reading(position?: string): Promise<SqlServerReading | { expired: string }>;
107
+ }
108
+
109
+ class SqlServerTable {
110
+ readonly name: string; // the reader name
111
+ readonly raw: string; // raw_<name>
112
+ }
113
+
114
+ class View {
115
+ constructor(options: {
116
+ into: SqlServerDestination;
117
+ name: string;
118
+ description: string;
119
+ columns: Readonly<Record<string, string>>;
120
+ sql: string;
121
+ });
122
+ }
123
+
124
+ class ComputedDataset<Input> {
125
+ constructor(
126
+ options: {
127
+ into: SqlServerDestination;
128
+ name: string;
129
+ description: string;
130
+ from: readonly (SqlServerTable | View)[];
131
+ indexes?: readonly SqlServerIndex[]; // of _warehouse.raw_<name>
132
+ } & (
133
+ | { compute: Rule<Input, RuleColumns> }
134
+ | {
135
+ compute: TableRule<Input, RuleColumns, RuleColumns>;
136
+ partIndexes?: readonly SqlServerIndex[]; // of _warehouse.raw_<name>_parts
137
+ }
138
+ ),
139
+ );
140
+ }
141
+
142
+ type SqlServerIndex =
143
+ | { type: 'nonclustered'; columns: readonly [string, ...string[]] }
144
+ | { type: 'vector'; column: string; metric: 'cosine' | 'euclidean' | 'dot' }; // DiskANN
145
+
146
+ class SqlServerCheckpointStore {
147
+ constructor(options: { login: SqlServerLogin });
148
+ }
149
+
150
+ class SqlServerSyncHistory {
151
+ constructor(options: { login: SqlServerLogin; readers?: readonly string[] });
152
+ }
153
+
154
+ type SqlServerReading = AsyncDisposable & {
155
+ readonly sql: SqlServerConnection; // of @eltjs/sdk-microsoft-sql-server: its own session, in a SNAPSHOT transaction
156
+ readonly position: string;
157
+ readonly at: Temporal.Instant; // when the commit of the position was made
158
+ };
159
+
160
+ function readPassStatus(login: SqlServerLogin): Promise<{
161
+ readonly pass: PassStatus | null;
162
+ readonly streams: readonly StreamPassStatus[];
163
+ }>;
164
+ ```
165
+
166
+ ## What each call does underneath
167
+
168
+ | Call | What happens |
169
+ | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
170
+ | `new SqlServerDestination(…)` | Checks the retention. No I/O |
171
+ | `pipeline.run()` → `load()` | Opens one connection. Checks the engine and the row versioning of the database. Opens a load in `_warehouse._elt_loads` |
172
+ | `prepare(…)` | Checks `_elt_writers` and the stored table. Makes a `#stage` table on the connection of the load. Evolves a stored table that no longer has the shape of the stream, in one commit, when the copy does not overwrite. Publishes no row of the stream |
173
+ | each operation | Goes to the stage in batches of at most 1,000 rows or 16 MiB, as one `NVARCHAR(MAX)` JSON parameter that `OPENJSON … WITH (…)` reads and casts for each column ([ADR 0011](./docs/adr/0011-a-batch-travels-as-one-json-parameter-that-openjson-reads.md)) |
174
+ | each commit | One transaction: inserts its commit row, takes `CAST(inserted.number AS bigint)` as its number ([ADR 0002](./docs/adr/0002-a-commit-number-is-the-rowversion-of-its-commit-row.md)), then one `INSERT` from the stage into the table for `append`, or one `MERGE` for `append_dedup`, whose `OUTPUT` puts each replaced or removed row into the history. The commit of an `overwrite` or `overwrite_dedup` run replaces the rows by difference instead: one `DELETE … OUTPUT deleted.*` into the history, then one `INSERT` ([ADR 0005](./docs/adr/0005-a-commit-is-one-merge-whose-output-keeps-the-replaced-rows-in-the-history.md)) |
175
+ | the first commit of a stream | Also makes the table, its clustered `_elt_row` key ([ADR 0003](./docs/adr/0003-a-raw-table-numbers-its-rows-with-an-identity-column.md)), the unique index on `_elt_key_hash` ([ADR 0004](./docs/adr/0004-a-key-is-indexed-by-its-hash.md)), the extended properties, the reader view and the writer |
176
+ | a change of shape or the swap of a reload | `SET LOCK_TIMEOUT 3000`, `ALTER TABLE` or two `sp_rename` calls, then `sp_refreshview` for each view over the table ([ADR 0006](./docs/adr/0006-a-change-of-a-table-refreshes-the-views-over-it.md)) |
177
+ | `positions(after)` | Reads `MIN_ACTIVE_ROWVERSION()`, then `_elt_commits` and `_elt_loads` in one `SNAPSHOT` transaction |
178
+ | `reading(position)` | Opens a connection, checks the engine and the row versioning, reads the high-water mark, begins a `SNAPSHOT` transaction, and sets `elt.position` with `sp_set_session_context … @read_only = 1` |
179
+ | a refresh of a `View` | Opens one connection. In one transaction under the lock of `marts`, reads `OBJECT_DEFINITION` of the view, and runs `CREATE OR ALTER VIEW` and `sp_refreshview` of each view over it only when the text differs. Writes the row of `derived_status` |
180
+ | a refresh of a `ComputedDataset` | Opens a load of the dataset. In one transaction, makes the raw table, its history and its reader view, or makes the table again when its declaration changed, and removes the rows whose input is gone. Then reconciles the declared indexes with `sys.indexes`, one statement at a time outside a transaction. Then calls the rule, and for each settled call runs one transaction: it reads the row hashes of the call's keys, runs one `MERGE` of the changed rows from `OPENJSON`, whose `OUTPUT` the history keeps, and takes the settled inputs off `pending`. A `vector(n)` value goes as the JSON text of `@zukhruf/vectors/sqlserver` and lands through `CAST(… AS VECTOR(n))` |
181
+ | `SqlServerCheckpointStore` acquire | One session lock of `sp_getapplock` for each copy `id`, through `SqlServerStore` of `@zukhruf/mutex/sqlserver` ([ADR 0009](./docs/adr/0009-each-lock-is-an-application-lock-of-sql-server.md)) |
182
+
183
+ ## Columns
184
+
185
+ Inferred columns follow the stream schema, as in Postgres, with the types of SQL Server:
186
+
187
+ | JSON Schema | SQL Server |
188
+ | ----------------------------------------------- | ----------------------------------------------------------------------------- |
189
+ | `string` | `NVARCHAR(MAX)` |
190
+ | `string` + `format: 'date'` | `DATE` |
191
+ | `string` + `format: 'date-time'`, precision ≤ 7 | `DATETIMEOFFSET(p)` |
192
+ | `string` + `format: 'date-time-local'`, ≤ 7 | `DATETIME2(p)` |
193
+ | `string` + `format: 'time-local'`, ≤ 7 | `TIME(p)` |
194
+ | any of those three at precision 8–9 | `NVARCHAR(MAX)`, compared with a binary collation |
195
+ | `string` + `format: 'int64'` | `BIGINT` |
196
+ | `string` + `format: 'decimal'`, precision ≤ 38 | `DECIMAL(precision, scale)`, or `NVARCHAR(MAX)` when not declared or above 38 |
197
+ | `string` + `contentEncoding: 'base64'` | `VARBINARY(MAX)` |
198
+ | `integer` | `BIGINT` |
199
+ | `number` | `FLOAT` (53 bits) |
200
+ | `boolean` | `BIT` |
201
+ | `array` | `JSON`, with `CHECK (ISJSON(column, ARRAY) = 1)` |
202
+ | a `vector(n)` column of a rule | `VECTOR(n)`, float32, n at most 1,998 |
203
+
204
+ - SQL Server keeps time to 100 nanoseconds. Thus the `datetime2(7)` of a SQL Server source loads as `DATETIME2(7)` with no loss. Postgres keeps it as text.
205
+ - A `DATETIMEOFFSET` keeps the offset that the value had. A Postgres `TIMESTAMPTZ` does not. Readers compare two values by their instant in both.
206
+ - `DATE` starts at the year 0001. A value of the year 0000 fails its commit, and the error names the column and the key.
207
+ - Text is `NVARCHAR` (UTF-16), as in the SQL Server destination of Fivetran. A cursor of text compares with the binary order of its UTF-8 bytes, as `COLLATE "C"` does in Postgres: the comparison casts to `VARCHAR(MAX) COLLATE Latin1_General_100_BIN2_UTF8`. The collation of the database applies to the reader's own `WHERE`.
208
+ - `VECTOR` has no comparison operators and cannot be a key column. The row hash counts it, so no SQL compares it ([ADR 0004](./docs/adr/0004-a-key-is-indexed-by-its-hash.md)).
209
+ - Explicit columns are `columns.text`, `integer`, `int64`, `real`, `boolean`, `date`, `timestamp`, `localTimestamp(field, precision)` and `blob(field)`, with `.notNull()` and `.primaryKey()`, as in Postgres.
210
+ - Table, view and column names follow the [naming rule](../../elt/README.md#names). SQL Server keeps 128 characters of a name. A longer name keeps its first 119 characters, then `_` and the first 8 hex digits of the SHA-256 of the field or the requested name. The destination compares names without case, as SQLite does, because the default collation of a database does so. It reserves `loaded_at`, `committed_in`, `replaced_in`, `_elt_row`, `_elt_row_hash` and `_elt_key_hash`.
211
+
212
+ ## Store vectors
213
+
214
+ The computed dataset of an [`Embedding`](../../embedding/README.md) rule keeps its vectors in a `VECTOR(n)` column. The rule says how many dimensions its model gives, as a pgai vectorizer does ([ADR 0007](./docs/adr/0007-a-rule-column-can-be-a-vector-with-a-fixed-number-of-dimensions.md)):
215
+
216
+ ```ts
217
+ import { openai } from '@ai-sdk/openai';
218
+
219
+ import { ComputedDataset } from '@eltjs/elt-sqlserver';
220
+ import { Embedding } from '@eltjs/embedding';
221
+
222
+ new ComputedDataset({
223
+ into: marts,
224
+ name: 'staff_passage_vectors',
225
+ description: 'The vector of each staff passage.',
226
+ from: [passages],
227
+ compute: new Embedding({
228
+ model: openai.embedding('text-embedding-3-small'),
229
+ dimensions: 1536,
230
+ clock: globalThis,
231
+ key: 'id',
232
+ fields: ['text'],
233
+ }),
234
+ indexes: [{ type: 'vector', column: 'vector', metric: 'cosine' }],
235
+ });
236
+ ```
237
+
238
+ - The raw table `_warehouse.raw_staff_passage_vectors` has the column `vector VECTOR(1536)`. A value is the JSON text of `toSql` of `@zukhruf/vectors/sqlserver`, with 9 significant digits, which keeps each float32 value; the commit casts it with `CAST(… AS VECTOR(1536))`.
239
+ - A rule column of more than 1,998 dimensions fails the constructor of `ComputedDataset` with a `TypeError` that names the column and the limit of a float32 `VECTOR`.
240
+ - The vector index is `CREATE VECTOR INDEX … ON [_warehouse].[raw_staff_passage_vectors] ([vector]) WITH (METRIC = 'cosine', TYPE = 'DiskANN')`, which each refresh makes when the table does not have it, also after a changed declaration makes the table again (see "Declare a computed dataset" below). An index made on an empty table starts as an exact index, and Azure SQL builds its graph after the rows come (`@zukhruf/vectors`, "SQL Server: what the probes found").
241
+ - SQL Server 2025 in Docker makes a vector index only with `PREVIEW_FEATURES` on and only on an `int` clustered key, so it refuses the index of a computed dataset (error 42217); the tests check the statement there and nothing more ([ADR 0013, probe](./docs/adr/0013-a-computed-dataset-declares-its-indexes-and-the-destination-reconciles-them.md#probe-2026-11-21-vector-indexes-in-sql-server-2025-in-docker)).
242
+ - Azure SQL Database makes the index with no preview feature, on the `bigint` `_elt_row` of the raw table, and the `MERGE` of each commit works on the indexed table ([ADR 0013, Azure probe](./docs/adr/0013-a-computed-dataset-declares-its-indexes-and-the-destination-reconciles-them.md#probe-2026-11-21-vector-indexes-on-azure-sql-database)).
243
+ - The search is the host's own SQL, with every feature of Azure SQL. `VECTOR_SEARCH` uses the index only on the raw table, and only when no join is in the same query block. Thus the nearest rows come from `_warehouse.raw_<name>` in a common table expression, and a join comes after it:
244
+
245
+ ```sql
246
+ DECLARE @qv VECTOR(1536) = CAST(@q AS VECTOR(1536));
247
+ WITH nearest AS (
248
+ SELECT TOP (10) WITH APPROXIMATE v.id, r.distance
249
+ FROM VECTOR_SEARCH(TABLE = _warehouse.raw_staff_passage_vectors AS v, COLUMN = vector, SIMILAR_TO = @qv, METRIC = 'cosine') AS r
250
+ WHERE v.status = 'computed'
251
+ ORDER BY r.distance)
252
+ SELECT p.id, p.text, n.distance
253
+ FROM nearest AS n JOIN marts.staff_passages AS p ON p.id = n.id
254
+ ORDER BY n.distance;
255
+ ```
256
+
257
+ - `VECTOR_SEARCH` through the reader view `marts.staff_passage_vectors`, a `UNION ALL` of the raw table and its history, gives the same rows with an exact scan, and no index.
258
+ - The raw table holds the rows of now. A search as of a position reads the reader view in a reading, with `VECTOR_DISTANCE` or `VECTOR_SEARCH`, and no index.
259
+ - A reader that searches needs `SELECT` on the raw table. The `readers` of `SqlServerSyncHistory` get `SELECT` on the numbering of `_warehouse` only.
260
+ - See the recipe [Chunk documents, embed each chunk and search them with a vector index](./docs/recipes/search-chunks-with-a-vector-index.md).
261
+
262
+ ## File bytes in the database
263
+
264
+ SQL Server takes the same `text().from(file).parse(parser)`, `text().from(file.store(storage))` and `blob().from(file)` declarations as SQLite and Postgres. See [stored files](../../elt/README.md#stored-files-and-document-parsing) in elt.
265
+
266
+ - Parsed content is `NVARCHAR(MAX)`. A reference to a file of a storage is `NVARCHAR(MAX)` too.
267
+ - The column of an original file holds a `UNIQUEIDENTIFIER`. Its bytes are in a table `_warehouse._elt_files_<hash>` for each column, as `(file UNIQUEIDENTIFIER, n BIGINT, bytes VARBINARY(MAX))` chunks of at most 4 MiB (`storedChunkSize` of elt). Readers read them through the view `marts._elt_files_<hash>`. The `MS_Description` of the column names that view, and the view has its own.
268
+ - The hash is the first 40 hex characters of the SHA-256 of `JSON.stringify([rawTableName, columnName])`.
269
+ - To read the original, join the view on `file` and concatenate `bytes` in the order of `n`.
270
+ - An empty file has one empty chunk. A file that is not available gives a null value.
271
+ - Each chunk goes to the server as base64 text in one JSON parameter, which `OPENJSON` reads as `varchar(max)`: `BASE64_DECODE` does not take `nvarchar` (error 8116, [ADR 0011](./docs/adr/0011-a-batch-travels-as-one-json-parameter-that-openjson-reads.md)).
272
+ - The chunks of a load wait in a `#` table of the connection until the commit of their rows moves them. A discard empties that table.
273
+ - The row hash counts the SHA-256 of the bytes in place of the id. Each load gives a file a new id, so a row whose file is the same stays as it is, and the commit removes the new chunks.
274
+ - After each merge, the commit removes the chunks that no row of the table, of its history, or of its hidden table while a reload is open, names. A row of the history keeps its file.
275
+ - A file column that the stream no longer has loses its file store and its view when its table evolves or a reload swaps it.
276
+ - A clear of a copy also removes its files.
277
+ - A plain `blob()` column with no `.from(file)` stores an inline `VARBINARY(MAX)` value.
278
+
279
+ ## Descriptions
280
+
281
+ The descriptions of the source become `MS_Description` extended properties of each table, view and column (`sp_addextendedproperty`, `sp_updateextendedproperty`), the property that SQL Server Management Studio and the SQL Server source of elt read. The rules are those of [elt-postgresql](../postgresql/README.md#descriptions). An extended property holds at most 7,500 bytes; a longer description fails the declaration of the copy before a connection reads.
282
+
283
+ ## Locks and commits
284
+
285
+ - As in Postgres, a load takes no schema lock, and no transaction stays open between commits. Each commit is one short transaction.
286
+ - Readers never wait for a load: with `READ_COMMITTED_SNAPSHOT` on, a reader reads the last committed version of each row. A `DELETE` or a `MERGE` that escalates to a table lock does not block them.
287
+ - Before the commits of a load, the writer makes the tables that the targets of one database share in `_warehouse`, when they are not there, in a transaction of its own that holds a transaction lock of `sp_getapplock` on `_warehouse`, as in Postgres. No commit holds that lock. The publication and the withdrawal of a view take the lock of `marts` ([ADR 0009](./docs/adr/0009-each-lock-is-an-application-lock-of-sql-server.md)).
288
+ - The open of a load takes a transaction lock of `sp_getapplock` on `_warehouse._elt_loads` before it inserts the row of the load, so the opens of a database take turns. The open closes each open load of its owner with an `UPDATE`, which reads each row of the table with an update lock, because the owner is `NVARCHAR(MAX)` and has no index. Without the lock, two connections of one pipeline that open their loads at the same time each hold the row that it inserted and wait for the row of the other, and SQL Server stops one of them with error 1205 ([ADR 0009](./docs/adr/0009-each-lock-is-an-application-lock-of-sql-server.md)).
289
+ - A deduplicating commit merges the stage of one stream into its table with one `MERGE` on `_elt_key_hash`. The result is the same as when the operations apply one at a time, as in Postgres. The `OUTPUT` of the `MERGE` gives each row that it updated or deleted, and the same statement inserts them into the history ([ADR 0005](./docs/adr/0005-a-commit-is-one-merge-whose-output-keeps-the-replaced-rows-in-the-history.md)). See "Deduplication" below.
290
+ - An overwrite replaces the rows by difference with `DELETE`, not `TRUNCATE`: `TRUNCATE` takes a schema modification lock, and readers wait for it. The rows of the table and the records of the commit match as multisets of row hashes: the k-th row with a hash matches the k-th record with that hash, by the order of `_elt_row` and of the stage. A row with no match goes, and the same statement keeps it in the history (`INSERT INTO <history> SELECT … FROM (DELETE … OUTPUT deleted.*)`). A row with a match stays with its `committed_in` and its `loaded_at`. A record with no match enters. An `overwrite_dedup` matches only the record of each key that its reduced stage keeps. A reset deletes its scope in the same way.
291
+ - A row is told apart by `_elt_row`, its identity number, where Postgres uses `ctid` and SQLite uses `_rowid_` ([ADR 0003](./docs/adr/0003-a-raw-table-numbers-its-rows-with-an-identity-column.md)).
292
+ - The writer of each table is in `_warehouse._elt_writers`, by the [target ownership](../../elt/README.md#target-ownership) rule.
293
+
294
+ Shape changes:
295
+
296
+ - When the columns, types or `NOT NULL`s of a stored table do not fit the stream, and the copy does not overwrite, the load evolves the table in place before it reads, in one commit of its own: `ALTER TABLE … ADD` a nullable column, `DROP COLUMN`, and `ALTER COLUMN … type`. Before a conversion, each stored value that is not null must convert back to itself (`TRY_CAST(TRY_CAST(column AS new) AS old) = column` for each row), so `1.5` in `FLOAT` fails the load to `BIGINT` and names the column. A conversion from `NVARCHAR(MAX)` is the parse that the load applies to each new value, so it has no check, and a value that it cannot parse fails the `ALTER`. A failure leaves the table as it was. The `CHECK` of an array column goes before an `ALTER` or a `DROP` of its column, and comes back for a column that is an array after the change. `VECTOR` converts only to and from text, so a change of the dimensions of a rule column makes the raw table again.
297
+ - A row with a value in a column that the evolve drops or converts loses or changes the value, so it is a new version in the commit of the evolve: the history keeps the row as it was, and the row hash of the row becomes `NULL`, which matches no record until a commit writes the row again. A column that the evolve adds is `NULL` in each row, which a row hash does not count, so it makes no new version. An evolve that keeps no row and expires no position takes back its commit row.
298
+ - The history follows the evolve: it takes each new column, nullable, keeps each dropped one, and converts its column for a widening, so each position stays readable. Any other change of type expires each position before the commit of the evolve (see "Commit numbers and positions" below).
299
+ - The stored key hashes cannot be made again from the stored rows. Thus an evolve of a table whose key or key types changed fails the copy, with no change to the table; clear the copy to load it again.
300
+ - A change waits at most 3 seconds for readers (`SET LOCK_TIMEOUT 3000`), because `ALTER COLUMN` takes a schema modification lock, which also blocks readers that use row versions. Then it fails the copy until the next run.
301
+ - A reload goes into a hidden table `_elt_next_<hash>`, which readers do not see, and no commit into it keeps a row in the history. The end of the reload gives its rows to the table by difference when the shapes agree, as an overwrite replaces its rows, and swaps it in when they do not. A reload that fails continues at the next run, and a new `RESET` starts it again.
302
+ - A `RESET` of a partition reloads only that partition, as Delta Lake's `replaceWhere` replaces only the rows of its condition ([elt ADR 0024](../../elt/docs/adr/0024-a-reset-of-a-partition-reloads-it-into-a-hidden-target.md)). The rows of the reloaded partitions go to the hidden table, the rows of the others go to the table, and a deletion whose key does not name its partition goes to both. `_warehouse._elt_reloads` keeps the partitions of an open reload, so a run that continues it reloads the same ones. A partition condition compares text in a binary collation, so `a` and `A` are two partitions.
303
+ - The swap is one transaction, and it waits for readers as a change does. It gives the rows of the hidden table the number of the commit, keeps each row of the old table in the history, renames the old table and then the hidden table with `sp_rename`, and drops the old table. The history takes each new column, nullable, and converts its column for a widening. Any other change of the type of a column that the history keeps drops the history and expires each position before the swap, because the history holds one type for each column. The swapped-in table takes the grants of the old one, as Snowflake's `COPY GRANTS` does, and `CREATE OR ALTER VIEW` defines the reader view again over the new columns, with its grants.
304
+ - The views over the table read the new table after the change: SQL Server finds the tables of a view by name, so the destination only runs `sp_refreshview` for each of them. A change that a view `WITH SCHEMABINDING` stops, such as a swap, fails, and the destination names the schema-bound views in its error; an added column goes through ([ADR 0006](./docs/adr/0006-a-change-of-a-table-refreshes-the-views-over-it.md)).
305
+
306
+ ## Deduplication
307
+
308
+ An `append_dedup` copy keeps one row for each key, as in [elt-postgresql](../postgresql/README.md#locks-and-commits). The key is the key of the stream, or the `primaryKey` of the copy.
309
+
310
+ - The stage keeps each operation in the order of the source: a record, or a deletion of its key (`_elt_op`). A deletion carries the key values and its key hash, and no row hash.
311
+ - The rows match by `_elt_key_hash`, the SHA-256 of the JSON array of the key values, with a unique index on it. Thus two keys that differ only in case, such as `A` and `a`, stay two rows ([ADR 0004](./docs/adr/0004-a-key-is-indexed-by-its-hash.md)).
312
+ - A commit first reduces its stage to one operation for each key hash, as if the operations applied one at a time: only the operations after the last deletion of a key count, and a deletion with no record after it removes the key. `replace` keeps the last record. `cursor_newer` keeps the first record with the greatest cursor.
313
+ - Then one statement changes the table and keeps the history: `INSERT INTO <history> SELECT … FROM (MERGE <table> USING <reduced stage> … OUTPUT deleted.*) WHERE committed_in IS NOT NULL` ([ADR 0005](./docs/adr/0005-a-commit-is-one-merge-whose-output-keeps-the-replaced-rows-in-the-history.md)). A deletion deletes its row. A record after a deletion of its key replaces the row. Another record replaces the row when its row hash differs and, for `cursor_newer`, when its cursor is greater than the cursor of the row. A record of a new key is inserted. An inserted row outputs a `deleted` row of nulls, which the history does not keep.
314
+ - A record with the row hash of the stored row leaves the row, its `loaded_at` and its `committed_in` as they are, and the history keeps nothing. A commit that changes no row takes back its commit row, so it gives no position.
315
+ - A text cursor compares by its UTF-8 bytes: `CONVERT(varchar(max), cursor COLLATE Latin1_General_100_BIN2_UTF8)`, as `COLLATE "C"` does in Postgres. Thus `B` comes before `a`, and `A` (U+FF21) comes before an emoji.
316
+ - The index of the key hash is named after the table, its key and the types of the key (`_elt_key_<hash>`). After a checkpoint reset that gives the copy another key, or a change of the type of a key field, the stored hashes are of the old key values, so the table no longer fits, and the copy fails with no change to the table; clear the copy to load it again.
317
+ - A stored row with a null key or cursor fails the load with `Existing deduplication keys and cursors must be non-null`, as in Postgres.
318
+
319
+ ## Commit numbers and positions
320
+
321
+ One numbering orders the commits of the whole database, in the schema `_warehouse`, as in Postgres. The tables and the rules are those of [elt-postgresql](../postgresql/README.md#commit-numbers-and-positions): `_elt_commits`, `_elt_loads`, `_elt_epoch` with `min_valid` and `expired_by`, positions `<epoch>:<number>`, the history of each table, the widening rule, retention, and the clear that forgets.
322
+
323
+ How SQL Server is different:
324
+
325
+ - A commit number is the `rowversion` of the commit's row in `_elt_commits`, which the commit inserts before it changes a row. `MIN_ACTIVE_ROWVERSION()` gives the lowest number that a transaction still holds, so each number below it is settled: the high-water mark is `MIN_ACTIVE_ROWVERSION() - 1`. No lock and no sequence is necessary ([ADR 0002](./docs/adr/0002-a-commit-number-is-the-rowversion-of-its-commit-row.md)).
326
+ - Numbers are not dense. Writes of other `rowversion` columns of the database and commits that rolled back are gaps, which a reader skips, as in Postgres.
327
+ - A transaction that changes no row deletes its commit row before it commits, so it takes no number that a reader sees.
328
+ - The widenings of SQL Server, which `ALTER COLUMN` converts with no loss: `BIGINT`, `BIT`, `DATE`, `DECIMAL`, `DATETIME2`, `DATETIMEOFFSET` and `TIME` to `NVARCHAR(MAX)`; `BIGINT` to a `DECIMAL(p, s)` with p − s ≥ 19; `DATE` to `DATETIME2`; `DATETIME2`, `DATETIMEOFFSET` and `TIME` to more digits of a second; `DECIMAL(p, s)` to a `DECIMAL(q, t)` with t ≥ s and q − t ≥ p − s. `FLOAT` does not widen to text, because the conversion keeps 6 digits; `VARBINARY` and `JSON` do not either. `BIGINT` does not widen to `FLOAT`, because past 2^53 it loses digits. Any other change of type sets `min_valid`, as in Postgres.
329
+
330
+ A reading as of a position:
331
+
332
+ - Each reader view reads its table as of the position in `SESSION_CONTEXT(N'elt.position')`: the rows of the table with `committed_in <= P`, and the rows of the history with `committed_in <= P < replaced_in`. A session that sets nothing reads the table as it is now. A view built on a reader view reads as of the same position ([ADR 0010](./docs/adr/0010-a-reading-sets-its-position-in-the-session-context-and-reads-row-versions.md)).
333
+ - `await destination.reading(position?)` gives `{ sql, position, at }` or `{ expired }`, as in Postgres. With no position, it reads the newest position. `sql` is a `SqlServerConnection` of its own session, in a `SNAPSHOT` transaction, so a commit after the reading begins shows in no read. Dispose the reading to end the session.
334
+ - The position is a read-only key of the session context, so the reader cannot change it: a change fails with error 15664. Below the newest commit that the snapshot sees, the key holds the number of the position. At the newest commit, it holds the largest `bigint`, which reads each table as it is now, because a key with no value cannot be read-only ([ADR 0010](./docs/adr/0010-a-reading-sets-its-position-in-the-session-context-and-reads-row-versions.md)).
335
+ - A position of another epoch, past the newest commit, or before `min_valid` gives `{ expired }` with the reason, such as a clear that forgot the rows. A commit inside a pass, or a text that is not a position, is a `TypeError`.
336
+ - A file of a row that the history kept reads through `marts._elt_files_<hash>` as of the position, because the chunks of a file stay while a row of the table or of its history names it.
337
+
338
+ ## Checkpoints
339
+
340
+ `SqlServerCheckpointStore({ login })` keeps the checkpoints of every connection in `_warehouse._elt_checkpoints`, beside the data, as in Postgres. The key is the id of the copy, so the copy ids of every connection of the database must be different. `binding` and `state` are `NVARCHAR(MAX)`, which keeps each UTF-16 code unit, also `\u0000` and lone surrogates; the `JSON` type would refuse such text. The store holds one session lock for each copy `id` through `SqlServerStore` of `@zukhruf/mutex/sqlserver`, as `SQLiteCheckpointStore` holds one through `SqliteStore` of `@zukhruf/mutex/sqlite`. A run that finds one ID in use fails with `Checkpoint <id> is in use by another run`. The server releases a session lock when the session ends.
341
+
342
+ ## Reader views
343
+
344
+ As in [elt-postgresql](../postgresql/README.md#reader-views): `table(name)` names the view `marts.<name>` over `_warehouse.raw_<name>`. `committed_in`, `_elt_row`, `_elt_row_hash`, `_elt_key_hash` and `replaced_in` are internal, and the reader view does not show them.
345
+
346
+ ## Declare a view
347
+
348
+ As in [elt-postgresql](../postgresql/README.md#declare-a-view). The view lives in `marts`, and the bare names of its SQL are found in `marts`, as in Postgres. Thus the SQL reads the reader views. SQL Server resolves a bare name inside a view through the schema of the view and then `dbo`, for each user; the default schema of the user takes no part ([probe of 2026-11-20](./docs/adr/0006-a-change-of-a-table-refreshes-the-views-over-it.md#probe-2026-11-20-a-bare-table-name-inside-a-view)). Name a relation of another schema with its schema, such as `_warehouse.raw_drive_items`. A missing relation is error 208 (`Invalid object name`), and the view waits only when the pipeline makes that relation: a reader view or a raw table of a copy into the same database, or a view declared before it. A view over a relation that the pipeline does not make, or over a view declared after it, fails the refresh with the error of SQL Server.
349
+
350
+ - A refresh compares the stored definition (`OBJECT_DEFINITION`) with the new text, from the name of the view to its end: SQL Server keeps `CREATE OR ALTER VIEW` as `CREATE VIEW` (probe of 2026-11-20). When they are the same, it leaves the view as it was. When they differ, `CREATE OR ALTER VIEW` changes the view in place, so it keeps its object and its grants, and `sp_refreshview` refreshes each view over it, so a view over `*` shows the new columns ([ADR 0006](./docs/adr/0006-a-change-of-a-table-refreshes-the-views-over-it.md)).
351
+ - A view that the refresh makes again must have exactly the columns that the declaration describes. The descriptions become `MS_Description` extended properties. A view over `*` gets a new column of its table when the change of the table refreshes it, and the next refresh of the view then writes the description of that column.
352
+ - While a view waits, its refresh drops the view of its name that an earlier declaration made, and each view over it, deepest first, as elt-postgresql drops a view with `CASCADE`. SQL Server keeps a view over a dropped view, and each read of it then fails with error 4413, so the refresh drops that view too. A view of the pipeline over it waits, or is made again by its own refresh. A view that someone made over it is gone.
353
+ - The publication and the withdrawal take the lock of `marts`, each in a transaction of its own.
354
+
355
+ ## Declare a computed dataset
356
+
357
+ As in [elt-postgresql](../postgresql/README.md#declare-a-computed-dataset). A computed dataset keeps one output row for each key, in the table `_warehouse.raw_<name>`, beside its history. Readers query the view `marts.<name>`. The columns are the key, `input_hash`, `rule`, `status`, `error`, and the columns of the rule. A `text` column of a rule is `NVARCHAR(MAX)`, an `integer` column is `BIGINT`, a `blob` column is `VARBINARY(MAX)`, and a `vector(n)` column is `VECTOR(n)`. The key is `NVARCHAR(MAX)`, and the table is unique on the hash of the key ([ADR 0004](./docs/adr/0004-a-key-is-indexed-by-its-hash.md)). The refresh, its commits, its positions and its status follow elt-postgresql.
358
+
359
+ - The raw table also has `_elt_row`, its clustered key ([ADR 0003](./docs/adr/0003-a-raw-table-numbers-its-rows-with-an-identity-column.md)), `committed_in`, `_elt_row_hash` and `_elt_key_hash`. The constructor refuses a rule column whose name is one of these, the key, an engine column or `replaced_in`, or the name of another rule column, without case, as SQL Server compares two names.
360
+ - Each settled call of the rule is one commit: one `MERGE` on `_elt_key_hash` of the rows whose row hash changed, whose `OUTPUT` the history keeps ([ADR 0005](./docs/adr/0005-a-commit-is-one-merge-whose-output-keeps-the-replaced-rows-in-the-history.md)). A row whose values did not change keeps its commit, and a call that changes no row takes no commit number. The row hash counts a vector by its JSON text.
361
+ - When the stored columns of the raw table differ from its declaration, for example a vector of other dimensions, the refresh makes the table again in one commit: the history keeps each old row and takes the new shape, `CREATE OR ALTER VIEW` keeps the reader view, and `sp_refreshview` refreshes each view over it ([ADR 0006](./docs/adr/0006-a-change-of-a-table-refreshes-the-views-over-it.md)). A column whose type the history cannot widen, such as `VECTOR(3)` to `VECTOR(4)`, expires each position before that commit, as a reload of a stream does.
362
+ - The dataset reads each relation of `from` through its reader view in one statement. A table of `from` must be loaded by a copy into the same database.
363
+ - `qualifiedName` is `marts.<name>`. A `View` declared after the dataset that reads it waits while the dataset waits.
364
+ - `indexes` declares the indexes of the raw table: `{ type: 'nonclustered', columns }` or `{ type: 'vector', column, metric }` (see "Store vectors" above). The constructor refuses, with a `TypeError`, a column that the table does not have, a vector index on a column that is not a vector, and an index key that SQL Server refuses: `NVARCHAR(MAX)`, `VARBINARY(MAX)` or `VECTOR` (errors 1919 and 1978). Thus of the engine columns only `committed_in` can be a key.
365
+ - The destination names each index `ix_elt_` and the first 16 hex digits of the SHA-256 of its declaration. Each refresh, before it writes a row, reads the indexes of the table with that start from `sys.indexes`, drops each one that is no longer declared and makes each declared one that is missing, one statement at a time and outside a transaction, since SQL Server makes a vector index only there. No row is written or computed again, and an index of another name, such as a person's own, stays ([ADR 0013](./docs/adr/0013-a-computed-dataset-declares-its-indexes-and-the-destination-reconciles-them.md)). A person's index is gone after a changed declaration makes the table again; a declared one is made again.
366
+
367
+ A `TableRule` gives many parts for one input, as in [elt-postgresql](../postgresql/README.md) and [elt ADR 0026](../../elt/docs/adr/0026-a-rule-can-give-many-parts-for-one-input.md). The raw table then keeps one output row for each key, with no rule column, and the parts table `_warehouse.raw_<name>_parts` keeps one row for each part. Readers query the view `marts.<name>_parts`: the key, `seq` (`INT`, from 0), `part_hash` (`NVARCHAR(MAX)`), the content columns and the place columns.
368
+
369
+ - The parts table also has `_elt_row`, its clustered key, `committed_in`, `_elt_row_hash` and `_elt_key_hash`. It is unique on `_elt_key_hash` and `seq`, because the key is `NVARCHAR(MAX)` (ADR 0004). The key hash of a part is the key hash of its output row.
370
+ - The commit of a settled call writes the output rows and their parts. A part whose row hash did not change keeps its commit. The changed parts are one `MERGE` on `_elt_key_hash` and `seq`, and the parts beyond the new count of their row are one `DELETE`; the history keeps each part that they replace or remove. A row that is `empty` or `failed` has no parts, and a removed input removes its parts in the commit that removes its output row.
371
+ - The two tables are one declaration: when either one differs from its declaration, the refresh makes both again in one commit, and `sp_refreshview` refreshes the views over each.
372
+ - `partIndexes` declares the indexes of the parts table, as `indexes` does for the raw table. Only a `TableRule` takes it.
373
+ - The constructor refuses a column of the rule named `seq` or `part_hash` as well, in any case. `validate` refuses a copy that makes `<name>_parts` or `raw_<name>_parts`.
374
+ - `derived_status` counts the output rows, one for each key, not the parts.
375
+
376
+ ## When something fails
377
+
378
+ As in [elt-postgresql](../postgresql/README.md#when-something-fails). Also:
379
+
380
+ - A database with no row versioning for readers fails the first load with the failure type `config`.
381
+ - An engine older than SQL Server 2025 fails the first load with the failure type `config`.
382
+ - A value that a column refuses fails its commit. The error names the column and the key.
383
+ - A vector of other dimensions than its rule column fails its output row, with an error that names both counts. The run and the other rows succeed.
384
+
385
+ ## Sync history
386
+
387
+ `SqlServerSyncHistory({ login, readers })` is the sync history of a pipeline that loads into SQL Server, with the tables, views and rules of [elt-postgresql](../postgresql/README.md#sync-history): `_warehouse.sync_attempts`, `extraction_coverage`, `stream_checks` and `derived_status`, and the views `sync_attempts`, `extraction_coverage`, `sync_status`, `stream_status`, `derived_status` and `catalog` in `marts`. `install()` makes them in one transaction. `readers` are database users or roles; each install grants them `SELECT` on the numbering and nothing else of `_warehouse`. Each attempt has a session of its own that holds a session lock of `sp_getapplock` on the attempt, so the next attempt of its connection finds a pass whose process stopped and records it as `failed` (elt-postgresql ADR 0009). The `catalog` reads the `MS_Description` extended properties. The sync history keeps no row versions, so a session that reads as of a position before the newest commit fails each of its views with error 245 and the message `The sync history cannot be read as of a position: it keeps no row versions`; a reading at the newest commit reads them. `readPassStatus(login)` gives the latest pass of the database and the latest outcome of each stream, as it does for Postgres. It reads them in one statement, so it needs no `SNAPSHOT` transaction, and it reads the failed passes of a database that keeps no row versions.
388
+
389
+ ## Derived status
390
+
391
+ As in [elt-postgresql](../postgresql/README.md#derived-status), from elt ADR 0025: one row of `_warehouse.derived_status` for each view and computed dataset, named with its schema, which readers see as `marts.derived_status`. The sync history makes the table and the view, and a dataset makes the table too when the database has none. A `View` writes its row through `DerivedStatus` of elt, in a transaction of its own that holds the lock of `_warehouse`; the row of `marts.record_count` has the dataset `marts.record_count`. `ComputedDataset` writes its rows in the same way, and takes the settled inputs off `pending` inside the transaction of each commit, with `GREATEST(pending - count, 0)`.
392
+
393
+ ## Errors
394
+
395
+ | Error | When | The caller's next step |
396
+ | --------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------- |
397
+ | `TypeError` | A declaration that the destination cannot hold, such as a reader name of a relation that the sync history publishes, or a `vector(n)` column with n > 1,998 | Change the declaration |
398
+ | `RangeError` | A negative retention | Give a retention of zero or more |
399
+ | `TargetOwnedError` of elt | A second writer of a table | Clear the target with the copy that owns it |
400
+ | `DerivedDatasetError` of elt | A dataset failure, such as SQL that SQL Server refuses | Fix the SQL of the view |
401
+ | `SqlServerSetupError`, with the failure type `config` | No row versioning for readers, an engine older than SQL Server 2025 | Run the `ALTER DATABASE` of the message, or use SQL Server 2025 |
402
+ | `SqlServerLoginError` of sdk-microsoft-sql-server, with the failure type `config` | A login that the server refuses: wrong credentials, a disabled login, a database that it may not open | Fix the login |
403
+ | An error 229, 230 or 262 of SQL Server, with the failure type `config` | A login that may not do what a statement needs, such as `CREATE TABLE` for the checkpoints | Grant the login the permission that the message names |
404
+
405
+ ## The same concepts in other systems
406
+
407
+ | Here | Airbyte destination-mssql | Fivetran SQL Server | dlt mssql | Debezium JDBC sink (SQL Server) | Meltano target-mssql (storebrand) |
408
+ | ------------------------------------ | ------------------------------------------------------------- | ------------------- | ------------------------------------- | ------------------------------- | --------------------------------- |
409
+ | one JSON batch through `OPENJSON` | `INSERT` statements, or `BULK INSERT` from Azure Blob Storage | not documented | `INSERT` of at most 1,000 rows | prepared statements | a bulk insert into a `#` table |
410
+ | `MERGE` on `_elt_key_hash` | deduped sync modes; the mechanism is not documented | not documented | `delete-insert` (default) or `upsert` | `MERGE` on the primary key | `MERGE` from the `#` table |
411
+ | text as `NVARCHAR(MAX)` | `VARCHAR(MAX)` for its own columns | `NVARCHAR` | JSON as `nvarchar` | `NVARCHAR` | not documented |
412
+ | `DATETIMEOFFSET` for an instant | `DATETIMEOFFSET` | `DATETIMEOFFSET` | not documented | `TIMESTAMP` | not documented |
413
+ | evolve: add, drop, cast with a check | not documented | not documented | adds columns | adds columns only | not documented |
414
+ | `VECTOR(n)` rule column | none | none | none | "Vector types are unsupported" | none |
415
+
416
+ Sources: [Airbyte](https://docs.airbyte.com/integrations/destinations/mssql), [Fivetran](https://fivetran.com/docs/destinations/sql-server), [dlt mssql](https://dlthub.com/docs/dlt-ecosystem/destinations/mssql) and [merge loading](https://dlthub.com/docs/general-usage/merge-loading), [Debezium JDBC sink](https://debezium.io/documentation/reference/stable/connectors/jdbc.html), [storebrand/target-mssql `sinks.py`](https://github.com/storebrand/target-mssql/blob/main/target_mssql/sinks.py), [pgai vectorizer](https://github.com/timescale/pgai/blob/main/docs/vectorizer/api-reference.md).
417
+
418
+ ## Why no package does the job
419
+
420
+ Checked on 2026-11-19 (Node.js 26.13.0; npm search for `sql server destination`, `mssql elt`, `mssql etl loader`, `singer target mssql`, `airbyte mssql`, `mssql vector`, `mssql bulk upsert`, `dlt mssql`, and the `@stdlib` scope for `sqlserver`):
421
+
422
+ | Candidate | Downloads a week | Why it does not do the job |
423
+ | -------------------------------------------- | ---------------- | -------------------------------------------------------------------------------------------- |
424
+ | Node.js built-ins | n/a | No SQL Server client, no TDS. A search of the exports of every built-in module found nothing |
425
+ | `mssql` 12.7.15 (on tedious 20.3.9) | 4,980,600 | The driver, not a destination. The destination uses it, as the SQL Server SDK of elt does |
426
+ | `faros-airbyte-cdk` 0.23.67 | 3,301 | A connector kit of Airbyte for JavaScript; it gives no SQL Server destination |
427
+ | `airbyte-faros-destination` 0.23.67 | 1,748 | It writes to the Faros API, not to SQL Server |
428
+ | `@prisma/adapter-mssql` 7.10.1 | 127,540 | A driver adapter of an ORM |
429
+ | Airbyte, dlt, Meltano, Debezium destinations | n/a | Java, Kotlin and Python programs; none keeps elt's positions, history or derived datasets |
430
+
431
+ `@zukhruf/vectors/sqlserver` is adopted for the vector form (its ADR 0007, "SQL Server gets JSON text with the digits that keep each value"), and `@zukhruf/mutex/sqlserver` for the checkpoint locks.
432
+
433
+ ## Who uses what
434
+
435
+ | Package or host | What it takes |
436
+ | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
437
+ | datahub, `apps/semantic-layer/elt` | `SqlServerDestination`, `View`, `ComputedDataset` with `Embedding` (backlog #4738), the history |
438
+ | `@eltjs/elt` | gives the `Destination` port, `Rule` with the `vector` column ([ADR 0007](./docs/adr/0007-a-rule-column-can-be-a-vector-with-a-fixed-number-of-dimensions.md)), and the rules that three destinations share ([ADR 0012](./docs/adr/0012-the-rules-that-the-destinations-share-move-into-elt-before-the-third-destination.md)) |
439
+ | `@eltjs/embedding` | gives `Vector` values of its `dimensions` |
440
+ | `@zukhruf/vectors` | `toSql` and the `VECTOR(n)` type of a column |
441
+ | `@zukhruf/mutex` | `SqlServerStore` for the checkpoint locks |
442
+ | `@zukhruf/testing` | `SqlServer`, the database of the tests (SQL Server 2025 in Docker; its DiskANN index is a preview that needs an `int` clustered key, so the tests run the exact search only) |
443
+
444
+ ## Ezz's rulings
445
+
446
+ Of 2026-11-19, except where a ruling gives another date.
447
+
448
+ 1. **The order.** The SQL Server destination is built last. First the rules that the destinations share move into elt, from the two copies and checked against this paper; then the open defects and the gaps ([ADR 0012](./docs/adr/0012-the-rules-that-the-destinations-share-move-into-elt-before-the-third-destination.md)).
449
+ 2. **The value of a vector column** is a `Vector` of `@zukhruf/vectors`, and `@eltjs/elt` takes that package ([ADR 0007](./docs/adr/0007-a-rule-column-can-be-a-vector-with-a-fixed-number-of-dimensions.md)).
450
+ 3. **The login.** The destination takes the `SqlServerDatabase` of `@eltjs/sdk-microsoft-sql-server`, and the SDK gets write sessions.
451
+ 4. **The type of a vector column.** `@zukhruf/vectors` gives the type for a kind of vector and a count of dimensions, such as `sqlType(Vector, 1536)`.
452
+ 5. **Indexes.** A computed dataset declares its indexes, and the destination reconciles them with `sys.indexes` at each refresh; `afterCreate` is removed (2026-11-21, [ADR 0013](./docs/adr/0013-a-computed-dataset-declares-its-indexes-and-the-destination-reconciles-them.md), which supersedes [ADR 0008](./docs/adr/0008-a-computed-dataset-runs-the-sql-of-the-host-each-time-it-makes-its-raw-table.md)).
453
+ 6. **Keys.** A unique index on `_elt_key_hash` ([ADR 0004](./docs/adr/0004-a-key-is-indexed-by-its-hash.md)).
454
+ 7. **The release.** The package is public after its tests pass, so the next green push of `main` publishes it, and datahub installs it from npm.
455
+
456
+ ## Probes
457
+
458
+ Each decision above that depends on the behavior of the server has a probe, whose output is in its ADR. The first seven ran on 2026-11-20, on Azure SQL Database (datahub BILLING) and on SQL Server 2025 (RTM-CU11) in Docker, except where a line says otherwise. Two results corrected a decision: ADR 0006 (a schema-bound view) and ADR 0002 (the mark of a database copy on Azure SQL Database).
459
+
460
+ - Done: `MIN_ACTIVE_ROWVERSION()` read before a `SNAPSHOT` transaction: a commit at or below the mark is always visible to the snapshot. Holds ([ADR 0002, probe](./docs/adr/0002-a-commit-number-is-the-rowversion-of-its-commit-row.md#probe-2026-11-20-the-high-water-mark-and-a-snapshot-transaction)).
461
+ - Done: `MERGE` with `OUTPUT` into the history, through composable DML, on a table with a `VECTOR` column and a DiskANN index. Holds; the index was on Azure SQL only, because SQL Server 2025 in Docker has none ([ADR 0005, probe](./docs/adr/0005-a-commit-is-one-merge-whose-output-keeps-the-replaced-rows-in-the-history.md#probe-2026-11-20)).
462
+ - Done: `OPENJSON … WITH (v NVARCHAR(MAX) '$.v')` with `CAST(v AS VECTOR(1536))` for a batch of 64 vectors. Holds; a base64 column must be `varchar(max)` for `BASE64_DECODE` ([ADR 0011, probe](./docs/adr/0011-a-batch-travels-as-one-json-parameter-that-openjson-reads.md#probe-2026-11-20)).
463
+ - Done: `sp_rename` of a table while a view reads it, then `sp_refreshview`; and the error of a `SCHEMABINDING` view. The refresh holds; a `SCHEMABINDING` view stops a swap, a drop and a change of a column that it reads, but not an `ADD` or a change of a column that it does not read, so ADR 0006 now says the destination names those views in its own error ([ADR 0006, probe](./docs/adr/0006-a-change-of-a-table-refreshes-the-views-over-it.md#probe-2026-11-20-sp_rename-sp_refreshview-and-schemabinding)).
464
+ - Done: `is_read_committed_snapshot_on` and `snapshot_isolation_state` of the datahub database. Both on ([ADR 0010, probe](./docs/adr/0010-a-reading-sets-its-position-in-the-session-context-and-reads-row-versions.md#probe-2026-11-20)).
465
+ - Done: How SQL Server resolves a bare table name inside a view in the schema `marts`, when the user that runs the query has another default schema. The schema of the view, then `dbo`, never the default schema of the user; the user with another default schema ran in Docker only ([ADR 0006, probe](./docs/adr/0006-a-change-of-a-table-refreshes-the-views-over-it.md#probe-2026-11-20-a-bare-table-name-inside-a-view)).
466
+ - Done: Whether a point-in-time restore of Azure SQL changes `recovery_fork_guid` in `sys.database_recovery_status`, so a restored database can start a new epoch. First in Docker (Ezz, 2026-11-19): on SQL Server 2025 each restore changes it and the numbers repeat after the restore point; Azure SQL Database has no `sys.database_recovery_status` ([ADR 0002, probe](./docs/adr/0002-a-commit-number-is-the-rowversion-of-its-commit-row.md#probe-2026-11-20-recovery_fork_guid-after-a-restore)). A point-in-time restore on Azure SQL Database (a scratch server in Ezz's own subscription, Ezz, 2026-11-20) changes `database_id`, `create_date` and `physical_database_name` of `sys.databases`, which a plain database user can read, and the numbers repeat; the rename to the original name keeps `database_id` and `physical_database_name`, and a change of service objective also changes `physical_database_name`. On Azure SQL Database the epoch keeps all three and starts a new epoch when any one changes, because a missed restore gives wrong rows and a needless new epoch only a full read ([ADR 0002, probe](./docs/adr/0002-a-commit-number-is-the-rowversion-of-its-commit-row.md#probe-2026-11-20-an-azure-sql-database-point-in-time-restore)).
467
+ - Done: A vector index of a computed dataset on Azure SQL Database (a scratch server in Ezz's own subscription, 2026-11-21), through a real pipeline: the DiskANN index on the `bigint` `_elt_row`, the `MERGE` and its `OUTPUT deleted.*` on the indexed table, the names in `sys.indexes`, a change of the metric on a filled table with no recompute, and a reading as of a position. Each holds. `VECTOR_SEARCH` through `marts.<name>` runs with no index, so the search reads the raw table ([ADR 0013, Azure probe](./docs/adr/0013-a-computed-dataset-declares-its-indexes-and-the-destination-reconciles-them.md#probe-2026-11-21-vector-indexes-on-azure-sql-database)).
468
+
469
+ ## Recipes
470
+
471
+ - [Load a stream into SQL Server and read it as of a position](./docs/recipes/load-a-stream-and-read-it-as-of-a-position.md)
472
+ - [Chunk documents, embed each chunk and search them with a vector index](./docs/recipes/search-chunks-with-a-vector-index.md)
473
+
474
+ ## Decisions
475
+
476
+ - [ADR 0001](./docs/adr/0001-the-sql-server-destination-takes-the-declarations-of-elt-postgresql.md): the SQL Server destination takes the declarations of elt-postgresql.
477
+ - [ADR 0002](./docs/adr/0002-a-commit-number-is-the-rowversion-of-its-commit-row.md): a commit number is the rowversion of its commit row.
478
+ - [ADR 0003](./docs/adr/0003-a-raw-table-numbers-its-rows-with-an-identity-column.md): a raw table numbers its rows with an identity column.
479
+ - [ADR 0004](./docs/adr/0004-a-key-is-indexed-by-its-hash.md): a key is indexed by its hash.
480
+ - [ADR 0005](./docs/adr/0005-a-commit-is-one-merge-whose-output-keeps-the-replaced-rows-in-the-history.md): a commit is one `MERGE` whose output keeps the replaced rows in the history.
481
+ - [ADR 0006](./docs/adr/0006-a-change-of-a-table-refreshes-the-views-over-it.md): a change of a table refreshes the views over it.
482
+ - [ADR 0007](./docs/adr/0007-a-rule-column-can-be-a-vector-with-a-fixed-number-of-dimensions.md): a rule column can be a vector with a fixed number of dimensions.
483
+ - [ADR 0008](./docs/adr/0008-a-computed-dataset-runs-the-sql-of-the-host-each-time-it-makes-its-raw-table.md): a computed dataset runs the SQL of the host each time it makes its raw table. Superseded by ADR 0013.
484
+ - [ADR 0009](./docs/adr/0009-each-lock-is-an-application-lock-of-sql-server.md): each lock is an application lock of SQL Server.
485
+ - [ADR 0010](./docs/adr/0010-a-reading-sets-its-position-in-the-session-context-and-reads-row-versions.md): a reading sets its position in the session context and reads row versions.
486
+ - [ADR 0011](./docs/adr/0011-a-batch-travels-as-one-json-parameter-that-openjson-reads.md): a batch travels as one JSON parameter that `OPENJSON` reads.
487
+ - [ADR 0012](./docs/adr/0012-the-rules-that-the-destinations-share-move-into-elt-before-the-third-destination.md): the rules that the destinations share move into elt before the third destination.
488
+ - [ADR 0013](./docs/adr/0013-a-computed-dataset-declares-its-indexes-and-the-destination-reconciles-them.md): a computed dataset declares its indexes and the destination reconciles them.
@@ -0,0 +1,31 @@
1
+ import { type Connection, DerivedDataset, type PipelineClock, type RefreshTrigger, type Rule, type RuleColumns, TableRule } from '@eltjs/elt';
2
+ import { SqlServerDestination } from './sql-server-destination.ts';
3
+ import { type SqlServerIndex } from './sql-server-indexes.ts';
4
+ import type { SqlServerTable } from './sql-server-table.ts';
5
+ import { View } from './view.ts';
6
+ export declare class ComputedDataset<Input> extends DerivedDataset<SqlServerTable> {
7
+ #private;
8
+ readonly name: string;
9
+ readonly into: SqlServerDestination;
10
+ constructor({ into, name, description, from, compute, indexes, partIndexes, }: {
11
+ into: SqlServerDestination;
12
+ name: string;
13
+ description: string;
14
+ from: readonly (SqlServerTable | View)[];
15
+ indexes?: readonly SqlServerIndex[];
16
+ } & ({
17
+ compute: Rule<Input, RuleColumns>;
18
+ partIndexes?: never;
19
+ } | {
20
+ compute: TableRule<Input, RuleColumns, RuleColumns>;
21
+ partIndexes?: readonly SqlServerIndex[];
22
+ }));
23
+ get qualifiedName(): string;
24
+ validate(connections: readonly Connection<SqlServerTable>[]): void;
25
+ refresh({ connections, clock, signal, trigger, }: {
26
+ readonly connections: readonly Connection<SqlServerTable>[];
27
+ readonly clock: PipelineClock;
28
+ readonly signal?: AbortSignal;
29
+ readonly trigger: RefreshTrigger;
30
+ }): Promise<void>;
31
+ }