@eltjs/source-microsoft-sql-server 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 ezzabuzaid
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -1,3 +1,217 @@
-# Temporary Holding Version
+# @eltjs/source-microsoft-sql-server
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+An ELT source for one SQL Server database. Each user table that the login can read becomes a stream named `<schema>.<table>`. The source needs SQL Server 2016 or later, Azure SQL Database or Azure SQL Managed Instance.
+
+The words in this README have one meaning each. See the glossary in [CONTEXT.md](./CONTEXT.md).
+
+```ts
+import { Connection, Pipeline } from '@eltjs/elt';
+import {
+  PostgresCheckpointStore,
+  PostgresDestination,
+  PostgresSyncHistory,
+} from '@eltjs/elt-postgresql';
+import { SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
+import { sqlServerCopies } from '@eltjs/source-microsoft-sql-server/sql-server-copies';
+import { SqlServerSource } from '@eltjs/source-microsoft-sql-server/sql-server-source';
+
+const url = 'postgres://warehouse:warehouse@127.0.0.1:55432/warehouse';
+const schema = 'shop';
+
+const database = new SqlServerDatabase(
+  'Server=sql.example.com,1433;Database=shop;User Id=reader;Password=...;Encrypt=true',
+);
+const source = await SqlServerSource.discover(database, { schemas: ['dbo'] });
+const destination = new PostgresDestination({ url, schema });
+
+await new Pipeline({
+  history: new PostgresSyncHistory({ url }),
+  connections: [
+    new Connection({
+      name: 'sql-server',
+      source,
+      destination,
+      checkpoints: new PostgresCheckpointStore({ url, schema }),
+      steps: sqlServerCopies(source, (stream) => destination.table(stream)),
+    }),
+  ],
+}).run();
+```
+
+## How to use it
+
+1. Make a `SqlServerDatabase` from an ADO.NET connection string. [mssql](https://github.com/tediousjs/node-mssql) parses it. It does not read an `mssql://` URL. The connection string must name a `Database`.
+2. Call `SqlServerSource.discover(database, { schemas })`. The streams are the tables of the database, so the source is discovered. A host discovers it again on each run.
+3. Call `sqlServerCopies` to make one copy for each discovered table, with the sync strategy that discovery selected.
+4. Connect the copies to a destination and a checkpoint store through `Connection` and `Pipeline`.
+
+`@eltjs/sdk-microsoft-sql-server` owns the database: the connection string, discovery, exact reads, Change Tracking and the values that a watch reads. The source owns the streams, their schemas and the sync strategy. The SDK and the source hold no configuration from environment variables or for a standalone app.
+
+`sqlServerCopies` makes each stream a copy:
+
+- A stream with a key gets an incremental `append_dedup` copy: Change Tracking, rowversion and snapshot streams.
+- A stream with no key gets a full-refresh `overwrite` copy.
+- The ID of each copy is `<source identity>/<schema>.<table>`, for example `sql-server:shop/dbo.orders`. The source identity is `sql-server:<database>`. Copy IDs must be different across a pipeline. Thus two databases that each have a table with the same name can be in one pipeline. See [ADR 0004](./docs/adr/0004-a-copy-id-carries-the-source-identity.md).
+
+A table whose columns changed gets to the pipeline as a stream whose fields changed. The copy keeps its rows and its checkpoint, and the destination changes its table in place (see shape changes in [elt](../../../elt/README.md#shape-changes)). The source fills in a field that the table gained. See the sync strategies.
+
+## Access
+
+A read-only login is sufficient: `db_datareader`, and `VIEW CHANGE TRACKING` on each tracked table whose changes and deletions it must read. You do not have to enable a feature on the server. The source uses Change Tracking and snapshot isolation where an administrator turned them on. It reads every other table by its rowversion, by snapshot or fully.
+
+`VIEW SERVER STATE` (`VIEW DATABASE STATE` on Azure SQL Database) is optional. With it, a watch sees each write to a table that the source reads by snapshot or fully, also to a column with no statistics. See [Watching](#watching).
+
+- A connection string that names no `Database` is refused. Thus a read never goes to the default database of the login.
+- A server that cannot be reached fails with `SqlServerUnavailableError`. It is down, it restarts or fails over, its address is wrong, or the network dropped the connection. This is a `system` failure, so a later attempt or the next check of a watch can succeed.
+- A login that the server refuses fails with `SqlServerLoginError`: the credentials are wrong, the login is disabled, or it cannot open the database. This is a `config` failure, which the user must fix. The login errors that SQL Server gives while it fails over or is busy (4060, 10928, 10929, 40197, 40501, 40613) are not refusals: tedious tries them again, and then they are `SqlServerUnavailableError`.
+- Both errors name the server and the database, but not the credentials.
+- A column that the login cannot read (a column-level `DENY`) is left out, and the table's description names it. A primary key with such a column identifies nothing, so its table reads fully.
+- A table with no readable column is not a stream.
+- When a grant is revoked after discovery, only the stream of that table fails, with `SqlServerPermissionError`, which names the grant (`SELECT ON [dbo].[orders]`, `VIEW CHANGE TRACKING ON [dbo].[orders]`).
+- Columns under dynamic data masking load as the login reads them: masked, unless the login has `UNMASK`. Their description says so.
+
+## Streams
+
+Discovery lists `sys.tables`: user tables. It does not list views, system tables, external tables, or the change tables that CDC keeps in `cdc`.
+
+- A host can keep only some schemas (`{ schemas }`). Write each name exactly as `sys.schemas` spells it. A schema in the list that holds no table that the login can read fails discovery. Thus a misspelling, a casing that the database does not use, or a missing grant never loads less without an error.
+- A column of an alias type (`CREATE TYPE ... FROM nvarchar(20)`) reads as the system type that it is built on, also when the login cannot see the alias type.
+- The stream key is the table's primary key in key order, plus the `_type` field of a `sql_variant` key column.
+- Descriptions come from `MS_Description` extended properties. When a column has none, the source makes one from the name and the type of the column. The table's description also tells how the stream reads the table, and which grant would let it read only changes.
+
+| SQL Server                                                                                 | Field                                                                                                               |
+| ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- |
+| `tinyint`, `smallint`, `int`                                                               | `integer`                                                                                                           |
+| `bigint`, `rowversion`                                                                     | `int64`                                                                                                             |
+| `bit`                                                                                      | `boolean`                                                                                                           |
+| `real`, `float`                                                                            | `number` (the IEEE value, exactly)                                                                                  |
+| `decimal(p, s)`, `numeric(p, s)`                                                           | `decimal`, precision `p`, scale `s`                                                                                 |
+| `money`, `smallmoney`                                                                      | `decimal` (19, 4) and (10, 4)                                                                                       |
+| `date`                                                                                     | `date`                                                                                                              |
+| `datetime2(p)`, `datetime`, `smalldatetime`                                                | `date-time-local`, precision `p`, 3 and 0                                                                           |
+| `datetimeoffset(p)`                                                                        | `date-time`, precision `p`: the UTC instant; beside it `<column>_offset`, the offset it was written with in minutes |
+| `time(p)`                                                                                  | `time-local`, precision `p`                                                                                         |
+| `char`, `varchar`, `nchar`, `nvarchar`, `text`, `ntext`, `xml`, `uniqueidentifier`, `json` | `string` (a uniqueidentifier in upper case)                                                                         |
+| `hierarchyid`                                                                              | `string`, its path (`/1/2/`)                                                                                        |
+| `geography`, `geometry`                                                                    | `string`, `SRID=<id>;` then the shape as text with its Z and M (`SRID=4326;POINT (-122.349 47.651 10 3)`)           |
+| `sql_variant`                                                                              | `string`, the value as text; beside it `<column>_type`, its base type                                               |
+| `binary`, `varbinary`, `image`, CLR types                                                  | `string`, base64                                                                                                    |
+
+The formats are the string formats of [elt](../../../elt/README.md#string-formats). The driver gives `decimal` and `money` as JavaScript numbers and `datetime2` as a millisecond `Date`, and it drops the offset of a `datetimeoffset`. Thus the source selects each such column as the text that SQL Server spells, and decodes it into its canonical form. See [ADR 0002](./docs/adr/0002-each-column-is-read-as-the-text-that-sql-server-spells.md).
+
+When a column of the table already has the name `<column>_offset` or `<column>_type`, the added field gets the next free name: `_1`, then `_2`. The description of the column names the field that holds its offset. See [ADR 0003](./docs/adr/0003-an-added-field-takes-the-next-free-name.md).
+
+## Sync strategies
+
+Discovery selects how each table reads. Each table reads with the first strategy that it can use:
+
+| Strategy        | When                                                                   | Copy                          |
+| --------------- | ---------------------------------------------------------------------- | ----------------------------- |
+| Change Tracking | the table has a key, tracks changes, and the login can see its changes | `incremental`, `append_dedup` |
+| Rowversion      | the table has a key and a `rowversion` column                          | `incremental`, `append_dedup` |
+| Snapshot        | all other tables that have a key                                       | `incremental`, `append_dedup` |
+| Full refresh    | the table has no key                                                   | `full_refresh`, `overwrite`   |
+
+**Change Tracking** is for a keyed table that tracks changes, when the login has `VIEW CHANGE TRACKING`.
+
+- The first read loads every row, 10,000 rows for each checkpoint, in primary key order. It starts from the position in the table's change history that it read before it began. A load that fails continues after its last checkpoint.
+- Later reads apply each key that changed since: its row as it is now, or a `DELETE`.
+- With `ALLOW_SNAPSHOT_ISOLATION` on, the position, the check and the changes come from one short snapshot transaction for each table. Without it, the source reads the position first. Thus a change made during the read is read again the next time, and the source checks the history again after the read.
+
+**Rowversion** is for a keyed table with a `rowversion` column.
+
+- Each read takes the rows above the last rowversion that it read, and below `MIN_ACTIVE_ROWVERSION()`. Thus a row that a transaction still writes waits for a later read.
+- The source does not see deletions. A deleted row stays loaded.
+- Its state records the columns that it read. A column that the table gained changes no rowversion. Thus the next read starts again from the first row, and keeps what is loaded.
+
+**Snapshot** is for a keyed table with no Change Tracking that the login can read and no `rowversion` column. It is a snapshot stream of [elt](../../../elt/CONTEXT.md):
+
+- Each read takes every row. The copy compares the rows with the snapshot in its checkpoint, writes each row that is new or changed, and deletes each row whose key the read did not give. The read gives every row, so a key that it does not give was deleted (the `ClosedWorld` deletion policy).
+- A read that fails deletes nothing.
+- A full-refresh copy of the stream replaces its target with every row.
+
+**Full refresh** is for a table with no key, or with a key that the login cannot read in full. Each read takes every row, and the copy replaces its target.
+
+A position in a table's change history is a Change Tracking version and the table's tracking generation. The generation is the table's `begin_version`, which a `TRUNCATE` moves to the current version, and the creation time of its change table, which enabling tracking again makes new. See [ADR 0001](./docs/adr/0001-a-change-tracking-position-carries-the-tracking-generation.md).
+
+- The history no longer covers a saved position when the position is below the table's minimum valid version, ahead of the database's current version, of a different generation, or at the version where its generation began. The stream then sends `RESET` and loads every row again into a hidden target, which replaces the table when the stream ends (see resets in [elt](../../../elt/README.md#resets)).
+- The stream does the same for a column that the table gained since the state was saved, because Change Tracking reports no change for the rows to which it was added.
+- The generation finds the case that the minimum valid version cannot find: a truncate just after a sync, with no change between them, leaves the minimum valid version equal to the saved one.
+- A truncate moves the generation to the current version. When the generation began at the saved version, a truncate leaves it as it was. Thus such a position can show no truncate, and the stream loads every row again.
+
+Pages compare the full primary key as a tuple. Each part is cast to the declared type and collation of its column. Thus a page does not skip or repeat rows that have the same leading key value, and a `varchar` key under a SQL collation orders as the column sorts.
+
+## Watching
+
+The source gives elt's pipeline one change signal for each stream, checked every 30 seconds (elt ADR 0020). The first check opens one session for the watch, and the pipeline closes it when the watch stops. Each sync strategy names the gauges of its stream, and the gauges of one stream are one group. On each check, each collector reads the values of all tables in one query, and the database version gives the Change Tracking version and the last rowversion in one query. A check wakes a stream when one gauge of its group changed. The first check wakes every stream. A check that wakes no stream makes no pass.
+
+| Sync strategy             | Gauges                                                             |
+| ------------------------- | ------------------------------------------------------------------ |
+| Change Tracking           | the Change Tracking version, the row count                         |
+| Rowversion                | the last rowversion                                                |
+| Snapshot and full refresh | the statistics, the row count, the last write (optional), the hour |
+
+- **The Change Tracking version** (`CHANGE_TRACKING_CURRENT_VERSION()`) of the database, one gauge for all streams. A change to a tracked table moves it, in the transaction of the change. A `TRUNCATE` does not move it.
+- **The last rowversion** (`@@DBTS`) of the database, one gauge for all streams. An insert or an update of a row with a `rowversion` column moves it.
+- **The statistics** of the table (`sys.dm_db_stats_properties`): their count, the rows changed in their columns, and the time of their last refresh. An insert, a delete, a `TRUNCATE` or an update of a column with statistics moves it. A restart keeps it. A refresh of statistics sets the count to 0 and changes the time. `db_datareader` can read it.
+- **The row count** of the table (`sys.partitions`). An insert, a delete or a `TRUNCATE` moves it, also when SQL Server has not built the statistics of the table. An update does not move it. `db_datareader` can read it. A Change Tracking stream watches it for a `TRUNCATE`. A `TRUNCATE` of an empty table moves nothing, and it removes nothing.
+- **The last write** of the table (`sys.dm_db_index_usage_stats`): the writes and the time of the last write. Each insert, update or delete moves it, also of a column with no statistics. A `TRUNCATE` does not move it. Discovery reads it one time. When the login cannot read it, the streams do not use it, and their description names the grant.
+- **The hour**: a period of one hour, the number of whole hours since the watch started. Each hour, it wakes the streams again, and the read finds a change that the other gauges did not see.
+
+The statistics, the row count and the last write are collectors: one key for each table, by its quoted name. A table that a collector does not give, for example a table dropped after discovery, changed, so its stream wakes and its read fails.
+
+The queries of one check have 10 seconds. When a gauge fails, for example because its query took longer or a restart of the SQL Server engine dropped the connection, the streams of the gauge fail, as a `system` failure, and the next check reads the gauge again. The other streams keep loading. A server that cannot be reached at the first check leaves the session closed, and the next check opens it again. A login that the server refuses fails every stream as `config`, and the pipeline reads the gauges no more in this watch.
+
+The database is current whenever the source reads it, so the source gives no `currentAsOf`.
+
+## Limitations
+
+- Rowversion tables do not see deletions. A deleted row stays.
+- Without the last write, an update of a column with no statistics loads only at the next hour.
+- SQL Server builds the statistics of a table when a query first needs them. Until then, they count nothing. In a table that no query has read by a column since it was created empty, an insert or a delete moves the row count, and an update waits for the last write or the hour. A table with no key and no index has no statistics at all, so the same is true for it.
+- The source does not read CDC (backlog #2315).
+- The source discovers tables only, not views (backlog #3253).
+- A Change Tracking stream whose position is at the version where its generation began loads every row again at each read, until a tracked change in the database moves the version. Tracking turned on after the last tracked change, and a truncate with no tracked change after it, give such a position. The count of the statistics moves on a truncate, but each tracked change and each refresh of statistics also move it. Thus it cannot show that the table was truncated.
+- A single value longer than the longest JavaScript string (about 512 million characters) fails its stream.
+
+## Live checks
+
+Run on 2026-10-06 against SQL Server 2022 CU27 (16.0.4295.3) Developer, in the compose `sqlserver` service on linux/amd64, emulated on Apple silicon, through mssql 12.7.4 and tedious 20.3.3:
+
+- Style 126 leaves out a zero fraction (`2025-01-02T03:04:05`) and keeps every digit of any other fraction (`.1200000`). `time(p)` keeps exactly `p` digits. Style 2 spells money with four digits.
+- `CHANGE_TRACKING_MIN_VALID_VERSION` gives 0, not NULL, to a login without `VIEW CHANGE TRACKING`. The `CHANGETABLE` read of that login fails with error 229. `sys.change_tracking_tables` and `MS_Description` stay visible to `db_datareader`.
+- `TRUNCATE` sets the table's minimum valid version and `begin_version` to the current version. Enabling tracking again makes a new `create_date` for the change table in `sys.internal_tables`, which `VIEW CHANGE TRACKING` makes visible.
+- A column-level `DENY` makes `HAS_PERMS_BY_NAME` report the full table as not readable. Thus the source checks readability for each column. A column of an alias type that the login cannot see disappears from an inner join with `sys.types`.
+- A key bound as an `nvarchar` parameter and compared with a `varchar` column under `SQL_Latin1_General_CP1_CI_AS` skips rows (`co-op` before `Coat`): the two sort hyphens differently.
+- `varbinary` keys ignore zero bytes at the end (`0x00`, `0x0000` and `0x` are one key).
+- tedious 20.0.0 refused to connect to an IP address over TLS. 20.3.3 connects.
+- `sql_variant` text under style 126 keeps times exact. Floating point needs style 3 for 17 digits.
+- An end-to-end check of the source and the pipeline loaded a seeded database, read by a `db_datareader` login. On a second run, it wrote nothing to the Change Tracking and rowversion tables. On a third run, it applied an update, a deletion, a truncate and an added column.
+
+Run on 2026-11-04 against three Azure SQL databases (General Purpose, Gen5), read only:
+
+- `sys.dm_db_index_usage_stats` gave the time of the last write to a table to the second of its `updatedAt` column. It also gave a write on 2026-10-29 to a table whose `DateUpdated` column the write did not change. It was empty for writes before the last restart of the engine, and the three engines restarted within 9 days.
+- `sys.dm_db_log_stats` moved its last log position in 15 seconds with no write by a user, so it is no signal of a table.
+
+Run on 2026-11-06 against SQL Server 2022 in a container, through a `db_datareader` login and a login with more grants:
+
+- A restart cleared the last writes and the row counts of `sys.dm_db_index_operational_stats`, and kept the count of `sys.dm_db_stats_properties`.
+- A `TRUNCATE` moved neither the last write nor the row counts. It moved the count of the statistics.
+- An update of 100 rows in a column with no statistics did not move the count of the statistics. `UPDATE STATISTICS` set the count to 0 and changed the time of the refresh.
+- An `UPDATE` of no row and an `UPDATE` that was rolled back moved the last write.
+- `db_datareader` could read the count of the statistics and the row counts of `sys.partitions`, and got error 297 for the last writes and the row counts of `sys.dm_db_index_operational_stats`.
+- Statistics that an index created on an empty table made had no count and no time of refresh until `UPDATE STATISTICS` built them. Rows inserted before then moved only the row count.
+- `CHECKSUM_AGG(BINARY_CHECKSUM(*))` read every page of the table. It did not change for a value changed from `' !'` to `''Q'`, for two rows that swapped a value, or for two equal rows added to a table with no key. A SHA-256 hash of each row found all three.
+
+Run on 2026-11-07 against SQL Server 2022 in a container:
+
+- A `TRUNCATE` of a tracked table with rows did not move `CHANGE_TRACKING_CURRENT_VERSION()`. It moved the row count to 0, and `begin_version` and the minimum valid version to the current version. It did not change the `partition_id` of the table.
+- A `SELECT` with no `WHERE` did not build the statistics of a table created empty. A `SELECT` with a `WHERE` on the key built them: the count went from none to 0, with a time of refresh. A `TRUNCATE` did not move statistics that were not built.
+- A disabled login and a login to a database that is `OFFLINE` both failed with `ELOGIN` that tedious did not mark as transient. The error log had one `Login failed` line for each attempt.
+
+## Decisions
+
+- [ADR 0001: A Change Tracking position carries the tracking generation](./docs/adr/0001-a-change-tracking-position-carries-the-tracking-generation.md)
+- [ADR 0002: Each column is read as the text that SQL Server spells](./docs/adr/0002-each-column-is-read-as-the-text-that-sql-server-spells.md)
+- [ADR 0003: An added field takes the next free name](./docs/adr/0003-an-added-field-takes-the-next-free-name.md)
+- [ADR 0004: A copy ID carries the source identity](./docs/adr/0004-a-copy-id-carries-the-source-identity.md)

package/dist/change-tracking-stream.d.ts

@@ -0,0 +1,11 @@
+import type { CopyConfiguration, ExtractionCoverage, SourceMessage } from '@eltjs/elt';
+import { type SqlServerSession, type SqlServerTable } from '@eltjs/sdk-microsoft-sql-server';
+import { SqlServerStream } from './sql-server-stream.ts';
+import type { WatchGauges } from './watch-gauges.ts';
+export declare class ChangeTrackingStream extends SqlServerStream {
+    #private;
+    constructor(table: SqlServerTable);
+    gauges(watch: WatchGauges): import("@eltjs/elt").Gauge<import("@zukhruf/async").Value>[];
+    coverage(): ExtractionCoverage;
+    extract(configuration: CopyConfiguration, state: unknown, session: SqlServerSession): AsyncGenerator<SourceMessage>;
+}

package/dist/change-tracking-stream.js

@@ -0,0 +1,149 @@
+import { ChangeHistoryExpiredError, } from '@eltjs/sdk-microsoft-sql-server';
+import { SqlServerStream, pageSize } from './sql-server-stream.js';
+function position(value) {
+    return typeof value === 'object' &&
+        value !== null &&
+        'version' in value &&
+        typeof value.version === 'string' &&
+        'generation' in value &&
+        typeof value.generation === 'string'
+        ? { version: value.version, generation: value.generation }
+        : undefined;
+}
+function parse(state) {
+    if (state === null)
+        return null;
+    const fields = typeof state === 'object' &&
+        'fields' in state &&
+        Array.isArray(state.fields) &&
+        state.fields.every((field) => typeof field === 'string')
+        ? state.fields
+        : undefined;
+    if (fields === undefined)
+        throw new TypeError(`Unknown Change Tracking state ${JSON.stringify(state)}`);
+    if (typeof state === 'object' && 'from' in state) {
+        const from = position(state.from);
+        if (from !== undefined)
+            return { from, fields };
+    }
+    if (typeof state === 'object' &&
+        'initial' in state &&
+        typeof state.initial === 'object' &&
+        state.initial !== null &&
+        'from' in state.initial &&
+        'after' in state.initial &&
+        Array.isArray(state.initial.after) &&
+        state.initial.after.every((part) => typeof part === 'string')) {
+        const from = position(state.initial.from);
+        if (from !== undefined)
+            return { initial: { from, after: state.initial.after }, fields };
+    }
+    throw new TypeError(`Unknown Change Tracking state ${JSON.stringify(state)}`);
+}
+// A table SQL Server tracks changes of, which the login may read: the first
+// read loads every row, page by page, from the position current before it
+// began; later reads apply what changed since, deletions included. When the
+// history no longer reaches back to the saved position (cleanup, a truncate,
+// tracking turned off and on), the stream starts over with a RESET and loads
+// every row again, as it does for a column the table gained: Change Tracking
+// reports no change for the rows it was added to.
+export class ChangeTrackingStream extends SqlServerStream {
+    constructor(table) {
+        super(table, {
+            supportedSyncModes: ['full_refresh', 'incremental'],
+            sourceDefinedCursor: true,
+            emitsDeletes: true,
+            strategy: 'Read through SQL Server Change Tracking: inserts, updates and deletions apply.',
+        });
+    }
+    // A TRUNCATE moves no Change Tracking version, but it moves the row count
+    // of a table that held rows.
+    gauges(watch) {
+        return [watch.changeTracking, watch.rowCounts.gauge(this.table.quoted)];
+    }
+    coverage() {
+        return {
+            description: `Every row of ${this.table.quoted} the login can read, then what Change Tracking reports changed`,
+            selection: { table: this.table.quoted },
+        };
+    }
+    async *extract(configuration, state, session) {
+        if (configuration.syncMode === 'full_refresh') {
+            yield* this.everyRow(session);
+            return;
+        }
+        const saved = parse(state);
+        if (saved === null) {
+            yield* this.#loadEveryRow(session);
+            return;
+        }
+        if (this.gained(saved.fields)) {
+            yield { type: 'RESET', stream: this.stream.name };
+            yield* this.#loadEveryRow(session);
+            return;
+        }
+        if ('initial' in saved) {
+            yield* this.#load(session, saved.initial.from, saved.initial.after);
+            return;
+        }
+        try {
+            yield* this.#changes(session, saved.from);
+        }
+        catch (error) {
+            if (!(error instanceof ChangeHistoryExpiredError))
+                throw error;
+            // Whatever the expired read already yielded, the reset drops.
+            yield { type: 'RESET', stream: this.stream.name };
+            yield* this.#loadEveryRow(session);
+        }
+    }
+    // Every row, from where the table's change history stands now.
+    async *#loadEveryRow(session) {
+        yield* this.#load(session, await session.changeTrackingPosition(this.table), null);
+    }
+    // Every row, from a position read before the first page, so a change made
+    // during the load is read again by the next delta.
+    async *#load(session, from, after) {
+        const { name } = this.stream;
+        const fields = this.fieldNames;
+        if (!this.table.pageable) {
+            yield* this.everyRow(session);
+            yield { type: 'STATE', stream: name, state: { from, fields } };
+            return;
+        }
+        let last = after;
+        for (;;) {
+            let count = 0;
+            for await (const row of session.page(this.table, last, pageSize)) {
+                yield this.message(row);
+                last = row.position;
+                count += 1;
+            }
+            if (count < pageSize)
+                break;
+            yield {
+                type: 'STATE',
+                stream: name,
+                state: { initial: { from, after: last }, fields },
+            };
+        }
+        yield { type: 'STATE', stream: name, state: { from, fields } };
+    }
+    async *#changes(session, since) {
+        const { name } = this.stream;
+        let reached;
+        {
+            await using set = await session.changes(this.table, since);
+            for await (const change of set)
+                yield change.type === 'upsert'
+                    ? this.message(change.row)
+                    : { type: 'DELETE', stream: name, key: this.fields.key(change.key) };
+            reached = set.position;
+        }
+        yield {
+            type: 'STATE',
+            stream: name,
+            state: { from: reached, fields: this.fieldNames },
+        };
+    }
+}

package/dist/full-refresh-stream.d.ts

@@ -0,0 +1,12 @@
+import type { CopyConfiguration, ExtractionCoverage, SourceMessage } from '@eltjs/elt';
+import type { SqlServerSession, SqlServerTable } from '@eltjs/sdk-microsoft-sql-server';
+import { SqlServerStream } from './sql-server-stream.ts';
+import type { WatchGauges } from './watch-gauges.ts';
+import type { WholeTableGauges } from './whole-table-gauges.ts';
+export declare class FullRefreshStream extends SqlServerStream {
+    #private;
+    constructor(table: SqlServerTable, whole: WholeTableGauges);
+    gauges(watch: WatchGauges): readonly import("@eltjs/elt").Gauge<import("@zukhruf/async").Value>[];
+    coverage(): ExtractionCoverage;
+    extract(_configuration: CopyConfiguration, _state: unknown, session: SqlServerSession): AsyncIterable<SourceMessage>;
+}

package/dist/full-refresh-stream.js

@@ -0,0 +1,25 @@
+import { SqlServerStream } from './sql-server-stream.js';
+// A table with no key the login can read in full, so no change can apply by
+// key: every read takes every row, and a copy replaces what it loaded before.
+export class FullRefreshStream extends SqlServerStream {
+    #whole;
+    constructor(table, whole) {
+        super(table, {
+            supportedSyncModes: ['full_refresh'],
+            strategy: `Read in full on every sync.${whole.hint}`,
+        });
+        this.#whole = whole;
+    }
+    gauges(watch) {
+        return this.#whole.gauges(this.table, watch);
+    }
+    coverage() {
+        return {
+            description: `Every row of ${this.table.quoted} the login can read`,
+            selection: { table: this.table.quoted },
+        };
+    }
+    extract(_configuration, _state, session) {
+        return this.everyRow(session);
+    }
+}

package/dist/rowversion-stream.d.ts

@@ -0,0 +1,11 @@
+import type { CopyConfiguration, ExtractionCoverage, SourceMessage } from '@eltjs/elt';
+import type { SqlServerSession, SqlServerTable } from '@eltjs/sdk-microsoft-sql-server';
+import { SqlServerStream } from './sql-server-stream.ts';
+import type { WatchGauges } from './watch-gauges.ts';
+export declare class RowversionStream extends SqlServerStream {
+    #private;
+    constructor(table: SqlServerTable);
+    gauges(watch: WatchGauges): import("@eltjs/elt").Gauge<import("@zukhruf/async").Value>[];
+    coverage(): ExtractionCoverage;
+    extract(configuration: CopyConfiguration, state: unknown, session: SqlServerSession): AsyncGenerator<SourceMessage>;
+}