@zukhruf/sqlite-changefeed 0.0.0-stage → 0.3.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +172 -2
- package/dist/change-event.d.ts +18 -0
- package/dist/change-event.d.ts.map +1 -0
- package/dist/change-event.js +2 -0
- package/dist/change-event.js.map +1 -0
- package/dist/changefeed.d.ts +23 -0
- package/dist/changefeed.d.ts.map +1 -0
- package/dist/changefeed.js +63 -0
- package/dist/changefeed.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -0
- package/dist/journal-mode-error.d.ts +7 -0
- package/dist/journal-mode-error.d.ts.map +1 -0
- package/dist/journal-mode-error.js +12 -0
- package/dist/journal-mode-error.js.map +1 -0
- package/dist/pass.d.ts +9 -0
- package/dist/pass.d.ts.map +1 -0
- package/dist/pass.js +43 -0
- package/dist/pass.js.map +1 -0
- package/dist/position-file.d.ts +11 -0
- package/dist/position-file.d.ts.map +1 -0
- package/dist/position-file.js +82 -0
- package/dist/position-file.js.map +1 -0
- package/dist/position-in-memory.d.ts +9 -0
- package/dist/position-in-memory.d.ts.map +1 -0
- package/dist/position-in-memory.js +20 -0
- package/dist/position-in-memory.js.map +1 -0
- package/dist/position.d.ts +15 -0
- package/dist/position.d.ts.map +1 -0
- package/dist/position.js +2 -0
- package/dist/position.js.map +1 -0
- package/dist/table-changes.d.ts +8 -0
- package/dist/table-changes.d.ts.map +1 -0
- package/dist/table-changes.js +48 -0
- package/dist/table-changes.js.map +1 -0
- package/dist/tables.d.ts +13 -0
- package/dist/tables.d.ts.map +1 -0
- package/dist/tables.js +68 -0
- package/dist/tables.js.map +1 -0
- package/package.json +52 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ezzabuzaid
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,173 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @zukhruf/sqlite-changefeed
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Some programs keep their data in a SQLite database, and write it while they run. The owner gives no change log and no API. Another program wants each change to the rows, so that it can copy them into a different store. This package gives a changefeed of the database: a change event for each changed row of each table, by the primary key of the row. It reads the database read-only, and it never writes to the file of the owner. It is change data capture by query, also known as a snapshot differential (Labio and Garcia-Molina, VLDB 1996).
|
|
4
|
+
|
|
5
|
+
The feed composes four packages:
|
|
6
|
+
|
|
7
|
+
- `@zukhruf/sqlite-data-version` tells when to read again. Its `DataVersion` changes when the owner commits, and it follows a replaced file.
|
|
8
|
+
- `@zukhruf/sqlite-snapshot` holds one moment for each pass. All tables of a pass come from the same moment.
|
|
9
|
+
- `@zukhruf/snapshot-differential` tells what changed since the last pass.
|
|
10
|
+
- The change watcher of `@zukhruf/async` polls the data version, and starts a pass only when it changes.
|
|
11
|
+
|
|
12
|
+
The words in these documents have one meaning each. See the glossary in [CONTEXT.md](./CONTEXT.md).
|
|
13
|
+
|
|
14
|
+
## Use it
|
|
15
|
+
|
|
16
|
+
This project is an experiment. It uses only `node:sqlite`.
|
|
17
|
+
|
|
18
|
+
```sh
|
|
19
|
+
npm install @zukhruf/sqlite-changefeed
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
The owner is a notes app. It keeps its database in WAL mode. The reader copies each table into its own store:
|
|
23
|
+
|
|
24
|
+
```ts
|
|
25
|
+
import { Changefeed } from '@zukhruf/sqlite-changefeed';
|
|
26
|
+
|
|
27
|
+
using feed = new Changefeed(
|
|
28
|
+
`${process.env.HOME}/Library/Application Support/Notes/notes.sqlite`,
|
|
29
|
+
`${process.env.HOME}/.notes-copy/notes.position.sqlite`,
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
const stop = new AbortController();
|
|
33
|
+
process.once('SIGINT', () => stop.abort());
|
|
34
|
+
|
|
35
|
+
for await (const batch of feed.watch({
|
|
36
|
+
interval: 1_000,
|
|
37
|
+
signal: stop.signal,
|
|
38
|
+
})) {
|
|
39
|
+
for (const event of batch) {
|
|
40
|
+
if (event.op === 'truncate') await store.clear(event.table);
|
|
41
|
+
else if (event.op === 'delete') await store.remove(event.table, event.key);
|
|
42
|
+
else await store.put(event.table, event.key, event.after);
|
|
43
|
+
}
|
|
44
|
+
// The feed saves the position of this batch when the loop asks for the next one.
|
|
45
|
+
}
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
This is a durable feed: it has a position file, and a new run goes on from the last saved batch. Without a position path, the feed observes from now. It reads each table one time with no events, and then gives only the changes after that moment:
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
using feed = new Changefeed(
|
|
52
|
+
`${process.env.HOME}/Library/Application Support/Notes/notes.sqlite`,
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
for await (const batch of feed.watch({
|
|
56
|
+
interval: 1_000,
|
|
57
|
+
signal: stop.signal,
|
|
58
|
+
})) {
|
|
59
|
+
for (const event of batch) console.log(event.table, event.op, event.key);
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
The first run of a durable feed gives a `read` event for each row of each table. Each later batch gives only the rows that changed:
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
{ table: 'notes', op: 'read', key: { id: 1 }, after: { id: 1, folder_id: 7, title: 'Plan', body: '…' } }
|
|
67
|
+
{ table: 'tags', op: 'read', key: { rowid: 3 }, after: { note_id: 1, tag: 'q4' } }
|
|
68
|
+
…
|
|
69
|
+
{ table: 'notes', op: 'update', key: { id: 1 }, after: { id: 1, folder_id: 7, title: 'Plan v2', body: '…' } }
|
|
70
|
+
{ table: 'tags', op: 'delete', key: { rowid: 3 } }
|
|
71
|
+
{ table: 'drafts', op: 'truncate' }
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
The table `tags` has no primary key, so its key is the `rowid`. The owner dropped the table `drafts`, so the feed gives a `truncate` for it.
|
|
75
|
+
|
|
76
|
+
## The rules
|
|
77
|
+
|
|
78
|
+
- The database must be in WAL mode. `new Changefeed(databasePath, positionPath?)` reads `PRAGMA journal_mode`, and throws `JournalModeError` for each other mode. Each pass reads it again, because the owner can replace the file. [ADR 0002](./docs/adr/0002-wal-is-required.md) tells why.
|
|
79
|
+
- The feed reads each table that `PRAGMA table_list` gives with type `table`, except a name that starts with `sqlite_`. It does not read an internal table of SQLite (`sqlite_schema`, `sqlite_sequence`, `sqlite_stat*`), a `view`, a `virtual` table, or a `shadow` table. A shadow table holds the data of a virtual table, for example the `notes_fts_data` of an FTS5 table. It is internal to the module of the virtual table. [ADR 0006](./docs/adr/0006-every-table-keyed-by-its-primary-key.md) tells why.
|
|
80
|
+
- The key of a row is the identity that SQLite makes unique and not `NULL`. The feed finds it from the schema of each table:
|
|
81
|
+
- A `WITHOUT ROWID` table: its declared primary key. SQLite makes each column of it `NOT NULL`, and `PRAGMA table_info` gives `notnull` 1 for each column of it.
|
|
82
|
+
- A rowid table with an `INTEGER PRIMARY KEY` that is an alias of the `rowid`, or with a primary key whose columns are all declared `NOT NULL`: that primary key. `id INTEGER PRIMARY KEY DESC` in a column definition is not an alias of the `rowid`, and can hold `NULL`.
|
|
83
|
+
- Each other rowid table: the key `{ rowid }`. A declared primary key of a rowid table can hold `NULL` in many rows, so it does not identify a row.
|
|
84
|
+
- The columns of a primary key are the columns of `PRAGMA table_info` with a `pk` above 0, in the order of `pk`.
|
|
85
|
+
- `VACUUM` can change the `rowid` of each row of a table with no primary key. The feed compares the rows by `rowid`, so it then gives an event for each `rowid` whose row changed: an `insert` for a new `rowid`, an `update` for a `rowid` that now holds a different row, and a `delete` for a `rowid` that is gone. The SQLite documentation advises an `INTEGER PRIMARY KEY` for a stable `rowid`.
|
|
86
|
+
- A pass copies each row into a plain object, because `node:sqlite` gives rows with no prototype. `after` keeps the values that SQLite gives: a number, a string, `null`, or a `Uint8Array` for a `BLOB`.
|
|
87
|
+
- A change event is `{ table, op, key, after? }`, or `{ table, op: 'truncate' }`. `op` is `read`, `insert`, `update`, `delete` or `truncate`. `after` is the full row for `read`, `insert` and `update`. There is no `before`. [ADR 0003](./docs/adr/0003-the-change-event.md) tells why.
|
|
88
|
+
- `read` is each row of a table that the position does not have: each row at the first run, and each row of a new table.
|
|
89
|
+
- `truncate` tells that the owner dropped a table, or changed the columns of its primary key. Remove all rows of that table from your copy. When the primary key changed, a `read` for each row comes after the `truncate`.
|
|
90
|
+
- A change to the other columns of a table, for example a new column, changes the `after` of its rows. Each row then gives an `update`.
|
|
91
|
+
- The feed owns its position. `positionPath` is optional, and it sets the kind of the feed. [ADR 0004](./docs/adr/0004-the-feed-owns-its-position.md) tells why.
|
|
92
|
+
- With `positionPath`, the feed is durable. It keeps its position in its own SQLite file at that path: one fingerprint for each row of each table. It creates the file when the file is missing. A missing or empty file is a first run, and the first pass gives each row as `read`. A later run goes on from the saved position. Give each durable feed its own position file.
|
|
93
|
+
- Without `positionPath`, the feed observes from now. The first pass of each `watch` is the baseline: it reads each table, keeps the fingerprints in memory, and gives no batch. Then it gives only the changes after the baseline. It writes nothing to disk. A new run takes a new baseline, so it does not see the changes that the owner made while the feed was stopped.
|
|
94
|
+
- `watch({ interval, signal })` gives batches. A batch is an array of change events. The feed makes a first pass at once. Then it polls the data version each `interval` milliseconds, and makes a pass only when the data version changed. A pass that finds no change gives no batch, and the feed saves its position at once: for example, the key columns of a new table with no rows.
|
|
95
|
+
- A durable feed saves the position of a batch when the loop asks for the next batch. It writes the position in one transaction. Apply each event of a batch before the loop goes on.
|
|
96
|
+
- Delivery of a durable feed is at least once. If the process stops before the loop asks for the next batch, the next run gives the same batch again. A loop that stops with `break`, an error or an abort also gives its last batch again at the next run. Apply each event so that a second apply has no effect: a `put` by key, a `remove` by key, a `clear` by table.
|
|
97
|
+
- An abort of `signal` stops the poll, and the loop rejects with `signal.reason`.
|
|
98
|
+
- A pass holds its read transaction only while it reads. It closes the snapshot before it gives the batch. The poll holds no read transaction. [ADR 0005](./docs/adr/0005-one-snapshot-for-each-pass.md) tells why.
|
|
99
|
+
- Dispose closes the connections of the feed. Use `using`, or call `[Symbol.dispose]()`.
|
|
100
|
+
|
|
101
|
+
## Limits
|
|
102
|
+
|
|
103
|
+
- You see only the net change between two passes. A row that the owner inserts and deletes between two passes gives no event. A row that the owner changes two times gives one `update`.
|
|
104
|
+
- Each commit of the owner makes a pass that reads each table in full. The cost of a pass grows with the size of the database, also when the commit changed one row. [ADR 0001](./docs/adr/0001-capture-by-snapshot-differential.md) names a later option that reads only the changed tables.
|
|
105
|
+
- The position keeps one fingerprint for each row of each table. It grows with the number of rows. A durable feed keeps it on disk. A feed that observes from now keeps it in memory.
|
|
106
|
+
- A feed that observes from now does not see the changes that the owner made while the feed was stopped.
|
|
107
|
+
- Precedent for scale: Fivetran's Teleport Sync, which also compares data sets, gives a warning when more than 50,000 records change between two syncs.
|
|
108
|
+
|
|
109
|
+
## Errors
|
|
110
|
+
|
|
111
|
+
| Error | When |
|
|
112
|
+
| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
113
|
+
| `UnavailableError` | SQLite cannot open the database: the file is missing, or the system denies access, a privacy denial too. `new Changefeed` throws it, and a pass throws it. `path` is the path. `cause` is the error of SQLite. |
|
|
114
|
+
| `JournalModeError` | The database is not in WAL mode. `new Changefeed` throws it, and a pass throws it. `path` is the path, and `journalMode` is the mode that `PRAGMA journal_mode` gave, for example `delete`. |
|
|
115
|
+
| The error of SQLite | All other failures, for example a position file that the feed cannot write. |
|
|
116
|
+
|
|
117
|
+
## The same concepts in other systems
|
|
118
|
+
|
|
119
|
+
| System | What it does |
|
|
120
|
+
| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
121
|
+
| CockroachDB `CREATE CHANGEFEED` | A changefeed for each table that it names. Each message is keyed by the primary key of the row. |
|
|
122
|
+
| Azure Cosmos DB change feed | A feed of the changes to the items of a container. A reader goes on from the last position that it read. |
|
|
123
|
+
| Debezium | Change data capture from the log of the database. It gives one topic for each table, keyed by the primary key. Its event has the envelope `before`, `after`, `op` (`c`, `u`, `d`, `r`, `t`), `source` and `ts_ms`. This package keeps `op` and `after`. Its `read` is the `r`, and its `truncate` is the `t`. The embedded engine keeps its offsets in a file that the host names (`offset.storage.file.filename`). |
|
|
124
|
+
| Kafka `auto.offset.reset` | It applies when a consumer has no committed offset. `earliest` reads each record again, and `latest` reads only the records that come after the consumer starts. Spring Cloud Stream gives `earliest` to a consumer with a group, which keeps committed offsets, and `latest` to a consumer with no group. A feed with a position path is the first case, and a feed with no path is the second. |
|
|
125
|
+
| Kafka consumer auto-commit | A poll commits the offsets of the records that the last poll gave. Delivery is at least once. The feed saves its position in the same way: at the next request for a batch. |
|
|
126
|
+
| Kafka Connect JDBC source | Change data capture by query. Its modes are `bulk`, `incrementing`, `timestamp` and `timestamp+incrementing`. It does not see a hard delete: a deleted row is not in the next query. This package compares the full table, so it sees each delete. |
|
|
127
|
+
| Fivetran Teleport Sync | It compares the data set of the source with the data set of the last sync, with no log. It needs only a read-only user and no timestamp columns. This package does the same for a SQLite file. |
|
|
128
|
+
| Labio and Garcia-Molina, VLDB 1996 | "Efficient Snapshot Differential Algorithms for Data Warehousing". A snapshot differential compares two snapshots of a source and gives inserts, updates and deletes. This package is a snapshot differential. |
|
|
129
|
+
| elt | The composition of `DataVersion`, `Snapshot` and `SnapshotDifferential` runs in production for 14 databases of macOS apps. This package moves it into one place. |
|
|
130
|
+
|
|
131
|
+
A join, a filter or a projection is not in the feed. The host makes it from the events of the tables, as a consumer of Debezium or of a CockroachDB changefeed does.
|
|
132
|
+
|
|
133
|
+
## Built on
|
|
134
|
+
|
|
135
|
+
This package needed four changes in other packages. All four are done:
|
|
136
|
+
|
|
137
|
+
1. `@zukhruf/snapshot-differential` gives `insert` and `update` in place of one `upsert`.
|
|
138
|
+
2. `@zukhruf/async` gives the change watcher `pollChanges`. It polls a value, and yields when the value changes.
|
|
139
|
+
3. `@zukhruf/snapshot-differential` fingerprints bytes (`Uint8Array`, `Buffer`) in a record and in a key. Thus a `BLOB` column can be in `after` and in a key.
|
|
140
|
+
4. `@zukhruf/sqlite-reader` opens the read-only connection and throws `UnavailableError`. This package exports the same `UnavailableError` class as `@zukhruf/sqlite-snapshot` and `@zukhruf/sqlite-data-version` (repository ADR 0001, rule 6).
|
|
141
|
+
|
|
142
|
+
## Documentation
|
|
143
|
+
|
|
144
|
+
- [0001 Capture by snapshot differential](./docs/adr/0001-capture-by-snapshot-differential.md)
|
|
145
|
+
- [0002 WAL is required](./docs/adr/0002-wal-is-required.md)
|
|
146
|
+
- [0003 The change event](./docs/adr/0003-the-change-event.md)
|
|
147
|
+
- [0004 The feed owns its position](./docs/adr/0004-the-feed-owns-its-position.md)
|
|
148
|
+
- [0005 One snapshot for each pass](./docs/adr/0005-one-snapshot-for-each-pass.md)
|
|
149
|
+
- [0006 Every table, keyed by its primary key](./docs/adr/0006-every-table-keyed-by-its-primary-key.md)
|
|
150
|
+
|
|
151
|
+
## Development
|
|
152
|
+
|
|
153
|
+
Run the commands from the workspace root.
|
|
154
|
+
|
|
155
|
+
```sh
|
|
156
|
+
npx nx run sqlite-changefeed:test # builds, then runs the tests
|
|
157
|
+
npx nx run sqlite-changefeed:typecheck # formats, lints, then type checks
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
```
|
|
161
|
+
src/
|
|
162
|
+
changefeed.ts Changefeed: the connections, the runs of watch, and the save of a batch
|
|
163
|
+
pass.ts one pass in one snapshot, and the WAL check
|
|
164
|
+
tables.ts the tables of PRAGMA table_list, the key of each table, and its rows
|
|
165
|
+
table-changes.ts the change events of one table, from the differential
|
|
166
|
+
position.ts the position, and what a pass changes in it
|
|
167
|
+
position-file.ts the position of a durable feed, in its SQLite file
|
|
168
|
+
position-in-memory.ts the position of an observing feed
|
|
169
|
+
change-event.ts the change event
|
|
170
|
+
journal-mode-error.ts JournalModeError
|
|
171
|
+
index.ts the public exports
|
|
172
|
+
changefeed.test.ts the tests
|
|
173
|
+
```
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { SQLOutputValue } from 'node:sqlite';
|
|
2
|
+
import type { KeyValue } from '@zukhruf/snapshot-differential';
|
|
3
|
+
export type Key = Readonly<Record<string, KeyValue>>;
|
|
4
|
+
export type Row = Readonly<Record<string, SQLOutputValue>>;
|
|
5
|
+
export type ChangeEvent = {
|
|
6
|
+
readonly table: string;
|
|
7
|
+
readonly op: 'read' | 'insert' | 'update';
|
|
8
|
+
readonly key: Key;
|
|
9
|
+
readonly after: Row;
|
|
10
|
+
} | {
|
|
11
|
+
readonly table: string;
|
|
12
|
+
readonly op: 'delete';
|
|
13
|
+
readonly key: Key;
|
|
14
|
+
} | {
|
|
15
|
+
readonly table: string;
|
|
16
|
+
readonly op: 'truncate';
|
|
17
|
+
};
|
|
18
|
+
//# sourceMappingURL=change-event.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"change-event.d.ts","sourceRoot":"","sources":["../src/change-event.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AAG/D,MAAM,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;AAGrD,MAAM,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;AAI3D,MAAM,MAAM,WAAW,GACnB;IACE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC1C,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC;IAClB,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC;CACrB,GACD;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAA;CAAE,GACpE;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,UAAU,CAAA;CAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"change-event.js","sourceRoot":"","sources":["../src/change-event.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { ChangeEvent } from './change-event.ts';
|
|
2
|
+
export type WatchOptions = {
|
|
3
|
+
/** Milliseconds between two polls of the data version. */
|
|
4
|
+
readonly interval: number;
|
|
5
|
+
/** Stops the poll. The loop then rejects with `signal.reason`. */
|
|
6
|
+
readonly signal: AbortSignal;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* A change event for each changed row of each table of a SQLite database
|
|
10
|
+
* that another process owns. With a position path the feed is durable and
|
|
11
|
+
* goes on from its last saved batch; without one it observes from now.
|
|
12
|
+
*/
|
|
13
|
+
export declare class Changefeed implements Disposable {
|
|
14
|
+
#private;
|
|
15
|
+
constructor(databasePath: string, positionPath?: string);
|
|
16
|
+
/**
|
|
17
|
+
* Batches of change events. The feed saves the position of a batch when
|
|
18
|
+
* the loop asks for the next one.
|
|
19
|
+
*/
|
|
20
|
+
watch({ interval, signal, }: WatchOptions): AsyncGenerator<ChangeEvent[], void, undefined>;
|
|
21
|
+
[Symbol.dispose](): void;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=changefeed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"changefeed.d.ts","sourceRoot":"","sources":["../src/changefeed.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAMrD,MAAM,MAAM,YAAY,GAAG;IACzB,0DAA0D;IAC1D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,kEAAkE;IAClE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;CAC9B,CAAC;AAMF;;;;GAIG;AACH,qBAAa,UAAW,YAAW,UAAU;;gBAM/B,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM;IAiBvD;;;OAGG;IACI,KAAK,CAAC,EACX,QAAQ,EACR,MAAM,GACP,EAAE,YAAY,GAAG,cAAc,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC;IAwBhE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { pollChanges } from '@zukhruf/async';
|
|
2
|
+
import { DataVersion } from '@zukhruf/sqlite-data-version';
|
|
3
|
+
import { Snapshot } from '@zukhruf/sqlite-snapshot';
|
|
4
|
+
import { pass, requireWal } from "./pass.js";
|
|
5
|
+
import { PositionFile } from "./position-file.js";
|
|
6
|
+
import { PositionInMemory } from "./position-in-memory.js";
|
|
7
|
+
/**
|
|
8
|
+
* A change event for each changed row of each table of a SQLite database
|
|
9
|
+
* that another process owns. With a position path the feed is durable and
|
|
10
|
+
* goes on from its last saved batch; without one it observes from now.
|
|
11
|
+
*/
|
|
12
|
+
export class Changefeed {
|
|
13
|
+
#path;
|
|
14
|
+
#version;
|
|
15
|
+
#run;
|
|
16
|
+
#resources;
|
|
17
|
+
constructor(databasePath, positionPath) {
|
|
18
|
+
using resources = new DisposableStack();
|
|
19
|
+
this.#path = databasePath;
|
|
20
|
+
this.#version = resources.use(new DataVersion(databasePath));
|
|
21
|
+
{
|
|
22
|
+
using snapshot = new Snapshot(databasePath);
|
|
23
|
+
requireWal(snapshot);
|
|
24
|
+
}
|
|
25
|
+
if (positionPath === undefined) {
|
|
26
|
+
this.#run = () => ({ position: new PositionInMemory(), baseline: true });
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
const file = resources.use(new PositionFile(positionPath));
|
|
30
|
+
this.#run = () => ({ position: file, baseline: false });
|
|
31
|
+
}
|
|
32
|
+
this.#resources = resources.move();
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Batches of change events. The feed saves the position of a batch when
|
|
36
|
+
* the loop asks for the next one.
|
|
37
|
+
*/
|
|
38
|
+
async *watch({ interval, signal, }) {
|
|
39
|
+
const { position, baseline } = this.#run();
|
|
40
|
+
let silent = baseline;
|
|
41
|
+
for await (const changed of pollChanges(new Map([['database', async () => this.#version.current]]), { interval, signal })) {
|
|
42
|
+
if (changed.size === 0)
|
|
43
|
+
continue;
|
|
44
|
+
const { events, update } = await pass(this.#path, position);
|
|
45
|
+
if (silent || events.length === 0) {
|
|
46
|
+
// A baseline, or a pass with no events, has no batch to deliver.
|
|
47
|
+
position.save(update);
|
|
48
|
+
silent = false;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
yield events;
|
|
52
|
+
// The loop may not have applied a batch that an abort stopped.
|
|
53
|
+
signal.throwIfAborted();
|
|
54
|
+
position.save(update);
|
|
55
|
+
}
|
|
56
|
+
// The poll ends only when the signal aborts.
|
|
57
|
+
signal.throwIfAborted();
|
|
58
|
+
}
|
|
59
|
+
[Symbol.dispose]() {
|
|
60
|
+
this.#resources.dispose();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=changefeed.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"changefeed.js","sourceRoot":"","sources":["../src/changefeed.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAGpD,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAc3D;;;;GAIG;AACH,MAAM,OAAO,UAAU;IACZ,KAAK,CAAS;IACd,QAAQ,CAAc;IACtB,IAAI,CAAY;IAChB,UAAU,CAAkB;IAErC,YAAY,YAAoB,EAAE,YAAqB;QACrD,MAAM,SAAS,GAAG,IAAI,eAAe,EAAE,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAC7D,CAAC;YACC,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC;YAC5C,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3E,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC;YAC3D,IAAI,CAAC,IAAI,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACrC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,CAAC,KAAK,CAAC,EACX,QAAQ,EACR,MAAM,GACO;QACb,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,MAAM,GAAG,QAAQ,CAAC;QACtB,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,WAAW,CACrC,IAAI,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAC1D,EAAE,QAAQ,EAAE,MAAM,EAAE,CACrB,EAAE,CAAC;YACF,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;gBAAE,SAAS;YACjC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAC5D,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClC,iEAAiE;gBACjE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM,GAAG,KAAK,CAAC;gBACf,SAAS;YACX,CAAC;YACD,MAAM,MAAM,CAAC;YACb,+DAA+D;YAC/D,MAAM,CAAC,cAAc,EAAE,CAAC;YACxB,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;QACD,6CAA6C;QAC7C,MAAM,CAAC,cAAc,EAAE,CAAC;IAC1B,CAAC;IAED,CAAC,MAAM,CAAC,OAAO,CAAC;QACd,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;IAC5B,CAAC;CACF"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export type { ChangeEvent, Key, Row } from './change-event.ts';
|
|
2
|
+
export { Changefeed, type WatchOptions } from './changefeed.ts';
|
|
3
|
+
export { JournalModeError } from './journal-mode-error.ts';
|
|
4
|
+
export { UnavailableError } from '@zukhruf/sqlite-reader';
|
|
5
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,WAAW,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAqB,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** The database is not in WAL mode, so a pass would stop the commits of its owner. */
|
|
2
|
+
export declare class JournalModeError extends Error {
|
|
3
|
+
readonly path: string;
|
|
4
|
+
readonly journalMode: string;
|
|
5
|
+
constructor(path: string, journalMode: string);
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=journal-mode-error.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"journal-mode-error.d.ts","sourceRoot":"","sources":["../src/journal-mode-error.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;gBAEjB,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM;CAQ9C"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** The database is not in WAL mode, so a pass would stop the commits of its owner. */
|
|
2
|
+
export class JournalModeError extends Error {
|
|
3
|
+
path;
|
|
4
|
+
journalMode;
|
|
5
|
+
constructor(path, journalMode) {
|
|
6
|
+
super(`The database at ${JSON.stringify(path)} is in journal mode ${journalMode}, not wal: a read would stop the commits of its owner. Only the owner can change the mode.`);
|
|
7
|
+
this.name = 'JournalModeError';
|
|
8
|
+
this.path = path;
|
|
9
|
+
this.journalMode = journalMode;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=journal-mode-error.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"journal-mode-error.js","sourceRoot":"","sources":["../src/journal-mode-error.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAChC,IAAI,CAAS;IACb,WAAW,CAAS;IAE7B,YAAY,IAAY,EAAE,WAAmB;QAC3C,KAAK,CACH,mBAAmB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,uBAAuB,WAAW,4FAA4F,CACtK,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;CACF"}
|
package/dist/pass.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Snapshot } from '@zukhruf/sqlite-snapshot';
|
|
2
|
+
import type { ChangeEvent } from './change-event.ts';
|
|
3
|
+
import type { Position, PositionUpdate } from './position.ts';
|
|
4
|
+
export declare function pass(path: string, position: Position): Promise<{
|
|
5
|
+
events: ChangeEvent[];
|
|
6
|
+
update: PositionUpdate;
|
|
7
|
+
}>;
|
|
8
|
+
export declare function requireWal(snapshot: Snapshot): void;
|
|
9
|
+
//# sourceMappingURL=pass.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pass.d.ts","sourceRoot":"","sources":["../src/pass.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAEpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAiB,MAAM,eAAe,CAAC;AAM7E,wBAAsB,IAAI,CACxB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,QAAQ,GACjB,OAAO,CAAC;IAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAAC,MAAM,EAAE,cAAc,CAAA;CAAE,CAAC,CA6B5D;AAGD,wBAAgB,UAAU,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAKnD"}
|
package/dist/pass.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { Snapshot } from '@zukhruf/sqlite-snapshot';
|
|
2
|
+
import { JournalModeError } from "./journal-mode-error.js";
|
|
3
|
+
import { tableChanges } from "./table-changes.js";
|
|
4
|
+
import { tablesOf } from "./tables.js";
|
|
5
|
+
// A pass reads each table in one snapshot, and compares it with the
|
|
6
|
+
// position. It holds the snapshot only while it reads.
|
|
7
|
+
export async function pass(path, position) {
|
|
8
|
+
using snapshot = new Snapshot(path);
|
|
9
|
+
requireWal(snapshot);
|
|
10
|
+
const known = position.keyColumns();
|
|
11
|
+
const events = [];
|
|
12
|
+
const tables = new Map();
|
|
13
|
+
const current = tablesOf(snapshot);
|
|
14
|
+
for (const table of current) {
|
|
15
|
+
const keyColumns = known.get(table.name);
|
|
16
|
+
const same = keyColumns !== undefined &&
|
|
17
|
+
JSON.stringify(keyColumns) === JSON.stringify(table.keyColumns);
|
|
18
|
+
if (keyColumns !== undefined && !same)
|
|
19
|
+
events.push({ table: table.name, op: 'truncate' });
|
|
20
|
+
const changes = await tableChanges(table, same ? position.state(table.name) : null);
|
|
21
|
+
events.push(...changes.events);
|
|
22
|
+
if (!same || changes.events.length > 0)
|
|
23
|
+
tables.set(table.name, {
|
|
24
|
+
keyColumns: table.keyColumns,
|
|
25
|
+
state: changes.state,
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
const names = new Set(current.map((table) => table.name));
|
|
29
|
+
const dropped = [...known.keys()].filter((table) => !names.has(table));
|
|
30
|
+
for (const table of dropped)
|
|
31
|
+
events.push({ table, op: 'truncate' });
|
|
32
|
+
return { events, update: { tables, dropped } };
|
|
33
|
+
}
|
|
34
|
+
// The owner can replace the file, and the new file can have another mode.
|
|
35
|
+
export function requireWal(snapshot) {
|
|
36
|
+
const [row] = snapshot.all('PRAGMA journal_mode');
|
|
37
|
+
if (row === undefined)
|
|
38
|
+
throw new TypeError('PRAGMA journal_mode gave no row');
|
|
39
|
+
const mode = String(row.journal_mode);
|
|
40
|
+
if (mode !== 'wal')
|
|
41
|
+
throw new JournalModeError(snapshot.path, mode);
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=pass.js.map
|
package/dist/pass.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pass.js","sourceRoot":"","sources":["../src/pass.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAGpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,oEAAoE;AACpE,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,IAAY,EACZ,QAAkB;IAElB,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IACpC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACrB,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAC;IACpC,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,GACR,UAAU,KAAK,SAAS;YACxB,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAClE,IAAI,UAAU,KAAK,SAAS,IAAI,CAAC,IAAI;YACnC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,MAAM,YAAY,CAChC,KAAK,EACL,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CACzC,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;YACpC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE;gBACrB,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC;IACP,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACvE,KAAK,MAAM,KAAK,IAAI,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;IACpE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC;AACjD,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,UAAU,CAAC,QAAkB;IAC3C,MAAM,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IAClD,IAAI,GAAG,KAAK,SAAS;QAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IAC9E,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACtC,IAAI,IAAI,KAAK,KAAK;QAAE,MAAM,IAAI,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AACtE,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { SnapshotState } from '@zukhruf/snapshot-differential';
|
|
2
|
+
import type { Position, PositionUpdate } from './position.ts';
|
|
3
|
+
export declare class PositionFile implements Position, Disposable {
|
|
4
|
+
#private;
|
|
5
|
+
constructor(path: string);
|
|
6
|
+
keyColumns(): ReadonlyMap<string, readonly string[]>;
|
|
7
|
+
state(table: string): SnapshotState;
|
|
8
|
+
save({ tables, dropped }: PositionUpdate): void;
|
|
9
|
+
[Symbol.dispose](): void;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=position-file.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"position-file.d.ts","sourceRoot":"","sources":["../src/position-file.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAGpE,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAK9D,qBAAa,YAAa,YAAW,QAAQ,EAAE,UAAU;;gBAG3C,IAAI,EAAE,MAAM;IAqBxB,UAAU,IAAI,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAYpD,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa;IAcnC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,cAAc,GAAG,IAAI;IA8B/C,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import { transaction } from '@zukhruf/sqlite-transaction';
|
|
3
|
+
// The position of a durable feed: a SQLite file of the feed, with the key
|
|
4
|
+
// columns of each table and one fingerprint for each (table, key). A missing
|
|
5
|
+
// or empty file is a position with no tables.
|
|
6
|
+
export class PositionFile {
|
|
7
|
+
#database;
|
|
8
|
+
constructor(path) {
|
|
9
|
+
this.#database = new DatabaseSync(path);
|
|
10
|
+
try {
|
|
11
|
+
this.#database.exec(`
|
|
12
|
+
CREATE TABLE IF NOT EXISTS tables (
|
|
13
|
+
name TEXT PRIMARY KEY,
|
|
14
|
+
key_columns TEXT NOT NULL
|
|
15
|
+
) STRICT, WITHOUT ROWID;
|
|
16
|
+
CREATE TABLE IF NOT EXISTS fingerprints (
|
|
17
|
+
table_name TEXT NOT NULL,
|
|
18
|
+
key TEXT NOT NULL,
|
|
19
|
+
fingerprint TEXT NOT NULL,
|
|
20
|
+
PRIMARY KEY (table_name, key)
|
|
21
|
+
) STRICT, WITHOUT ROWID;
|
|
22
|
+
`);
|
|
23
|
+
}
|
|
24
|
+
catch (cause) {
|
|
25
|
+
this.#database.close();
|
|
26
|
+
throw cause;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
keyColumns() {
|
|
30
|
+
return new Map(this.#database
|
|
31
|
+
.prepare('SELECT name, key_columns FROM tables')
|
|
32
|
+
.all()
|
|
33
|
+
.map(({ name, key_columns }) => [
|
|
34
|
+
String(name),
|
|
35
|
+
columnsOf(String(key_columns)),
|
|
36
|
+
]));
|
|
37
|
+
}
|
|
38
|
+
state(table) {
|
|
39
|
+
return {
|
|
40
|
+
snapshot: Object.fromEntries(this.#database
|
|
41
|
+
.prepare('SELECT key, fingerprint FROM fingerprints WHERE table_name = ?')
|
|
42
|
+
.all(table)
|
|
43
|
+
.map(({ key, fingerprint }) => [String(key), String(fingerprint)])),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
// One transaction for the position of one batch.
|
|
47
|
+
save({ tables, dropped }) {
|
|
48
|
+
const forgetTable = this.#database.prepare('DELETE FROM tables WHERE name = ?');
|
|
49
|
+
const forgetRows = this.#database.prepare('DELETE FROM fingerprints WHERE table_name = ?');
|
|
50
|
+
const keepTable = this.#database.prepare('INSERT INTO tables (name, key_columns) VALUES (?, ?)');
|
|
51
|
+
const keepRow = this.#database.prepare('INSERT INTO fingerprints (table_name, key, fingerprint) VALUES (?, ?, ?)');
|
|
52
|
+
transaction(this.#database, () => {
|
|
53
|
+
for (const table of [...dropped, ...tables.keys()]) {
|
|
54
|
+
forgetTable.run(table);
|
|
55
|
+
forgetRows.run(table);
|
|
56
|
+
}
|
|
57
|
+
for (const [table, { keyColumns, state }] of tables) {
|
|
58
|
+
keepTable.run(table, JSON.stringify(keyColumns));
|
|
59
|
+
for (const [key, entry] of Object.entries(state.snapshot))
|
|
60
|
+
keepRow.run(table, key, fingerprintOf(entry));
|
|
61
|
+
}
|
|
62
|
+
}, { mode: 'immediate' });
|
|
63
|
+
}
|
|
64
|
+
[Symbol.dispose]() {
|
|
65
|
+
this.#database.close();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function columnsOf(json) {
|
|
69
|
+
const columns = JSON.parse(json);
|
|
70
|
+
if (!Array.isArray(columns) ||
|
|
71
|
+
!columns.every((column) => typeof column === 'string'))
|
|
72
|
+
throw new TypeError(`The position file holds ${json} as key columns`);
|
|
73
|
+
return columns;
|
|
74
|
+
}
|
|
75
|
+
// The feed gives the differential no expiresBy, so each entry is a
|
|
76
|
+
// fingerprint alone.
|
|
77
|
+
function fingerprintOf(entry) {
|
|
78
|
+
if (typeof entry !== 'string')
|
|
79
|
+
throw new TypeError('The position keeps no expiry');
|
|
80
|
+
return entry;
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=position-file.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"position-file.js","sourceRoot":"","sources":["../src/position-file.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAI1D,0EAA0E;AAC1E,6EAA6E;AAC7E,8CAA8C;AAC9C,MAAM,OAAO,YAAY;IACd,SAAS,CAAe;IAEjC,YAAY,IAAY;QACtB,IAAI,CAAC,SAAS,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC;YACH,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;;;;;;;;;;;OAWnB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YACvB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,UAAU;QACR,OAAO,IAAI,GAAG,CACZ,IAAI,CAAC,SAAS;aACX,OAAO,CAAC,sCAAsC,CAAC;aAC/C,GAAG,EAAE;aACL,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC;YACZ,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;SAC/B,CAAC,CACL,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAa;QACjB,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,WAAW,CAC1B,IAAI,CAAC,SAAS;iBACX,OAAO,CACN,gEAAgE,CACjE;iBACA,GAAG,CAAC,KAAK,CAAC;iBACV,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CACrE;SACF,CAAC;IACJ,CAAC;IAED,iDAAiD;IACjD,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAkB;QACtC,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACxC,mCAAmC,CACpC,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACvC,+CAA+C,CAChD,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACtC,sDAAsD,CACvD,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACpC,0EAA0E,CAC3E,CAAC;QACF,WAAW,CACT,IAAI,CAAC,SAAS,EACd,GAAG,EAAE;YACH,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBACnD,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACvB,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;YACD,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC;gBACpD,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC;gBACjD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;oBACvD,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;YAClD,CAAC;QACH,CAAC,EACD,EAAE,IAAI,EAAE,WAAW,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,CAAC,MAAM,CAAC,OAAO,CAAC;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,OAAO,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1C,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QACvB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,CAAC;QAEtD,MAAM,IAAI,SAAS,CAAC,2BAA2B,IAAI,iBAAiB,CAAC,CAAC;IACxE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,mEAAmE;AACnE,qBAAqB;AACrB,SAAS,aAAa,CAAC,KAAwC;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAC3B,MAAM,IAAI,SAAS,CAAC,8BAA8B,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { SnapshotState } from '@zukhruf/snapshot-differential';
|
|
2
|
+
import type { Position, PositionUpdate } from './position.ts';
|
|
3
|
+
export declare class PositionInMemory implements Position {
|
|
4
|
+
#private;
|
|
5
|
+
keyColumns(): ReadonlyMap<string, readonly string[]>;
|
|
6
|
+
state(table: string): SnapshotState;
|
|
7
|
+
save({ tables, dropped }: PositionUpdate): void;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=position-in-memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"position-in-memory.d.ts","sourceRoot":"","sources":["../src/position-in-memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAEpE,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAiB,MAAM,eAAe,CAAC;AAG7E,qBAAa,gBAAiB,YAAW,QAAQ;;IAG/C,UAAU,IAAI,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAMpD,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa;IAOnC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,cAAc,GAAG,IAAI;CAIhD"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// The position of an observing feed: in memory, for one run.
|
|
2
|
+
export class PositionInMemory {
|
|
3
|
+
#tables = new Map();
|
|
4
|
+
keyColumns() {
|
|
5
|
+
return new Map([...this.#tables].map(([table, { keyColumns }]) => [table, keyColumns]));
|
|
6
|
+
}
|
|
7
|
+
state(table) {
|
|
8
|
+
const position = this.#tables.get(table);
|
|
9
|
+
if (position === undefined)
|
|
10
|
+
throw new RangeError(`The position has no table ${table}`);
|
|
11
|
+
return position.state;
|
|
12
|
+
}
|
|
13
|
+
save({ tables, dropped }) {
|
|
14
|
+
for (const table of dropped)
|
|
15
|
+
this.#tables.delete(table);
|
|
16
|
+
for (const [table, position] of tables)
|
|
17
|
+
this.#tables.set(table, position);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=position-in-memory.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"position-in-memory.js","sourceRoot":"","sources":["../src/position-in-memory.ts"],"names":[],"mappings":"AAIA,6DAA6D;AAC7D,MAAM,OAAO,gBAAgB;IAClB,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEpD,UAAU;QACR,OAAO,IAAI,GAAG,CACZ,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CACxE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAa;QACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,QAAQ,KAAK,SAAS;YACxB,MAAM,IAAI,UAAU,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC,KAAK,CAAC;IACxB,CAAC;IAED,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAkB;QACtC,KAAK,MAAM,KAAK,IAAI,OAAO;YAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxD,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,MAAM;YAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC5E,CAAC;CACF"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { SnapshotState } from '@zukhruf/snapshot-differential';
|
|
2
|
+
export type TablePosition = {
|
|
3
|
+
readonly keyColumns: readonly string[];
|
|
4
|
+
readonly state: SnapshotState;
|
|
5
|
+
};
|
|
6
|
+
export type PositionUpdate = {
|
|
7
|
+
readonly tables: ReadonlyMap<string, TablePosition>;
|
|
8
|
+
readonly dropped: readonly string[];
|
|
9
|
+
};
|
|
10
|
+
export interface Position {
|
|
11
|
+
keyColumns(): ReadonlyMap<string, readonly string[]>;
|
|
12
|
+
state(table: string): SnapshotState;
|
|
13
|
+
save(update: PositionUpdate): void;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=position.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"position.d.ts","sourceRoot":"","sources":["../src/position.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAIpE,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;CAC/B,CAAC;AAIF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACpD,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC,CAAC;AAGF,MAAM,WAAW,QAAQ;IAEvB,UAAU,IAAI,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IAErD,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC;IACpC,IAAI,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC;CACpC"}
|
package/dist/position.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"position.js","sourceRoot":"","sources":["../src/position.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type SnapshotState } from '@zukhruf/snapshot-differential';
|
|
2
|
+
import type { ChangeEvent } from './change-event.ts';
|
|
3
|
+
import type { Table } from './tables.ts';
|
|
4
|
+
export declare function tableChanges(table: Table, saved: SnapshotState | null): Promise<{
|
|
5
|
+
events: ChangeEvent[];
|
|
6
|
+
state: SnapshotState;
|
|
7
|
+
}>;
|
|
8
|
+
//# sourceMappingURL=table-changes.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"table-changes.d.ts","sourceRoot":"","sources":["../src/table-changes.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,gCAAgC,CAAC;AAExC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAY,KAAK,EAAE,MAAM,aAAa,CAAC;AAKnD,wBAAsB,YAAY,CAChC,KAAK,EAAE,KAAK,EACZ,KAAK,EAAE,aAAa,GAAG,IAAI,GAC1B,OAAO,CAAC;IAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAAC,KAAK,EAAE,aAAa,CAAA;CAAE,CAAC,CAoC1D"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { ClosedWorld, SnapshotDifferential, } from '@zukhruf/snapshot-differential';
|
|
2
|
+
// The change events of one table against its snapshot in the position, and
|
|
3
|
+
// its new snapshot. null as the snapshot: the position does not have the
|
|
4
|
+
// table, so each row is a read.
|
|
5
|
+
export async function tableChanges(table, saved) {
|
|
6
|
+
const differential = new SnapshotDifferential({
|
|
7
|
+
name: table.name,
|
|
8
|
+
primaryKey: table.keyColumns,
|
|
9
|
+
key: (row) => table.keyColumns.map((column) => keyValueOf(row, column)),
|
|
10
|
+
// A pass reads each row of the table, so a key it does not find was
|
|
11
|
+
// deleted.
|
|
12
|
+
policy: new ClosedWorld(),
|
|
13
|
+
});
|
|
14
|
+
const events = [];
|
|
15
|
+
for await (const change of differential.diff(table.rows(), saved)) {
|
|
16
|
+
switch (change.type) {
|
|
17
|
+
case 'insert':
|
|
18
|
+
events.push({
|
|
19
|
+
table: table.name,
|
|
20
|
+
op: saved === null ? 'read' : 'insert',
|
|
21
|
+
key: change.record.key,
|
|
22
|
+
after: change.record.after,
|
|
23
|
+
});
|
|
24
|
+
break;
|
|
25
|
+
case 'update':
|
|
26
|
+
events.push({
|
|
27
|
+
table: table.name,
|
|
28
|
+
op: 'update',
|
|
29
|
+
key: change.record.key,
|
|
30
|
+
after: change.record.after,
|
|
31
|
+
});
|
|
32
|
+
break;
|
|
33
|
+
case 'delete':
|
|
34
|
+
events.push({ table: table.name, op: 'delete', key: change.key });
|
|
35
|
+
break;
|
|
36
|
+
case 'state':
|
|
37
|
+
return { events, state: change.state };
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
throw new TypeError(`The differential of ${table.name} gave no snapshot`);
|
|
41
|
+
}
|
|
42
|
+
function keyValueOf(row, column) {
|
|
43
|
+
const value = row.key[column];
|
|
44
|
+
if (value === undefined)
|
|
45
|
+
throw new TypeError(`A row has no value for the key column ${column}`);
|
|
46
|
+
return value;
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=table-changes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"table-changes.js","sourceRoot":"","sources":["../src/table-changes.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EACX,oBAAoB,GAErB,MAAM,gCAAgC,CAAC;AAKxC,2EAA2E;AAC3E,yEAAyE;AACzE,gCAAgC;AAChC,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAY,EACZ,KAA2B;IAE3B,MAAM,YAAY,GAAG,IAAI,oBAAoB,CAAW;QACtD,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACvE,oEAAoE;QACpE,WAAW;QACX,MAAM,EAAE,IAAI,WAAW,EAAE;KAC1B,CAAC,CAAC;IACH,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC;QAClE,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,QAAQ;gBACX,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,KAAK,CAAC,IAAI;oBACjB,EAAE,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;oBACtC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;oBACtB,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;iBAC3B,CAAC,CAAC;gBACH,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,KAAK,CAAC,IAAI;oBACjB,EAAE,EAAE,QAAQ;oBACZ,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;oBACtB,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;iBAC3B,CAAC,CAAC;gBACH,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;gBAClE,MAAM;YACR,KAAK,OAAO;gBACV,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,MAAM,IAAI,SAAS,CAAC,uBAAuB,KAAK,CAAC,IAAI,mBAAmB,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,UAAU,CAAC,GAAa,EAAE,MAAc;IAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,KAAK,KAAK,SAAS;QACrB,MAAM,IAAI,SAAS,CAAC,yCAAyC,MAAM,EAAE,CAAC,CAAC;IACzE,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/dist/tables.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Snapshot } from '@zukhruf/sqlite-snapshot';
|
|
2
|
+
import type { Key, Row } from './change-event.ts';
|
|
3
|
+
export type KeyedRow = {
|
|
4
|
+
readonly key: Key;
|
|
5
|
+
readonly after: Row;
|
|
6
|
+
};
|
|
7
|
+
export type Table = {
|
|
8
|
+
readonly name: string;
|
|
9
|
+
readonly keyColumns: readonly string[];
|
|
10
|
+
rows(): Iterable<KeyedRow>;
|
|
11
|
+
};
|
|
12
|
+
export declare function tablesOf(snapshot: Snapshot): Table[];
|
|
13
|
+
//# sourceMappingURL=tables.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../src/tables.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAEzD,OAAO,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAGlD,MAAM,MAAM,QAAQ,GAAG;IAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAA;CAAE,CAAC;AAIlE,MAAM,MAAM,KAAK,GAAG;IAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,IAAI,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC;CAC5B,CAAC;AAKF,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,KAAK,EAAE,CAuCpD"}
|
package/dist/tables.js
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// Each entry of PRAGMA table_list with type table, except the internal
|
|
2
|
+
// tables of SQLite. Views, virtual tables and shadow tables have no rows of
|
|
3
|
+
// their own in the file.
|
|
4
|
+
export function tablesOf(snapshot) {
|
|
5
|
+
return snapshot
|
|
6
|
+
.all(`SELECT name FROM pragma_table_list
|
|
7
|
+
WHERE schema = 'main' AND type = 'table'
|
|
8
|
+
AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\'
|
|
9
|
+
ORDER BY name`)
|
|
10
|
+
.map(({ name }) => {
|
|
11
|
+
const table = text(name);
|
|
12
|
+
const primaryKey = identifyingPrimaryKey(snapshot, table);
|
|
13
|
+
const from = `FROM "${table.replaceAll('"', '""')}"`;
|
|
14
|
+
if (primaryKey === null)
|
|
15
|
+
return {
|
|
16
|
+
name: table,
|
|
17
|
+
keyColumns: ['rowid'],
|
|
18
|
+
rows: () => snapshot
|
|
19
|
+
.iterate(`SELECT rowid AS rowid, * ${from}`)
|
|
20
|
+
.map(({ rowid, ...after }) => ({
|
|
21
|
+
key: { rowid: keyValue(rowid) },
|
|
22
|
+
after,
|
|
23
|
+
})),
|
|
24
|
+
};
|
|
25
|
+
return {
|
|
26
|
+
name: table,
|
|
27
|
+
keyColumns: primaryKey,
|
|
28
|
+
rows: () => snapshot.iterate(`SELECT * ${from}`).map((row) => {
|
|
29
|
+
const after = { ...row };
|
|
30
|
+
return {
|
|
31
|
+
key: Object.fromEntries(primaryKey.map((column) => [column, keyValue(after[column])])),
|
|
32
|
+
after,
|
|
33
|
+
};
|
|
34
|
+
}),
|
|
35
|
+
};
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
// The columns of the declared primary key when SQLite makes it unique and
|
|
39
|
+
// not NULL: as an alias of the rowid, or when each of its columns is NOT
|
|
40
|
+
// NULL. PRAGMA table_info gives NOT NULL for each primary key column of a
|
|
41
|
+
// WITHOUT ROWID table, because SQLite enforces it there. null when the rowid identifies the row: a table
|
|
42
|
+
// with no primary key, or one whose declared primary key can hold NULL in
|
|
43
|
+
// many rows.
|
|
44
|
+
function identifyingPrimaryKey(snapshot, table) {
|
|
45
|
+
const columns = snapshot.all('SELECT name, "notnull" FROM pragma_table_info(?) WHERE pk > 0 ORDER BY pk', table);
|
|
46
|
+
if (columns.length === 0)
|
|
47
|
+
return null;
|
|
48
|
+
// SQLite makes an index for each primary key but an alias of the rowid.
|
|
49
|
+
// The type alone does not tell it: INTEGER PRIMARY KEY DESC in a column
|
|
50
|
+
// definition is no alias, and can hold NULL.
|
|
51
|
+
const rowidAlias = snapshot.all("SELECT 1 FROM pragma_index_list(?) WHERE origin = 'pk'", table).length === 0;
|
|
52
|
+
if (rowidAlias || columns.every((column) => column.notnull === 1))
|
|
53
|
+
return columns.map((column) => text(column.name));
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
function text(value) {
|
|
57
|
+
if (typeof value !== 'string')
|
|
58
|
+
throw new TypeError(`SQLite gave ${String(value)} for a name`);
|
|
59
|
+
return value;
|
|
60
|
+
}
|
|
61
|
+
// The schema rules above keep NULL out of a key, and node:sqlite gives a
|
|
62
|
+
// bigint only to a statement that asks for it.
|
|
63
|
+
function keyValue(value) {
|
|
64
|
+
if (value === null || value === undefined || typeof value === 'bigint')
|
|
65
|
+
throw new TypeError(`SQLite gave ${String(value)} for a key value`);
|
|
66
|
+
return value;
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=tables.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tables.js","sourceRoot":"","sources":["../src/tables.ts"],"names":[],"mappings":"AAkBA,uEAAuE;AACvE,4EAA4E;AAC5E,yBAAyB;AACzB,MAAM,UAAU,QAAQ,CAAC,QAAkB;IACzC,OAAO,QAAQ;SACZ,GAAG,CACF;;;qBAGe,CAChB;SACA,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACzB,MAAM,UAAU,GAAG,qBAAqB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,SAAS,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;QACrD,IAAI,UAAU,KAAK,IAAI;YACrB,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,UAAU,EAAE,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,GAAG,EAAE,CACT,QAAQ;qBACL,OAAO,CAAC,4BAA4B,IAAI,EAAE,CAAC;qBAC3C,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;oBAC7B,GAAG,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE;oBAC/B,KAAK;iBACN,CAAC,CAAC;aACR,CAAC;QACJ,OAAO;YACL,IAAI,EAAE,KAAK;YACX,UAAU,EAAE,UAAU;YACtB,IAAI,EAAE,GAAG,EAAE,CACT,QAAQ,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/C,MAAM,KAAK,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC;gBACzB,OAAO;oBACL,GAAG,EAAE,MAAM,CAAC,WAAW,CACrB,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAC9D;oBACD,KAAK;iBACN,CAAC;YACJ,CAAC,CAAC;SACL,CAAC;IACJ,CAAC,CAAC,CAAC;AACP,CAAC;AAED,0EAA0E;AAC1E,yEAAyE;AACzE,0EAA0E;AAC1E,yGAAyG;AACzG,0EAA0E;AAC1E,aAAa;AACb,SAAS,qBAAqB,CAC5B,QAAkB,EAClB,KAAa;IAEb,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAC1B,2EAA2E,EAC3E,KAAK,CACN,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,wEAAwE;IACxE,wEAAwE;IACxE,6CAA6C;IAC7C,MAAM,UAAU,GACd,QAAQ,CAAC,GAAG,CACV,wDAAwD,EACxD,KAAK,CACN,CAAC,MAAM,KAAK,CAAC,CAAC;IACjB,IAAI,UAAU,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACpD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,IAAI,CAAC,KAAiC;IAC7C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAC3B,MAAM,IAAI,SAAS,CAAC,eAAe,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACjE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,yEAAyE;AACzE,+CAA+C;AAC/C,SAAS,QAAQ,CAAC,KAAiC;IACjD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,QAAQ;QACpE,MAAM,IAAI,SAAS,CAAC,eAAe,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtE,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,54 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/sqlite-changefeed",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.3.53",
|
|
4
|
+
"description": "A changefeed of a SQLite database that another process owns and writes: a change event for each changed row of each table, keyed by the primary key of the row, by a snapshot differential over a read-only connection. The feed keeps its own position, and needs WAL mode.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "https://github.com/JanuaryLabs/zukhruf",
|
|
9
|
+
"directory": "packages/sqlite/changefeed"
|
|
10
|
+
},
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/JanuaryLabs/zukhruf/issues"
|
|
13
|
+
},
|
|
14
|
+
"homepage": "https://github.com/JanuaryLabs/zukhruf/tree/main/packages/sqlite/changefeed#readme",
|
|
15
|
+
"keywords": [
|
|
16
|
+
"sqlite",
|
|
17
|
+
"node-sqlite",
|
|
18
|
+
"changefeed",
|
|
19
|
+
"change-data-capture",
|
|
20
|
+
"cdc",
|
|
21
|
+
"snapshot-differential",
|
|
22
|
+
"wal"
|
|
23
|
+
],
|
|
24
|
+
"type": "module",
|
|
25
|
+
"exports": {
|
|
26
|
+
"./package.json": "./package.json",
|
|
27
|
+
".": {
|
|
28
|
+
"@zukhruf/source": "./src/index.ts",
|
|
29
|
+
"types": "./dist/index.d.ts",
|
|
30
|
+
"import": "./dist/index.js",
|
|
31
|
+
"default": "./dist/index.js"
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
"files": [
|
|
35
|
+
"dist",
|
|
36
|
+
"!dist/spec/**",
|
|
37
|
+
"!**/*.tsbuildinfo",
|
|
38
|
+
"!**/*.test.*"
|
|
39
|
+
],
|
|
40
|
+
"publishConfig": {
|
|
41
|
+
"access": "public"
|
|
42
|
+
},
|
|
43
|
+
"dependencies": {
|
|
44
|
+
"@zukhruf/async": "^0.3.53",
|
|
45
|
+
"@zukhruf/snapshot-differential": "^0.3.53",
|
|
46
|
+
"@zukhruf/sqlite-data-version": "^0.3.53",
|
|
47
|
+
"@zukhruf/sqlite-reader": "^0.3.53",
|
|
48
|
+
"@zukhruf/sqlite-snapshot": "^0.3.53",
|
|
49
|
+
"@zukhruf/sqlite-transaction": "^0.3.53"
|
|
50
|
+
},
|
|
51
|
+
"devDependencies": {
|
|
52
|
+
"@zukhruf/testing": "^0.3.53"
|
|
53
|
+
}
|
|
54
|
+
}
|