@syncular/client 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/apply.d.ts +1 -1
- package/dist/apply.js +18 -6
- package/dist/schema.d.ts +10 -0
- package/dist/schema.js +91 -0
- package/package.json +3 -3
- package/src/apply.ts +18 -6
- package/src/schema.ts +148 -0
package/README.md
CHANGED
|
@@ -3,6 +3,17 @@
|
|
|
3
3
|
The TypeScript client protocol core (SPEC.md §§3–8, client side) plus its
|
|
4
4
|
browser platform bindings.
|
|
5
5
|
|
|
6
|
+
## Client-local FTS5 projections
|
|
7
|
+
|
|
8
|
+
Generated schemas may attach `ftsIndexes` to a synced table (RFC 0005). The
|
|
9
|
+
client materializes each as a contentful local FTS5 table with a private stable
|
|
10
|
+
source identity and insert/update/delete triggers. Existing visible rows are
|
|
11
|
+
bulk-indexed on first creation; schema reset recreates the projection. The FTS
|
|
12
|
+
table is a read-only application query surface: it is never synced, subscribed,
|
|
13
|
+
or mutated. A missing FTS5 build fails schema setup explicitly—Syncular does
|
|
14
|
+
not substitute an unbounded `LIKE` scan. Indexed fields must be non-encrypted
|
|
15
|
+
strings.
|
|
16
|
+
|
|
6
17
|
## Browser modes — there are exactly two
|
|
7
18
|
|
|
8
19
|
**Persistent worker mode is THE mode** (REVISE Direction decision 2,
|
package/dist/apply.d.ts
CHANGED
|
@@ -61,7 +61,7 @@ export interface SqliteSegmentDescriptor {
|
|
|
61
61
|
* the in-file metadata against the descriptor, validate the data table's
|
|
62
62
|
* column names/order against the generated schema, run the §5.6
|
|
63
63
|
* first-page clear when fresh, then copy every row with a single
|
|
64
|
-
*
|
|
64
|
+
* one primary-key upsert — `_syncular_version` lands in
|
|
65
65
|
* `_sync_version` exactly like a rows segment's per-row `serverVersion`.
|
|
66
66
|
*/
|
|
67
67
|
export declare function applySqliteSegment(db: ClientDatabase, schema: CompiledClientSchema, table: CompiledClientTable, bytes: Uint8Array, descriptor: SqliteSegmentDescriptor, options: {
|
package/dist/apply.js
CHANGED
|
@@ -12,7 +12,12 @@ function upsertSql(table) {
|
|
|
12
12
|
quoteIdent(SYNC_VERSION_COLUMN),
|
|
13
13
|
];
|
|
14
14
|
const placeholders = names.map(() => '?').join(', ');
|
|
15
|
-
|
|
15
|
+
const updates = names
|
|
16
|
+
.filter((name) => name !== quoteIdent(table.primaryKey))
|
|
17
|
+
.map((name) => `${name}=excluded.${name}`)
|
|
18
|
+
.join(', ');
|
|
19
|
+
return `INSERT INTO ${quoteIdent(table.name)} (${names.join(', ')}) VALUES (${placeholders})
|
|
20
|
+
ON CONFLICT (${quoteIdent(table.primaryKey)}) DO UPDATE SET ${updates}`;
|
|
16
21
|
}
|
|
17
22
|
export function upsertLocalRow(db, table, values, syncVersion) {
|
|
18
23
|
db.exec(upsertSql(table), [...values.map(toSqlValue), syncVersion]);
|
|
@@ -182,7 +187,7 @@ function imageInvalid(detail) {
|
|
|
182
187
|
* the in-file metadata against the descriptor, validate the data table's
|
|
183
188
|
* column names/order against the generated schema, run the §5.6
|
|
184
189
|
* first-page clear when fresh, then copy every row with a single
|
|
185
|
-
*
|
|
190
|
+
* one primary-key upsert — `_syncular_version` lands in
|
|
186
191
|
* `_sync_version` exactly like a rows segment's per-row `serverVersion`.
|
|
187
192
|
*/
|
|
188
193
|
export function applySqliteSegment(db, schema, table, bytes, descriptor, options) {
|
|
@@ -239,16 +244,23 @@ export function applySqliteSegment(db, schema, table, bytes, descriptor, options
|
|
|
239
244
|
expected.some((name, index) => actual[index] !== name)) {
|
|
240
245
|
throw new ClientSyncError('sync.schema_mismatch', `sqlite segment for ${JSON.stringify(table.name)} does not match the generated schema: columns [${actual.join(', ')}] (§5.3)`);
|
|
241
246
|
}
|
|
242
|
-
// 3. One transaction: fresh-bootstrap clear, then
|
|
247
|
+
// 3. One transaction: fresh-bootstrap clear, then primary-key upsert.
|
|
243
248
|
const names = table.columns.map((column) => quoteIdent(column.name));
|
|
249
|
+
const insertNames = [...names, quoteIdent(SYNC_VERSION_COLUMN)];
|
|
250
|
+
const updates = insertNames
|
|
251
|
+
.filter((name) => name !== quoteIdent(table.primaryKey))
|
|
252
|
+
.map((name) => `${name}=excluded.${name}`)
|
|
253
|
+
.join(', ');
|
|
244
254
|
return (options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
245
255
|
if (options.clearFirst) {
|
|
246
256
|
deleteScopedRows(db, table, options.effective);
|
|
247
257
|
}
|
|
248
|
-
db.exec(`INSERT
|
|
249
|
-
(${
|
|
258
|
+
db.exec(`INSERT INTO ${quoteIdent(table.name)}
|
|
259
|
+
(${insertNames.join(', ')})
|
|
250
260
|
SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')}
|
|
251
|
-
FROM ${IMAGE_ALIAS}.${quoteIdent(table.name)}
|
|
261
|
+
FROM ${IMAGE_ALIAS}.${quoteIdent(table.name)}
|
|
262
|
+
WHERE true
|
|
263
|
+
ON CONFLICT (${quoteIdent(table.primaryKey)}) DO UPDATE SET ${updates}`);
|
|
252
264
|
const counted = db.query(`SELECT count(*) AS n FROM ${IMAGE_ALIAS}.${quoteIdent(table.name)}`)[0];
|
|
253
265
|
const applied = Number(counted?.n ?? 0);
|
|
254
266
|
if (applied !== descriptor.rowCount) {
|
package/dist/schema.d.ts
CHANGED
|
@@ -17,6 +17,12 @@ export interface ClientIndexSpec {
|
|
|
17
17
|
readonly columns: readonly string[];
|
|
18
18
|
readonly unique: boolean;
|
|
19
19
|
}
|
|
20
|
+
/** One client-local contentful FTS5 projection (RFC 0005). */
|
|
21
|
+
export interface ClientFtsIndexSpec {
|
|
22
|
+
readonly name: string;
|
|
23
|
+
readonly columns: readonly string[];
|
|
24
|
+
readonly tokenize: string;
|
|
25
|
+
}
|
|
20
26
|
export interface ClientTableSchema {
|
|
21
27
|
readonly name: string;
|
|
22
28
|
/** Columns in schema-IR declaration order (the row-codec order, §2.4). */
|
|
@@ -27,6 +33,8 @@ export interface ClientTableSchema {
|
|
|
27
33
|
/** Local secondary indexes; absent in the generated schema when a table
|
|
28
34
|
* declares none (typegen omits the key for index-free tables). */
|
|
29
35
|
readonly indexes?: readonly ClientIndexSpec[];
|
|
36
|
+
/** Local FTS5 search projections; omitted when a table declares none. */
|
|
37
|
+
readonly ftsIndexes?: readonly ClientFtsIndexSpec[];
|
|
30
38
|
}
|
|
31
39
|
export interface ClientSchema {
|
|
32
40
|
readonly version: number;
|
|
@@ -56,6 +64,8 @@ export interface CompiledClientTable {
|
|
|
56
64
|
/** Local secondary indexes to create on the mirror table (declaration
|
|
57
65
|
* order); empty when the table declares none. */
|
|
58
66
|
readonly indexes: readonly ClientIndexSpec[];
|
|
67
|
+
/** Client-local FTS5 projections, in declaration order. */
|
|
68
|
+
readonly ftsIndexes: readonly ClientFtsIndexSpec[];
|
|
59
69
|
/** §5.11: true when any column is `encrypted`. Drives the encrypt/decrypt
|
|
60
70
|
* seam (skipped entirely when false) and the local-plaintext DDL. */
|
|
61
71
|
readonly hasEncryptedColumns: boolean;
|
package/dist/schema.js
CHANGED
|
@@ -1,12 +1,25 @@
|
|
|
1
1
|
import { ClientSyncError } from './errors.js';
|
|
2
2
|
import { snakeToCamel } from './naming.js';
|
|
3
3
|
const PATTERN_RE = /^([^{}]+):\{([^{}:]+)\}$/;
|
|
4
|
+
const ALLOWED_FTS_TOKENIZERS = new Set([
|
|
5
|
+
'unicode61',
|
|
6
|
+
'unicode61 remove_diacritics 0',
|
|
7
|
+
'unicode61 remove_diacritics 1',
|
|
8
|
+
'unicode61 remove_diacritics 2',
|
|
9
|
+
'porter unicode61',
|
|
10
|
+
'trigram',
|
|
11
|
+
]);
|
|
4
12
|
export function compileClientSchema(schema) {
|
|
5
13
|
const tables = new Map();
|
|
14
|
+
const schemaObjectNames = new Set();
|
|
6
15
|
for (const table of schema.tables) {
|
|
7
16
|
if (tables.has(table.name)) {
|
|
8
17
|
throw new Error(`duplicate table ${JSON.stringify(table.name)}`);
|
|
9
18
|
}
|
|
19
|
+
if (schemaObjectNames.has(table.name)) {
|
|
20
|
+
throw new Error(`table ${JSON.stringify(table.name)} conflicts with an index or FTS projection`);
|
|
21
|
+
}
|
|
22
|
+
schemaObjectNames.add(table.name);
|
|
10
23
|
const columnIndex = new Map();
|
|
11
24
|
table.columns.forEach((column, index) => {
|
|
12
25
|
if (columnIndex.has(column.name)) {
|
|
@@ -59,12 +72,46 @@ export function compileClientSchema(schema) {
|
|
|
59
72
|
columnIndexByCamel.delete(alias);
|
|
60
73
|
const indexes = table.indexes ?? [];
|
|
61
74
|
for (const index of indexes) {
|
|
75
|
+
if (schemaObjectNames.has(index.name)) {
|
|
76
|
+
throw new Error(`table ${table.name}: index ${JSON.stringify(index.name)} conflicts with another schema object`);
|
|
77
|
+
}
|
|
78
|
+
schemaObjectNames.add(index.name);
|
|
62
79
|
for (const column of index.columns) {
|
|
63
80
|
if (!columnIndex.has(column)) {
|
|
64
81
|
throw new Error(`table ${table.name}: index ${JSON.stringify(index.name)} names unknown column ${JSON.stringify(column)}`);
|
|
65
82
|
}
|
|
66
83
|
}
|
|
67
84
|
}
|
|
85
|
+
const ftsIndexes = table.ftsIndexes ?? [];
|
|
86
|
+
for (const index of ftsIndexes) {
|
|
87
|
+
if (schemaObjectNames.has(index.name)) {
|
|
88
|
+
throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} conflicts with another schema object`);
|
|
89
|
+
}
|
|
90
|
+
schemaObjectNames.add(index.name);
|
|
91
|
+
if (index.columns.length === 0 || index.columns.length > 32) {
|
|
92
|
+
throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} needs between 1 and 32 columns`);
|
|
93
|
+
}
|
|
94
|
+
const seenColumns = new Set();
|
|
95
|
+
for (const columnName of index.columns) {
|
|
96
|
+
if (seenColumns.has(columnName)) {
|
|
97
|
+
throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} repeats column ${JSON.stringify(columnName)}`);
|
|
98
|
+
}
|
|
99
|
+
seenColumns.add(columnName);
|
|
100
|
+
const column = table.columns.find((candidate) => candidate.name === columnName);
|
|
101
|
+
if (column === undefined) {
|
|
102
|
+
throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} names unknown column ${JSON.stringify(columnName)}`);
|
|
103
|
+
}
|
|
104
|
+
if (localColumnType(column) !== 'string') {
|
|
105
|
+
throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} column ${JSON.stringify(columnName)} must have string type`);
|
|
106
|
+
}
|
|
107
|
+
if (column.encrypted === true) {
|
|
108
|
+
throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} cannot index encrypted column ${JSON.stringify(columnName)}`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
if (!ALLOWED_FTS_TOKENIZERS.has(index.tokenize)) {
|
|
112
|
+
throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} tokenizer ${JSON.stringify(index.tokenize)} is not allowlisted`);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
68
115
|
tables.set(table.name, {
|
|
69
116
|
name: table.name,
|
|
70
117
|
columns: table.columns,
|
|
@@ -75,6 +122,7 @@ export function compileClientSchema(schema) {
|
|
|
75
122
|
scopeColumnByVariable,
|
|
76
123
|
scopePrefixByVariable,
|
|
77
124
|
indexes,
|
|
125
|
+
ftsIndexes,
|
|
78
126
|
hasEncryptedColumns: table.columns.some((c) => c.encrypted === true),
|
|
79
127
|
});
|
|
80
128
|
}
|
|
@@ -167,6 +215,31 @@ function createSyncedTable(db, table) {
|
|
|
167
215
|
db.exec(`CREATE ${unique}INDEX IF NOT EXISTS ${quoteIdent(index.name)} ON ${quoteIdent(table.name)} (${cols})`);
|
|
168
216
|
}
|
|
169
217
|
}
|
|
218
|
+
const FTS_SOURCE_ID_COLUMN = '_syncular_source_id';
|
|
219
|
+
function createFtsProjection(db, table, index) {
|
|
220
|
+
const existed = db.query("SELECT 1 AS present FROM sqlite_master WHERE type='table' AND name=?", [index.name]).length > 0;
|
|
221
|
+
const indexedColumns = index.columns.map(quoteIdent);
|
|
222
|
+
const tokenizer = index.tokenize.replaceAll("'", "''");
|
|
223
|
+
db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${quoteIdent(index.name)} USING fts5(${quoteIdent(FTS_SOURCE_ID_COLUMN)} UNINDEXED, ${indexedColumns.join(', ')}, tokenize='${tokenizer}')`);
|
|
224
|
+
const sourceId = `CAST(${quoteIdent(table.primaryKey)} AS TEXT)`;
|
|
225
|
+
const newSourceId = `CAST(new.${quoteIdent(table.primaryKey)} AS TEXT)`;
|
|
226
|
+
const oldSourceId = `CAST(old.${quoteIdent(table.primaryKey)} AS TEXT)`;
|
|
227
|
+
const projectionColumns = [
|
|
228
|
+
quoteIdent(FTS_SOURCE_ID_COLUMN),
|
|
229
|
+
...indexedColumns,
|
|
230
|
+
].join(', ');
|
|
231
|
+
const newValues = [
|
|
232
|
+
newSourceId,
|
|
233
|
+
...index.columns.map((column) => `new.${quoteIdent(column)}`),
|
|
234
|
+
].join(', ');
|
|
235
|
+
const deleteFor = (value) => `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} = ${value}`;
|
|
236
|
+
db.exec(`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`);
|
|
237
|
+
db.exec(`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_ad`)} AFTER DELETE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; END`);
|
|
238
|
+
db.exec(`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`);
|
|
239
|
+
if (!existed) {
|
|
240
|
+
db.exec(`INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) SELECT ${sourceId}, ${indexedColumns.join(', ')} FROM ${quoteIdent(table.name)}`);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
170
243
|
/**
|
|
171
244
|
* Create the synced tables plus client bookkeeping tables (outbox,
|
|
172
245
|
* subscription state, meta). Idempotent.
|
|
@@ -176,6 +249,11 @@ export function ensureLocalSchema(db, schema) {
|
|
|
176
249
|
for (const table of schema.tables.values()) {
|
|
177
250
|
createSyncedTable(db, table);
|
|
178
251
|
}
|
|
252
|
+
for (const table of schema.tables.values()) {
|
|
253
|
+
for (const index of table.ftsIndexes) {
|
|
254
|
+
createFtsProjection(db, table, index);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
179
257
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
|
|
180
258
|
key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
|
|
181
259
|
db.exec(`INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`);
|
|
@@ -249,6 +327,14 @@ const RESERVED_TABLE_PREFIX = '_syncular_';
|
|
|
249
327
|
* transaction and the subscription-state reset (state.ts).
|
|
250
328
|
*/
|
|
251
329
|
export function dropAndRecreateSyncedTables(db, schema) {
|
|
330
|
+
// Drop virtual tables first. SQLite then removes their shadow tables as one
|
|
331
|
+
// unit, so the generic discovery below never tears an FTS5 projection apart.
|
|
332
|
+
const virtualTables = db.query(`SELECT name FROM sqlite_master WHERE type = 'table'
|
|
333
|
+
AND sql LIKE 'CREATE VIRTUAL TABLE%'
|
|
334
|
+
AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'`);
|
|
335
|
+
for (const row of virtualTables) {
|
|
336
|
+
db.exec(`DROP TABLE IF EXISTS ${quoteIdent(String(row.name))}`);
|
|
337
|
+
}
|
|
252
338
|
const existing = db.query(`SELECT name FROM sqlite_master WHERE type = 'table'
|
|
253
339
|
AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'
|
|
254
340
|
AND name NOT LIKE 'sqlite_%'`);
|
|
@@ -258,6 +344,11 @@ export function dropAndRecreateSyncedTables(db, schema) {
|
|
|
258
344
|
for (const table of schema.tables.values()) {
|
|
259
345
|
createSyncedTable(db, table);
|
|
260
346
|
}
|
|
347
|
+
for (const table of schema.tables.values()) {
|
|
348
|
+
for (const index of table.ftsIndexes) {
|
|
349
|
+
createFtsProjection(db, table, index);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
261
352
|
}
|
|
262
353
|
// ---------------------------------------------------------------------------
|
|
263
354
|
// Value conversion
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -81,7 +81,7 @@
|
|
|
81
81
|
},
|
|
82
82
|
"dependencies": {
|
|
83
83
|
"@sqlite.org/sqlite-wasm": "^3.53.0-build1",
|
|
84
|
-
"@syncular/core": "0.
|
|
84
|
+
"@syncular/core": "0.15.0"
|
|
85
85
|
},
|
|
86
86
|
"peerDependencies": {
|
|
87
87
|
"better-sqlite3": ">=11"
|
|
@@ -92,7 +92,7 @@
|
|
|
92
92
|
}
|
|
93
93
|
},
|
|
94
94
|
"devDependencies": {
|
|
95
|
-
"@syncular/server": "0.
|
|
95
|
+
"@syncular/server": "0.15.0",
|
|
96
96
|
"@types/better-sqlite3": "^7.6.13",
|
|
97
97
|
"better-sqlite3": "^12.11.1"
|
|
98
98
|
}
|
package/src/apply.ts
CHANGED
|
@@ -31,7 +31,12 @@ function upsertSql(table: CompiledClientTable): string {
|
|
|
31
31
|
quoteIdent(SYNC_VERSION_COLUMN),
|
|
32
32
|
];
|
|
33
33
|
const placeholders = names.map(() => '?').join(', ');
|
|
34
|
-
|
|
34
|
+
const updates = names
|
|
35
|
+
.filter((name) => name !== quoteIdent(table.primaryKey))
|
|
36
|
+
.map((name) => `${name}=excluded.${name}`)
|
|
37
|
+
.join(', ');
|
|
38
|
+
return `INSERT INTO ${quoteIdent(table.name)} (${names.join(', ')}) VALUES (${placeholders})
|
|
39
|
+
ON CONFLICT (${quoteIdent(table.primaryKey)}) DO UPDATE SET ${updates}`;
|
|
35
40
|
}
|
|
36
41
|
|
|
37
42
|
export function upsertLocalRow(
|
|
@@ -285,7 +290,7 @@ function imageInvalid(detail: string): never {
|
|
|
285
290
|
* the in-file metadata against the descriptor, validate the data table's
|
|
286
291
|
* column names/order against the generated schema, run the §5.6
|
|
287
292
|
* first-page clear when fresh, then copy every row with a single
|
|
288
|
-
*
|
|
293
|
+
* one primary-key upsert — `_syncular_version` lands in
|
|
289
294
|
* `_sync_version` exactly like a rows segment's per-row `serverVersion`.
|
|
290
295
|
*/
|
|
291
296
|
export function applySqliteSegment(
|
|
@@ -365,17 +370,24 @@ export function applySqliteSegment(
|
|
|
365
370
|
);
|
|
366
371
|
}
|
|
367
372
|
|
|
368
|
-
// 3. One transaction: fresh-bootstrap clear, then
|
|
373
|
+
// 3. One transaction: fresh-bootstrap clear, then primary-key upsert.
|
|
369
374
|
const names = table.columns.map((column) => quoteIdent(column.name));
|
|
375
|
+
const insertNames = [...names, quoteIdent(SYNC_VERSION_COLUMN)];
|
|
376
|
+
const updates = insertNames
|
|
377
|
+
.filter((name) => name !== quoteIdent(table.primaryKey))
|
|
378
|
+
.map((name) => `${name}=excluded.${name}`)
|
|
379
|
+
.join(', ');
|
|
370
380
|
return (options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
371
381
|
if (options.clearFirst) {
|
|
372
382
|
deleteScopedRows(db, table, options.effective);
|
|
373
383
|
}
|
|
374
384
|
db.exec(
|
|
375
|
-
`INSERT
|
|
376
|
-
(${
|
|
385
|
+
`INSERT INTO ${quoteIdent(table.name)}
|
|
386
|
+
(${insertNames.join(', ')})
|
|
377
387
|
SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')}
|
|
378
|
-
FROM ${IMAGE_ALIAS}.${quoteIdent(table.name)}
|
|
388
|
+
FROM ${IMAGE_ALIAS}.${quoteIdent(table.name)}
|
|
389
|
+
WHERE true
|
|
390
|
+
ON CONFLICT (${quoteIdent(table.primaryKey)}) DO UPDATE SET ${updates}`,
|
|
379
391
|
);
|
|
380
392
|
const counted = db.query(
|
|
381
393
|
`SELECT count(*) AS n FROM ${IMAGE_ALIAS}.${quoteIdent(table.name)}`,
|
package/src/schema.ts
CHANGED
|
@@ -19,6 +19,13 @@ export interface ClientIndexSpec {
|
|
|
19
19
|
readonly unique: boolean;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
/** One client-local contentful FTS5 projection (RFC 0005). */
|
|
23
|
+
export interface ClientFtsIndexSpec {
|
|
24
|
+
readonly name: string;
|
|
25
|
+
readonly columns: readonly string[];
|
|
26
|
+
readonly tokenize: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
22
29
|
export interface ClientTableSchema {
|
|
23
30
|
readonly name: string;
|
|
24
31
|
/** Columns in schema-IR declaration order (the row-codec order, §2.4). */
|
|
@@ -29,6 +36,8 @@ export interface ClientTableSchema {
|
|
|
29
36
|
/** Local secondary indexes; absent in the generated schema when a table
|
|
30
37
|
* declares none (typegen omits the key for index-free tables). */
|
|
31
38
|
readonly indexes?: readonly ClientIndexSpec[];
|
|
39
|
+
/** Local FTS5 search projections; omitted when a table declares none. */
|
|
40
|
+
readonly ftsIndexes?: readonly ClientFtsIndexSpec[];
|
|
32
41
|
}
|
|
33
42
|
|
|
34
43
|
export interface ClientSchema {
|
|
@@ -60,6 +69,8 @@ export interface CompiledClientTable {
|
|
|
60
69
|
/** Local secondary indexes to create on the mirror table (declaration
|
|
61
70
|
* order); empty when the table declares none. */
|
|
62
71
|
readonly indexes: readonly ClientIndexSpec[];
|
|
72
|
+
/** Client-local FTS5 projections, in declaration order. */
|
|
73
|
+
readonly ftsIndexes: readonly ClientFtsIndexSpec[];
|
|
63
74
|
/** §5.11: true when any column is `encrypted`. Drives the encrypt/decrypt
|
|
64
75
|
* seam (skipped entirely when false) and the local-plaintext DDL. */
|
|
65
76
|
readonly hasEncryptedColumns: boolean;
|
|
@@ -71,15 +82,30 @@ export interface CompiledClientSchema {
|
|
|
71
82
|
}
|
|
72
83
|
|
|
73
84
|
const PATTERN_RE = /^([^{}]+):\{([^{}:]+)\}$/;
|
|
85
|
+
const ALLOWED_FTS_TOKENIZERS = new Set([
|
|
86
|
+
'unicode61',
|
|
87
|
+
'unicode61 remove_diacritics 0',
|
|
88
|
+
'unicode61 remove_diacritics 1',
|
|
89
|
+
'unicode61 remove_diacritics 2',
|
|
90
|
+
'porter unicode61',
|
|
91
|
+
'trigram',
|
|
92
|
+
]);
|
|
74
93
|
|
|
75
94
|
export function compileClientSchema(
|
|
76
95
|
schema: ClientSchema,
|
|
77
96
|
): CompiledClientSchema {
|
|
78
97
|
const tables = new Map<string, CompiledClientTable>();
|
|
98
|
+
const schemaObjectNames = new Set<string>();
|
|
79
99
|
for (const table of schema.tables) {
|
|
80
100
|
if (tables.has(table.name)) {
|
|
81
101
|
throw new Error(`duplicate table ${JSON.stringify(table.name)}`);
|
|
82
102
|
}
|
|
103
|
+
if (schemaObjectNames.has(table.name)) {
|
|
104
|
+
throw new Error(
|
|
105
|
+
`table ${JSON.stringify(table.name)} conflicts with an index or FTS projection`,
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
schemaObjectNames.add(table.name);
|
|
83
109
|
const columnIndex = new Map<string, number>();
|
|
84
110
|
table.columns.forEach((column, index) => {
|
|
85
111
|
if (columnIndex.has(column.name)) {
|
|
@@ -142,6 +168,12 @@ export function compileClientSchema(
|
|
|
142
168
|
for (const alias of ambiguous) columnIndexByCamel.delete(alias);
|
|
143
169
|
const indexes = table.indexes ?? [];
|
|
144
170
|
for (const index of indexes) {
|
|
171
|
+
if (schemaObjectNames.has(index.name)) {
|
|
172
|
+
throw new Error(
|
|
173
|
+
`table ${table.name}: index ${JSON.stringify(index.name)} conflicts with another schema object`,
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
schemaObjectNames.add(index.name);
|
|
145
177
|
for (const column of index.columns) {
|
|
146
178
|
if (!columnIndex.has(column)) {
|
|
147
179
|
throw new Error(
|
|
@@ -150,6 +182,52 @@ export function compileClientSchema(
|
|
|
150
182
|
}
|
|
151
183
|
}
|
|
152
184
|
}
|
|
185
|
+
const ftsIndexes = table.ftsIndexes ?? [];
|
|
186
|
+
for (const index of ftsIndexes) {
|
|
187
|
+
if (schemaObjectNames.has(index.name)) {
|
|
188
|
+
throw new Error(
|
|
189
|
+
`table ${table.name}: FTS projection ${JSON.stringify(index.name)} conflicts with another schema object`,
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
schemaObjectNames.add(index.name);
|
|
193
|
+
if (index.columns.length === 0 || index.columns.length > 32) {
|
|
194
|
+
throw new Error(
|
|
195
|
+
`table ${table.name}: FTS projection ${JSON.stringify(index.name)} needs between 1 and 32 columns`,
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
const seenColumns = new Set<string>();
|
|
199
|
+
for (const columnName of index.columns) {
|
|
200
|
+
if (seenColumns.has(columnName)) {
|
|
201
|
+
throw new Error(
|
|
202
|
+
`table ${table.name}: FTS projection ${JSON.stringify(index.name)} repeats column ${JSON.stringify(columnName)}`,
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
seenColumns.add(columnName);
|
|
206
|
+
const column = table.columns.find(
|
|
207
|
+
(candidate) => candidate.name === columnName,
|
|
208
|
+
);
|
|
209
|
+
if (column === undefined) {
|
|
210
|
+
throw new Error(
|
|
211
|
+
`table ${table.name}: FTS projection ${JSON.stringify(index.name)} names unknown column ${JSON.stringify(columnName)}`,
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
if (localColumnType(column) !== 'string') {
|
|
215
|
+
throw new Error(
|
|
216
|
+
`table ${table.name}: FTS projection ${JSON.stringify(index.name)} column ${JSON.stringify(columnName)} must have string type`,
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
if (column.encrypted === true) {
|
|
220
|
+
throw new Error(
|
|
221
|
+
`table ${table.name}: FTS projection ${JSON.stringify(index.name)} cannot index encrypted column ${JSON.stringify(columnName)}`,
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (!ALLOWED_FTS_TOKENIZERS.has(index.tokenize)) {
|
|
226
|
+
throw new Error(
|
|
227
|
+
`table ${table.name}: FTS projection ${JSON.stringify(index.name)} tokenizer ${JSON.stringify(index.tokenize)} is not allowlisted`,
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
153
231
|
tables.set(table.name, {
|
|
154
232
|
name: table.name,
|
|
155
233
|
columns: table.columns,
|
|
@@ -160,6 +238,7 @@ export function compileClientSchema(
|
|
|
160
238
|
scopeColumnByVariable,
|
|
161
239
|
scopePrefixByVariable,
|
|
162
240
|
indexes,
|
|
241
|
+
ftsIndexes,
|
|
163
242
|
hasEncryptedColumns: table.columns.some((c) => c.encrypted === true),
|
|
164
243
|
});
|
|
165
244
|
}
|
|
@@ -266,6 +345,55 @@ function createSyncedTable(
|
|
|
266
345
|
}
|
|
267
346
|
}
|
|
268
347
|
|
|
348
|
+
const FTS_SOURCE_ID_COLUMN = '_syncular_source_id';
|
|
349
|
+
|
|
350
|
+
function createFtsProjection(
|
|
351
|
+
db: ClientDatabase,
|
|
352
|
+
table: CompiledClientTable,
|
|
353
|
+
index: ClientFtsIndexSpec,
|
|
354
|
+
): void {
|
|
355
|
+
const existed =
|
|
356
|
+
db.query(
|
|
357
|
+
"SELECT 1 AS present FROM sqlite_master WHERE type='table' AND name=?",
|
|
358
|
+
[index.name],
|
|
359
|
+
).length > 0;
|
|
360
|
+
const indexedColumns = index.columns.map(quoteIdent);
|
|
361
|
+
const tokenizer = index.tokenize.replaceAll("'", "''");
|
|
362
|
+
db.exec(
|
|
363
|
+
`CREATE VIRTUAL TABLE IF NOT EXISTS ${quoteIdent(index.name)} USING fts5(${quoteIdent(FTS_SOURCE_ID_COLUMN)} UNINDEXED, ${indexedColumns.join(', ')}, tokenize='${tokenizer}')`,
|
|
364
|
+
);
|
|
365
|
+
|
|
366
|
+
const sourceId = `CAST(${quoteIdent(table.primaryKey)} AS TEXT)`;
|
|
367
|
+
const newSourceId = `CAST(new.${quoteIdent(table.primaryKey)} AS TEXT)`;
|
|
368
|
+
const oldSourceId = `CAST(old.${quoteIdent(table.primaryKey)} AS TEXT)`;
|
|
369
|
+
const projectionColumns = [
|
|
370
|
+
quoteIdent(FTS_SOURCE_ID_COLUMN),
|
|
371
|
+
...indexedColumns,
|
|
372
|
+
].join(', ');
|
|
373
|
+
const newValues = [
|
|
374
|
+
newSourceId,
|
|
375
|
+
...index.columns.map((column) => `new.${quoteIdent(column)}`),
|
|
376
|
+
].join(', ');
|
|
377
|
+
const deleteFor = (value: string) =>
|
|
378
|
+
`DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} = ${value}`;
|
|
379
|
+
|
|
380
|
+
db.exec(
|
|
381
|
+
`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`,
|
|
382
|
+
);
|
|
383
|
+
db.exec(
|
|
384
|
+
`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_ad`)} AFTER DELETE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; END`,
|
|
385
|
+
);
|
|
386
|
+
db.exec(
|
|
387
|
+
`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`,
|
|
388
|
+
);
|
|
389
|
+
|
|
390
|
+
if (!existed) {
|
|
391
|
+
db.exec(
|
|
392
|
+
`INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) SELECT ${sourceId}, ${indexedColumns.join(', ')} FROM ${quoteIdent(table.name)}`,
|
|
393
|
+
);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
269
397
|
/**
|
|
270
398
|
* Create the synced tables plus client bookkeeping tables (outbox,
|
|
271
399
|
* subscription state, meta). Idempotent.
|
|
@@ -278,6 +406,11 @@ export function ensureLocalSchema(
|
|
|
278
406
|
for (const table of schema.tables.values()) {
|
|
279
407
|
createSyncedTable(db, table);
|
|
280
408
|
}
|
|
409
|
+
for (const table of schema.tables.values()) {
|
|
410
|
+
for (const index of table.ftsIndexes) {
|
|
411
|
+
createFtsProjection(db, table, index);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
281
414
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
|
|
282
415
|
key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
|
|
283
416
|
db.exec(
|
|
@@ -359,6 +492,16 @@ export function dropAndRecreateSyncedTables(
|
|
|
359
492
|
db: ClientDatabase,
|
|
360
493
|
schema: CompiledClientSchema,
|
|
361
494
|
): void {
|
|
495
|
+
// Drop virtual tables first. SQLite then removes their shadow tables as one
|
|
496
|
+
// unit, so the generic discovery below never tears an FTS5 projection apart.
|
|
497
|
+
const virtualTables = db.query(
|
|
498
|
+
`SELECT name FROM sqlite_master WHERE type = 'table'
|
|
499
|
+
AND sql LIKE 'CREATE VIRTUAL TABLE%'
|
|
500
|
+
AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'`,
|
|
501
|
+
);
|
|
502
|
+
for (const row of virtualTables) {
|
|
503
|
+
db.exec(`DROP TABLE IF EXISTS ${quoteIdent(String(row.name))}`);
|
|
504
|
+
}
|
|
362
505
|
const existing = db.query(
|
|
363
506
|
`SELECT name FROM sqlite_master WHERE type = 'table'
|
|
364
507
|
AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'
|
|
@@ -370,6 +513,11 @@ export function dropAndRecreateSyncedTables(
|
|
|
370
513
|
for (const table of schema.tables.values()) {
|
|
371
514
|
createSyncedTable(db, table);
|
|
372
515
|
}
|
|
516
|
+
for (const table of schema.tables.values()) {
|
|
517
|
+
for (const index of table.ftsIndexes) {
|
|
518
|
+
createFtsProjection(db, table, index);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
373
521
|
}
|
|
374
522
|
|
|
375
523
|
// ---------------------------------------------------------------------------
|