@syncular/client 0.30.10 → 0.30.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/apply.d.ts +5 -1
- package/dist/apply.js +58 -58
- package/dist/blob.js +13 -39
- package/dist/client.js +178 -161
- package/dist/failed-overlay.d.ts +10 -5
- package/dist/failed-overlay.js +118 -36
- package/dist/outbox.d.ts +2 -1
- package/dist/outbox.js +20 -40
- package/dist/outcomes.d.ts +8 -0
- package/dist/outcomes.js +22 -30
- package/dist/previous-version.js +14 -28
- package/dist/schema.d.ts +3 -0
- package/dist/schema.js +89 -133
- package/dist/state.d.ts +6 -9
- package/dist/state.js +66 -32
- package/dist/wasm-database.js +40 -24
- package/dist/worker-host.js +37 -11
- package/package.json +3 -3
- package/src/apply.ts +95 -72
- package/src/blob.ts +17 -39
- package/src/client.ts +312 -214
- package/src/failed-overlay.ts +197 -48
- package/src/outbox.ts +25 -44
- package/src/outcomes.ts +40 -35
- package/src/previous-version.ts +14 -28
- package/src/schema.ts +154 -166
- package/src/state.ts +90 -33
- package/src/wasm-database.ts +56 -29
- package/src/worker-host.ts +34 -12
package/README.md
CHANGED
|
@@ -18,6 +18,13 @@ Ordinary commits use wire version 1 until the caller supplies an acquired
|
|
|
18
18
|
partition `logEpoch`. Set `logEpoch` after a restore rotation requires epoch
|
|
19
19
|
validation.
|
|
20
20
|
|
|
21
|
+
Applied and cached acknowledgements drain the send queue while protected local
|
|
22
|
+
intent stays visible until the same row's server image arrives at that commit
|
|
23
|
+
sequence or later. Empty pulls and restart preserve it; later edits stack above
|
|
24
|
+
it. This behavior is automatic, independent of `retainFailedCommits`. The core
|
|
25
|
+
requests a following pull without relying on an origin realtime notification.
|
|
26
|
+
See [optimistic writes](https://syncular.dev/concepts-conflicts/#the-optimistic-outbox).
|
|
27
|
+
|
|
21
28
|
## Client-local FTS5 projections
|
|
22
29
|
|
|
23
30
|
Generated schemas may attach `ftsIndexes` to a synced table. The
|
|
@@ -389,6 +396,14 @@ and scope revocation remove whole retained aggregates. If a sparse conflict
|
|
|
389
396
|
loses its server base, the client keeps the sparse operation and conflict
|
|
390
397
|
evidence without materializing a local row from the saved before-image.
|
|
391
398
|
|
|
399
|
+
A retained insert whose unique key belongs to another server primary key remains
|
|
400
|
+
journal intent while reads show the server winner. `retainedRows[].uniqueConflicts`
|
|
401
|
+
contains each matching index, its columns, competing `rowId`, authorized
|
|
402
|
+
`serverRow` and `serverVersion`. Keep-mine patches that competing ID at its
|
|
403
|
+
current version and links the replacement through `superseded`. Edit can choose
|
|
404
|
+
a free unique key. Take-server uses `resolved_keep_server`. Revocation and local
|
|
405
|
+
security purge erase the retained aggregate's protected journal payloads.
|
|
406
|
+
|
|
392
407
|
## Application-authorized local security purge
|
|
393
408
|
|
|
394
409
|
`purgeLocalData({ purgeId, targets })` is the narrow local-storage primitive
|
package/dist/apply.d.ts
CHANGED
|
@@ -23,7 +23,7 @@ export declare function deleteLocalRow(db: ClientDatabase, table: CompiledClient
|
|
|
23
23
|
* (`client.decrypt_failed`) aborts before any local write, so the commit
|
|
24
24
|
* never half-applies.
|
|
25
25
|
*/
|
|
26
|
-
export declare function applyCommitFrame(db: ClientDatabase, schema: CompiledClientSchema, frame: CommitFrame, encryption?: EncryptionConfig, transaction?:
|
|
26
|
+
export declare function applyCommitFrame(db: ClientDatabase, schema: CompiledClientSchema, frame: CommitFrame, encryption?: EncryptionConfig, transaction?: (fn: () => readonly string[] | undefined) => unknown): Promise<void>;
|
|
27
27
|
/**
|
|
28
28
|
* §5.2: the segment's column table must match the generated schema for
|
|
29
29
|
* (table, schemaVersion) — order, names, types, nullability. A mismatch is
|
|
@@ -37,6 +37,10 @@ export declare function validateSegmentColumns(schema: CompiledClientSchema, tab
|
|
|
37
37
|
* (`sync.scope_revoked`) when the table has no local mapping for a key:
|
|
38
38
|
* precision or nothing, never clear-the-table.
|
|
39
39
|
*/
|
|
40
|
+
export declare function localScopePredicate(table: CompiledClientTable, effective: ScopeMap): {
|
|
41
|
+
sql: string;
|
|
42
|
+
params: string[];
|
|
43
|
+
} | undefined;
|
|
40
44
|
export declare function deleteScopedRows(db: ClientDatabase, table: CompiledClientTable, effective: ScopeMap): void;
|
|
41
45
|
/**
|
|
42
46
|
* §4.8 window eviction: delete rows matching a departing unit's effective
|
package/dist/apply.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { RETAINED_ROWS } from './failed-overlay.js';
|
|
2
|
+
import { retainedBaseWrite, restoreFailedBases } from './failed-overlay.js';
|
|
2
3
|
/**
|
|
3
4
|
* Local application of server data: `COMMIT` frames (§4.5), rows segments
|
|
4
5
|
* (§5.2, §5.6), and the scope-matched delete shared by the §3.3 purge
|
|
@@ -6,7 +7,7 @@ import { retainedBaseWrite } from './failed-overlay.js';
|
|
|
6
7
|
*/
|
|
7
8
|
import { decodeRow, } from '@syncular/core';
|
|
8
9
|
import { ClientSyncError, invalidRequest } from './errors.js';
|
|
9
|
-
import { quoteIdent, SYNC_VERSION_COLUMN, toSqlValue, rowValueToJson, } from './schema.js';
|
|
10
|
+
import { quoteIdent, matchesLocalScopes, SYNC_VERSION_COLUMN, toSqlValue, rowValueToJson, } from './schema.js';
|
|
10
11
|
function upsertSql(table) {
|
|
11
12
|
const names = [
|
|
12
13
|
...table.columns.map((column) => quoteIdent(column.name)),
|
|
@@ -17,21 +18,20 @@ function upsertSql(table) {
|
|
|
17
18
|
.filter((name) => name !== quoteIdent(table.primaryKey))
|
|
18
19
|
.map((name) => `${name}=excluded.${name}`)
|
|
19
20
|
.join(', ');
|
|
20
|
-
return `INSERT INTO ${quoteIdent(table.name)}
|
|
21
|
-
ON CONFLICT (${quoteIdent(table.primaryKey)}) DO UPDATE SET ${updates}`;
|
|
21
|
+
return `INSERT INTO ${quoteIdent(table.name)}(${names.join(', ')})VALUES(${placeholders})ON CONFLICT(${quoteIdent(table.primaryKey)})DO UPDATE SET ${updates}`;
|
|
22
22
|
}
|
|
23
23
|
export function upsertLocalRow(db, table, values, syncVersion, serverWrite = true) {
|
|
24
|
+
db.exec(upsertSql(table), [...values.map(toSqlValue), syncVersion]);
|
|
24
25
|
if (serverWrite)
|
|
25
|
-
retainedBaseWrite(db, table
|
|
26
|
+
retainedBaseWrite(db, table, String(values[table.primaryKeyIndex]), Object.fromEntries(table.columns.map((column, index) => [
|
|
26
27
|
column.name,
|
|
27
28
|
rowValueToJson(values[index] ?? null),
|
|
28
29
|
])), syncVersion);
|
|
29
|
-
db.exec(upsertSql(table), [...values.map(toSqlValue), syncVersion]);
|
|
30
30
|
}
|
|
31
31
|
export function deleteLocalRow(db, table, rowId, serverWrite = true) {
|
|
32
32
|
if (serverWrite)
|
|
33
|
-
retainedBaseWrite(db, table
|
|
34
|
-
db.exec(`DELETE FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(table.primaryKey)}
|
|
33
|
+
retainedBaseWrite(db, table, rowId);
|
|
34
|
+
db.exec(`DELETE FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(table.primaryKey)}=?`, [rowId]);
|
|
35
35
|
}
|
|
36
36
|
/**
|
|
37
37
|
* Apply one `COMMIT` frame in one local transaction (§1.4 rule 4).
|
|
@@ -76,7 +76,14 @@ export async function applyCommitFrame(db, schema, frame, encryption, transactio
|
|
|
76
76
|
});
|
|
77
77
|
}
|
|
78
78
|
transaction(() => {
|
|
79
|
-
|
|
79
|
+
const rowKeys = frame.tables.some((name) => schema.tables.get(name)?.indexes.some((index) => index.unique))
|
|
80
|
+
? undefined
|
|
81
|
+
: frame.changes.map((change) => JSON.stringify([frame.tables[change.tableIndex], change.rowId]));
|
|
82
|
+
restoreFailedBases(db, schema, false, rowKeys);
|
|
83
|
+
for (const [index, change] of resolved.entries()) {
|
|
84
|
+
const rowId = frame.changes[index].rowId;
|
|
85
|
+
db.exec("UPDATE _syncular_outbox_before_images SET delivery_seq=MAX(COALESCE(delivery_seq,0),?) WHERE (client_commit_id,op_index) IN(SELECT client_commit_id,json_each.key FROM _syncular_outbox,json_each(operations) WHERE json_extract(value,'$.table')=? AND json_extract(value,'$.rowId')=?)", [frame.commitSeq, change.table.name, rowId]);
|
|
86
|
+
db.exec(`DELETE FROM ${RETAINED_ROWS} WHERE tbl=? AND id=? AND commit_seq<=?`, [change.table.name, rowId, frame.commitSeq]);
|
|
80
87
|
if (change.op === 'delete') {
|
|
81
88
|
deleteLocalRow(db, change.table, change.rowId);
|
|
82
89
|
}
|
|
@@ -84,6 +91,7 @@ export async function applyCommitFrame(db, schema, frame, encryption, transactio
|
|
|
84
91
|
upsertLocalRow(db, change.table, change.values, change.rowVersion);
|
|
85
92
|
}
|
|
86
93
|
}
|
|
94
|
+
return rowKeys;
|
|
87
95
|
});
|
|
88
96
|
}
|
|
89
97
|
/**
|
|
@@ -123,26 +131,31 @@ export function validateSegmentColumns(schema, table, segment) {
|
|
|
123
131
|
* (`sync.scope_revoked`) when the table has no local mapping for a key:
|
|
124
132
|
* precision or nothing, never clear-the-table.
|
|
125
133
|
*/
|
|
126
|
-
export function
|
|
134
|
+
export function localScopePredicate(table, effective) {
|
|
127
135
|
const entries = Object.entries(effective);
|
|
128
|
-
if (entries.length
|
|
136
|
+
if (!entries.length || entries.some(([, values]) => !values.length))
|
|
129
137
|
return;
|
|
130
|
-
const clauses = [];
|
|
131
138
|
const params = [];
|
|
132
|
-
|
|
139
|
+
const sql = entries
|
|
140
|
+
.map(([variable, values]) => {
|
|
133
141
|
const column = table.scopeColumnByVariable.get(variable);
|
|
134
|
-
if (column === undefined)
|
|
135
|
-
throw new ClientSyncError('sync.scope_revoked',
|
|
136
|
-
}
|
|
137
|
-
if (values.length === 0)
|
|
138
|
-
return;
|
|
139
|
-
clauses.push(`${quoteIdent(column)} IN (${values.map(() => '?').join(', ')})`);
|
|
142
|
+
if (column === undefined)
|
|
143
|
+
throw new ClientSyncError('sync.scope_revoked', 'table has no local scope-column mapping');
|
|
140
144
|
params.push(...values);
|
|
145
|
+
return `${quoteIdent(column)} IN (${values.map(() => '?').join(', ')})`;
|
|
146
|
+
})
|
|
147
|
+
.join(' AND ');
|
|
148
|
+
return { sql, params };
|
|
149
|
+
}
|
|
150
|
+
export function deleteScopedRows(db, table, effective) {
|
|
151
|
+
const scope = localScopePredicate(table, effective);
|
|
152
|
+
if (!scope)
|
|
153
|
+
return;
|
|
154
|
+
const { sql, params } = scope;
|
|
155
|
+
for (const row of db.query(`SELECT ${quoteIdent(table.primaryKey)} AS id FROM ${quoteIdent(table.name)} WHERE ${sql}`, params)) {
|
|
156
|
+
retainedBaseWrite(db, table, String(row.id));
|
|
141
157
|
}
|
|
142
|
-
|
|
143
|
-
retainedBaseWrite(db, table.name, String(row.id));
|
|
144
|
-
}
|
|
145
|
-
db.exec(`DELETE FROM ${quoteIdent(table.name)} WHERE ${clauses.join(' AND ')}`, params);
|
|
158
|
+
db.exec(`DELETE FROM ${quoteIdent(table.name)} WHERE ${sql}`, params);
|
|
146
159
|
}
|
|
147
160
|
/**
|
|
148
161
|
* §4.8 window eviction: delete rows matching a departing unit's effective
|
|
@@ -155,21 +168,10 @@ export function deleteScopedRows(db, table, effective) {
|
|
|
155
168
|
* cache), which is automatic since the version column is per-row.
|
|
156
169
|
*/
|
|
157
170
|
export function evictScopedRows(db, table, effective, pinnedRowIds) {
|
|
158
|
-
const
|
|
159
|
-
if (
|
|
171
|
+
const scope = localScopePredicate(table, effective);
|
|
172
|
+
if (!scope)
|
|
160
173
|
return { remaining: false, deferred: false };
|
|
161
|
-
const
|
|
162
|
-
const params = [];
|
|
163
|
-
for (const [variable, values] of entries) {
|
|
164
|
-
const column = table.scopeColumnByVariable.get(variable);
|
|
165
|
-
if (column === undefined) {
|
|
166
|
-
throw new ClientSyncError('sync.scope_revoked', `table ${JSON.stringify(table.name)} has no local scope-column mapping for ${JSON.stringify(variable)} (§4.8/§3.3 fail-closed)`);
|
|
167
|
-
}
|
|
168
|
-
if (values.length === 0)
|
|
169
|
-
return { remaining: false, deferred: false };
|
|
170
|
-
clauses.push(`${quoteIdent(column)} IN (${values.map(() => '?').join(', ')})`);
|
|
171
|
-
params.push(...values);
|
|
172
|
-
}
|
|
174
|
+
const { sql, params } = scope;
|
|
173
175
|
const pk = quoteIdent(table.primaryKey);
|
|
174
176
|
let pinnedClause = '';
|
|
175
177
|
if (pinnedRowIds.size > 0) {
|
|
@@ -178,12 +180,16 @@ export function evictScopedRows(db, table, effective, pinnedRowIds) {
|
|
|
178
180
|
params.push(...ids);
|
|
179
181
|
}
|
|
180
182
|
const target = quoteIdent(table.name);
|
|
181
|
-
db.
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
183
|
+
for (const row of db.query(`SELECT id,intent,base FROM ${RETAINED_ROWS} WHERE commit_seq IS NOT NULL AND tbl=?`, [table.name])) {
|
|
184
|
+
if (!pinnedRowIds.has(row.id) &&
|
|
185
|
+
[row.intent, row.base].some((json) => json !== null &&
|
|
186
|
+
matchesLocalScopes(table, effective, JSON.parse(json))))
|
|
187
|
+
db.exec(`DELETE FROM ${RETAINED_ROWS} WHERE tbl=? AND id=? AND commit_seq IS NOT NULL`, [table.name, row.id]);
|
|
188
|
+
}
|
|
189
|
+
db.exec(`DELETE FROM ${target} WHERE ${pk} IN(SELECT ${pk} FROM ${target} WHERE ${sql}${pinnedClause} LIMIT 1024)`, params);
|
|
190
|
+
const remaining = db.query(`SELECT 1 FROM ${target} WHERE ${sql}${pinnedClause} LIMIT 1`, params).length > 0;
|
|
185
191
|
const deferred = remaining ||
|
|
186
|
-
db.query(`SELECT 1 FROM ${target} WHERE ${
|
|
192
|
+
db.query(`SELECT 1 FROM ${target} WHERE ${sql} LIMIT 1`, params.slice(0, params.length - pinnedRowIds.size)).length > 0;
|
|
187
193
|
return { remaining, deferred };
|
|
188
194
|
}
|
|
189
195
|
const IMAGE_ALIAS = 'syncular_image';
|
|
@@ -201,7 +207,7 @@ function imageInvalid(detail) {
|
|
|
201
207
|
export async function applySqliteSegment(db, schema, table, bytes, descriptor, options) {
|
|
202
208
|
const withImage = db.withSqliteImage?.bind(db);
|
|
203
209
|
if (withImage === undefined) {
|
|
204
|
-
throw invalidRequest('
|
|
210
|
+
throw invalidRequest('database cannot import images; disable accept bit 2 (§4.2)');
|
|
205
211
|
}
|
|
206
212
|
if (descriptor.table !== table.name) {
|
|
207
213
|
imageInvalid(`descriptor table ${JSON.stringify(descriptor.table)}`);
|
|
@@ -211,10 +217,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
211
217
|
// SQLite database or lacks the metadata table fails right here.
|
|
212
218
|
let meta;
|
|
213
219
|
try {
|
|
214
|
-
meta = db.query(`SELECT format,
|
|
215
|
-
"asOfCommitSeq" AS pin, "scopeDigest" AS sd,
|
|
216
|
-
"rowCount" AS rc
|
|
217
|
-
FROM ${IMAGE_ALIAS}."_syncular_segment"`);
|
|
220
|
+
meta = db.query(`SELECT format,"table" AS tbl,"schemaVersion" AS sv,"asOfCommitSeq" AS pin,"scopeDigest" AS sd,"rowCount" AS rc FROM ${IMAGE_ALIAS}."_syncular_segment"`);
|
|
218
221
|
}
|
|
219
222
|
catch {
|
|
220
223
|
imageInvalid('bytes are not a SQLite image with _syncular_segment');
|
|
@@ -256,7 +259,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
256
259
|
if (primaryKeys.length !== 1 || primaryKeys[0]?.name !== table.primaryKey)
|
|
257
260
|
imageInvalid('image primary key does not match generated schema');
|
|
258
261
|
const source = `${IMAGE_ALIAS}.${quoteIdent(table.name)}`;
|
|
259
|
-
const count = Number(db.query(`SELECT count(*)
|
|
262
|
+
const count = Number(db.query(`SELECT count(*)AS n FROM ${source}`)[0]?.n);
|
|
260
263
|
if (count !== descriptor.rowCount)
|
|
261
264
|
imageInvalid('image row count does not match descriptor');
|
|
262
265
|
const names = table.columns.map((column) => quoteIdent(column.name));
|
|
@@ -271,9 +274,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
271
274
|
for (;;) {
|
|
272
275
|
if (options.isCurrent?.() === false)
|
|
273
276
|
return applied;
|
|
274
|
-
const boundary = db.query(`SELECT ${primaryKey} AS boundary FROM ${source}
|
|
275
|
-
${after === undefined ? '' : `WHERE ${primaryKey} > ?`}
|
|
276
|
-
ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`, after === undefined ? [] : [after])[0]?.boundary;
|
|
277
|
+
const boundary = db.query(`SELECT ${primaryKey} AS boundary FROM ${source} ${after === undefined ? '' : `WHERE ${primaryKey} > ?`} ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`, after === undefined ? [] : [after])[0]?.boundary;
|
|
277
278
|
if (boundary === null)
|
|
278
279
|
imageInvalid('image primary key must not be null');
|
|
279
280
|
const predicates = [];
|
|
@@ -287,15 +288,13 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
287
288
|
params.push(boundary);
|
|
288
289
|
}
|
|
289
290
|
(options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
291
|
+
restoreFailedBases(db, schema);
|
|
290
292
|
if (after === undefined && options.clearFirst)
|
|
291
293
|
deleteScopedRows(db, table, options.effective);
|
|
292
|
-
db.exec(`INSERT INTO ${quoteIdent(table.name)}
|
|
293
|
-
|
|
294
|
-
FROM ${source} WHERE ${predicates.join(' AND ') || 'true'}
|
|
295
|
-
ON CONFLICT (${primaryKey}) DO UPDATE SET ${updates}`, params);
|
|
296
|
-
applied += Number(db.query('SELECT changes() AS n')[0]?.n ?? 0);
|
|
294
|
+
db.exec(`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${predicates.join(' AND ') || 'true'} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates}`, params);
|
|
295
|
+
applied += Number(db.query('SELECT changes()AS n')[0]?.n ?? 0);
|
|
297
296
|
for (const row of db.query(`SELECT * FROM ${source} WHERE ${predicates.join(' AND ') || 'true'}`, params)) {
|
|
298
|
-
retainedBaseWrite(db, table
|
|
297
|
+
retainedBaseWrite(db, table, String(row[table.primaryKey]), Object.fromEntries(table.columns.map((column) => [
|
|
299
298
|
column.name,
|
|
300
299
|
rowValueToJson(row[column.name]),
|
|
301
300
|
])), Number(row._syncular_version));
|
|
@@ -348,11 +347,12 @@ export async function applyRowsSegment(db, schema, table, segment, options, encr
|
|
|
348
347
|
if (!clearThisBlock && rows.length === 0)
|
|
349
348
|
continue;
|
|
350
349
|
(options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
350
|
+
const protectedRows = restoreFailedBases(db, schema);
|
|
351
351
|
if (clearThisBlock) {
|
|
352
352
|
deleteScopedRows(db, table, options.effective);
|
|
353
353
|
}
|
|
354
354
|
for (const row of rows) {
|
|
355
|
-
upsertLocalRow(db, table, row.values, row.serverVersion);
|
|
355
|
+
upsertLocalRow(db, table, row.values, row.serverVersion, protectedRows);
|
|
356
356
|
applied += 1;
|
|
357
357
|
if (applied % 1024 === 0)
|
|
358
358
|
options.onProgress?.(applied);
|
package/dist/blob.js
CHANGED
|
@@ -20,7 +20,7 @@ export async function computeBlobId(bytes) {
|
|
|
20
20
|
return `sha256:${hex}`;
|
|
21
21
|
}
|
|
22
22
|
export function ensureBlobSchema(db) {
|
|
23
|
-
const existing = db.query("SELECT 1 FROM sqlite_master WHERE type
|
|
23
|
+
const existing = db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name='_syncular_blobs'");
|
|
24
24
|
if (existing.length > 0) {
|
|
25
25
|
const columns = db
|
|
26
26
|
.query('PRAGMA table_info(_syncular_blobs)')
|
|
@@ -39,24 +39,16 @@ export function ensureBlobSchema(db) {
|
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
41
|
else {
|
|
42
|
-
db.exec(
|
|
43
|
-
blob_id TEXT PRIMARY KEY,
|
|
44
|
-
bytes BLOB NOT NULL,
|
|
45
|
-
byte_length INTEGER NOT NULL,
|
|
46
|
-
media_type TEXT,
|
|
47
|
-
created_at_ms INTEGER NOT NULL)`);
|
|
42
|
+
db.exec('CREATE TABLE _syncular_blobs(blob_id TEXT PRIMARY KEY,bytes BLOB NOT NULL,byte_length INTEGER NOT NULL,media_type TEXT,created_at_ms INTEGER NOT NULL)');
|
|
48
43
|
}
|
|
49
|
-
db.exec(
|
|
50
|
-
blob_id TEXT PRIMARY KEY, media_type TEXT, created_at_ms INTEGER NOT NULL)`);
|
|
44
|
+
db.exec('CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(blob_id TEXT PRIMARY KEY,media_type TEXT,created_at_ms INTEGER NOT NULL)');
|
|
51
45
|
}
|
|
52
46
|
/** Put bytes into the content-addressed cache without rewriting an existing body. */
|
|
53
47
|
export function putCachedBlob(db, blobId, bytes, nowMs, mediaType) {
|
|
54
|
-
db.exec(
|
|
55
|
-
blob_id, bytes, byte_length, media_type, created_at_ms)
|
|
56
|
-
VALUES (?,?,?,?,?) ON CONFLICT(blob_id) DO NOTHING`, [blobId, bytes, bytes.length, mediaType ?? null, nowMs]);
|
|
48
|
+
db.exec('INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)VALUES(?,?,?,?,?)ON CONFLICT(blob_id)DO NOTHING', [blobId, bytes, bytes.length, mediaType ?? null, nowMs]);
|
|
57
49
|
}
|
|
58
50
|
export function getCachedBlob(db, blobId) {
|
|
59
|
-
const rows = db.query('SELECT bytes,
|
|
51
|
+
const rows = db.query('SELECT bytes,byte_length,media_type FROM _syncular_blobs WHERE blob_id=?', [blobId]);
|
|
60
52
|
const row = rows[0];
|
|
61
53
|
if (row === undefined)
|
|
62
54
|
return undefined;
|
|
@@ -73,23 +65,19 @@ export function getCachedBlob(db, blobId) {
|
|
|
73
65
|
* until back under the cap. Returns the evicted blobIds.
|
|
74
66
|
*/
|
|
75
67
|
export function enforceBlobCacheCap(db, schema, maxBytes) {
|
|
76
|
-
const totalRow = db.query('SELECT COALESCE(SUM(byte_length),
|
|
68
|
+
const totalRow = db.query('SELECT COALESCE(SUM(byte_length),0)AS total FROM _syncular_blobs')[0];
|
|
77
69
|
let total = Number(totalRow?.total ?? 0);
|
|
78
70
|
if (total <= maxBytes)
|
|
79
71
|
return [];
|
|
80
72
|
// Oldest eligible body first, with blob ID as a stable tie.
|
|
81
|
-
const candidates = db.query(`SELECT blob_id,
|
|
82
|
-
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
83
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
84
|
-
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})
|
|
85
|
-
ORDER BY created_at_ms ASC, blob_id ASC`);
|
|
73
|
+
const candidates = db.query(`SELECT blob_id,byte_length FROM _syncular_blobs WHERE blob_id NOT IN(SELECT blob_id FROM _syncular_blob_uploads)AND blob_id NOT IN(SELECT blob_id FROM _syncular_blob_commit_refs)AND blob_id NOT IN(${visibleBlobIdsSql(schema)})ORDER BY created_at_ms ASC,blob_id ASC`);
|
|
86
74
|
const evicted = [];
|
|
87
75
|
db.transaction(() => {
|
|
88
76
|
for (const row of candidates) {
|
|
89
77
|
if (total <= maxBytes)
|
|
90
78
|
break;
|
|
91
79
|
const blobId = row.blob_id;
|
|
92
|
-
db.exec('DELETE FROM _syncular_blobs WHERE blob_id
|
|
80
|
+
db.exec('DELETE FROM _syncular_blobs WHERE blob_id=?', [blobId]);
|
|
93
81
|
total -= Number(row.byte_length);
|
|
94
82
|
evicted.push(blobId);
|
|
95
83
|
}
|
|
@@ -98,13 +86,7 @@ export function enforceBlobCacheCap(db, schema, maxBytes) {
|
|
|
98
86
|
}
|
|
99
87
|
export function listPendingUploads(db) {
|
|
100
88
|
return db
|
|
101
|
-
.query(
|
|
102
|
-
SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
|
|
103
|
-
UNION ALL
|
|
104
|
-
SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
|
|
105
|
-
FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
|
|
106
|
-
WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
|
|
107
|
-
) ORDER BY created_at_ms, blob_id`)
|
|
89
|
+
.query('SELECT blob_id,media_type FROM(SELECT blob_id,media_type,created_at_ms FROM _syncular_blob_uploads UNION ALL SELECT DISTINCT r.blob_id,b.media_type,9223372036854775807 AS created_at_ms FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id=r.blob_id WHERE NOT EXISTS(SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id=r.blob_id))ORDER BY created_at_ms,blob_id')
|
|
108
90
|
.map((row) => {
|
|
109
91
|
if (typeof row.blob_id !== 'string' ||
|
|
110
92
|
(row.media_type !== null && typeof row.media_type !== 'string')) {
|
|
@@ -117,11 +99,10 @@ export function listPendingUploads(db) {
|
|
|
117
99
|
});
|
|
118
100
|
}
|
|
119
101
|
export function recordPendingUpload(db, blobId, nowMs, mediaType) {
|
|
120
|
-
db.exec(
|
|
121
|
-
VALUES (?,?,?)`, [blobId, mediaType ?? null, nowMs]);
|
|
102
|
+
db.exec('INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id,media_type,created_at_ms)VALUES(?,?,?)', [blobId, mediaType ?? null, nowMs]);
|
|
122
103
|
}
|
|
123
104
|
export function clearPendingUpload(db, blobId) {
|
|
124
|
-
db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id
|
|
105
|
+
db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id=?', [blobId]);
|
|
125
106
|
}
|
|
126
107
|
/**
|
|
127
108
|
* All `blob_ref` column names per table.
|
|
@@ -149,11 +130,7 @@ function visibleBlobIdsSql(schema) {
|
|
|
149
130
|
for (const [tableName, columns] of blobRefColumnsBySchema(schema)) {
|
|
150
131
|
for (const column of columns) {
|
|
151
132
|
const identifier = quoteIdent(column);
|
|
152
|
-
selects.push(`SELECT json_extract(${identifier},
|
|
153
|
-
FROM ${quoteIdent(tableName)}
|
|
154
|
-
WHERE typeof(${identifier}) = 'text'
|
|
155
|
-
AND json_valid(${identifier})
|
|
156
|
-
AND json_type(${identifier}, '$.blobId') = 'text'`);
|
|
133
|
+
selects.push(`SELECT json_extract(${identifier},'$.blobId')AS blob_id FROM ${quoteIdent(tableName)} WHERE typeof(${identifier})='text' AND json_valid(${identifier})AND json_type(${identifier},'$.blobId')='text'`);
|
|
157
134
|
}
|
|
158
135
|
}
|
|
159
136
|
return selects.length === 0
|
|
@@ -161,9 +138,6 @@ function visibleBlobIdsSql(schema) {
|
|
|
161
138
|
: selects.join(' UNION ');
|
|
162
139
|
}
|
|
163
140
|
export function deleteUnreferencedCachedBlobs(db, schema) {
|
|
164
|
-
db.exec(`DELETE FROM _syncular_blobs
|
|
165
|
-
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
166
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
167
|
-
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})`);
|
|
141
|
+
db.exec(`DELETE FROM _syncular_blobs WHERE blob_id NOT IN(SELECT blob_id FROM _syncular_blob_uploads)AND blob_id NOT IN(SELECT blob_id FROM _syncular_blob_commit_refs)AND blob_id NOT IN(${visibleBlobIdsSql(schema)})`);
|
|
168
142
|
}
|
|
169
143
|
export { parseBlobRef, serializeBlobRef };
|