@syncular/client 0.30.13 → 0.30.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apply.d.ts +3 -2
- package/dist/apply.js +63 -15
- package/dist/blob.d.ts +1 -0
- package/dist/blob.js +4 -3
- package/dist/client.js +51 -38
- package/dist/failed-overlay.d.ts +6 -3
- package/dist/failed-overlay.js +50 -48
- package/dist/outbox.d.ts +4 -1
- package/dist/outbox.js +18 -12
- package/dist/outcomes.d.ts +1 -0
- package/dist/outcomes.js +12 -7
- package/dist/previous-version.js +3 -3
- package/dist/schema.d.ts +1 -0
- package/dist/schema.js +23 -14
- package/dist/state.d.ts +2 -0
- package/dist/state.js +15 -15
- package/dist/window.d.ts +2 -0
- package/dist/window.js +13 -10
- package/package.json +3 -3
- package/src/apply.ts +106 -37
- package/src/blob.ts +4 -3
- package/src/client.ts +77 -48
- package/src/failed-overlay.ts +96 -78
- package/src/outbox.ts +19 -16
- package/src/outcomes.ts +12 -12
- package/src/previous-version.ts +3 -3
- package/src/schema.ts +24 -18
- package/src/state.ts +15 -15
- package/src/window.ts +13 -15
package/src/blob.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export const BLOB_COMMIT_REFS = '_syncular_blob_commit_refs';
|
|
1
2
|
/**
|
|
2
3
|
* Client-side blob cache + transport (SPEC.md §5.9.7).
|
|
3
4
|
*
|
|
@@ -187,7 +188,7 @@ export function enforceBlobCacheCap(
|
|
|
187
188
|
if (total <= maxBytes) return [];
|
|
188
189
|
// Oldest eligible body first, with blob ID as a stable tie.
|
|
189
190
|
const candidates = db.query(
|
|
190
|
-
`SELECT blob_id,byte_length FROM _syncular_blobs WHERE blob_id NOT IN(SELECT blob_id FROM _syncular_blob_uploads)AND blob_id NOT IN(SELECT blob_id FROM
|
|
191
|
+
`SELECT blob_id,byte_length FROM _syncular_blobs WHERE blob_id NOT IN(SELECT blob_id FROM _syncular_blob_uploads)AND blob_id NOT IN(SELECT blob_id FROM ${BLOB_COMMIT_REFS})AND blob_id NOT IN(${visibleBlobIdsSql(schema)})ORDER BY created_at_ms ASC,blob_id ASC`,
|
|
191
192
|
);
|
|
192
193
|
const evicted: string[] = [];
|
|
193
194
|
db.transaction(() => {
|
|
@@ -207,7 +208,7 @@ export function listPendingUploads(
|
|
|
207
208
|
): { blobId: string; mediaType?: string }[] {
|
|
208
209
|
return db
|
|
209
210
|
.query(
|
|
210
|
-
|
|
211
|
+
`SELECT blob_id,media_type FROM(SELECT blob_id,media_type,created_at_ms FROM _syncular_blob_uploads UNION ALL SELECT DISTINCT r.blob_id,b.media_type,9223372036854775807 AS created_at_ms FROM ${BLOB_COMMIT_REFS} r LEFT JOIN _syncular_blobs b ON b.blob_id=r.blob_id WHERE NOT EXISTS(SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id=r.blob_id))ORDER BY created_at_ms,blob_id`,
|
|
211
212
|
)
|
|
212
213
|
.map((row) => {
|
|
213
214
|
if (
|
|
@@ -285,7 +286,7 @@ export function deleteUnreferencedCachedBlobs(
|
|
|
285
286
|
schema: CompiledClientSchema,
|
|
286
287
|
): void {
|
|
287
288
|
db.exec(
|
|
288
|
-
`DELETE FROM _syncular_blobs WHERE blob_id NOT IN(SELECT blob_id FROM _syncular_blob_uploads)AND blob_id NOT IN(SELECT blob_id FROM
|
|
289
|
+
`DELETE FROM _syncular_blobs WHERE blob_id NOT IN(SELECT blob_id FROM _syncular_blob_uploads)AND blob_id NOT IN(SELECT blob_id FROM ${BLOB_COMMIT_REFS})AND blob_id NOT IN(${visibleBlobIdsSql(schema)})`,
|
|
289
290
|
);
|
|
290
291
|
}
|
|
291
292
|
|
package/src/client.ts
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
|
+
import { BLOB_COMMIT_REFS } from './blob';
|
|
2
|
+
import { hasUniqueIndex } from './schema';
|
|
3
|
+
import { OUTCOMES_TABLE } from './outcomes';
|
|
4
|
+
import { OUTBOX_TABLE } from './outbox';
|
|
1
5
|
import { RETAINED_ROWS } from './failed-overlay';
|
|
2
6
|
import {
|
|
3
7
|
dropFailedRows,
|
|
4
8
|
failedOverlayCommits,
|
|
5
9
|
retainFailedRows,
|
|
6
10
|
restoreFailedBases,
|
|
11
|
+
type OverlayScope,
|
|
7
12
|
uniqueConflicts,
|
|
8
13
|
} from './failed-overlay';
|
|
9
14
|
import {
|
|
@@ -1168,6 +1173,13 @@ export class SyncClient {
|
|
|
1168
1173
|
|
|
1169
1174
|
/** §2.1 reset after the server reports a different log continuity. */
|
|
1170
1175
|
#runLogEpochReset(logEpoch: string): string[] {
|
|
1176
|
+
if (getMeta(this.#db, LOG_EPOCH_META_KEY) === undefined) {
|
|
1177
|
+
this.#applyBatch(() => setMeta(this.#db, LOG_EPOCH_META_KEY, logEpoch));
|
|
1178
|
+
this.#setSyncNeeded(true);
|
|
1179
|
+
this.#emitSyncNeeded('startup');
|
|
1180
|
+
this.#emitSyncIntent({ kind: 'interactive' });
|
|
1181
|
+
return [];
|
|
1182
|
+
}
|
|
1171
1183
|
const subscriptions = loadSubscriptions(this.#db);
|
|
1172
1184
|
const pending = listOutbox(this.#db);
|
|
1173
1185
|
this.#setUpgrading(true);
|
|
@@ -1917,11 +1929,11 @@ export class SyncClient {
|
|
|
1917
1929
|
);
|
|
1918
1930
|
const outboxBytes = Number(
|
|
1919
1931
|
this.#db.query(
|
|
1920
|
-
|
|
1932
|
+
`SELECT COALESCE(SUM(LENGTH(operations)),0)AS bytes FROM ${OUTBOX_TABLE}`,
|
|
1921
1933
|
)[0]?.bytes ?? 0,
|
|
1922
1934
|
);
|
|
1923
1935
|
const outcome = this.#db.query(
|
|
1924
|
-
|
|
1936
|
+
`SELECT COUNT(*)AS entries,COALESCE(SUM(LENGTH(results)+ COALESCE(LENGTH(operations),0)),0)AS bytes FROM ${OUTCOMES_TABLE}`,
|
|
1925
1937
|
)[0];
|
|
1926
1938
|
const blobBytes = this.#hasBlobs
|
|
1927
1939
|
? Number(
|
|
@@ -2894,7 +2906,7 @@ export class SyncClient {
|
|
|
2894
2906
|
continue;
|
|
2895
2907
|
}
|
|
2896
2908
|
this.#db.exec(
|
|
2897
|
-
|
|
2909
|
+
`INSERT OR IGNORE INTO ${BLOB_COMMIT_REFS}(commit_id,blob_id)SELECT ?,? WHERE EXISTS(SELECT 1 FROM _syncular_blobs WHERE blob_id=?)`,
|
|
2898
2910
|
[clientCommitId, blobId, blobId],
|
|
2899
2911
|
);
|
|
2900
2912
|
}
|
|
@@ -3295,7 +3307,7 @@ export class SyncClient {
|
|
|
3295
3307
|
// Pin before the first encryption await: mutations can append while a
|
|
3296
3308
|
// round is encoding, and belong to the next request.
|
|
3297
3309
|
const bounds = this.#db.query(
|
|
3298
|
-
|
|
3310
|
+
`SELECT COUNT(*)AS count,MAX(seq)AS last_seq FROM ${OUTBOX_TABLE}`,
|
|
3299
3311
|
)[0]!;
|
|
3300
3312
|
const pendingCount = bounds.count as number;
|
|
3301
3313
|
const throughSeq = (bounds.last_seq as number | null) ?? 0;
|
|
@@ -4398,13 +4410,7 @@ export class SyncClient {
|
|
|
4398
4410
|
if (errorFrame !== undefined) break;
|
|
4399
4411
|
}
|
|
4400
4412
|
} finally {
|
|
4401
|
-
|
|
4402
|
-
// §7.1: local reads see outbox state applied optimistically — replay
|
|
4403
|
-
// the still-pending commits on top of the freshly applied server state.
|
|
4404
|
-
this.#replayOutbox();
|
|
4405
|
-
} finally {
|
|
4406
|
-
this.#endDiagnosticsDeferral();
|
|
4407
|
-
}
|
|
4413
|
+
this.#endDiagnosticsDeferral();
|
|
4408
4414
|
}
|
|
4409
4415
|
|
|
4410
4416
|
if (errorFrame !== undefined) throw errorFrame;
|
|
@@ -4558,6 +4564,15 @@ export class SyncClient {
|
|
|
4558
4564
|
listOutboxBeforeImages(this.#db, commit.clientCommitId),
|
|
4559
4565
|
);
|
|
4560
4566
|
this.#rollbackFailedCommit(commit, batch);
|
|
4567
|
+
this.#replayOutbox(
|
|
4568
|
+
commit.operations.map((operation) =>
|
|
4569
|
+
JSON.stringify(
|
|
4570
|
+
hasUniqueIndex(this.#table(operation.table))
|
|
4571
|
+
? [operation.table]
|
|
4572
|
+
: [operation.table, operation.rowId],
|
|
4573
|
+
),
|
|
4574
|
+
),
|
|
4575
|
+
);
|
|
4561
4576
|
});
|
|
4562
4577
|
batch.status();
|
|
4563
4578
|
summary.rejected.push(frame.clientCommitId);
|
|
@@ -4567,7 +4582,7 @@ export class SyncClient {
|
|
|
4567
4582
|
#outboxCommitExists(clientCommitId: string): boolean {
|
|
4568
4583
|
return (
|
|
4569
4584
|
this.#db.query(
|
|
4570
|
-
|
|
4585
|
+
`SELECT 1 FROM ${OUTBOX_TABLE} WHERE client_commit_id=? LIMIT 1`,
|
|
4571
4586
|
[clientCommitId],
|
|
4572
4587
|
).length > 0
|
|
4573
4588
|
);
|
|
@@ -4621,7 +4636,7 @@ export class SyncClient {
|
|
|
4621
4636
|
this.#applyBatch((batch) => {
|
|
4622
4637
|
this.#recordCommitChanges(batch, frame);
|
|
4623
4638
|
const result = fn();
|
|
4624
|
-
this.#replayOutbox(result);
|
|
4639
|
+
this.#replayOutbox(result, false);
|
|
4625
4640
|
return result;
|
|
4626
4641
|
}),
|
|
4627
4642
|
);
|
|
@@ -4704,7 +4719,7 @@ export class SyncClient {
|
|
|
4704
4719
|
)
|
|
4705
4720
|
batch.table(table.name);
|
|
4706
4721
|
const result = fn();
|
|
4707
|
-
this.#replayOutbox();
|
|
4722
|
+
this.#replayOutbox(result, false);
|
|
4708
4723
|
return result;
|
|
4709
4724
|
}),
|
|
4710
4725
|
};
|
|
@@ -4888,14 +4903,17 @@ export class SyncClient {
|
|
|
4888
4903
|
}),
|
|
4889
4904
|
);
|
|
4890
4905
|
if (covered.length) {
|
|
4891
|
-
|
|
4906
|
+
const scope = covered.map((row) =>
|
|
4907
|
+
JSON.stringify([sub.table, String(row.id)]),
|
|
4908
|
+
);
|
|
4909
|
+
restoreFailedBases(this.#db, this.#schema, scope);
|
|
4892
4910
|
for (const row of covered)
|
|
4893
4911
|
this.#db.exec(
|
|
4894
4912
|
`DELETE FROM ${RETAINED_ROWS} WHERE tbl=? AND id=? AND commit_seq<=?`,
|
|
4895
4913
|
[sub.table, String(row.id), nextCursor],
|
|
4896
4914
|
);
|
|
4897
4915
|
batch.table(sub.table);
|
|
4898
|
-
this.#replayOutbox();
|
|
4916
|
+
this.#replayOutbox(scope, false);
|
|
4899
4917
|
}
|
|
4900
4918
|
}
|
|
4901
4919
|
if (completed && registered !== undefined) {
|
|
@@ -4945,13 +4963,15 @@ export class SyncClient {
|
|
|
4945
4963
|
Object.keys(lastEffective).length > 0
|
|
4946
4964
|
) {
|
|
4947
4965
|
try {
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
4951
|
-
|
|
4952
|
-
name
|
|
4953
|
-
|
|
4954
|
-
|
|
4966
|
+
const scope = [
|
|
4967
|
+
JSON.stringify([table.name]),
|
|
4968
|
+
...this.#dropRetainedRows(
|
|
4969
|
+
batch,
|
|
4970
|
+
(name, values) =>
|
|
4971
|
+
name === table.name &&
|
|
4972
|
+
matchesLocalScopes(table, lastEffective, values),
|
|
4973
|
+
),
|
|
4974
|
+
];
|
|
4955
4975
|
deleteScopedRows(this.#db, table, lastEffective);
|
|
4956
4976
|
batch.scopeMap(table, lastEffective);
|
|
4957
4977
|
const pendingById = new Map(
|
|
@@ -4997,6 +5017,7 @@ export class SyncClient {
|
|
|
4997
5017
|
batch.rejections();
|
|
4998
5018
|
batch.outcomes();
|
|
4999
5019
|
}
|
|
5020
|
+
this.#replayOutbox(scope);
|
|
5000
5021
|
this.#deleteUnreferencedBlobs();
|
|
5001
5022
|
} catch (error) {
|
|
5002
5023
|
if (
|
|
@@ -5217,7 +5238,7 @@ export class SyncClient {
|
|
|
5217
5238
|
: (local[SYNC_VERSION_COLUMN] as number);
|
|
5218
5239
|
if (
|
|
5219
5240
|
replay &&
|
|
5220
|
-
table
|
|
5241
|
+
hasUniqueIndex(table) &&
|
|
5221
5242
|
uniqueConflicts(
|
|
5222
5243
|
this.#db,
|
|
5223
5244
|
table,
|
|
@@ -5245,8 +5266,9 @@ export class SyncClient {
|
|
|
5245
5266
|
table: string,
|
|
5246
5267
|
values: Readonly<Record<string, JsonRowValue>>,
|
|
5247
5268
|
) => boolean,
|
|
5248
|
-
):
|
|
5249
|
-
|
|
5269
|
+
): string[] {
|
|
5270
|
+
const scope: string[] = [];
|
|
5271
|
+
for (const commit of dropFailedRows(this.#db, this.#schema, matches)) {
|
|
5250
5272
|
batch.conflicts();
|
|
5251
5273
|
batch.rejections();
|
|
5252
5274
|
batch.outcomes();
|
|
@@ -5256,32 +5278,39 @@ export class SyncClient {
|
|
|
5256
5278
|
this.#rejections = this.#rejections.filter(
|
|
5257
5279
|
(record) => record.clientCommitId !== commit.clientCommitId,
|
|
5258
5280
|
);
|
|
5259
|
-
for (const op of commit.operations)
|
|
5281
|
+
for (const op of commit.operations) {
|
|
5282
|
+
batch.table(op.table);
|
|
5283
|
+
scope.push(JSON.stringify([op.table, op.rowId]));
|
|
5284
|
+
}
|
|
5260
5285
|
}
|
|
5286
|
+
return scope;
|
|
5261
5287
|
}
|
|
5262
5288
|
|
|
5263
|
-
#replayOutbox(
|
|
5264
|
-
|
|
5265
|
-
const pending = [
|
|
5266
|
-
...failedOverlayCommits(this.#db, false, rowKeys),
|
|
5267
|
-
...listOutbox(this.#db, rowKeys),
|
|
5268
|
-
].sort((a, b) => a.createdAtMs - b.createdAtMs || a.seq - b.seq);
|
|
5269
|
-
if (pending.length === 0 && acknowledged.length === 0) return;
|
|
5289
|
+
#replayOutbox(scope?: OverlayScope, restoreBases = true): void {
|
|
5290
|
+
if (scope?.length === 0) return;
|
|
5270
5291
|
this.#applyBatch((batch) => {
|
|
5271
|
-
this.#db
|
|
5272
|
-
|
|
5273
|
-
|
|
5274
|
-
|
|
5275
|
-
|
|
5276
|
-
|
|
5277
|
-
|
|
5278
|
-
|
|
5279
|
-
|
|
5280
|
-
|
|
5281
|
-
|
|
5282
|
-
|
|
5283
|
-
|
|
5284
|
-
|
|
5292
|
+
if (restoreBases) restoreFailedBases(this.#db, this.#schema, scope);
|
|
5293
|
+
const acknowledged = failedOverlayCommits(
|
|
5294
|
+
this.#db,
|
|
5295
|
+
true,
|
|
5296
|
+
scope,
|
|
5297
|
+
this.#schema,
|
|
5298
|
+
);
|
|
5299
|
+
const pending = [
|
|
5300
|
+
...failedOverlayCommits(this.#db, false, scope, this.#schema),
|
|
5301
|
+
...listOutbox(this.#db, scope),
|
|
5302
|
+
].sort((a, b) => a.createdAtMs - b.createdAtMs || a.seq - b.seq);
|
|
5303
|
+
for (const commit of [...acknowledged, ...pending])
|
|
5304
|
+
this.#applyOperationsLocally(
|
|
5305
|
+
commit.operations.filter(
|
|
5306
|
+
(op) =>
|
|
5307
|
+
scope === undefined ||
|
|
5308
|
+
scope.includes(JSON.stringify([op.table, op.rowId])) ||
|
|
5309
|
+
scope.includes(JSON.stringify([op.table])),
|
|
5310
|
+
),
|
|
5311
|
+
batch,
|
|
5312
|
+
true,
|
|
5313
|
+
);
|
|
5285
5314
|
});
|
|
5286
5315
|
}
|
|
5287
5316
|
|
package/src/failed-overlay.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { hasUniqueIndex } from './schema';
|
|
2
|
+
import { OUTCOMES_TABLE } from './outcomes';
|
|
3
|
+
import { OUTBOX_TABLE } from './outbox';
|
|
1
4
|
export const RETAINED_ROWS = '_syncular_failed_rows';
|
|
2
5
|
import { ClientSyncError } from './errors';
|
|
3
6
|
import type { ClientDatabase } from './database';
|
|
@@ -48,7 +51,7 @@ export function retainedBaseWrite(
|
|
|
48
51
|
table.name,
|
|
49
52
|
rowId,
|
|
50
53
|
]);
|
|
51
|
-
if (!table
|
|
54
|
+
if (!hasUniqueIndex(table)) return;
|
|
52
55
|
for (const retained of db.query(
|
|
53
56
|
`SELECT commit_id,idx,intent,unique_conflicts FROM ${RETAINED_ROWS} WHERE tbl=? AND commit_seq IS NULL`,
|
|
54
57
|
[table.name],
|
|
@@ -164,90 +167,51 @@ export function uniqueConflicts(
|
|
|
164
167
|
);
|
|
165
168
|
}
|
|
166
169
|
|
|
167
|
-
|
|
170
|
+
/** Exact row keys are JSON [table,id]; table scopes include unique-key peers. */
|
|
171
|
+
export type OverlayScope = readonly string[];
|
|
172
|
+
|
|
173
|
+
export function overlayScopePredicate(table: string, id: string): string {
|
|
174
|
+
return `?1 IS NULL OR json_array(${table},${id}) IN(SELECT value FROM json_each(?1)) OR json_array(${table}) IN(SELECT value FROM json_each(?1))`;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function retainedRows(
|
|
178
|
+
db: ClientDatabase,
|
|
179
|
+
scope?: OverlayScope,
|
|
180
|
+
acknowledged?: boolean,
|
|
181
|
+
) {
|
|
168
182
|
return db.query(
|
|
169
|
-
`SELECT * FROM ${RETAINED_ROWS} WHERE
|
|
170
|
-
[
|
|
183
|
+
`SELECT * FROM ${RETAINED_ROWS} WHERE (${overlayScopePredicate('tbl', 'id')})${acknowledged === undefined ? '' : ` AND commit_seq IS${acknowledged ? ' NOT' : ''} NULL`} ORDER BY commit_seq,at,commit_id,idx`,
|
|
184
|
+
[scope ? JSON.stringify(scope) : null],
|
|
171
185
|
);
|
|
172
186
|
}
|
|
173
187
|
|
|
188
|
+
function retainedTable(
|
|
189
|
+
schema: CompiledClientSchema,
|
|
190
|
+
name: string,
|
|
191
|
+
): CompiledClientTable {
|
|
192
|
+
const table = schema.tables.get(name);
|
|
193
|
+
if (!table)
|
|
194
|
+
throw new ClientSyncError(
|
|
195
|
+
'sync.unknown_table',
|
|
196
|
+
'retained table is unknown',
|
|
197
|
+
);
|
|
198
|
+
return table;
|
|
199
|
+
}
|
|
200
|
+
|
|
174
201
|
export function restoreFailedBases(
|
|
175
202
|
db: ClientDatabase,
|
|
176
203
|
schema: CompiledClientSchema,
|
|
177
|
-
|
|
178
|
-
rowKeys?: readonly string[],
|
|
204
|
+
scope?: OverlayScope,
|
|
179
205
|
): boolean {
|
|
180
206
|
const seen = new Set<string>();
|
|
181
|
-
for (const row of retainedRows(db,
|
|
207
|
+
for (const row of retainedRows(db, scope)) {
|
|
182
208
|
const key = JSON.stringify([row.tbl, row.id]);
|
|
183
|
-
if (
|
|
209
|
+
if (seen.has(key)) continue;
|
|
184
210
|
seen.add(key);
|
|
185
|
-
const table = schema
|
|
186
|
-
if (
|
|
187
|
-
throw new ClientSyncError(
|
|
188
|
-
'sync.unknown_table',
|
|
189
|
-
'retained table is unknown',
|
|
190
|
-
);
|
|
191
|
-
const json = absentIntent ? row.intent : row.base;
|
|
192
|
-
if (absentIntent && row.intent !== null) {
|
|
193
|
-
const values: Record<string, JsonRowValue> = JSON.parse(
|
|
194
|
-
row.intent as string,
|
|
195
|
-
);
|
|
196
|
-
const conflicts = uniqueConflicts(
|
|
197
|
-
db,
|
|
198
|
-
table,
|
|
199
|
-
mapRowValues(values, jsonToRowValue),
|
|
200
|
-
);
|
|
201
|
-
// Hydrate a pre-0.30.11 failure after its winner was already imported.
|
|
202
|
-
// Pending and retained overlays cannot supply authoritative row evidence.
|
|
203
|
-
if (row.commit_seq === null && row.unique_conflicts === null) {
|
|
204
|
-
const authorized = conflicts.filter(
|
|
205
|
-
(conflict) =>
|
|
206
|
-
conflict.serverVersion >= 0 &&
|
|
207
|
-
!db.query(
|
|
208
|
-
`SELECT 1 FROM ${RETAINED_ROWS} WHERE tbl=? AND id=? UNION ALL SELECT 1 FROM _syncular_outbox,json_each(operations)WHERE json_extract(json_each.value,'$.table')=? AND json_extract(json_each.value,'$.rowId')=? LIMIT 1`,
|
|
209
|
-
[table.name, conflict.rowId, table.name, conflict.rowId],
|
|
210
|
-
).length,
|
|
211
|
-
);
|
|
212
|
-
if (authorized.length)
|
|
213
|
-
db.exec(
|
|
214
|
-
`UPDATE ${RETAINED_ROWS} SET unique_conflicts=? WHERE commit_id=? AND idx=?`,
|
|
215
|
-
[
|
|
216
|
-
JSON.stringify(
|
|
217
|
-
authorized.map((conflict) => ({
|
|
218
|
-
...conflict,
|
|
219
|
-
serverRow: mapRowValues(conflict.serverRow, rowValueToJson),
|
|
220
|
-
})),
|
|
221
|
-
),
|
|
222
|
-
row.commit_id as string,
|
|
223
|
-
row.idx as number,
|
|
224
|
-
],
|
|
225
|
-
);
|
|
226
|
-
}
|
|
227
|
-
if (conflicts.length) continue;
|
|
228
|
-
}
|
|
229
|
-
if (
|
|
230
|
-
absentIntent &&
|
|
231
|
-
(json === null ||
|
|
232
|
-
db.query(
|
|
233
|
-
`SELECT 1 FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(table.primaryKey)}=?`,
|
|
234
|
-
[row.id as string],
|
|
235
|
-
).length)
|
|
236
|
-
)
|
|
237
|
-
continue;
|
|
238
|
-
if (json === null) deleteLocalRow(db, table, row.id as string, false);
|
|
211
|
+
const table = retainedTable(schema, row.tbl as string);
|
|
212
|
+
if (row.base === null) deleteLocalRow(db, table, row.id as string, false);
|
|
239
213
|
else {
|
|
240
|
-
const values: Record<string, JsonRowValue> = JSON.parse(String(
|
|
241
|
-
if (absentIntent) {
|
|
242
|
-
const operation: OutboxOperation = JSON.parse(row.op as string);
|
|
243
|
-
if (
|
|
244
|
-
row.commit_seq === null &&
|
|
245
|
-
!table.columns.every((column) =>
|
|
246
|
-
Object.hasOwn(operation.values ?? {}, column.name),
|
|
247
|
-
)
|
|
248
|
-
)
|
|
249
|
-
continue;
|
|
250
|
-
}
|
|
214
|
+
const values: Record<string, JsonRowValue> = JSON.parse(String(row.base));
|
|
251
215
|
upsertLocalRow(
|
|
252
216
|
db,
|
|
253
217
|
table,
|
|
@@ -265,7 +229,8 @@ export function restoreFailedBases(
|
|
|
265
229
|
export function failedOverlayCommits(
|
|
266
230
|
db: ClientDatabase,
|
|
267
231
|
acknowledged = false,
|
|
268
|
-
|
|
232
|
+
scope?: OverlayScope,
|
|
233
|
+
schema?: CompiledClientSchema,
|
|
269
234
|
): OutboxCommit[] {
|
|
270
235
|
const commits = new Map<
|
|
271
236
|
string,
|
|
@@ -276,10 +241,55 @@ export function failedOverlayCommits(
|
|
|
276
241
|
operations: OutboxOperation[];
|
|
277
242
|
}
|
|
278
243
|
>();
|
|
279
|
-
for (const row of retainedRows(db,
|
|
280
|
-
if ((row.commit_seq !== null) !== acknowledged) continue;
|
|
244
|
+
for (const row of retainedRows(db, scope, acknowledged)) {
|
|
281
245
|
const id = row.commit_id as string;
|
|
282
|
-
|
|
246
|
+
let operation: OutboxOperation = JSON.parse(row.op as string);
|
|
247
|
+
if (schema && operation.op === 'upsert') {
|
|
248
|
+
const table = retainedTable(schema, operation.table);
|
|
249
|
+
if (acknowledged) {
|
|
250
|
+
if (
|
|
251
|
+
!db.query(
|
|
252
|
+
`SELECT 1 FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(table.primaryKey)}=?`,
|
|
253
|
+
[operation.rowId],
|
|
254
|
+
).length
|
|
255
|
+
)
|
|
256
|
+
operation = {
|
|
257
|
+
...operation,
|
|
258
|
+
values: JSON.parse(row.intent as string),
|
|
259
|
+
};
|
|
260
|
+
} else if (row.unique_conflicts === null && row.intent !== null) {
|
|
261
|
+
// Hydrate legacy failure evidence from authoritative unique-key peers.
|
|
262
|
+
const values: Record<string, JsonRowValue> = JSON.parse(
|
|
263
|
+
row.intent as string,
|
|
264
|
+
);
|
|
265
|
+
const authorized = uniqueConflicts(
|
|
266
|
+
db,
|
|
267
|
+
table,
|
|
268
|
+
mapRowValues(values, jsonToRowValue),
|
|
269
|
+
).filter(
|
|
270
|
+
(conflict) =>
|
|
271
|
+
conflict.serverVersion >= 0 &&
|
|
272
|
+
!db.query(
|
|
273
|
+
`SELECT 1 FROM ${RETAINED_ROWS} WHERE tbl=? AND id=? UNION ALL SELECT 1 FROM ${OUTBOX_TABLE},json_each(operations)WHERE json_extract(json_each.value,'$.table')=? AND json_extract(json_each.value,'$.rowId')=? LIMIT 1`,
|
|
274
|
+
[table.name, conflict.rowId, table.name, conflict.rowId],
|
|
275
|
+
).length,
|
|
276
|
+
);
|
|
277
|
+
if (authorized.length)
|
|
278
|
+
db.exec(
|
|
279
|
+
`UPDATE ${RETAINED_ROWS} SET unique_conflicts=? WHERE commit_id=? AND idx=?`,
|
|
280
|
+
[
|
|
281
|
+
JSON.stringify(
|
|
282
|
+
authorized.map((conflict) => ({
|
|
283
|
+
...conflict,
|
|
284
|
+
serverRow: mapRowValues(conflict.serverRow, rowValueToJson),
|
|
285
|
+
})),
|
|
286
|
+
),
|
|
287
|
+
row.commit_id as string,
|
|
288
|
+
row.idx as number,
|
|
289
|
+
],
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
283
293
|
const prior = commits.get(id);
|
|
284
294
|
if (prior) prior.operations.push(operation);
|
|
285
295
|
else
|
|
@@ -295,6 +305,7 @@ export function failedOverlayCommits(
|
|
|
295
305
|
|
|
296
306
|
export function dropFailedRows(
|
|
297
307
|
db: ClientDatabase,
|
|
308
|
+
schema: CompiledClientSchema,
|
|
298
309
|
matches: (
|
|
299
310
|
table: string,
|
|
300
311
|
values: Readonly<Record<string, JsonRowValue>>,
|
|
@@ -312,10 +323,17 @@ export function dropFailedRows(
|
|
|
312
323
|
...failedOverlayCommits(db),
|
|
313
324
|
...failedOverlayCommits(db, true),
|
|
314
325
|
].filter((commit) => ids.has(commit.clientCommitId));
|
|
326
|
+
restoreFailedBases(
|
|
327
|
+
db,
|
|
328
|
+
schema,
|
|
329
|
+
dropped.flatMap((commit) =>
|
|
330
|
+
commit.operations.map((op) => JSON.stringify([op.table, op.rowId])),
|
|
331
|
+
),
|
|
332
|
+
);
|
|
315
333
|
for (const id of ids) {
|
|
316
334
|
db.exec(`DELETE FROM ${RETAINED_ROWS} WHERE commit_id=?`, [id]);
|
|
317
335
|
db.exec(
|
|
318
|
-
|
|
336
|
+
`UPDATE ${OUTCOMES_TABLE} SET operations=NULL,results='[]' WHERE client_commit_id=?`,
|
|
319
337
|
[id],
|
|
320
338
|
);
|
|
321
339
|
}
|
package/src/outbox.ts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import { BLOB_COMMIT_REFS } from './blob';
|
|
2
|
+
export const OUTBOX_TABLE = '_syncular_outbox';
|
|
3
|
+
export const OUTBOX_IMAGES = '_syncular_outbox_before_images';
|
|
4
|
+
import { overlayScopePredicate, type OverlayScope } from './failed-overlay';
|
|
1
5
|
/**
|
|
2
6
|
* The durable outbox (SPEC.md §7.1) with encode-at-send (the §0 binary-push
|
|
3
7
|
* outbox rule): local mutations are persisted in a schema-agnostic JSON
|
|
@@ -68,7 +72,7 @@ export function appendOutboxCommit(
|
|
|
68
72
|
);
|
|
69
73
|
}
|
|
70
74
|
db.exec(
|
|
71
|
-
|
|
75
|
+
`INSERT INTO ${OUTBOX_TABLE}(client_commit_id,created_at_ms,operations)VALUES(?,?,?)`,
|
|
72
76
|
[clientCommitId, nowMs, JSON.stringify(operations)],
|
|
73
77
|
);
|
|
74
78
|
replaceOutboxBeforeImages(db, clientCommitId, beforeImages);
|
|
@@ -77,14 +81,14 @@ export function appendOutboxCommit(
|
|
|
77
81
|
/** Pending commits in FIFO creation order (§7.1). Full reads serve replay and the public listing. */
|
|
78
82
|
export function listOutbox(
|
|
79
83
|
db: ClientDatabase,
|
|
80
|
-
|
|
84
|
+
scope?: OverlayScope,
|
|
81
85
|
): OutboxCommit[] {
|
|
82
86
|
return db
|
|
83
87
|
.query(
|
|
84
|
-
|
|
85
|
-
?
|
|
86
|
-
:
|
|
87
|
-
|
|
88
|
+
scope === undefined
|
|
89
|
+
? `SELECT * FROM ${OUTBOX_TABLE} ORDER BY seq ASC`
|
|
90
|
+
: `SELECT * FROM ${OUTBOX_TABLE} WHERE EXISTS(SELECT 1 FROM json_each(operations) WHERE ${overlayScopePredicate("json_extract(value,'$.table')", "json_extract(value,'$.rowId')")}) ORDER BY seq`,
|
|
91
|
+
scope && [JSON.stringify(scope)],
|
|
88
92
|
)
|
|
89
93
|
.map(decodeOutboxRow);
|
|
90
94
|
}
|
|
@@ -106,7 +110,7 @@ export function* iterateOutbox(
|
|
|
106
110
|
let afterSeq = 0;
|
|
107
111
|
while (afterSeq < throughSeq) {
|
|
108
112
|
const rows = db.query(
|
|
109
|
-
|
|
113
|
+
`SELECT * FROM ${OUTBOX_TABLE} WHERE seq > ? AND seq <= ? ORDER BY seq ASC LIMIT 32`,
|
|
110
114
|
[afterSeq, throughSeq],
|
|
111
115
|
);
|
|
112
116
|
if (rows.length === 0) return;
|
|
@@ -120,7 +124,7 @@ export function* iterateOutbox(
|
|
|
120
124
|
|
|
121
125
|
/** Routine status reads never load operation bodies. */
|
|
122
126
|
export function countOutbox(db: ClientDatabase): number {
|
|
123
|
-
return db.query(
|
|
127
|
+
return db.query(`SELECT COUNT(*)AS count FROM ${OUTBOX_TABLE}`)[0]!
|
|
124
128
|
.count as number;
|
|
125
129
|
}
|
|
126
130
|
|
|
@@ -128,14 +132,13 @@ export function deleteOutboxCommit(
|
|
|
128
132
|
db: ClientDatabase,
|
|
129
133
|
clientCommitId: string,
|
|
130
134
|
): void {
|
|
131
|
-
db.exec(
|
|
135
|
+
db.exec(`DELETE FROM ${BLOB_COMMIT_REFS} WHERE commit_id=?`, [
|
|
132
136
|
clientCommitId,
|
|
133
137
|
]);
|
|
134
|
-
db.exec(
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
db.exec('DELETE FROM _syncular_outbox WHERE client_commit_id=?', [
|
|
138
|
+
db.exec(`DELETE FROM ${OUTBOX_IMAGES} WHERE client_commit_id=?`, [
|
|
139
|
+
clientCommitId,
|
|
140
|
+
]);
|
|
141
|
+
db.exec(`DELETE FROM ${OUTBOX_TABLE} WHERE client_commit_id=?`, [
|
|
139
142
|
clientCommitId,
|
|
140
143
|
]);
|
|
141
144
|
}
|
|
@@ -146,7 +149,7 @@ export function listOutboxBeforeImages(
|
|
|
146
149
|
): OutboxBeforeImage[] {
|
|
147
150
|
return db
|
|
148
151
|
.query(
|
|
149
|
-
|
|
152
|
+
`SELECT op_index,existed,sync_version,values_json,delivery_seq FROM ${OUTBOX_IMAGES} WHERE client_commit_id=? ORDER BY op_index`,
|
|
150
153
|
[clientCommitId],
|
|
151
154
|
)
|
|
152
155
|
.map((row) => ({
|
|
@@ -175,7 +178,7 @@ export function replaceOutboxBeforeImages(
|
|
|
175
178
|
): void {
|
|
176
179
|
for (const image of replacements) {
|
|
177
180
|
db.exec(
|
|
178
|
-
|
|
181
|
+
`INSERT INTO ${OUTBOX_IMAGES}(client_commit_id,op_index,existed,sync_version,values_json)VALUES(?,?,?,?,?) ON CONFLICT(client_commit_id,op_index)DO UPDATE SET existed=excluded.existed,sync_version=excluded.sync_version,values_json=excluded.values_json`,
|
|
179
182
|
[
|
|
180
183
|
clientCommitId,
|
|
181
184
|
image.opIndex,
|
package/src/outcomes.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
export const OUTCOMES_TABLE = '_syncular_commit_outcomes';
|
|
2
|
+
const SELECT_OUTCOME = `SELECT * FROM ${OUTCOMES_TABLE}`;
|
|
1
3
|
import { RETAINED_ROWS } from './failed-overlay';
|
|
2
4
|
/**
|
|
3
5
|
* Durable per-client commit outcomes.
|
|
@@ -233,7 +235,7 @@ export function recordCommitOutcome(
|
|
|
233
235
|
outcome: Omit<CommitOutcome, 'sequence' | 'resolution'>,
|
|
234
236
|
): CommitOutcome {
|
|
235
237
|
db.exec(
|
|
236
|
-
|
|
238
|
+
`INSERT INTO ${OUTCOMES_TABLE}(client_commit_id,status,recorded_at_ms,results,operations,resolution)VALUES(?,?,?,?,?,'active')`,
|
|
237
239
|
[
|
|
238
240
|
outcome.clientCommitId,
|
|
239
241
|
outcome.status,
|
|
@@ -251,10 +253,9 @@ export function commitOutcome(
|
|
|
251
253
|
db: ClientDatabase,
|
|
252
254
|
clientCommitId: string,
|
|
253
255
|
): CommitOutcome | undefined {
|
|
254
|
-
const row = db.query(
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
)[0];
|
|
256
|
+
const row = db.query(`${SELECT_OUTCOME} WHERE client_commit_id=?`, [
|
|
257
|
+
clientCommitId,
|
|
258
|
+
])[0];
|
|
258
259
|
return row === undefined ? undefined : parseOutcome(db, row);
|
|
259
260
|
}
|
|
260
261
|
|
|
@@ -273,7 +274,7 @@ export function listCommitOutcomes(
|
|
|
273
274
|
? "WHERE resolution = 'active' AND status IN ('conflict', 'rejected')"
|
|
274
275
|
: '';
|
|
275
276
|
const rows = db.query(
|
|
276
|
-
|
|
277
|
+
`${SELECT_OUTCOME} ${where} ORDER BY seq DESC${limit === undefined ? '' : ' LIMIT ?'}`,
|
|
277
278
|
limit === undefined ? [] : [limit],
|
|
278
279
|
);
|
|
279
280
|
return rows.map((row) => parseOutcome(db, row));
|
|
@@ -285,7 +286,7 @@ export function persistCommitOutcomeResolution(
|
|
|
285
286
|
nowMs: number,
|
|
286
287
|
): CommitOutcome | undefined {
|
|
287
288
|
db.exec(
|
|
288
|
-
|
|
289
|
+
`UPDATE ${OUTCOMES_TABLE} SET resolution=?,resolved_at_ms=?,replacement_client_commit_id=? WHERE client_commit_id=? AND resolution='active'`,
|
|
289
290
|
[
|
|
290
291
|
input.resolution,
|
|
291
292
|
nowMs,
|
|
@@ -310,17 +311,16 @@ export function pruneCommitOutcomes(
|
|
|
310
311
|
'outcome retention maxEntries must be a positive safe integer',
|
|
311
312
|
);
|
|
312
313
|
}
|
|
313
|
-
const count = db.query(
|
|
314
|
-
|
|
315
|
-
)[0]?.count as number | undefined;
|
|
314
|
+
const count = db.query(`SELECT COUNT(*) AS count FROM ${OUTCOMES_TABLE}`)[0]
|
|
315
|
+
?.count as number | undefined;
|
|
316
316
|
const excess = Math.max(0, (count ?? 0) - maxEntries);
|
|
317
317
|
if (excess === 0) return 0;
|
|
318
318
|
const candidates = db.query(
|
|
319
|
-
|
|
319
|
+
`SELECT seq FROM ${OUTCOMES_TABLE} WHERE status IN('applied','cached')OR resolution !='active' ORDER BY seq ASC LIMIT ?`,
|
|
320
320
|
[excess],
|
|
321
321
|
);
|
|
322
322
|
for (const candidate of candidates) {
|
|
323
|
-
db.exec(
|
|
323
|
+
db.exec(`DELETE FROM ${OUTCOMES_TABLE} WHERE seq=?`, [
|
|
324
324
|
candidate.seq as number,
|
|
325
325
|
]);
|
|
326
326
|
}
|