@syncular/server 0.34.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blob-handlers.js +10 -3
- package/dist/blob-store.d.ts +9 -1
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +1 -0
- package/dist/bucket-blob-store.js +22 -16
- package/dist/d1-storage.d.ts +3 -1
- package/dist/d1-storage.js +25 -1
- package/dist/erase.d.ts +28 -0
- package/dist/erase.js +122 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/postgres-storage.d.ts +3 -1
- package/dist/postgres-storage.js +40 -0
- package/dist/push.d.ts +8 -2
- package/dist/push.js +1 -1
- package/dist/sqlite-blob-store.d.ts +1 -0
- package/dist/sqlite-blob-store.js +14 -2
- package/dist/sqlite-dialect.d.ts +11 -1
- package/dist/sqlite-dialect.js +35 -0
- package/dist/sqlite-storage.d.ts +3 -1
- package/dist/sqlite-storage.js +26 -1
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +2 -0
- package/dist/storage.d.ts +26 -0
- package/package.json +2 -2
- package/src/blob-handlers.ts +13 -5
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +26 -20
- package/src/d1-storage.ts +45 -0
- package/src/erase.ts +173 -0
- package/src/index.ts +1 -0
- package/src/postgres-storage.ts +59 -0
- package/src/push.ts +1 -1
- package/src/sqlite-blob-store.ts +17 -1
- package/src/sqlite-dialect.ts +43 -0
- package/src/sqlite-storage.ts +39 -0
- package/src/storage-errors.ts +7 -1
- package/src/storage.ts +35 -0
package/dist/storage-errors.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export declare class StorageConstraintError extends Error {
|
|
|
9
9
|
constructor(cause: unknown, opIndex?: number);
|
|
10
10
|
}
|
|
11
11
|
/** Stable, privacy-safe failures for trusted server storage queries. */
|
|
12
|
-
export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor' | 'sync.storage.checkpoint_not_declared' | 'sync.storage.checkpoint_unsupported' | 'sync.storage.checkpoint_fence_missing' | 'sync.storage.checkpoint_incomplete' | 'sync.storage.stored_layout_mismatch' | 'sync.storage.physical_layout_mismatch' | 'sync.storage.transaction_query_unsupported' | 'sync.storage.query_over_staged_writes';
|
|
12
|
+
export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor' | 'sync.storage.checkpoint_not_declared' | 'sync.storage.checkpoint_unsupported' | 'sync.storage.checkpoint_fence_missing' | 'sync.storage.checkpoint_incomplete' | 'sync.storage.stored_layout_mismatch' | 'sync.storage.physical_layout_mismatch' | 'sync.storage.transaction_query_unsupported' | 'sync.storage.query_over_staged_writes' | 'sync.storage.erase_unsupported' | 'sync.storage.invalid_erase_query';
|
|
13
13
|
/**
|
|
14
14
|
* Host-only query error. Messages never include identifiers, values, SQL,
|
|
15
15
|
* paths, or row data; callers branch on `code`, never message text. The
|
package/dist/storage-errors.js
CHANGED
|
@@ -32,6 +32,8 @@ const STORAGE_QUERY_MESSAGES = {
|
|
|
32
32
|
'sync.storage.index_not_materialized': 'trusted row lookup requires a materialized relational table',
|
|
33
33
|
'sync.storage.index_value_count_mismatch': 'trusted row lookup requires one exact value per index column',
|
|
34
34
|
'sync.storage.invalid_limit': 'trusted row lookup limit must be an integer from 1 through 1,000',
|
|
35
|
+
'sync.storage.erase_unsupported': 'this storage backend cannot erase a scope value',
|
|
36
|
+
'sync.storage.invalid_erase_query': 'erasure requires a declared scope variable, a non-empty value, and a row budget within the operation cap',
|
|
35
37
|
};
|
|
36
38
|
/**
|
|
37
39
|
* Host-only query error. Messages never include identifiers, values, SQL,
|
package/dist/storage.d.ts
CHANGED
|
@@ -701,6 +701,22 @@ export interface ServerStorage {
|
|
|
701
701
|
*/
|
|
702
702
|
listRowsReferencingBlob?(partition: string, blobId: string): Promise<BlobReferencingRow[]>;
|
|
703
703
|
listReferencedBlobIds?(partition: string): Promise<string[]>;
|
|
704
|
+
/**
|
|
705
|
+
* §4.6.1 scope-value erasure — ADDITIVE, optional; `eraseScopeValue`
|
|
706
|
+
* requires both.
|
|
707
|
+
*
|
|
708
|
+
* `listScopeValueRows`: up to `limit` rowIds of `table` whose stored scope
|
|
709
|
+
* binds `variable` to `value`, ordered by rowId (the row scope index).
|
|
710
|
+
*
|
|
711
|
+
* `eraseScopeValueHistory`: under the partition write lock, removes up to
|
|
712
|
+
* `limit` upsert change records of `table` at or below `throughSeq` whose
|
|
713
|
+
* scopes bind `variable` to `value`, then up to `limit` change scope index
|
|
714
|
+
* entries of that binding whose commit holds no such change any more.
|
|
715
|
+
* Delete change records stay: clients behind the erasure still need them.
|
|
716
|
+
* Returns the number of records removed.
|
|
717
|
+
*/
|
|
718
|
+
listScopeValueRows?(partition: string, query: ScopeValueQuery): Promise<string[]>;
|
|
719
|
+
eraseScopeValueHistory?(partition: string, query: ScopeValueHistoryQuery): Promise<number>;
|
|
704
720
|
/**
|
|
705
721
|
* Admin/console read surface (`SyncularAdmin`): ADDITIVE,
|
|
706
722
|
* optional. All read-only, partition-scoped, and JSON-able. A backend
|
|
@@ -736,6 +752,16 @@ export interface ServerStorage {
|
|
|
736
752
|
listPushResults?(partition: string, clientCommitId: string): Promise<ClientPushResult[]>;
|
|
737
753
|
listPartitions(): Promise<string[]>;
|
|
738
754
|
}
|
|
755
|
+
/** One scope binding of one table, budgeted (§4.6.1). */
|
|
756
|
+
export interface ScopeValueQuery {
|
|
757
|
+
readonly table: string;
|
|
758
|
+
readonly variable: string;
|
|
759
|
+
readonly value: string;
|
|
760
|
+
readonly limit: number;
|
|
761
|
+
}
|
|
762
|
+
export interface ScopeValueHistoryQuery extends ScopeValueQuery {
|
|
763
|
+
readonly throughSeq: number;
|
|
764
|
+
}
|
|
739
765
|
/** A row referencing a blob, with the scopes needed to authorize download. */
|
|
740
766
|
export interface BlobReferencingRow {
|
|
741
767
|
readonly table: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.35.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.35.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/blob-handlers.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* `blobId` is never a bearer capability. Framework-free; the hono/runtime
|
|
7
7
|
* adapters map these onto `PUT/GET <mount>/blobs/{blobId}`.
|
|
8
8
|
*/
|
|
9
|
-
import { blobIdFor, isBlobId } from './blob-store';
|
|
9
|
+
import { type BlobRecord, blobIdFor, isBlobId } from './blob-store';
|
|
10
10
|
import type { SyncRequestContext } from './context';
|
|
11
11
|
import {
|
|
12
12
|
clockOf,
|
|
@@ -180,7 +180,7 @@ async function downloadBlob(
|
|
|
180
180
|
// egress path — it only needs to know the object EXISTS (a HEAD/`has`), not
|
|
181
181
|
// the bytes. Inline delivery loads the bytes; presigned delivery does not.
|
|
182
182
|
const presignConfigured = ctx.blobSignedUrls !== undefined;
|
|
183
|
-
let entry: { bytes: Uint8Array } | undefined;
|
|
183
|
+
let entry: { record: BlobRecord; bytes: Uint8Array } | undefined;
|
|
184
184
|
if (presignConfigured) {
|
|
185
185
|
if (!(await store.has(ctx.partition, blobId))) {
|
|
186
186
|
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
@@ -277,12 +277,20 @@ async function downloadBlob(
|
|
|
277
277
|
// Unreachable: presign not configured ⇒ `entry` was loaded above.
|
|
278
278
|
throw syncError('blob.not_found', 'blob bytes unavailable (§5.9.5)');
|
|
279
279
|
}
|
|
280
|
-
// §5.9.5: inline bytes
|
|
281
|
-
//
|
|
280
|
+
// §5.9.5: inline bytes carry the stored media type, so an <img> or <a>
|
|
281
|
+
// pointed at the route renders them. A JSON type stays octet-stream: a
|
|
282
|
+
// JSON body is reserved for the signed-URL envelope.
|
|
283
|
+
const mediaType = entry?.record.mediaType;
|
|
282
284
|
return {
|
|
283
285
|
bytes,
|
|
284
286
|
headers: {
|
|
285
|
-
'Content-Type':
|
|
287
|
+
'Content-Type':
|
|
288
|
+
mediaType === undefined ||
|
|
289
|
+
mediaType.toLowerCase().includes('application/json')
|
|
290
|
+
? 'application/octet-stream'
|
|
291
|
+
: mediaType,
|
|
292
|
+
'X-Content-Type-Options': 'nosniff',
|
|
293
|
+
'Content-Security-Policy': "sandbox; default-src 'none'",
|
|
286
294
|
ETag: `"${blobId}"`,
|
|
287
295
|
'Cache-Control': 'private, max-age=0',
|
|
288
296
|
Vary: 'Authorization, Accept-Encoding',
|
package/src/blob-store.ts
CHANGED
|
Binary file
|
package/src/bucket-blob-store.ts
CHANGED
|
@@ -218,29 +218,31 @@ export class BucketBlobStore implements BlobStore {
|
|
|
218
218
|
// blob once, and preserve the FIRST createdAtMs so the sweep's grace clock
|
|
219
219
|
// is not reset by a re-upload (§5.9.2 upload age).
|
|
220
220
|
const preexisting = await this.#bucket.head(objectKey);
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
}
|
|
221
|
+
const storedMeta = preexisting?.metadata[BLOB_META_KEY];
|
|
222
|
+
const existing =
|
|
223
|
+
storedMeta === undefined
|
|
224
|
+
? undefined
|
|
225
|
+
: parseMetaJson(base64urlToUtf8(storedMeta), blobId, this.#label);
|
|
226
|
+
if (
|
|
227
|
+
existing !== undefined &&
|
|
228
|
+
(existing.mediaType !== undefined || mediaType === undefined)
|
|
229
|
+
) {
|
|
230
|
+
return {
|
|
231
|
+
blobId,
|
|
232
|
+
partition,
|
|
233
|
+
byteLength: existing.byteLength,
|
|
234
|
+
...(existing.mediaType !== undefined
|
|
235
|
+
? { mediaType: existing.mediaType }
|
|
236
|
+
: {}),
|
|
237
|
+
createdAtMs: existing.createdAtMs,
|
|
238
|
+
};
|
|
239
239
|
}
|
|
240
|
+
// A re-upload that names the media type an earlier upload left out
|
|
241
|
+
// rewrites the object under the same precondition, keeping its age.
|
|
240
242
|
const meta: BlobMeta = {
|
|
241
243
|
byteLength: bytes.length,
|
|
242
244
|
...(mediaType !== undefined ? { mediaType } : {}),
|
|
243
|
-
createdAtMs: nowMs,
|
|
245
|
+
createdAtMs: existing?.createdAtMs ?? nowMs,
|
|
244
246
|
};
|
|
245
247
|
// Conditional write: of two concurrent uploads of the same blob exactly
|
|
246
248
|
// one lands, so the first createdAtMs survives and the blob counts once.
|
|
@@ -256,7 +258,11 @@ export class BucketBlobStore implements BlobStore {
|
|
|
256
258
|
if (!written) return this.put(partition, blobId, bytes, nowMs, mediaType);
|
|
257
259
|
if (preexisting === undefined)
|
|
258
260
|
await this.#bumpStats({ count: 1, bytes: bytes.length });
|
|
259
|
-
return record;
|
|
261
|
+
return { ...record, createdAtMs: meta.createdAtMs };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async delete(partition: string, blobId: string): Promise<void> {
|
|
265
|
+
await this.#bucket.delete(this.objectKeyFor(partition, blobId));
|
|
260
266
|
}
|
|
261
267
|
|
|
262
268
|
async has(partition: string, blobId: string): Promise<boolean> {
|
package/src/d1-storage.ts
CHANGED
|
@@ -92,6 +92,9 @@ import {
|
|
|
92
92
|
collectCommitWindowPage,
|
|
93
93
|
deserializePushResult,
|
|
94
94
|
placeholders,
|
|
95
|
+
ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL,
|
|
96
|
+
eraseScopeValueHistoryParams,
|
|
97
|
+
LIST_SCOPE_VALUE_ROWS_SQLITE_SQL,
|
|
95
98
|
type SqliteChangeRecord,
|
|
96
99
|
type SqliteCommitWindowRecord,
|
|
97
100
|
type SqliteRowRecord,
|
|
@@ -130,6 +133,8 @@ import type {
|
|
|
130
133
|
RowScanQuery,
|
|
131
134
|
ScopeActivityQuery,
|
|
132
135
|
ScopeCommitActivity,
|
|
136
|
+
ScopeValueHistoryQuery,
|
|
137
|
+
ScopeValueQuery,
|
|
133
138
|
ServerStorage,
|
|
134
139
|
ServeGate,
|
|
135
140
|
StorageTransaction,
|
|
@@ -2473,6 +2478,46 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
2473
2478
|
return results.map((r) => r.blob_id);
|
|
2474
2479
|
}
|
|
2475
2480
|
|
|
2481
|
+
async listScopeValueRows(
|
|
2482
|
+
partition: string,
|
|
2483
|
+
query: ScopeValueQuery,
|
|
2484
|
+
): Promise<string[]> {
|
|
2485
|
+
const { results } = await this.#db
|
|
2486
|
+
.prepare(LIST_SCOPE_VALUE_ROWS_SQLITE_SQL)
|
|
2487
|
+
.bind(partition, query.table, query.variable, query.value, query.limit)
|
|
2488
|
+
.all<{ row_id: string }>();
|
|
2489
|
+
return results.map((row) => row.row_id);
|
|
2490
|
+
}
|
|
2491
|
+
|
|
2492
|
+
async eraseScopeValueHistory(
|
|
2493
|
+
partition: string,
|
|
2494
|
+
query: ScopeValueHistoryQuery,
|
|
2495
|
+
): Promise<number> {
|
|
2496
|
+
if (!this.#pushApplySerialized)
|
|
2497
|
+
throw new Error(
|
|
2498
|
+
'D1 erasure requires externally serialized partition writes',
|
|
2499
|
+
);
|
|
2500
|
+
const params = eraseScopeValueHistoryParams(partition, query);
|
|
2501
|
+
const results = await this.#db.batch(
|
|
2502
|
+
ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL.map((sql) =>
|
|
2503
|
+
this.#db.prepare(sql).bind(...params),
|
|
2504
|
+
),
|
|
2505
|
+
);
|
|
2506
|
+
let removed = 0;
|
|
2507
|
+
for (const result of results) {
|
|
2508
|
+
if (
|
|
2509
|
+
typeof result !== 'object' ||
|
|
2510
|
+
result === null ||
|
|
2511
|
+
!('results' in result) ||
|
|
2512
|
+
!Array.isArray(result.results)
|
|
2513
|
+
) {
|
|
2514
|
+
throw new Error('D1 erasure returned an invalid batch result');
|
|
2515
|
+
}
|
|
2516
|
+
removed += result.results.length;
|
|
2517
|
+
}
|
|
2518
|
+
return removed;
|
|
2519
|
+
}
|
|
2520
|
+
|
|
2476
2521
|
// -- admin/console read surface --------------------------------------------
|
|
2477
2522
|
|
|
2478
2523
|
async listClientRecords(partition: string): Promise<ClientRecord[]> {
|
package/src/erase.ts
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scope-value erasure (SPEC.md §4.6.1): remove every row whose stored scope
|
|
3
|
+
* binds one variable to one value, the upsert change records that carried
|
|
4
|
+
* those rows, and the blob bytes no remaining row references.
|
|
5
|
+
*
|
|
6
|
+
* Live rows leave through one ordinary delete commit per call, applied on
|
|
7
|
+
* the shared push path (partition lock, tombstones, blob reference index,
|
|
8
|
+
* realtime fanout). The commit advances the binding's scoped change seq, so
|
|
9
|
+
* no cached segment taken before the erasure is reused, and every client
|
|
10
|
+
* holding the rows receives the deletes. Once no live row is left, each call
|
|
11
|
+
* scrubs the change log of the binding. Hosts loop until `done`.
|
|
12
|
+
*/
|
|
13
|
+
import { decodeRow, type PushOperation } from '@syncular/core';
|
|
14
|
+
import { limitsOf, type SyncServerConfig } from './context';
|
|
15
|
+
import { SyncError } from './errors';
|
|
16
|
+
import { blobIdsInRow, processPushOperationsWithTrace } from './push';
|
|
17
|
+
import { compileSchema } from './schema';
|
|
18
|
+
import { StorageQueryError } from './storage-errors';
|
|
19
|
+
|
|
20
|
+
export interface EraseScopeValueOptions {
|
|
21
|
+
readonly partition: string;
|
|
22
|
+
/** The scope variable as declared by the schema's scope patterns. */
|
|
23
|
+
readonly variable: string;
|
|
24
|
+
readonly value: string;
|
|
25
|
+
/**
|
|
26
|
+
* Rows deleted per call, and change records removed per table per call
|
|
27
|
+
* (default 200, at most the push operation cap).
|
|
28
|
+
*/
|
|
29
|
+
readonly maxRows?: number;
|
|
30
|
+
/** Actor recorded on the erasure commits (default `syncular.erase`). */
|
|
31
|
+
readonly actorId?: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface EraseScopeValueResult {
|
|
35
|
+
/** Rows the erasure commit of this call deleted. */
|
|
36
|
+
readonly rowsErased: number;
|
|
37
|
+
/** The erasure commit of this call, when it deleted rows. */
|
|
38
|
+
readonly commitSeq?: number;
|
|
39
|
+
/** Change log records this call removed. */
|
|
40
|
+
readonly historyErased: number;
|
|
41
|
+
/** Blobs whose last reference this call removed; their bytes are gone. */
|
|
42
|
+
readonly blobsErased: readonly string[];
|
|
43
|
+
/** Nothing of the binding is left; further calls change nothing. */
|
|
44
|
+
readonly done: boolean;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Erase one scope binding of one partition, one budgeted step per call. */
|
|
48
|
+
export async function eraseScopeValue(
|
|
49
|
+
config: SyncServerConfig,
|
|
50
|
+
options: EraseScopeValueOptions,
|
|
51
|
+
): Promise<EraseScopeValueResult> {
|
|
52
|
+
const { storage } = config;
|
|
53
|
+
const { partition, variable, value } = options;
|
|
54
|
+
const maxRows = options.maxRows ?? 200;
|
|
55
|
+
const listRows = storage.listScopeValueRows?.bind(storage);
|
|
56
|
+
const eraseHistory = storage.eraseScopeValueHistory?.bind(storage);
|
|
57
|
+
const listReferences = storage.listRowsReferencingBlob?.bind(storage);
|
|
58
|
+
if (
|
|
59
|
+
listRows === undefined ||
|
|
60
|
+
eraseHistory === undefined ||
|
|
61
|
+
listReferences === undefined
|
|
62
|
+
) {
|
|
63
|
+
throw new StorageQueryError('sync.storage.erase_unsupported');
|
|
64
|
+
}
|
|
65
|
+
const schema = compileSchema(config.schema);
|
|
66
|
+
const tables = [...schema.tables.values()].filter((table) =>
|
|
67
|
+
table.scopePatterns.some((pattern) => pattern.variable === variable),
|
|
68
|
+
);
|
|
69
|
+
if (
|
|
70
|
+
tables.length === 0 ||
|
|
71
|
+
value.length === 0 ||
|
|
72
|
+
!Number.isSafeInteger(maxRows) ||
|
|
73
|
+
maxRows < 1 ||
|
|
74
|
+
maxRows > limitsOf(config).maxOperationsPerRequest
|
|
75
|
+
) {
|
|
76
|
+
throw new StorageQueryError('sync.storage.invalid_erase_query');
|
|
77
|
+
}
|
|
78
|
+
if ((await storage.getPartitionLogEpoch(partition)) === undefined) {
|
|
79
|
+
return { rowsErased: 0, historyErased: 0, blobsErased: [], done: true };
|
|
80
|
+
}
|
|
81
|
+
await storage.ensureSchema(schema, config.checkpoints);
|
|
82
|
+
|
|
83
|
+
const targets: { table: string; rowId: string }[] = [];
|
|
84
|
+
for (const table of tables) {
|
|
85
|
+
const limit = maxRows - targets.length;
|
|
86
|
+
if (limit === 0) break;
|
|
87
|
+
const query = { table: table.name, variable, value, limit };
|
|
88
|
+
for (const rowId of await listRows(partition, query)) {
|
|
89
|
+
targets.push({ table: table.name, rowId });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if (targets.length > 0) {
|
|
94
|
+
// Erasure is the operator's act: app validators do not gate it.
|
|
95
|
+
const { validators, commitValidator, ...trusted } = config;
|
|
96
|
+
const blobIds = new Set<string>();
|
|
97
|
+
let rowsErased = 0;
|
|
98
|
+
const processed = await processPushOperationsWithTrace(
|
|
99
|
+
{ ...trusted, partition, actorId: options.actorId ?? 'syncular.erase' },
|
|
100
|
+
schema,
|
|
101
|
+
{
|
|
102
|
+
ok: true,
|
|
103
|
+
allowed: Object.fromEntries(
|
|
104
|
+
[...schema.tables.values()].flatMap((table) =>
|
|
105
|
+
table.scopePatterns.map((pattern) => [pattern.variable, ['*']]),
|
|
106
|
+
),
|
|
107
|
+
),
|
|
108
|
+
},
|
|
109
|
+
'syncular.erase',
|
|
110
|
+
crypto.randomUUID(),
|
|
111
|
+
async (tx) => {
|
|
112
|
+
const operations: PushOperation[] = [];
|
|
113
|
+
for (const target of targets) {
|
|
114
|
+
// Re-read under the partition lock: a row may have moved or gone.
|
|
115
|
+
const stored = await tx.getRow(target.table, target.rowId);
|
|
116
|
+
if (stored?.scopes[variable] !== value) continue;
|
|
117
|
+
const table = schema.tables.get(target.table)!;
|
|
118
|
+
for (const blobId of blobIdsInRow(
|
|
119
|
+
table,
|
|
120
|
+
decodeRow(table.columns, stored.payload),
|
|
121
|
+
)) {
|
|
122
|
+
blobIds.add(blobId);
|
|
123
|
+
}
|
|
124
|
+
operations.push({
|
|
125
|
+
table: target.table,
|
|
126
|
+
op: 'delete',
|
|
127
|
+
rowId: target.rowId,
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
rowsErased = operations.length;
|
|
131
|
+
return operations;
|
|
132
|
+
},
|
|
133
|
+
);
|
|
134
|
+
const { frame } = processed;
|
|
135
|
+
if (frame.status !== 'applied') {
|
|
136
|
+
const failed = frame.results.find((result) => 'code' in result);
|
|
137
|
+
throw failed !== undefined && 'code' in failed
|
|
138
|
+
? new SyncError(failed.code, failed.message)
|
|
139
|
+
: new Error('eraseScopeValue: the erasure commit was not applied');
|
|
140
|
+
}
|
|
141
|
+
const blobsErased: string[] = [];
|
|
142
|
+
for (const blobId of blobIds) {
|
|
143
|
+
if ((await listReferences(partition, blobId)).length > 0) continue;
|
|
144
|
+
await config.blobs?.delete(partition, blobId);
|
|
145
|
+
blobsErased.push(blobId);
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
rowsErased,
|
|
149
|
+
...(frame.commitSeq !== undefined ? { commitSeq: frame.commitSeq } : {}),
|
|
150
|
+
historyErased: 0,
|
|
151
|
+
blobsErased,
|
|
152
|
+
done: false,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const throughSeq = await storage.getMaxCommitSeq(partition);
|
|
157
|
+
let historyErased = 0;
|
|
158
|
+
for (const table of tables) {
|
|
159
|
+
historyErased += await eraseHistory(partition, {
|
|
160
|
+
table: table.name,
|
|
161
|
+
variable,
|
|
162
|
+
value,
|
|
163
|
+
throughSeq,
|
|
164
|
+
limit: maxRows,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
return {
|
|
168
|
+
rowsErased: 0,
|
|
169
|
+
historyErased,
|
|
170
|
+
blobsErased: [],
|
|
171
|
+
done: historyErased === 0,
|
|
172
|
+
};
|
|
173
|
+
}
|
package/src/index.ts
CHANGED
package/src/postgres-storage.ts
CHANGED
|
@@ -121,6 +121,8 @@ import type {
|
|
|
121
121
|
RowScanQuery,
|
|
122
122
|
ScopeActivityQuery,
|
|
123
123
|
ScopeCommitActivity,
|
|
124
|
+
ScopeValueHistoryQuery,
|
|
125
|
+
ScopeValueQuery,
|
|
124
126
|
ServerStorage,
|
|
125
127
|
ServeGate,
|
|
126
128
|
StorageTransaction,
|
|
@@ -2567,6 +2569,63 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
2567
2569
|
return rows.map((r) => r.blob_id);
|
|
2568
2570
|
}
|
|
2569
2571
|
|
|
2572
|
+
async listScopeValueRows(
|
|
2573
|
+
partition: string,
|
|
2574
|
+
query: ScopeValueQuery,
|
|
2575
|
+
): Promise<string[]> {
|
|
2576
|
+
const { rows } = await this.#exec.query<{ row_id: string }>(
|
|
2577
|
+
`SELECT row_id FROM sync_row_scopes
|
|
2578
|
+
WHERE partition=$1 AND tbl=$2 AND var=$3 AND value=$4
|
|
2579
|
+
ORDER BY row_id LIMIT $5`,
|
|
2580
|
+
[partition, query.table, query.variable, query.value, query.limit],
|
|
2581
|
+
);
|
|
2582
|
+
return rows.map((row) => row.row_id);
|
|
2583
|
+
}
|
|
2584
|
+
|
|
2585
|
+
async eraseScopeValueHistory(
|
|
2586
|
+
partition: string,
|
|
2587
|
+
query: ScopeValueHistoryQuery,
|
|
2588
|
+
): Promise<number> {
|
|
2589
|
+
const params = [
|
|
2590
|
+
partition,
|
|
2591
|
+
query.table,
|
|
2592
|
+
query.variable,
|
|
2593
|
+
query.value,
|
|
2594
|
+
query.throughSeq,
|
|
2595
|
+
query.limit,
|
|
2596
|
+
];
|
|
2597
|
+
return this.#exec.transaction(async (client) => {
|
|
2598
|
+
await lockPartitionOn(client, partition);
|
|
2599
|
+
// Upsert change records of the binding, then the change scope entries
|
|
2600
|
+
// whose commit no longer holds a change of the binding.
|
|
2601
|
+
const changes = await client.query(
|
|
2602
|
+
`DELETE FROM sync_changes c USING (
|
|
2603
|
+
SELECT c2.commit_seq, c2.idx FROM sync_change_scopes s
|
|
2604
|
+
JOIN sync_changes c2 ON c2.partition=s.partition
|
|
2605
|
+
AND c2.commit_seq=s.commit_seq AND c2.tbl=s.tbl
|
|
2606
|
+
WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
|
|
2607
|
+
AND s.commit_seq<=$5 AND c2.op=1 AND c2.scopes->>$3=s.value
|
|
2608
|
+
LIMIT $6) d
|
|
2609
|
+
WHERE c.partition=$1 AND c.commit_seq=d.commit_seq AND c.idx=d.idx`,
|
|
2610
|
+
params,
|
|
2611
|
+
);
|
|
2612
|
+
const scopes = await client.query(
|
|
2613
|
+
`DELETE FROM sync_change_scopes cs USING (
|
|
2614
|
+
SELECT s.commit_seq FROM sync_change_scopes s
|
|
2615
|
+
WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
|
|
2616
|
+
AND s.commit_seq<=$5 AND NOT EXISTS (
|
|
2617
|
+
SELECT 1 FROM sync_changes c
|
|
2618
|
+
WHERE c.partition=s.partition AND c.commit_seq=s.commit_seq
|
|
2619
|
+
AND c.tbl=s.tbl AND c.scopes->>$3=s.value)
|
|
2620
|
+
LIMIT $6) d
|
|
2621
|
+
WHERE cs.partition=$1 AND cs.tbl=$2 AND cs.var=$3 AND cs.value=$4
|
|
2622
|
+
AND cs.commit_seq=d.commit_seq`,
|
|
2623
|
+
params,
|
|
2624
|
+
);
|
|
2625
|
+
return changes.rowCount + scopes.rowCount;
|
|
2626
|
+
});
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2570
2629
|
// -- admin/console read surface --------------------------------------------
|
|
2571
2630
|
|
|
2572
2631
|
async listClientRecords(partition: string): Promise<ClientRecord[]> {
|
package/src/push.ts
CHANGED
|
@@ -75,7 +75,7 @@ import {
|
|
|
75
75
|
* columns (§5.9.4), skipping NULLs. Malformed BlobRefs already failed at
|
|
76
76
|
* row-codec decode (§5.9.1), so `parseBlobRef` here is total.
|
|
77
77
|
*/
|
|
78
|
-
function blobIdsInRow(
|
|
78
|
+
export function blobIdsInRow(
|
|
79
79
|
table: CompiledTable,
|
|
80
80
|
values: readonly RowValue[],
|
|
81
81
|
): string[] {
|
package/src/sqlite-blob-store.ts
CHANGED
|
@@ -33,7 +33,17 @@ export class SqliteBlobStore implements BlobStore {
|
|
|
33
33
|
mediaType?: string,
|
|
34
34
|
): Promise<BlobRecord> {
|
|
35
35
|
const existing = await this.get(partition, blobId);
|
|
36
|
-
if (existing !== undefined)
|
|
36
|
+
if (existing !== undefined) {
|
|
37
|
+
if (existing.record.mediaType !== undefined || mediaType === undefined)
|
|
38
|
+
return existing.record;
|
|
39
|
+
// A re-upload names the media type an earlier upload left out.
|
|
40
|
+
this.db
|
|
41
|
+
.query(
|
|
42
|
+
'UPDATE sync_blobs SET media_type=? WHERE partition=? AND blob_id=? AND media_type IS NULL',
|
|
43
|
+
)
|
|
44
|
+
.run(mediaType, partition, blobId);
|
|
45
|
+
return { ...existing.record, mediaType };
|
|
46
|
+
}
|
|
37
47
|
const record: BlobRecord = {
|
|
38
48
|
blobId,
|
|
39
49
|
partition,
|
|
@@ -51,6 +61,12 @@ export class SqliteBlobStore implements BlobStore {
|
|
|
51
61
|
return record;
|
|
52
62
|
}
|
|
53
63
|
|
|
64
|
+
async delete(partition: string, blobId: string): Promise<void> {
|
|
65
|
+
this.db
|
|
66
|
+
.query('DELETE FROM sync_blobs WHERE partition=? AND blob_id=?')
|
|
67
|
+
.run(partition, blobId);
|
|
68
|
+
}
|
|
69
|
+
|
|
54
70
|
async has(partition: string, blobId: string): Promise<boolean> {
|
|
55
71
|
const row = this.db
|
|
56
72
|
.query<{ n: number }, [string, string]>(
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import type { PushOperationResult, ScopeMap } from '@syncular/core';
|
|
19
19
|
import { matchesEffective } from './scopes';
|
|
20
20
|
import type {
|
|
21
|
+
ScopeValueHistoryQuery,
|
|
21
22
|
StoredChange,
|
|
22
23
|
StoredCommit,
|
|
23
24
|
StoredPushResult,
|
|
@@ -151,6 +152,48 @@ export function sqliteDdlStatements(): string[] {
|
|
|
151
152
|
.filter((statement) => statement.length > 0);
|
|
152
153
|
}
|
|
153
154
|
|
|
155
|
+
/** §4.6.1: rowIds of one table bound to one scope value, budgeted. */
|
|
156
|
+
export const LIST_SCOPE_VALUE_ROWS_SQLITE_SQL =
|
|
157
|
+
'SELECT row_id FROM sync_row_scopes WHERE partition=? AND tbl=? AND var=? AND value=? ORDER BY row_id LIMIT ?';
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* §4.6.1 history scrub, bound by `eraseScopeValueHistoryParams`: upsert change
|
|
161
|
+
* records of the binding first, then the change scope entries whose commit
|
|
162
|
+
* no longer holds a change of the binding. Each returns one row per removed
|
|
163
|
+
* record (D1 batch results carry no reliable change count).
|
|
164
|
+
*/
|
|
165
|
+
export const ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL = [
|
|
166
|
+
`DELETE FROM sync_changes WHERE rowid IN (
|
|
167
|
+
SELECT c.rowid FROM sync_change_scopes s JOIN sync_changes c
|
|
168
|
+
ON c.partition=s.partition AND c.commit_seq=s.commit_seq AND c.tbl=s.tbl
|
|
169
|
+
WHERE s.partition=?1 AND s.tbl=?2 AND s.var=?3 AND s.value=?4
|
|
170
|
+
AND s.commit_seq<=?5 AND c.op=1 AND json_extract(c.scopes, ?6)=s.value
|
|
171
|
+
LIMIT ?7) RETURNING 1 AS removed`,
|
|
172
|
+
`DELETE FROM sync_change_scopes WHERE rowid IN (
|
|
173
|
+
SELECT s.rowid FROM sync_change_scopes s
|
|
174
|
+
WHERE s.partition=?1 AND s.tbl=?2 AND s.var=?3 AND s.value=?4
|
|
175
|
+
AND s.commit_seq<=?5 AND NOT EXISTS (
|
|
176
|
+
SELECT 1 FROM sync_changes c
|
|
177
|
+
WHERE c.partition=s.partition AND c.commit_seq=s.commit_seq
|
|
178
|
+
AND c.tbl=s.tbl AND json_extract(c.scopes, ?6)=s.value)
|
|
179
|
+
LIMIT ?7) RETURNING 1 AS removed`,
|
|
180
|
+
] as const;
|
|
181
|
+
|
|
182
|
+
export function eraseScopeValueHistoryParams(
|
|
183
|
+
partition: string,
|
|
184
|
+
query: ScopeValueHistoryQuery,
|
|
185
|
+
): [string, string, string, string, number, string, number] {
|
|
186
|
+
return [
|
|
187
|
+
partition,
|
|
188
|
+
query.table,
|
|
189
|
+
query.variable,
|
|
190
|
+
query.value,
|
|
191
|
+
query.throughSeq,
|
|
192
|
+
`$.${JSON.stringify(query.variable)}`,
|
|
193
|
+
query.limit,
|
|
194
|
+
];
|
|
195
|
+
}
|
|
196
|
+
|
|
154
197
|
/** `?,?,…` for an `IN (…)` clause of `count` positional parameters. */
|
|
155
198
|
export function placeholders(count: number): string {
|
|
156
199
|
return new Array(count).fill('?').join(',');
|
package/src/sqlite-storage.ts
CHANGED
|
@@ -56,6 +56,9 @@ import { matchesEffective } from './scopes';
|
|
|
56
56
|
import {
|
|
57
57
|
collectCommitWindowPage,
|
|
58
58
|
deserializePushResult,
|
|
59
|
+
ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL,
|
|
60
|
+
eraseScopeValueHistoryParams,
|
|
61
|
+
LIST_SCOPE_VALUE_ROWS_SQLITE_SQL,
|
|
59
62
|
placeholders,
|
|
60
63
|
SQLITE_DDL,
|
|
61
64
|
type SqliteChangeRecord,
|
|
@@ -100,6 +103,8 @@ import type {
|
|
|
100
103
|
RowScanQuery,
|
|
101
104
|
ScopeActivityQuery,
|
|
102
105
|
ScopeCommitActivity,
|
|
106
|
+
ScopeValueHistoryQuery,
|
|
107
|
+
ScopeValueQuery,
|
|
103
108
|
ServerStorage,
|
|
104
109
|
ServeGate,
|
|
105
110
|
StorageTransaction,
|
|
@@ -1925,6 +1930,40 @@ END`);
|
|
|
1925
1930
|
return rows.map((r) => r.blob_id);
|
|
1926
1931
|
}
|
|
1927
1932
|
|
|
1933
|
+
async listScopeValueRows(
|
|
1934
|
+
partition: string,
|
|
1935
|
+
query: ScopeValueQuery,
|
|
1936
|
+
): Promise<string[]> {
|
|
1937
|
+
return this.db
|
|
1938
|
+
.query<{ row_id: string }, [string, string, string, string, number]>(
|
|
1939
|
+
LIST_SCOPE_VALUE_ROWS_SQLITE_SQL,
|
|
1940
|
+
)
|
|
1941
|
+
.all(partition, query.table, query.variable, query.value, query.limit)
|
|
1942
|
+
.map((row) => row.row_id);
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
async eraseScopeValueHistory(
|
|
1946
|
+
partition: string,
|
|
1947
|
+
query: ScopeValueHistoryQuery,
|
|
1948
|
+
): Promise<number> {
|
|
1949
|
+
const params = eraseScopeValueHistoryParams(partition, query);
|
|
1950
|
+
return this.#serializeWrite(() => {
|
|
1951
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
1952
|
+
try {
|
|
1953
|
+
let removed = 0;
|
|
1954
|
+
for (const sql of ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL) {
|
|
1955
|
+
removed += this.db.query(sql).all(...params).length;
|
|
1956
|
+
}
|
|
1957
|
+
this.db.exec('COMMIT');
|
|
1958
|
+
return removed;
|
|
1959
|
+
} catch (error) {
|
|
1960
|
+
rollbackAfterError(() => {
|
|
1961
|
+
this.db.exec('ROLLBACK');
|
|
1962
|
+
}, error);
|
|
1963
|
+
}
|
|
1964
|
+
});
|
|
1965
|
+
}
|
|
1966
|
+
|
|
1928
1967
|
// -- admin/console read surface --------------------------------------------
|
|
1929
1968
|
|
|
1930
1969
|
async listClientRecords(partition: string): Promise<ClientRecord[]> {
|