@syncular/server 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blob-handlers.js +10 -3
- package/dist/blob-store.d.ts +9 -1
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +1 -0
- package/dist/bucket-blob-store.js +22 -16
- package/dist/d1-storage.d.ts +3 -1
- package/dist/d1-storage.js +27 -3
- package/dist/erase.d.ts +28 -0
- package/dist/erase.js +122 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/operations.d.ts +20 -3
- package/dist/operations.js +23 -3
- package/dist/postgres-storage.d.ts +3 -1
- package/dist/postgres-storage.js +40 -0
- package/dist/push.d.ts +8 -2
- package/dist/push.js +1 -1
- package/dist/relational-rows.d.ts +6 -3
- package/dist/relational-rows.js +8 -9
- package/dist/sqlite-blob-store.d.ts +1 -0
- package/dist/sqlite-blob-store.js +14 -2
- package/dist/sqlite-dialect.d.ts +11 -1
- package/dist/sqlite-dialect.js +35 -0
- package/dist/sqlite-storage.d.ts +3 -1
- package/dist/sqlite-storage.js +29 -9
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +2 -0
- package/dist/storage.d.ts +26 -0
- package/package.json +2 -2
- package/src/blob-handlers.ts +13 -5
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +26 -20
- package/src/d1-storage.ts +48 -3
- package/src/erase.ts +173 -0
- package/src/index.ts +1 -0
- package/src/operations.ts +52 -5
- package/src/postgres-storage.ts +59 -0
- package/src/push.ts +1 -1
- package/src/relational-rows.ts +8 -11
- package/src/sqlite-blob-store.ts +17 -1
- package/src/sqlite-dialect.ts +43 -0
- package/src/sqlite-storage.ts +42 -10
- package/src/storage-errors.ts +7 -1
- package/src/storage.ts +35 -0
package/dist/blob-handlers.js
CHANGED
|
@@ -189,12 +189,19 @@ async function downloadBlob(ctx, blobId) {
|
|
|
189
189
|
// Unreachable: presign not configured ⇒ `entry` was loaded above.
|
|
190
190
|
throw syncError('blob.not_found', 'blob bytes unavailable (§5.9.5)');
|
|
191
191
|
}
|
|
192
|
-
// §5.9.5: inline bytes
|
|
193
|
-
//
|
|
192
|
+
// §5.9.5: inline bytes carry the stored media type, so an <img> or <a>
|
|
193
|
+
// pointed at the route renders them. A JSON type stays octet-stream: a
|
|
194
|
+
// JSON body is reserved for the signed-URL envelope.
|
|
195
|
+
const mediaType = entry?.record.mediaType;
|
|
194
196
|
return {
|
|
195
197
|
bytes,
|
|
196
198
|
headers: {
|
|
197
|
-
'Content-Type':
|
|
199
|
+
'Content-Type': mediaType === undefined ||
|
|
200
|
+
mediaType.toLowerCase().includes('application/json')
|
|
201
|
+
? 'application/octet-stream'
|
|
202
|
+
: mediaType,
|
|
203
|
+
'X-Content-Type-Options': 'nosniff',
|
|
204
|
+
'Content-Security-Policy': "sandbox; default-src 'none'",
|
|
198
205
|
ETag: `"${blobId}"`,
|
|
199
206
|
'Cache-Control': 'private, max-age=0',
|
|
200
207
|
Vary: 'Authorization, Accept-Encoding',
|
package/dist/blob-store.d.ts
CHANGED
|
@@ -37,7 +37,9 @@ export interface BlobStoreStats {
|
|
|
37
37
|
export interface BlobStore {
|
|
38
38
|
/**
|
|
39
39
|
* Store blob bytes under `blobId` (idempotent — same id ⇒ no-op success,
|
|
40
|
-
* §5.9.3).
|
|
40
|
+
* §5.9.3). A re-upload that declares a media type records it on a blob
|
|
41
|
+
* stored without one; a recorded media type never changes. The caller has
|
|
42
|
+
* already verified the content address matches.
|
|
41
43
|
*/
|
|
42
44
|
put(partition: string, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): Promise<BlobRecord>;
|
|
43
45
|
/** Presence check for the push existence rule (§5.9.6) and download. */
|
|
@@ -53,6 +55,11 @@ export interface BlobStore {
|
|
|
53
55
|
* Returns the deleted blobIds.
|
|
54
56
|
*/
|
|
55
57
|
sweepOrphans(partition: string, olderThanMs: number, referencedBlobIds: ReadonlySet<string>): Promise<string[]>;
|
|
58
|
+
/**
|
|
59
|
+
* §4.6.1 erasure: delete one blob whose last reference is gone. Deleting a
|
|
60
|
+
* missing blob is a no-op.
|
|
61
|
+
*/
|
|
62
|
+
delete(partition: string, blobId: string): Promise<void>;
|
|
56
63
|
/** Admin/console counters (work item §2.5) — ADDITIVE, optional. */
|
|
57
64
|
stats?(partition: string): Promise<BlobStoreStats>;
|
|
58
65
|
}
|
|
@@ -68,6 +75,7 @@ export declare class MemoryBlobStore implements BlobStore {
|
|
|
68
75
|
bytes: Uint8Array;
|
|
69
76
|
} | undefined>;
|
|
70
77
|
sweepOrphans(partition: string, olderThanMs: number, referencedBlobIds: ReadonlySet<string>): Promise<string[]>;
|
|
78
|
+
delete(partition: string, blobId: string): Promise<void>;
|
|
71
79
|
stats(partition: string): Promise<BlobStoreStats>;
|
|
72
80
|
}
|
|
73
81
|
export interface SweepOrphanBlobsOptions {
|
package/dist/blob-store.js
CHANGED
|
Binary file
|
|
@@ -51,6 +51,7 @@ export declare class BucketBlobStore implements BlobStore {
|
|
|
51
51
|
*/
|
|
52
52
|
objectKeyFor(partition: string, blobId: string): string;
|
|
53
53
|
put(partition: string, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): Promise<BlobRecord>;
|
|
54
|
+
delete(partition: string, blobId: string): Promise<void>;
|
|
54
55
|
has(partition: string, blobId: string): Promise<boolean>;
|
|
55
56
|
get(partition: string, blobId: string): Promise<{
|
|
56
57
|
record: BlobRecord;
|
|
@@ -141,25 +141,28 @@ export class BucketBlobStore {
|
|
|
141
141
|
// blob once, and preserve the FIRST createdAtMs so the sweep's grace clock
|
|
142
142
|
// is not reset by a re-upload (§5.9.2 upload age).
|
|
143
143
|
const preexisting = await this.#bucket.head(objectKey);
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
144
|
+
const storedMeta = preexisting?.metadata[BLOB_META_KEY];
|
|
145
|
+
const existing = storedMeta === undefined
|
|
146
|
+
? undefined
|
|
147
|
+
: parseMetaJson(base64urlToUtf8(storedMeta), blobId, this.#label);
|
|
148
|
+
if (existing !== undefined &&
|
|
149
|
+
(existing.mediaType !== undefined || mediaType === undefined)) {
|
|
150
|
+
return {
|
|
151
|
+
blobId,
|
|
152
|
+
partition,
|
|
153
|
+
byteLength: existing.byteLength,
|
|
154
|
+
...(existing.mediaType !== undefined
|
|
155
|
+
? { mediaType: existing.mediaType }
|
|
156
|
+
: {}),
|
|
157
|
+
createdAtMs: existing.createdAtMs,
|
|
158
|
+
};
|
|
158
159
|
}
|
|
160
|
+
// A re-upload that names the media type an earlier upload left out
|
|
161
|
+
// rewrites the object under the same precondition, keeping its age.
|
|
159
162
|
const meta = {
|
|
160
163
|
byteLength: bytes.length,
|
|
161
164
|
...(mediaType !== undefined ? { mediaType } : {}),
|
|
162
|
-
createdAtMs: nowMs,
|
|
165
|
+
createdAtMs: existing?.createdAtMs ?? nowMs,
|
|
163
166
|
};
|
|
164
167
|
// Conditional write: of two concurrent uploads of the same blob exactly
|
|
165
168
|
// one lands, so the first createdAtMs survives and the blob counts once.
|
|
@@ -176,7 +179,10 @@ export class BucketBlobStore {
|
|
|
176
179
|
return this.put(partition, blobId, bytes, nowMs, mediaType);
|
|
177
180
|
if (preexisting === undefined)
|
|
178
181
|
await this.#bumpStats({ count: 1, bytes: bytes.length });
|
|
179
|
-
return record;
|
|
182
|
+
return { ...record, createdAtMs: meta.createdAtMs };
|
|
183
|
+
}
|
|
184
|
+
async delete(partition, blobId) {
|
|
185
|
+
await this.#bucket.delete(this.objectKeyFor(partition, blobId));
|
|
180
186
|
}
|
|
181
187
|
async has(partition, blobId) {
|
|
182
188
|
if (!BLOB_ID_PATTERN.test(blobId))
|
package/dist/d1-storage.d.ts
CHANGED
|
@@ -42,7 +42,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
|
42
42
|
*/
|
|
43
43
|
import { type ScopeMap } from '@syncular/core';
|
|
44
44
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
45
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
45
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
46
46
|
export interface D1PreparedStatement {
|
|
47
47
|
bind(...values: unknown[]): D1PreparedStatement;
|
|
48
48
|
first<T = Record<string, unknown>>(): Promise<T | null>;
|
|
@@ -134,6 +134,8 @@ export declare class D1ServerStorage implements ServerStorage {
|
|
|
134
134
|
readonly scopes: Record<string, string>;
|
|
135
135
|
}[]>;
|
|
136
136
|
listReferencedBlobIds(partition: string): Promise<string[]>;
|
|
137
|
+
listScopeValueRows(partition: string, query: ScopeValueQuery): Promise<string[]>;
|
|
138
|
+
eraseScopeValueHistory(partition: string, query: ScopeValueHistoryQuery): Promise<number>;
|
|
137
139
|
listClientRecords(partition: string): Promise<ClientRecord[]>;
|
|
138
140
|
listCommitMetadata(partition: string, query: CommitMetadataQuery): Promise<CommitMetadata[]>;
|
|
139
141
|
getCommit(partition: string, commitSeq: number): Promise<CommitDetail | undefined>;
|
package/dist/d1-storage.js
CHANGED
|
@@ -45,9 +45,9 @@ import { StorageQueryError } from './storage-errors.js';
|
|
|
45
45
|
import { decodeRow } from '@syncular/core';
|
|
46
46
|
import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
47
47
|
import { syncError } from './errors.js';
|
|
48
|
-
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql,
|
|
48
|
+
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, upsertSql, upsertValues, } from './relational-rows.js';
|
|
49
49
|
import { matchesEffective } from './scopes.js';
|
|
50
|
-
import { asUint8Array, collectCommitWindowPage, deserializePushResult, placeholders, serializePushResult, sqliteDdlStatements, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
|
|
50
|
+
import { asUint8Array, collectCommitWindowPage, deserializePushResult, placeholders, ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL, eraseScopeValueHistoryParams, LIST_SCOPE_VALUE_ROWS_SQLITE_SQL, serializePushResult, sqliteDdlStatements, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
|
|
51
51
|
import { isD1ConstraintError, StorageConstraintError } from './storage-errors.js';
|
|
52
52
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
53
53
|
function toStoredReaction(record) {
|
|
@@ -1456,7 +1456,7 @@ export class D1ServerStorage {
|
|
|
1456
1456
|
if (firstVariable === undefined || values.length === 0)
|
|
1457
1457
|
return 0;
|
|
1458
1458
|
// Concurrent reads of a pull share one `db.batch` round trip.
|
|
1459
|
-
const [row] = await this.#batchedAll(
|
|
1459
|
+
const [row] = await this.#batchedAll(LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, [partition, query.table, firstVariable, JSON.stringify(values)]);
|
|
1460
1460
|
return row?.seq ?? 0;
|
|
1461
1461
|
}
|
|
1462
1462
|
async readCommitWindow(partition, query) {
|
|
@@ -1654,6 +1654,30 @@ export class D1ServerStorage {
|
|
|
1654
1654
|
.all();
|
|
1655
1655
|
return results.map((r) => r.blob_id);
|
|
1656
1656
|
}
|
|
1657
|
+
async listScopeValueRows(partition, query) {
|
|
1658
|
+
const { results } = await this.#db
|
|
1659
|
+
.prepare(LIST_SCOPE_VALUE_ROWS_SQLITE_SQL)
|
|
1660
|
+
.bind(partition, query.table, query.variable, query.value, query.limit)
|
|
1661
|
+
.all();
|
|
1662
|
+
return results.map((row) => row.row_id);
|
|
1663
|
+
}
|
|
1664
|
+
async eraseScopeValueHistory(partition, query) {
|
|
1665
|
+
if (!this.#pushApplySerialized)
|
|
1666
|
+
throw new Error('D1 erasure requires externally serialized partition writes');
|
|
1667
|
+
const params = eraseScopeValueHistoryParams(partition, query);
|
|
1668
|
+
const results = await this.#db.batch(ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL.map((sql) => this.#db.prepare(sql).bind(...params)));
|
|
1669
|
+
let removed = 0;
|
|
1670
|
+
for (const result of results) {
|
|
1671
|
+
if (typeof result !== 'object' ||
|
|
1672
|
+
result === null ||
|
|
1673
|
+
!('results' in result) ||
|
|
1674
|
+
!Array.isArray(result.results)) {
|
|
1675
|
+
throw new Error('D1 erasure returned an invalid batch result');
|
|
1676
|
+
}
|
|
1677
|
+
removed += result.results.length;
|
|
1678
|
+
}
|
|
1679
|
+
return removed;
|
|
1680
|
+
}
|
|
1657
1681
|
// -- admin/console read surface --------------------------------------------
|
|
1658
1682
|
async listClientRecords(partition) {
|
|
1659
1683
|
const { results } = await this.#db
|
package/dist/erase.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type SyncServerConfig } from './context.js';
|
|
2
|
+
export interface EraseScopeValueOptions {
|
|
3
|
+
readonly partition: string;
|
|
4
|
+
/** The scope variable as declared by the schema's scope patterns. */
|
|
5
|
+
readonly variable: string;
|
|
6
|
+
readonly value: string;
|
|
7
|
+
/**
|
|
8
|
+
* Rows deleted per call, and change records removed per table per call
|
|
9
|
+
* (default 200, at most the push operation cap).
|
|
10
|
+
*/
|
|
11
|
+
readonly maxRows?: number;
|
|
12
|
+
/** Actor recorded on the erasure commits (default `syncular.erase`). */
|
|
13
|
+
readonly actorId?: string;
|
|
14
|
+
}
|
|
15
|
+
export interface EraseScopeValueResult {
|
|
16
|
+
/** Rows the erasure commit of this call deleted. */
|
|
17
|
+
readonly rowsErased: number;
|
|
18
|
+
/** The erasure commit of this call, when it deleted rows. */
|
|
19
|
+
readonly commitSeq?: number;
|
|
20
|
+
/** Change log records this call removed. */
|
|
21
|
+
readonly historyErased: number;
|
|
22
|
+
/** Blobs whose last reference this call removed; their bytes are gone. */
|
|
23
|
+
readonly blobsErased: readonly string[];
|
|
24
|
+
/** Nothing of the binding is left; further calls change nothing. */
|
|
25
|
+
readonly done: boolean;
|
|
26
|
+
}
|
|
27
|
+
/** Erase one scope binding of one partition, one budgeted step per call. */
|
|
28
|
+
export declare function eraseScopeValue(config: SyncServerConfig, options: EraseScopeValueOptions): Promise<EraseScopeValueResult>;
|
package/dist/erase.js
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scope-value erasure (SPEC.md §4.6.1): remove every row whose stored scope
|
|
3
|
+
* binds one variable to one value, the upsert change records that carried
|
|
4
|
+
* those rows, and the blob bytes no remaining row references.
|
|
5
|
+
*
|
|
6
|
+
* Live rows leave through one ordinary delete commit per call, applied on
|
|
7
|
+
* the shared push path (partition lock, tombstones, blob reference index,
|
|
8
|
+
* realtime fanout). The commit advances the binding's scoped change seq, so
|
|
9
|
+
* no cached segment taken before the erasure is reused, and every client
|
|
10
|
+
* holding the rows receives the deletes. Once no live row is left, each call
|
|
11
|
+
* scrubs the change log of the binding. Hosts loop until `done`.
|
|
12
|
+
*/
|
|
13
|
+
import { decodeRow } from '@syncular/core';
|
|
14
|
+
import { limitsOf } from './context.js';
|
|
15
|
+
import { SyncError } from './errors.js';
|
|
16
|
+
import { blobIdsInRow, processPushOperationsWithTrace } from './push.js';
|
|
17
|
+
import { compileSchema } from './schema.js';
|
|
18
|
+
import { StorageQueryError } from './storage-errors.js';
|
|
19
|
+
/** Erase one scope binding of one partition, one budgeted step per call. */
|
|
20
|
+
export async function eraseScopeValue(config, options) {
|
|
21
|
+
const { storage } = config;
|
|
22
|
+
const { partition, variable, value } = options;
|
|
23
|
+
const maxRows = options.maxRows ?? 200;
|
|
24
|
+
const listRows = storage.listScopeValueRows?.bind(storage);
|
|
25
|
+
const eraseHistory = storage.eraseScopeValueHistory?.bind(storage);
|
|
26
|
+
const listReferences = storage.listRowsReferencingBlob?.bind(storage);
|
|
27
|
+
if (listRows === undefined ||
|
|
28
|
+
eraseHistory === undefined ||
|
|
29
|
+
listReferences === undefined) {
|
|
30
|
+
throw new StorageQueryError('sync.storage.erase_unsupported');
|
|
31
|
+
}
|
|
32
|
+
const schema = compileSchema(config.schema);
|
|
33
|
+
const tables = [...schema.tables.values()].filter((table) => table.scopePatterns.some((pattern) => pattern.variable === variable));
|
|
34
|
+
if (tables.length === 0 ||
|
|
35
|
+
value.length === 0 ||
|
|
36
|
+
!Number.isSafeInteger(maxRows) ||
|
|
37
|
+
maxRows < 1 ||
|
|
38
|
+
maxRows > limitsOf(config).maxOperationsPerRequest) {
|
|
39
|
+
throw new StorageQueryError('sync.storage.invalid_erase_query');
|
|
40
|
+
}
|
|
41
|
+
if ((await storage.getPartitionLogEpoch(partition)) === undefined) {
|
|
42
|
+
return { rowsErased: 0, historyErased: 0, blobsErased: [], done: true };
|
|
43
|
+
}
|
|
44
|
+
await storage.ensureSchema(schema, config.checkpoints);
|
|
45
|
+
const targets = [];
|
|
46
|
+
for (const table of tables) {
|
|
47
|
+
const limit = maxRows - targets.length;
|
|
48
|
+
if (limit === 0)
|
|
49
|
+
break;
|
|
50
|
+
const query = { table: table.name, variable, value, limit };
|
|
51
|
+
for (const rowId of await listRows(partition, query)) {
|
|
52
|
+
targets.push({ table: table.name, rowId });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (targets.length > 0) {
|
|
56
|
+
// Erasure is the operator's act: app validators do not gate it.
|
|
57
|
+
const { validators, commitValidator, ...trusted } = config;
|
|
58
|
+
const blobIds = new Set();
|
|
59
|
+
let rowsErased = 0;
|
|
60
|
+
const processed = await processPushOperationsWithTrace({ ...trusted, partition, actorId: options.actorId ?? 'syncular.erase' }, schema, {
|
|
61
|
+
ok: true,
|
|
62
|
+
allowed: Object.fromEntries([...schema.tables.values()].flatMap((table) => table.scopePatterns.map((pattern) => [pattern.variable, ['*']]))),
|
|
63
|
+
}, 'syncular.erase', crypto.randomUUID(), async (tx) => {
|
|
64
|
+
const operations = [];
|
|
65
|
+
for (const target of targets) {
|
|
66
|
+
// Re-read under the partition lock: a row may have moved or gone.
|
|
67
|
+
const stored = await tx.getRow(target.table, target.rowId);
|
|
68
|
+
if (stored?.scopes[variable] !== value)
|
|
69
|
+
continue;
|
|
70
|
+
const table = schema.tables.get(target.table);
|
|
71
|
+
for (const blobId of blobIdsInRow(table, decodeRow(table.columns, stored.payload))) {
|
|
72
|
+
blobIds.add(blobId);
|
|
73
|
+
}
|
|
74
|
+
operations.push({
|
|
75
|
+
table: target.table,
|
|
76
|
+
op: 'delete',
|
|
77
|
+
rowId: target.rowId,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
rowsErased = operations.length;
|
|
81
|
+
return operations;
|
|
82
|
+
});
|
|
83
|
+
const { frame } = processed;
|
|
84
|
+
if (frame.status !== 'applied') {
|
|
85
|
+
const failed = frame.results.find((result) => 'code' in result);
|
|
86
|
+
throw failed !== undefined && 'code' in failed
|
|
87
|
+
? new SyncError(failed.code, failed.message)
|
|
88
|
+
: new Error('eraseScopeValue: the erasure commit was not applied');
|
|
89
|
+
}
|
|
90
|
+
const blobsErased = [];
|
|
91
|
+
for (const blobId of blobIds) {
|
|
92
|
+
if ((await listReferences(partition, blobId)).length > 0)
|
|
93
|
+
continue;
|
|
94
|
+
await config.blobs?.delete(partition, blobId);
|
|
95
|
+
blobsErased.push(blobId);
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
rowsErased,
|
|
99
|
+
...(frame.commitSeq !== undefined ? { commitSeq: frame.commitSeq } : {}),
|
|
100
|
+
historyErased: 0,
|
|
101
|
+
blobsErased,
|
|
102
|
+
done: false,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
const throughSeq = await storage.getMaxCommitSeq(partition);
|
|
106
|
+
let historyErased = 0;
|
|
107
|
+
for (const table of tables) {
|
|
108
|
+
historyErased += await eraseHistory(partition, {
|
|
109
|
+
table: table.name,
|
|
110
|
+
variable,
|
|
111
|
+
value,
|
|
112
|
+
throughSeq,
|
|
113
|
+
limit: maxRows,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
return {
|
|
117
|
+
rowsErased: 0,
|
|
118
|
+
historyErased,
|
|
119
|
+
blobsErased: [],
|
|
120
|
+
done: historyErased === 0,
|
|
121
|
+
};
|
|
122
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -17,6 +17,7 @@ export * from './content-encoding.js';
|
|
|
17
17
|
export * from './context.js';
|
|
18
18
|
export * from './crdt-merger.js';
|
|
19
19
|
export * from './d1-storage.js';
|
|
20
|
+
export * from './erase.js';
|
|
20
21
|
export * from './errors.js';
|
|
21
22
|
export * from './events.js';
|
|
22
23
|
export * from './events-ring.js';
|
package/dist/index.js
CHANGED
|
@@ -17,6 +17,7 @@ export * from './content-encoding.js';
|
|
|
17
17
|
export * from './context.js';
|
|
18
18
|
export * from './crdt-merger.js';
|
|
19
19
|
export * from './d1-storage.js';
|
|
20
|
+
export * from './erase.js';
|
|
20
21
|
export * from './errors.js';
|
|
21
22
|
export * from './events.js';
|
|
22
23
|
export * from './events-ring.js';
|
package/dist/operations.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type AuthoritativeRelationPlan } from './authoritative-query.js';
|
|
2
|
-
import { type RemoteOperationResponse, type RowValue } from '@syncular/core';
|
|
3
|
-
import type { SyncRequestContext } from './context.js';
|
|
2
|
+
import { type RemoteOperationRequest, type RemoteOperationResponse, type RowValue } from '@syncular/core';
|
|
3
|
+
import type { ResolveScopes, SyncRequestContext } from './context.js';
|
|
4
4
|
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, AuthoritativeQueryValue } from './storage.js';
|
|
5
5
|
import { type ValidateRow } from './validate.js';
|
|
6
6
|
export interface RemoteQueryDependency {
|
|
@@ -101,5 +101,22 @@ export declare class RemoteOperationRegistry {
|
|
|
101
101
|
constructor(operations: readonly RegisteredRemoteOperation[]);
|
|
102
102
|
get(id: string): RegisteredRemoteOperation | undefined;
|
|
103
103
|
}
|
|
104
|
-
|
|
104
|
+
/**
|
|
105
|
+
* The registry and identity one remote operation request runs with
|
|
106
|
+
* (REMOTE.md §5.1). `actorId` is the commit's actor and defaults to the
|
|
107
|
+
* authenticated actor; `resolveScopes` replaces the configured resolver for
|
|
108
|
+
* this request.
|
|
109
|
+
*/
|
|
110
|
+
export interface RemoteOperationBinding {
|
|
111
|
+
readonly registry: RemoteOperationRegistry;
|
|
112
|
+
readonly actorId?: string;
|
|
113
|
+
readonly resolveScopes?: ResolveScopes;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Chooses the binding of one decoded request (REMOTE.md §5.1). `ctx` carries
|
|
117
|
+
* the authenticated actor; the resolver runs after the client-id binding
|
|
118
|
+
* check and may throw a `SyncError` or a `ValidationRejection`.
|
|
119
|
+
*/
|
|
120
|
+
export type RemoteOperationResolver = (request: RemoteOperationRequest, ctx: SyncRequestContext) => RemoteOperationBinding | Promise<RemoteOperationBinding>;
|
|
121
|
+
export declare function handleRemoteOperation(bytes: Uint8Array, authenticated: SyncRequestContext, operations: RemoteOperationRegistry | RemoteOperationResolver): Promise<Uint8Array>;
|
|
105
122
|
export {};
|
package/dist/operations.js
CHANGED
|
@@ -6,7 +6,7 @@ import { processPushOperationsWithTrace } from './push.js';
|
|
|
6
6
|
import { compileSchema } from './schema.js';
|
|
7
7
|
import { authorizeWrite, renderScopeValue, } from './scopes.js';
|
|
8
8
|
import { transactionQuery } from './storage-query.js';
|
|
9
|
-
import { toValidateRow } from './validate.js';
|
|
9
|
+
import { toValidateRow, ValidationRejection, } from './validate.js';
|
|
10
10
|
function scopeAllowed(allowed, variable, value) {
|
|
11
11
|
const values = allowed[variable];
|
|
12
12
|
return (values !== undefined && (values.includes('*') || values.includes(value)));
|
|
@@ -353,6 +353,15 @@ export class RemoteOperationRegistry {
|
|
|
353
353
|
}
|
|
354
354
|
}
|
|
355
355
|
function encodeRemoteOperationError(error, fallbackCode) {
|
|
356
|
+
if (error instanceof ValidationRejection) {
|
|
357
|
+
return encodeRemoteOperationResponse({
|
|
358
|
+
revision: 1,
|
|
359
|
+
kind: 'error',
|
|
360
|
+
code: error.code,
|
|
361
|
+
message: error.message,
|
|
362
|
+
retryable: false,
|
|
363
|
+
});
|
|
364
|
+
}
|
|
356
365
|
const sync = error instanceof SyncError
|
|
357
366
|
? error
|
|
358
367
|
: syncError(fallbackCode, fallbackCode === 'operation.invalid_request'
|
|
@@ -366,7 +375,7 @@ function encodeRemoteOperationError(error, fallbackCode) {
|
|
|
366
375
|
retryable: sync.retryable,
|
|
367
376
|
});
|
|
368
377
|
}
|
|
369
|
-
export async function handleRemoteOperation(bytes,
|
|
378
|
+
export async function handleRemoteOperation(bytes, authenticated, operations) {
|
|
370
379
|
let request;
|
|
371
380
|
try {
|
|
372
381
|
request = decodeRemoteOperationRequest(bytes);
|
|
@@ -384,6 +393,7 @@ export async function handleRemoteOperation(bytes, ctx, registry) {
|
|
|
384
393
|
catch (error) {
|
|
385
394
|
return encodeRemoteOperationError(error, 'operation.invalid_request');
|
|
386
395
|
}
|
|
396
|
+
let ctx = authenticated;
|
|
387
397
|
try {
|
|
388
398
|
await touchAuthenticatedPartition(ctx);
|
|
389
399
|
if (request.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
|
|
@@ -397,6 +407,16 @@ export async function handleRemoteOperation(bytes, ctx, registry) {
|
|
|
397
407
|
(typeof request.requestId !== 'string' || request.requestId.length === 0)) {
|
|
398
408
|
throw syncError('operation.invalid_request');
|
|
399
409
|
}
|
|
410
|
+
let registry = operations;
|
|
411
|
+
if (typeof registry === 'function') {
|
|
412
|
+
const binding = await registry(request, ctx);
|
|
413
|
+
registry = binding.registry;
|
|
414
|
+
ctx = {
|
|
415
|
+
...ctx,
|
|
416
|
+
actorId: binding.actorId ?? ctx.actorId,
|
|
417
|
+
resolveScopes: binding.resolveScopes ?? ctx.resolveScopes,
|
|
418
|
+
};
|
|
419
|
+
}
|
|
400
420
|
const operation = registry.get(request.operationId);
|
|
401
421
|
if (operation === undefined) {
|
|
402
422
|
throw syncError('operation.unknown');
|
|
@@ -411,7 +431,7 @@ export async function handleRemoteOperation(bytes, ctx, registry) {
|
|
|
411
431
|
return encodeRemoteOperationResponse(await operation.run(ctx, request.clientId, request.requestId, request.params));
|
|
412
432
|
}
|
|
413
433
|
catch (error) {
|
|
414
|
-
if (!(error instanceof SyncError)) {
|
|
434
|
+
if (!(error instanceof SyncError || error instanceof ValidationRejection)) {
|
|
415
435
|
reportError(error, ctx.onError, 'operations');
|
|
416
436
|
}
|
|
417
437
|
return encodeRemoteOperationError(error, 'operation.execution_failed');
|
|
@@ -39,7 +39,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
|
39
39
|
import type { ScopeMap } from '@syncular/core';
|
|
40
40
|
import { type PgExecutor } from './pg-executor.js';
|
|
41
41
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
42
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
42
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
43
43
|
/**
|
|
44
44
|
* Schema DDL. Applied by `PostgresServerStorage.migrate()` (idempotent).
|
|
45
45
|
*
|
|
@@ -134,6 +134,8 @@ export declare class PostgresServerStorage implements ServerStorage {
|
|
|
134
134
|
readonly scopes: Record<string, string>;
|
|
135
135
|
}[]>;
|
|
136
136
|
listReferencedBlobIds(partition: string): Promise<string[]>;
|
|
137
|
+
listScopeValueRows(partition: string, query: ScopeValueQuery): Promise<string[]>;
|
|
138
|
+
eraseScopeValueHistory(partition: string, query: ScopeValueHistoryQuery): Promise<number>;
|
|
137
139
|
listClientRecords(partition: string): Promise<ClientRecord[]>;
|
|
138
140
|
listCommitMetadata(partition: string, query: CommitMetadataQuery): Promise<CommitMetadata[]>;
|
|
139
141
|
getCommit(partition: string, commitSeq: number): Promise<CommitDetail | undefined>;
|
package/dist/postgres-storage.js
CHANGED
|
@@ -1644,6 +1644,46 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
1644
1644
|
const { rows } = await this.#exec.query('SELECT DISTINCT blob_id FROM sync_blob_refs WHERE partition=$1', [partition]);
|
|
1645
1645
|
return rows.map((r) => r.blob_id);
|
|
1646
1646
|
}
|
|
1647
|
+
async listScopeValueRows(partition, query) {
|
|
1648
|
+
const { rows } = await this.#exec.query(`SELECT row_id FROM sync_row_scopes
|
|
1649
|
+
WHERE partition=$1 AND tbl=$2 AND var=$3 AND value=$4
|
|
1650
|
+
ORDER BY row_id LIMIT $5`, [partition, query.table, query.variable, query.value, query.limit]);
|
|
1651
|
+
return rows.map((row) => row.row_id);
|
|
1652
|
+
}
|
|
1653
|
+
async eraseScopeValueHistory(partition, query) {
|
|
1654
|
+
const params = [
|
|
1655
|
+
partition,
|
|
1656
|
+
query.table,
|
|
1657
|
+
query.variable,
|
|
1658
|
+
query.value,
|
|
1659
|
+
query.throughSeq,
|
|
1660
|
+
query.limit,
|
|
1661
|
+
];
|
|
1662
|
+
return this.#exec.transaction(async (client) => {
|
|
1663
|
+
await lockPartitionOn(client, partition);
|
|
1664
|
+
// Upsert change records of the binding, then the change scope entries
|
|
1665
|
+
// whose commit no longer holds a change of the binding.
|
|
1666
|
+
const changes = await client.query(`DELETE FROM sync_changes c USING (
|
|
1667
|
+
SELECT c2.commit_seq, c2.idx FROM sync_change_scopes s
|
|
1668
|
+
JOIN sync_changes c2 ON c2.partition=s.partition
|
|
1669
|
+
AND c2.commit_seq=s.commit_seq AND c2.tbl=s.tbl
|
|
1670
|
+
WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
|
|
1671
|
+
AND s.commit_seq<=$5 AND c2.op=1 AND c2.scopes->>$3=s.value
|
|
1672
|
+
LIMIT $6) d
|
|
1673
|
+
WHERE c.partition=$1 AND c.commit_seq=d.commit_seq AND c.idx=d.idx`, params);
|
|
1674
|
+
const scopes = await client.query(`DELETE FROM sync_change_scopes cs USING (
|
|
1675
|
+
SELECT s.commit_seq FROM sync_change_scopes s
|
|
1676
|
+
WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
|
|
1677
|
+
AND s.commit_seq<=$5 AND NOT EXISTS (
|
|
1678
|
+
SELECT 1 FROM sync_changes c
|
|
1679
|
+
WHERE c.partition=s.partition AND c.commit_seq=s.commit_seq
|
|
1680
|
+
AND c.tbl=s.tbl AND c.scopes->>$3=s.value)
|
|
1681
|
+
LIMIT $6) d
|
|
1682
|
+
WHERE cs.partition=$1 AND cs.tbl=$2 AND cs.var=$3 AND cs.value=$4
|
|
1683
|
+
AND cs.commit_seq=d.commit_seq`, params);
|
|
1684
|
+
return changes.rowCount + scopes.rowCount;
|
|
1685
|
+
});
|
|
1686
|
+
}
|
|
1647
1687
|
// -- admin/console read surface --------------------------------------------
|
|
1648
1688
|
async listClientRecords(partition) {
|
|
1649
1689
|
const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC', [partition]);
|
package/dist/push.d.ts
CHANGED
|
@@ -17,11 +17,17 @@
|
|
|
17
17
|
* merge); a validator throw rejects the whole commit atomically with a
|
|
18
18
|
* host code.
|
|
19
19
|
*/
|
|
20
|
-
import { type PushCommitFrame, type PushOperation, type PushResultFrame } from '@syncular/core';
|
|
20
|
+
import { type PushCommitFrame, type PushOperation, type PushResultFrame, type RowValue } from '@syncular/core';
|
|
21
21
|
import type { SyncRequestContext } from './context.js';
|
|
22
|
-
import type { CompiledSchema } from './schema.js';
|
|
22
|
+
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
23
23
|
import type { ResolvedScopes } from './scopes.js';
|
|
24
24
|
import type { StorageTransaction, StoredCommit } from './storage.js';
|
|
25
|
+
/**
|
|
26
|
+
* Extract the blobIds a decoded row references through its `blob_ref`
|
|
27
|
+
* columns (§5.9.4), skipping NULLs. Malformed BlobRefs already failed at
|
|
28
|
+
* row-codec decode (§5.9.1), so `parseBlobRef` here is total.
|
|
29
|
+
*/
|
|
30
|
+
export declare function blobIdsInRow(table: CompiledTable, values: readonly RowValue[]): string[];
|
|
25
31
|
export interface ProcessedPushCommit {
|
|
26
32
|
readonly frame: PushResultFrame;
|
|
27
33
|
/** True when this request observed an already-recorded idempotency outcome. */
|
package/dist/push.js
CHANGED
|
@@ -35,7 +35,7 @@ import { CommitValidationRejection, toValidateRow, ValidationRejection, } from '
|
|
|
35
35
|
* columns (§5.9.4), skipping NULLs. Malformed BlobRefs already failed at
|
|
36
36
|
* row-codec decode (§5.9.1), so `parseBlobRef` here is total.
|
|
37
37
|
*/
|
|
38
|
-
function blobIdsInRow(table, values) {
|
|
38
|
+
export function blobIdsInRow(table, values) {
|
|
39
39
|
const ids = [];
|
|
40
40
|
for (const index of table.blobRefColumnIndices) {
|
|
41
41
|
const value = values[index];
|
|
@@ -259,11 +259,14 @@ export declare function prefetchRowsSql(table: CompiledTable): string;
|
|
|
259
259
|
* SQLite/D1: the newest `commit_seq` in the inverted change-scope index for
|
|
260
260
|
* one variable's values, as one index-only max per value (a `(partition,
|
|
261
261
|
* tbl, var, value)` prefix), so the cost is independent of the table's
|
|
262
|
-
* history.
|
|
262
|
+
* history. The values arrive as one JSON array, so the statement has no
|
|
263
|
+
* compound SELECT and four bound parameters for any value count (D1 caps a
|
|
264
|
+
* compound SELECT at 5 terms and a statement at 100 parameters).
|
|
265
|
+
* Params: [partition, tbl, var, values as JSON array text].
|
|
263
266
|
*/
|
|
264
|
-
export declare
|
|
267
|
+
export declare const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = "SELECT max((SELECT max(commit_seq) FROM sync_change_scopes\n WHERE partition=? AND tbl=? AND var=? AND value=v.value)) AS seq\n FROM json_each(?) AS v";
|
|
265
268
|
/**
|
|
266
|
-
* PostgreSQL set-based form of `
|
|
269
|
+
* PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
|
|
267
270
|
* subscriptions of one table in a pull. `$2` is a JSON array of
|
|
268
271
|
* `{i, var, vals}` records. Params: [partition, specs (JSON text), tbl].
|
|
269
272
|
*/
|
package/dist/relational-rows.js
CHANGED
|
@@ -610,17 +610,16 @@ export function prefetchRowsSql(table) {
|
|
|
610
610
|
* SQLite/D1: the newest `commit_seq` in the inverted change-scope index for
|
|
611
611
|
* one variable's values, as one index-only max per value (a `(partition,
|
|
612
612
|
* tbl, var, value)` prefix), so the cost is independent of the table's
|
|
613
|
-
* history.
|
|
613
|
+
* history. The values arrive as one JSON array, so the statement has no
|
|
614
|
+
* compound SELECT and four bound parameters for any value count (D1 caps a
|
|
615
|
+
* compound SELECT at 5 terms and a statement at 100 parameters).
|
|
616
|
+
* Params: [partition, tbl, var, values as JSON array text].
|
|
614
617
|
*/
|
|
615
|
-
export
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
perValue.push('SELECT (SELECT max(commit_seq) FROM sync_change_scopes WHERE partition=? AND tbl=? AND var=? AND value=?) AS seq');
|
|
619
|
-
}
|
|
620
|
-
return `SELECT max(seq) AS seq FROM (${perValue.join(' UNION ALL ')})`;
|
|
621
|
-
}
|
|
618
|
+
export const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = `SELECT max((SELECT max(commit_seq) FROM sync_change_scopes
|
|
619
|
+
WHERE partition=? AND tbl=? AND var=? AND value=v.value)) AS seq
|
|
620
|
+
FROM json_each(?) AS v`;
|
|
622
621
|
/**
|
|
623
|
-
* PostgreSQL set-based form of `
|
|
622
|
+
* PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
|
|
624
623
|
* subscriptions of one table in a pull. `$2` is a JSON array of
|
|
625
624
|
* `{i, var, vals}` records. Params: [partition, specs (JSON text), tbl].
|
|
626
625
|
*/
|
|
@@ -7,6 +7,7 @@ export declare class SqliteBlobStore implements BlobStore {
|
|
|
7
7
|
readonly db: SqliteDatabase;
|
|
8
8
|
constructor(db?: SqliteDatabase | string);
|
|
9
9
|
put(partition: string, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): Promise<BlobRecord>;
|
|
10
|
+
delete(partition: string, blobId: string): Promise<void>;
|
|
10
11
|
has(partition: string, blobId: string): Promise<boolean>;
|
|
11
12
|
get(partition: string, blobId: string): Promise<{
|
|
12
13
|
record: BlobRecord;
|