@syncular/server 0.0.1-100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobs/adapters/database.d.ts +83 -0
- package/dist/blobs/adapters/database.d.ts.map +1 -0
- package/dist/blobs/adapters/database.js +202 -0
- package/dist/blobs/adapters/database.js.map +1 -0
- package/dist/blobs/adapters/s3.d.ts +82 -0
- package/dist/blobs/adapters/s3.d.ts.map +1 -0
- package/dist/blobs/adapters/s3.js +170 -0
- package/dist/blobs/adapters/s3.js.map +1 -0
- package/dist/blobs/index.d.ts +9 -0
- package/dist/blobs/index.d.ts.map +1 -0
- package/dist/blobs/index.js +9 -0
- package/dist/blobs/index.js.map +1 -0
- package/dist/blobs/manager.d.ts +195 -0
- package/dist/blobs/manager.d.ts.map +1 -0
- package/dist/blobs/manager.js +440 -0
- package/dist/blobs/manager.js.map +1 -0
- package/dist/blobs/migrate.d.ts +27 -0
- package/dist/blobs/migrate.d.ts.map +1 -0
- package/dist/blobs/migrate.js +119 -0
- package/dist/blobs/migrate.js.map +1 -0
- package/dist/blobs/types.d.ts +54 -0
- package/dist/blobs/types.d.ts.map +1 -0
- package/dist/blobs/types.js +5 -0
- package/dist/blobs/types.js.map +1 -0
- package/dist/clients.d.ts +14 -0
- package/dist/clients.d.ts.map +1 -0
- package/dist/clients.js +7 -0
- package/dist/clients.js.map +1 -0
- package/dist/compaction.d.ts +27 -0
- package/dist/compaction.d.ts.map +1 -0
- package/dist/compaction.js +49 -0
- package/dist/compaction.js.map +1 -0
- package/dist/dialect/base.d.ts +83 -0
- package/dist/dialect/base.d.ts.map +1 -0
- package/dist/dialect/base.js +144 -0
- package/dist/dialect/base.js.map +1 -0
- package/dist/dialect/helpers.d.ts +10 -0
- package/dist/dialect/helpers.d.ts.map +1 -0
- package/dist/dialect/helpers.js +59 -0
- package/dist/dialect/helpers.js.map +1 -0
- package/dist/dialect/index.d.ts +7 -0
- package/dist/dialect/index.d.ts.map +1 -0
- package/dist/dialect/index.js +7 -0
- package/dist/dialect/index.js.map +1 -0
- package/dist/dialect/types.d.ts +149 -0
- package/dist/dialect/types.d.ts.map +1 -0
- package/dist/dialect/types.js +8 -0
- package/dist/dialect/types.js.map +1 -0
- package/dist/helpers/conflict.d.ts +52 -0
- package/dist/helpers/conflict.d.ts.map +1 -0
- package/dist/helpers/conflict.js +49 -0
- package/dist/helpers/conflict.js.map +1 -0
- package/dist/helpers/emitted-change.d.ts +56 -0
- package/dist/helpers/emitted-change.d.ts.map +1 -0
- package/dist/helpers/emitted-change.js +46 -0
- package/dist/helpers/emitted-change.js.map +1 -0
- package/dist/helpers/index.d.ts +10 -0
- package/dist/helpers/index.d.ts.map +1 -0
- package/dist/helpers/index.js +10 -0
- package/dist/helpers/index.js.map +1 -0
- package/dist/helpers/paginate.d.ts +49 -0
- package/dist/helpers/paginate.d.ts.map +1 -0
- package/dist/helpers/paginate.js +54 -0
- package/dist/helpers/paginate.js.map +1 -0
- package/dist/helpers/scope-strings.d.ts +74 -0
- package/dist/helpers/scope-strings.d.ts.map +1 -0
- package/dist/helpers/scope-strings.js +82 -0
- package/dist/helpers/scope-strings.js.map +1 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +27 -0
- package/dist/index.js.map +1 -0
- package/dist/migrate.d.ts +14 -0
- package/dist/migrate.d.ts.map +1 -0
- package/dist/migrate.js +13 -0
- package/dist/migrate.js.map +1 -0
- package/dist/proxy/handler.d.ts +42 -0
- package/dist/proxy/handler.d.ts.map +1 -0
- package/dist/proxy/handler.js +102 -0
- package/dist/proxy/handler.js.map +1 -0
- package/dist/proxy/index.d.ts +9 -0
- package/dist/proxy/index.d.ts.map +1 -0
- package/dist/proxy/index.js +14 -0
- package/dist/proxy/index.js.map +1 -0
- package/dist/proxy/mutation-detector.d.ts +35 -0
- package/dist/proxy/mutation-detector.d.ts.map +1 -0
- package/dist/proxy/mutation-detector.js +246 -0
- package/dist/proxy/mutation-detector.js.map +1 -0
- package/dist/proxy/oplog.d.ts +30 -0
- package/dist/proxy/oplog.d.ts.map +1 -0
- package/dist/proxy/oplog.js +110 -0
- package/dist/proxy/oplog.js.map +1 -0
- package/dist/proxy/registry.d.ts +35 -0
- package/dist/proxy/registry.d.ts.map +1 -0
- package/dist/proxy/registry.js +49 -0
- package/dist/proxy/registry.js.map +1 -0
- package/dist/proxy/types.d.ts +44 -0
- package/dist/proxy/types.d.ts.map +1 -0
- package/dist/proxy/types.js +7 -0
- package/dist/proxy/types.js.map +1 -0
- package/dist/prune.d.ts +37 -0
- package/dist/prune.d.ts.map +1 -0
- package/dist/prune.js +112 -0
- package/dist/prune.js.map +1 -0
- package/dist/pull.d.ts +31 -0
- package/dist/pull.d.ts.map +1 -0
- package/dist/pull.js +608 -0
- package/dist/pull.js.map +1 -0
- package/dist/push.d.ts +33 -0
- package/dist/push.d.ts.map +1 -0
- package/dist/push.js +412 -0
- package/dist/push.js.map +1 -0
- package/dist/realtime/in-memory.d.ts +13 -0
- package/dist/realtime/in-memory.d.ts.map +1 -0
- package/dist/realtime/in-memory.js +28 -0
- package/dist/realtime/in-memory.js.map +1 -0
- package/dist/realtime/index.d.ts +3 -0
- package/dist/realtime/index.d.ts.map +1 -0
- package/dist/realtime/index.js +2 -0
- package/dist/realtime/index.js.map +1 -0
- package/dist/realtime/types.d.ts +50 -0
- package/dist/realtime/types.d.ts.map +1 -0
- package/dist/realtime/types.js +7 -0
- package/dist/realtime/types.js.map +1 -0
- package/dist/schema.d.ts +164 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +10 -0
- package/dist/schema.js.map +1 -0
- package/dist/shapes/create-handler.d.ts +119 -0
- package/dist/shapes/create-handler.d.ts.map +1 -0
- package/dist/shapes/create-handler.js +327 -0
- package/dist/shapes/create-handler.js.map +1 -0
- package/dist/shapes/index.d.ts +4 -0
- package/dist/shapes/index.d.ts.map +1 -0
- package/dist/shapes/index.js +4 -0
- package/dist/shapes/index.js.map +1 -0
- package/dist/shapes/registry.d.ts +20 -0
- package/dist/shapes/registry.d.ts.map +1 -0
- package/dist/shapes/registry.js +88 -0
- package/dist/shapes/registry.js.map +1 -0
- package/dist/shapes/types.d.ts +204 -0
- package/dist/shapes/types.d.ts.map +1 -0
- package/dist/shapes/types.js +2 -0
- package/dist/shapes/types.js.map +1 -0
- package/dist/snapshot-chunks/adapters/s3.d.ts +74 -0
- package/dist/snapshot-chunks/adapters/s3.d.ts.map +1 -0
- package/dist/snapshot-chunks/adapters/s3.js +50 -0
- package/dist/snapshot-chunks/adapters/s3.js.map +1 -0
- package/dist/snapshot-chunks/db-metadata.d.ts +38 -0
- package/dist/snapshot-chunks/db-metadata.d.ts.map +1 -0
- package/dist/snapshot-chunks/db-metadata.js +324 -0
- package/dist/snapshot-chunks/db-metadata.js.map +1 -0
- package/dist/snapshot-chunks/index.d.ts +9 -0
- package/dist/snapshot-chunks/index.d.ts.map +1 -0
- package/dist/snapshot-chunks/index.js +9 -0
- package/dist/snapshot-chunks/index.js.map +1 -0
- package/dist/snapshot-chunks/types.d.ts +78 -0
- package/dist/snapshot-chunks/types.d.ts.map +1 -0
- package/dist/snapshot-chunks/types.js +8 -0
- package/dist/snapshot-chunks/types.js.map +1 -0
- package/dist/snapshot-chunks.d.ts +60 -0
- package/dist/snapshot-chunks.d.ts.map +1 -0
- package/dist/snapshot-chunks.js +223 -0
- package/dist/snapshot-chunks.js.map +1 -0
- package/dist/stats.d.ts +19 -0
- package/dist/stats.d.ts.map +1 -0
- package/dist/stats.js +57 -0
- package/dist/stats.js.map +1 -0
- package/dist/subscriptions/index.d.ts +2 -0
- package/dist/subscriptions/index.d.ts.map +1 -0
- package/dist/subscriptions/index.js +2 -0
- package/dist/subscriptions/index.js.map +1 -0
- package/dist/subscriptions/resolve.d.ts +35 -0
- package/dist/subscriptions/resolve.d.ts.map +1 -0
- package/dist/subscriptions/resolve.js +134 -0
- package/dist/subscriptions/resolve.js.map +1 -0
- package/package.json +80 -0
- package/src/blobs/adapters/database.test.ts +67 -0
- package/src/blobs/adapters/database.ts +315 -0
- package/src/blobs/adapters/s3.ts +271 -0
- package/src/blobs/index.ts +9 -0
- package/src/blobs/manager.ts +600 -0
- package/src/blobs/migrate.ts +150 -0
- package/src/blobs/types.ts +70 -0
- package/src/clients.ts +21 -0
- package/src/compaction.ts +77 -0
- package/src/dialect/base.ts +292 -0
- package/src/dialect/helpers.ts +61 -0
- package/src/dialect/index.ts +7 -0
- package/src/dialect/types.ts +197 -0
- package/src/helpers/conflict.ts +64 -0
- package/src/helpers/emitted-change.ts +69 -0
- package/src/helpers/index.ts +10 -0
- package/src/helpers/paginate.ts +82 -0
- package/src/helpers/scope-strings.ts +101 -0
- package/src/index.ts +28 -0
- package/src/migrate.ts +20 -0
- package/src/proxy/handler.test.ts +120 -0
- package/src/proxy/handler.ts +159 -0
- package/src/proxy/index.ts +18 -0
- package/src/proxy/mutation-detector.test.ts +71 -0
- package/src/proxy/mutation-detector.ts +281 -0
- package/src/proxy/oplog.ts +146 -0
- package/src/proxy/registry.ts +56 -0
- package/src/proxy/types.ts +46 -0
- package/src/prune.ts +200 -0
- package/src/pull.ts +858 -0
- package/src/push.ts +583 -0
- package/src/realtime/in-memory.ts +33 -0
- package/src/realtime/index.ts +5 -0
- package/src/realtime/types.ts +55 -0
- package/src/schema.ts +172 -0
- package/src/shapes/create-handler.ts +590 -0
- package/src/shapes/index.ts +3 -0
- package/src/shapes/registry.ts +109 -0
- package/src/shapes/types.ts +267 -0
- package/src/snapshot-chunks/adapters/s3.ts +68 -0
- package/src/snapshot-chunks/db-metadata.test.ts +100 -0
- package/src/snapshot-chunks/db-metadata.ts +466 -0
- package/src/snapshot-chunks/index.ts +9 -0
- package/src/snapshot-chunks/types.ts +103 -0
- package/src/snapshot-chunks.ts +329 -0
- package/src/stats.ts +104 -0
- package/src/subscriptions/index.ts +1 -0
- package/src/subscriptions/resolve.ts +185 -0
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @syncular/server - Database-backed metadata store for snapshot chunks
|
|
3
|
+
*
|
|
4
|
+
* Stores chunk metadata in sync_snapshot_chunks_metadata table,
|
|
5
|
+
* body content in blob storage adapter.
|
|
6
|
+
*/
|
|
7
|
+
import { createHash } from 'node:crypto';
|
|
8
|
+
import { SYNC_SNAPSHOT_CHUNK_COMPRESSION, SYNC_SNAPSHOT_CHUNK_ENCODING, } from '@syncular/core';
|
|
9
|
+
/**
|
|
10
|
+
* Create a snapshot chunk storage that uses:
|
|
11
|
+
* - Database for metadata (scope, commit seq, etc.)
|
|
12
|
+
* - Blob adapter for body content
|
|
13
|
+
*/
|
|
14
|
+
export function createDbMetadataChunkStorage(options) {
|
|
15
|
+
const { db, blobAdapter, chunkIdPrefix = 'chunk_' } = options;
|
|
16
|
+
// Generate deterministic blob hash from chunk identity metadata.
|
|
17
|
+
function computeBlobHash(metadata) {
|
|
18
|
+
const digest = createHash('sha256')
|
|
19
|
+
.update(`${metadata.encoding}:${metadata.compression}:${metadata.sha256}`)
|
|
20
|
+
.digest('hex');
|
|
21
|
+
return `sha256:${digest}`;
|
|
22
|
+
}
|
|
23
|
+
function bytesToStream(bytes) {
|
|
24
|
+
return new ReadableStream({
|
|
25
|
+
start(controller) {
|
|
26
|
+
controller.enqueue(bytes);
|
|
27
|
+
controller.close();
|
|
28
|
+
},
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
async function streamToBytes(stream) {
|
|
32
|
+
const reader = stream.getReader();
|
|
33
|
+
try {
|
|
34
|
+
const chunks = [];
|
|
35
|
+
let total = 0;
|
|
36
|
+
while (true) {
|
|
37
|
+
const { done, value } = await reader.read();
|
|
38
|
+
if (done)
|
|
39
|
+
break;
|
|
40
|
+
if (!value)
|
|
41
|
+
continue;
|
|
42
|
+
chunks.push(value);
|
|
43
|
+
total += value.length;
|
|
44
|
+
}
|
|
45
|
+
const out = new Uint8Array(total);
|
|
46
|
+
let offset = 0;
|
|
47
|
+
for (const chunk of chunks) {
|
|
48
|
+
out.set(chunk, offset);
|
|
49
|
+
offset += chunk.length;
|
|
50
|
+
}
|
|
51
|
+
return out;
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
reader.releaseLock();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
async function streamByteLength(stream) {
|
|
58
|
+
const reader = stream.getReader();
|
|
59
|
+
try {
|
|
60
|
+
let total = 0;
|
|
61
|
+
while (true) {
|
|
62
|
+
const { done, value } = await reader.read();
|
|
63
|
+
if (done)
|
|
64
|
+
break;
|
|
65
|
+
if (!value)
|
|
66
|
+
continue;
|
|
67
|
+
total += value.length;
|
|
68
|
+
}
|
|
69
|
+
return total;
|
|
70
|
+
}
|
|
71
|
+
finally {
|
|
72
|
+
reader.releaseLock();
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
// Generate unique chunk ID
|
|
76
|
+
function generateChunkId() {
|
|
77
|
+
return `${chunkIdPrefix}${Date.now()}_${Math.random().toString(36).slice(2, 11)}`;
|
|
78
|
+
}
|
|
79
|
+
async function readStoredRef(args) {
|
|
80
|
+
const nowIso = args.nowIso ?? new Date().toISOString();
|
|
81
|
+
const rowCursorKey = args.rowCursor ?? '';
|
|
82
|
+
const baseQuery = db
|
|
83
|
+
.selectFrom('sync_snapshot_chunks')
|
|
84
|
+
.select(['chunk_id', 'sha256', 'byte_length', 'encoding', 'compression'])
|
|
85
|
+
.where('partition_id', '=', args.partitionId)
|
|
86
|
+
.where('scope_key', '=', args.scopeKey)
|
|
87
|
+
.where('scope', '=', args.scope)
|
|
88
|
+
.where('as_of_commit_seq', '=', args.asOfCommitSeq)
|
|
89
|
+
.where('row_cursor', '=', rowCursorKey)
|
|
90
|
+
.where('row_limit', '=', args.rowLimit)
|
|
91
|
+
.where('encoding', '=', args.encoding)
|
|
92
|
+
.where('compression', '=', args.compression);
|
|
93
|
+
const row = await (args.includeExpired
|
|
94
|
+
? baseQuery.executeTakeFirst()
|
|
95
|
+
: baseQuery.where('expires_at', '>', nowIso).executeTakeFirst());
|
|
96
|
+
if (!row)
|
|
97
|
+
return null;
|
|
98
|
+
if (row.encoding !== SYNC_SNAPSHOT_CHUNK_ENCODING) {
|
|
99
|
+
throw new Error(`Unexpected snapshot chunk encoding: ${String(row.encoding)}`);
|
|
100
|
+
}
|
|
101
|
+
if (row.compression !== SYNC_SNAPSHOT_CHUNK_COMPRESSION) {
|
|
102
|
+
throw new Error(`Unexpected snapshot chunk compression: ${String(row.compression)}`);
|
|
103
|
+
}
|
|
104
|
+
return {
|
|
105
|
+
id: row.chunk_id,
|
|
106
|
+
sha256: row.sha256,
|
|
107
|
+
byteLength: Number(row.byte_length ?? 0),
|
|
108
|
+
encoding: row.encoding,
|
|
109
|
+
compression: row.compression,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
async function readBlobHash(chunkId) {
|
|
113
|
+
const row = await db
|
|
114
|
+
.selectFrom('sync_snapshot_chunks')
|
|
115
|
+
.select(['blob_hash'])
|
|
116
|
+
.where('chunk_id', '=', chunkId)
|
|
117
|
+
.executeTakeFirst();
|
|
118
|
+
return row?.blob_hash ?? null;
|
|
119
|
+
}
|
|
120
|
+
async function upsertChunkMetadata(metadata, args) {
|
|
121
|
+
const chunkId = generateChunkId();
|
|
122
|
+
const now = new Date().toISOString();
|
|
123
|
+
await db
|
|
124
|
+
.insertInto('sync_snapshot_chunks')
|
|
125
|
+
.values({
|
|
126
|
+
chunk_id: chunkId,
|
|
127
|
+
partition_id: metadata.partitionId,
|
|
128
|
+
scope_key: metadata.scopeKey,
|
|
129
|
+
scope: metadata.scope,
|
|
130
|
+
as_of_commit_seq: metadata.asOfCommitSeq,
|
|
131
|
+
row_cursor: metadata.rowCursor ?? '',
|
|
132
|
+
row_limit: metadata.rowLimit,
|
|
133
|
+
encoding: metadata.encoding,
|
|
134
|
+
compression: metadata.compression,
|
|
135
|
+
sha256: metadata.sha256,
|
|
136
|
+
byte_length: args.byteLength,
|
|
137
|
+
blob_hash: args.blobHash,
|
|
138
|
+
expires_at: metadata.expiresAt,
|
|
139
|
+
created_at: now,
|
|
140
|
+
})
|
|
141
|
+
.onConflict((oc) => oc
|
|
142
|
+
.columns([
|
|
143
|
+
'partition_id',
|
|
144
|
+
'scope_key',
|
|
145
|
+
'scope',
|
|
146
|
+
'as_of_commit_seq',
|
|
147
|
+
'row_cursor',
|
|
148
|
+
'row_limit',
|
|
149
|
+
'encoding',
|
|
150
|
+
'compression',
|
|
151
|
+
])
|
|
152
|
+
.doUpdateSet({
|
|
153
|
+
expires_at: metadata.expiresAt,
|
|
154
|
+
blob_hash: args.blobHash,
|
|
155
|
+
sha256: metadata.sha256,
|
|
156
|
+
byte_length: args.byteLength,
|
|
157
|
+
row_cursor: metadata.rowCursor ?? '',
|
|
158
|
+
}))
|
|
159
|
+
.execute();
|
|
160
|
+
}
|
|
161
|
+
async function readChunkStreamById(chunkId) {
|
|
162
|
+
const blobHash = await readBlobHash(chunkId);
|
|
163
|
+
if (!blobHash)
|
|
164
|
+
return null;
|
|
165
|
+
if (blobAdapter.getStream) {
|
|
166
|
+
return blobAdapter.getStream(blobHash);
|
|
167
|
+
}
|
|
168
|
+
if (blobAdapter.get) {
|
|
169
|
+
const bytes = await blobAdapter.get(blobHash);
|
|
170
|
+
return bytes ? bytesToStream(bytes) : null;
|
|
171
|
+
}
|
|
172
|
+
throw new Error(`Blob adapter ${blobAdapter.name} does not support direct get() for snapshot chunks`);
|
|
173
|
+
}
|
|
174
|
+
return {
|
|
175
|
+
name: `db-metadata+${blobAdapter.name}`,
|
|
176
|
+
async storeChunk(metadata) {
|
|
177
|
+
const { body, ...metaWithoutBody } = metadata;
|
|
178
|
+
const blobHash = computeBlobHash(metaWithoutBody);
|
|
179
|
+
// Check if blob already exists (content-addressed dedup)
|
|
180
|
+
const blobExists = await blobAdapter.exists(blobHash);
|
|
181
|
+
if (!blobExists) {
|
|
182
|
+
// Store body in blob adapter
|
|
183
|
+
if (blobAdapter.put) {
|
|
184
|
+
await blobAdapter.put(blobHash, body);
|
|
185
|
+
}
|
|
186
|
+
else if (blobAdapter.putStream) {
|
|
187
|
+
await blobAdapter.putStream(blobHash, bytesToStream(body));
|
|
188
|
+
}
|
|
189
|
+
else {
|
|
190
|
+
throw new Error(`Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
await upsertChunkMetadata(metaWithoutBody, {
|
|
194
|
+
blobHash,
|
|
195
|
+
byteLength: body.length,
|
|
196
|
+
});
|
|
197
|
+
const storedRef = await readStoredRef({
|
|
198
|
+
partitionId: metaWithoutBody.partitionId,
|
|
199
|
+
scopeKey: metaWithoutBody.scopeKey,
|
|
200
|
+
scope: metaWithoutBody.scope,
|
|
201
|
+
asOfCommitSeq: metaWithoutBody.asOfCommitSeq,
|
|
202
|
+
rowCursor: metaWithoutBody.rowCursor,
|
|
203
|
+
rowLimit: metaWithoutBody.rowLimit,
|
|
204
|
+
encoding: metaWithoutBody.encoding,
|
|
205
|
+
compression: metaWithoutBody.compression,
|
|
206
|
+
includeExpired: true,
|
|
207
|
+
});
|
|
208
|
+
if (!storedRef) {
|
|
209
|
+
throw new Error('Failed to read stored snapshot chunk reference');
|
|
210
|
+
}
|
|
211
|
+
return storedRef;
|
|
212
|
+
},
|
|
213
|
+
async storeChunkStream(metadata) {
|
|
214
|
+
const { bodyStream, byteLength, ...metaWithoutBody } = metadata;
|
|
215
|
+
const blobHash = computeBlobHash(metaWithoutBody);
|
|
216
|
+
const blobExists = await blobAdapter.exists(blobHash);
|
|
217
|
+
let observedByteLength;
|
|
218
|
+
if (!blobExists) {
|
|
219
|
+
if (blobAdapter.putStream) {
|
|
220
|
+
const [uploadStream, countStream] = bodyStream.tee();
|
|
221
|
+
const uploadPromise = typeof byteLength === 'number'
|
|
222
|
+
? blobAdapter.putStream(blobHash, uploadStream, {
|
|
223
|
+
byteLength,
|
|
224
|
+
contentLength: byteLength,
|
|
225
|
+
})
|
|
226
|
+
: blobAdapter.putStream(blobHash, uploadStream);
|
|
227
|
+
const countPromise = streamByteLength(countStream);
|
|
228
|
+
const [, countedByteLength] = await Promise.all([
|
|
229
|
+
uploadPromise,
|
|
230
|
+
countPromise,
|
|
231
|
+
]);
|
|
232
|
+
observedByteLength = countedByteLength;
|
|
233
|
+
}
|
|
234
|
+
else if (blobAdapter.put) {
|
|
235
|
+
const body = await streamToBytes(bodyStream);
|
|
236
|
+
await blobAdapter.put(blobHash, body);
|
|
237
|
+
observedByteLength = body.length;
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
throw new Error(`Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
else if (typeof byteLength === 'number') {
|
|
244
|
+
observedByteLength = byteLength;
|
|
245
|
+
await bodyStream.cancel();
|
|
246
|
+
}
|
|
247
|
+
else if (blobAdapter.getMetadata) {
|
|
248
|
+
const metadata = await blobAdapter.getMetadata(blobHash);
|
|
249
|
+
if (!metadata) {
|
|
250
|
+
throw new Error(`Blob metadata missing for existing chunk ${blobHash}`);
|
|
251
|
+
}
|
|
252
|
+
observedByteLength = metadata.size;
|
|
253
|
+
await bodyStream.cancel();
|
|
254
|
+
}
|
|
255
|
+
else {
|
|
256
|
+
observedByteLength = await streamByteLength(bodyStream);
|
|
257
|
+
}
|
|
258
|
+
if (typeof byteLength === 'number' &&
|
|
259
|
+
Number.isFinite(byteLength) &&
|
|
260
|
+
observedByteLength !== byteLength) {
|
|
261
|
+
throw new Error(`Snapshot chunk byte length mismatch: expected ${byteLength}, got ${observedByteLength}`);
|
|
262
|
+
}
|
|
263
|
+
await upsertChunkMetadata(metaWithoutBody, {
|
|
264
|
+
blobHash,
|
|
265
|
+
byteLength: observedByteLength,
|
|
266
|
+
});
|
|
267
|
+
const storedRef = await readStoredRef({
|
|
268
|
+
partitionId: metaWithoutBody.partitionId,
|
|
269
|
+
scopeKey: metaWithoutBody.scopeKey,
|
|
270
|
+
scope: metaWithoutBody.scope,
|
|
271
|
+
asOfCommitSeq: metaWithoutBody.asOfCommitSeq,
|
|
272
|
+
rowCursor: metaWithoutBody.rowCursor,
|
|
273
|
+
rowLimit: metaWithoutBody.rowLimit,
|
|
274
|
+
encoding: metaWithoutBody.encoding,
|
|
275
|
+
compression: metaWithoutBody.compression,
|
|
276
|
+
includeExpired: true,
|
|
277
|
+
});
|
|
278
|
+
if (!storedRef) {
|
|
279
|
+
throw new Error('Failed to read stored snapshot chunk reference');
|
|
280
|
+
}
|
|
281
|
+
return storedRef;
|
|
282
|
+
},
|
|
283
|
+
async readChunk(chunkId) {
|
|
284
|
+
const stream = await readChunkStreamById(chunkId);
|
|
285
|
+
if (!stream)
|
|
286
|
+
return null;
|
|
287
|
+
return streamToBytes(stream);
|
|
288
|
+
},
|
|
289
|
+
async readChunkStream(chunkId) {
|
|
290
|
+
return readChunkStreamById(chunkId);
|
|
291
|
+
},
|
|
292
|
+
async findChunk(pageKey) {
|
|
293
|
+
return readStoredRef(pageKey);
|
|
294
|
+
},
|
|
295
|
+
async cleanupExpired(beforeIso) {
|
|
296
|
+
// Find expired chunks
|
|
297
|
+
const expiredRows = await db
|
|
298
|
+
.selectFrom('sync_snapshot_chunks')
|
|
299
|
+
.select(['chunk_id', 'blob_hash'])
|
|
300
|
+
.where('expires_at', '<=', beforeIso)
|
|
301
|
+
.execute();
|
|
302
|
+
if (expiredRows.length === 0)
|
|
303
|
+
return 0;
|
|
304
|
+
// Delete from blob storage (best effort)
|
|
305
|
+
for (const row of expiredRows) {
|
|
306
|
+
try {
|
|
307
|
+
await blobAdapter.delete(row.blob_hash);
|
|
308
|
+
}
|
|
309
|
+
catch {
|
|
310
|
+
// Ignore deletion errors - blob may be shared or already deleted
|
|
311
|
+
// Log for observability but don't fail the cleanup
|
|
312
|
+
console.warn(`Failed to delete blob ${row.blob_hash} for chunk ${row.chunk_id}, may be already deleted or shared`);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
// Delete metadata from database
|
|
316
|
+
const result = await db
|
|
317
|
+
.deleteFrom('sync_snapshot_chunks')
|
|
318
|
+
.where('expires_at', '<=', beforeIso)
|
|
319
|
+
.executeTakeFirst();
|
|
320
|
+
return Number(result.numDeletedRows ?? 0);
|
|
321
|
+
},
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
//# sourceMappingURL=db-metadata.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"db-metadata.js","sourceRoot":"","sources":["../../src/snapshot-chunks/db-metadata.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAEL,+BAA+B,EAC/B,4BAA4B,GAI7B,MAAM,gBAAgB,CAAC;AAcxB;;;;GAIG;AACH,MAAM,UAAU,4BAA4B,CAC1C,OAA8C,EA4B9C;IACA,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,aAAa,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC;IAE9D,iEAAiE;IACjE,SAAS,eAAe,CAAC,QAIxB,EAAU;QACT,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;aAChC,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;aACzE,MAAM,CAAC,KAAK,CAAC,CAAC;QACjB,OAAO,UAAU,MAAM,EAAE,CAAC;IAAA,CAC3B;IAED,SAAS,aAAa,CAAC,KAAiB,EAA8B;QACpE,OAAO,IAAI,cAAc,CAAa;YACpC,KAAK,CAAC,UAAU,EAAE;gBAChB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC1B,UAAU,CAAC,KAAK,EAAE,CAAC;YAAA,CACpB;SACF,CAAC,CAAC;IAAA,CACJ;IAED,KAAK,UAAU,aAAa,CAC1B,MAAkC,EACb;QACrB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,KAAK,GAAG,CAAC,CAAC;YAEd,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI;oBAAE,MAAM;gBAChB,IAAI,CAAC,KAAK;oBAAE,SAAS;gBACrB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACxB,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;YAClC,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC;IAAA,CACF;IAED,KAAK,UAAU,gBAAgB,CAC7B,MAAkC,EACjB;QACjB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI;oBAAE,MAAM;gBAChB,IAAI,CAAC,KAAK;oBAAE,SAAS;gBACrB,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACxB,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC;IAAA,CACF;IAED,2BAA2B;IAC3B,SAAS,eAAe,GAAW;QACjC,OAAO,GAAG,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAAA,CACnF;IAED,KAAK,UAAU,aAAa,CAAC,IAW5B,EAAwC;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,EAAE;aACjB,UAAU,CAAC,sBAAsB,CAAC;aAClC,MAAM,CAAC,CAAC,UAAU,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;aACxE,KAAK,CAAC,cAAc,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC;aAC5C,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC;aACtC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC;aAC/B,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC;aAClD,KAAK,CAAC,YAAY,EAAE,GAAG,EAAE,YAAY,CAAC;aACtC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC;aACtC,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC;aACrC,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc;YACpC,CAAC,CAAC,SAAS,CAAC,gBAAgB,EAAE;YAC9B,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAEnE,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,IAAI,GAAG,CAAC,QAAQ,KAAK,4BAA4B,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CACb,uCAAuC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC9D,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,KAAK,+BAA+B,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CACb,0CAA0C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CACpE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,QAAQ;YAChB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;YACxC,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,WAAW,EAAE,GAAG,CAAC,WAAW;SAC7B,CAAC;IAAA,CACH;IAED,KAAK,UAAU,YAAY,CAAC,OAAe,EAA0B;QACnE,MAAM,GAAG,GAAG,MAAM,EAAE;aACjB,UAAU,CAAC,sBAAsB,CAAC;aAClC,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC;aACrB,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,OAAO,CAAC;aAC/B,gBAAgB,EAAE,CAAC;QACtB,OAAO,GAAG,EAAE,SAAS,IAAI,IAAI,CAAC;IAAA,CAC/B;IAED,KAAK,UAAU,mBAAmB,CAChC,QAGC,EACD,IAA8C,EAC/B;QACf,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAErC,MAAM,EAAE;aACL,UAAU,CAAC,sBAAsB,CAAC;aAClC,MAAM,CAAC;YACN,QAAQ,EAAE,OAAO;YACjB,YAAY,EAAE,QAAQ,CAAC,WAAW;YAClC,SAAS,EAAE,QAAQ,CAAC,QAAQ;YAC5B,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,gBAAgB,EAAE,QAAQ,CAAC,aAAa;YACxC,UAAU,EAAE,QAAQ,CAAC,SAAS,IAAI,EAAE;YACpC,SAAS,EAAE,QAAQ,CAAC,QAAQ;YAC5B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,IAAI,CAAC,UAAU;YAC5B,SAAS,EAAE,IAAI,CAAC,QAAQ;YACxB,UAAU,EAAE,QAAQ,CAAC,SAAS;YAC9B,UAAU,EAAE,GAAG;SAChB,CAAC;aACD,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CACjB,EAAE;aACC,OAAO,CAAC;YACP,cAAc;YACd,WAAW;YACX,OAAO;YACP,kBAAkB;YAClB,YAAY;YACZ,WAAW;YACX,UAAU;YACV,aAAa;SACd,CAAC;aACD,WAAW,CAAC;YACX,UAAU,EAAE,QAAQ,CAAC,SAAS;YAC9B,SAAS,EAAE,IAAI,CAAC,QAAQ;YACxB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,IAAI,CAAC,UAAU;YAC5B,UAAU,EAAE,QAAQ,CAAC,SAAS,IAAI,EAAE;SACrC,CAAC,CACL;aACA,OAAO,EAAE,CAAC;IAAA,CACd;IAED,KAAK,UAAU,mBAAmB,CAChC,OAAe,EAC6B;QAC5C,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;YAC1B,OAAO,WAAW,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC9C,OAAO,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7C,CAAC;QAED,MAAM,IAAI,KAAK,CACb,gBAAgB,WAAW,CAAC,IAAI,oDAAoD,CACrF,CAAC;IAAA,CACH;IAED,OAAO;QACL,IAAI,EAAE,eAAe,WAAW,CAAC,IAAI,EAAE;QAEvC,KAAK,CAAC,UAAU,CACd,QAKC,EAC8B;YAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,eAAe,EAAE,GAAG,QAAQ,CAAC;YAC9C,MAAM,QAAQ,GAAG,eAAe,CAAC,eAAe,CAAC,CAAC;YAElD,yDAAyD;YACzD,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEtD,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,6BAA6B;gBAC7B,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;oBACpB,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBACxC,CAAC;qBAAM,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;oBACjC,MAAM,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC7D,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CACb,gBAAgB,WAAW,CAAC,IAAI,oDAAoD,CACrF,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,mBAAmB,CAAC,eAAe,EAAE;gBACzC,QAAQ;gBACR,UAAU,EAAE,IAAI,CAAC,MAAM;aACxB,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC;gBACpC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,KAAK,EAAE,eAAe,CAAC,KAAK;gBAC5B,aAAa,EAAE,eAAe,CAAC,aAAa;gBAC5C,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACpE,CAAC;YAED,OAAO,SAAS,CAAC;QAAA,CAClB;QAED,KAAK,CAAC,gBAAgB,CACpB,QAMC,EAC8B;YAC/B,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,eAAe,EAAE,GAAG,QAAQ,CAAC;YAChE,MAAM,QAAQ,GAAG,eAAe,CAAC,eAAe,CAAC,CAAC;YAElD,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACtD,IAAI,kBAA0B,CAAC;YAE/B,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;oBAC1B,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC;oBACrD,MAAM,aAAa,GACjB,OAAO,UAAU,KAAK,QAAQ;wBAC5B,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE;4BAC5C,UAAU;4BACV,aAAa,EAAE,UAAU;yBAC1B,CAAC;wBACJ,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;oBACpD,MAAM,YAAY,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;oBAEnD,MAAM,CAAC,EAAE,iBAAiB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;wBAC9C,aAAa;wBACb,YAAY;qBACb,CAAC,CAAC;oBACH,kBAAkB,GAAG,iBAAiB,CAAC;gBACzC,CAAC;qBAAM,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;oBAC3B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,UAAU,CAAC,CAAC;oBAC7C,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;oBACtC,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CACb,gBAAgB,WAAW,CAAC,IAAI,oDAAoD,CACrF,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;gBAC1C,kBAAkB,GAAG,UAAU,CAAC;gBAChC,MAAM,UAAU,CAAC,MAAM,EAAE,CAAC;YAC5B,CAAC;iBAAM,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;gBACnC,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACzD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,KAAK,CACb,4CAA4C,QAAQ,EAAE,CACvD,CAAC;gBACJ,CAAC;gBACD,kBAAkB,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACnC,MAAM,UAAU,CAAC,MAAM,EAAE,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,kBAAkB,GAAG,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAC1D,CAAC;YAED,IACE,OAAO,UAAU,KAAK,QAAQ;gBAC9B,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAC3B,kBAAkB,KAAK,UAAU,EACjC,CAAC;gBACD,MAAM,IAAI,KAAK,CACb,iDAAiD,UAAU,SAAS,kBAAkB,EAAE,CACzF,CAAC;YACJ,CAAC;YAED,MAAM,mBAAmB,CAAC,eAAe,EAAE;gBACzC,QAAQ;gBACR,UAAU,EAAE,kBAAkB;aAC/B,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC;gBACpC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,KAAK,EAAE,eAAe,CAAC,KAAK;gBAC5B,aAAa,EAAE,eAAe,CAAC,aAAa;gBAC5C,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACpE,CAAC;YAED,OAAO,SAAS,CAAC;QAAA,CAClB;QAED,KAAK,CAAC,SAAS,CAAC,OAAe,EAA8B;YAC3D,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,CAAC,MAAM;gBAAE,OAAO,IAAI,CAAC;YACzB,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;QAAA,CAC9B;QAED,KAAK,CAAC,eAAe,CACnB,OAAe,EAC6B;YAC5C,OAAO,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAAA,CACrC;QAED,KAAK,CAAC,SAAS,CACb,OAA6B,EACS;YACtC,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;QAAA,CAC/B;QAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAmB;YACvD,sBAAsB;YACtB,MAAM,WAAW,GAAG,MAAM,EAAE;iBACzB,UAAU,CAAC,sBAAsB,CAAC;iBAClC,MAAM,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;iBACjC,KAAK,CAAC,YAAY,EAAE,IAAI,EAAE,SAAS,CAAC;iBACpC,OAAO,EAAE,CAAC;YAEb,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,CAAC,CAAC;YAEvC,yCAAyC;YACzC,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC9B,IAAI,CAAC;oBACH,MAAM,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;gBAAC,MAAM,CAAC;oBACP,iEAAiE;oBACjE,mDAAmD;oBACnD,OAAO,CAAC,IAAI,CACV,yBAAyB,GAAG,CAAC,SAAS,cAAc,GAAG,CAAC,QAAQ,oCAAoC,CACrG,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,gCAAgC;YAChC,MAAM,MAAM,GAAG,MAAM,EAAE;iBACpB,UAAU,CAAC,sBAAsB,CAAC;iBAClC,KAAK,CAAC,YAAY,EAAE,IAAI,EAAE,SAAS,CAAC;iBACpC,gBAAgB,EAAE,CAAC;YAEtB,OAAO,MAAM,CAAC,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC;QAAA,CAC3C;KACF,CAAC;AAAA,CACH"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/snapshot-chunks/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,SAAS,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @syncular/server - Snapshot chunk storage
|
|
3
|
+
*
|
|
4
|
+
* Separates chunk metadata (database) from body content (blob storage).
|
|
5
|
+
*/
|
|
6
|
+
export * from './adapters/s3.js';
|
|
7
|
+
export * from './db-metadata.js';
|
|
8
|
+
export * from './types.js';
|
|
9
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/snapshot-chunks/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,SAAS,CAAC"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @syncular/server - Snapshot chunk storage types
|
|
3
|
+
*
|
|
4
|
+
* Separates chunk metadata (in database) from chunk body (in blob storage).
|
|
5
|
+
* Enables flexible storage backends (database, S3, R2, etc.)
|
|
6
|
+
*/
|
|
7
|
+
import type { SyncSnapshotChunkCompression, SyncSnapshotChunkEncoding, SyncSnapshotChunkRef } from '@syncular/core';
|
|
8
|
+
/**
|
|
9
|
+
* Page key for identifying a specific chunk
|
|
10
|
+
*/
|
|
11
|
+
export interface SnapshotChunkPageKey {
|
|
12
|
+
partitionId: string;
|
|
13
|
+
scopeKey: string;
|
|
14
|
+
scope: string;
|
|
15
|
+
asOfCommitSeq: number;
|
|
16
|
+
rowCursor: string | null;
|
|
17
|
+
rowLimit: number;
|
|
18
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
19
|
+
compression: SyncSnapshotChunkCompression;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Metadata stored in the database for each chunk
|
|
23
|
+
*/
|
|
24
|
+
export interface SnapshotChunkMetadata {
|
|
25
|
+
chunkId: string;
|
|
26
|
+
partitionId: string;
|
|
27
|
+
scopeKey: string;
|
|
28
|
+
scope: string;
|
|
29
|
+
asOfCommitSeq: number;
|
|
30
|
+
rowCursor: string | null;
|
|
31
|
+
rowLimit: number;
|
|
32
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
33
|
+
compression: SyncSnapshotChunkCompression;
|
|
34
|
+
sha256: string;
|
|
35
|
+
byteLength: number;
|
|
36
|
+
blobHash: string;
|
|
37
|
+
expiresAt: string;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Storage interface for snapshot chunks
|
|
41
|
+
*/
|
|
42
|
+
export interface SnapshotChunkStorage {
|
|
43
|
+
/** Storage adapter name */
|
|
44
|
+
readonly name: string;
|
|
45
|
+
/**
|
|
46
|
+
* Store a chunk. Returns chunk reference.
|
|
47
|
+
* If chunk with same content already exists (by hash), returns existing reference.
|
|
48
|
+
*/
|
|
49
|
+
storeChunk(metadata: Omit<SnapshotChunkMetadata, 'chunkId' | 'byteLength' | 'blobHash'> & {
|
|
50
|
+
body: Uint8Array;
|
|
51
|
+
}): Promise<SyncSnapshotChunkRef>;
|
|
52
|
+
/**
|
|
53
|
+
* Store a chunk from a stream.
|
|
54
|
+
* Preferred for large payloads to avoid full buffering in memory.
|
|
55
|
+
*/
|
|
56
|
+
storeChunkStream?(metadata: Omit<SnapshotChunkMetadata, 'chunkId' | 'byteLength' | 'blobHash'> & {
|
|
57
|
+
bodyStream: ReadableStream<Uint8Array>;
|
|
58
|
+
byteLength?: number;
|
|
59
|
+
}): Promise<SyncSnapshotChunkRef>;
|
|
60
|
+
/**
|
|
61
|
+
* Read chunk body by chunk ID
|
|
62
|
+
*/
|
|
63
|
+
readChunk(chunkId: string): Promise<Uint8Array | null>;
|
|
64
|
+
/**
|
|
65
|
+
* Read chunk body as a stream.
|
|
66
|
+
* Preferred for large payloads to avoid full buffering in memory.
|
|
67
|
+
*/
|
|
68
|
+
readChunkStream?(chunkId: string): Promise<ReadableStream<Uint8Array> | null>;
|
|
69
|
+
/**
|
|
70
|
+
* Find existing chunk by page key
|
|
71
|
+
*/
|
|
72
|
+
findChunk(pageKey: SnapshotChunkPageKey): Promise<SyncSnapshotChunkRef | null>;
|
|
73
|
+
/**
|
|
74
|
+
* Delete expired chunks. Returns number deleted.
|
|
75
|
+
*/
|
|
76
|
+
cleanupExpired(beforeIso: string): Promise<number>;
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/snapshot-chunks/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,4BAA4B,EAC5B,yBAAyB,EACzB,oBAAoB,EACrB,MAAM,gBAAgB,CAAC;AAExB;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,WAAW,EAAE,4BAA4B,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,WAAW,EAAE,4BAA4B,CAAC;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,2BAA2B;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,UAAU,CACR,QAAQ,EAAE,IAAI,CACZ,qBAAqB,EACrB,SAAS,GAAG,YAAY,GAAG,UAAU,CACtC,GAAG;QACF,IAAI,EAAE,UAAU,CAAC;KAClB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;;OAGG;IACH,gBAAgB,CAAC,CACf,QAAQ,EAAE,IAAI,CACZ,qBAAqB,EACrB,SAAS,GAAG,YAAY,GAAG,UAAU,CACtC,GAAG;QACF,UAAU,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACvC,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAEvD;;;OAGG;IACH,eAAe,CAAC,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;IAE9E;;OAEG;IACH,SAAS,CACP,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IAExC;;OAEG;IACH,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/snapshot-chunks/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @syncular/server - Encoded snapshot chunk cache (server-side)
|
|
3
|
+
*
|
|
4
|
+
* Used for efficiently serving large bootstrap snapshots (e.g. catalogs)
|
|
5
|
+
* without embedding huge JSON payloads into pull responses.
|
|
6
|
+
*/
|
|
7
|
+
import { type SyncSnapshotChunkCompression, type SyncSnapshotChunkEncoding, type SyncSnapshotChunkRef } from '@syncular/core';
|
|
8
|
+
import { type Kysely } from 'kysely';
|
|
9
|
+
import type { SyncCoreDb } from './schema';
|
|
10
|
+
export interface SnapshotChunkPageKey {
|
|
11
|
+
partitionId: string;
|
|
12
|
+
scopeKey: string;
|
|
13
|
+
scope: string;
|
|
14
|
+
asOfCommitSeq: number;
|
|
15
|
+
rowCursor: string | null;
|
|
16
|
+
rowLimit: number;
|
|
17
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
18
|
+
compression: SyncSnapshotChunkCompression;
|
|
19
|
+
}
|
|
20
|
+
export interface SnapshotChunkRow {
|
|
21
|
+
chunkId: string;
|
|
22
|
+
partitionId: string;
|
|
23
|
+
scopeKey: string;
|
|
24
|
+
scope: string;
|
|
25
|
+
asOfCommitSeq: number;
|
|
26
|
+
rowCursor: string;
|
|
27
|
+
rowLimit: number;
|
|
28
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
29
|
+
compression: SyncSnapshotChunkCompression;
|
|
30
|
+
sha256: string;
|
|
31
|
+
byteLength: number;
|
|
32
|
+
body: Uint8Array | ReadableStream<Uint8Array>;
|
|
33
|
+
expiresAt: string;
|
|
34
|
+
}
|
|
35
|
+
export declare function readSnapshotChunkRefByPageKey<DB extends SyncCoreDb>(db: Kysely<DB>, args: SnapshotChunkPageKey & {
|
|
36
|
+
nowIso?: string;
|
|
37
|
+
}): Promise<SyncSnapshotChunkRef | null>;
|
|
38
|
+
export declare function insertSnapshotChunk<DB extends SyncCoreDb>(db: Kysely<DB>, args: {
|
|
39
|
+
chunkId: string;
|
|
40
|
+
partitionId: string;
|
|
41
|
+
scopeKey: string;
|
|
42
|
+
scope: string;
|
|
43
|
+
asOfCommitSeq: number;
|
|
44
|
+
rowCursor: string | null;
|
|
45
|
+
rowLimit: number;
|
|
46
|
+
encoding: SyncSnapshotChunkEncoding;
|
|
47
|
+
compression: SyncSnapshotChunkCompression;
|
|
48
|
+
sha256: string;
|
|
49
|
+
body: Uint8Array;
|
|
50
|
+
expiresAt: string;
|
|
51
|
+
}): Promise<SyncSnapshotChunkRef>;
|
|
52
|
+
export declare function readSnapshotChunk<DB extends SyncCoreDb>(db: Kysely<DB>, chunkId: string, options?: {
|
|
53
|
+
/** External chunk storage for reading from S3/R2/etc */
|
|
54
|
+
chunkStorage?: {
|
|
55
|
+
readChunk(chunkId: string): Promise<Uint8Array | null>;
|
|
56
|
+
readChunkStream?(chunkId: string): Promise<ReadableStream<Uint8Array> | null>;
|
|
57
|
+
};
|
|
58
|
+
}): Promise<SnapshotChunkRow | null>;
|
|
59
|
+
export declare function deleteExpiredSnapshotChunks<DB extends SyncCoreDb>(db: Kysely<DB>, nowIso?: string): Promise<number>;
|
|
60
|
+
//# sourceMappingURL=snapshot-chunks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"snapshot-chunks.d.ts","sourceRoot":"","sources":["../src/snapshot-chunks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAGL,KAAK,4BAA4B,EACjC,KAAK,yBAAyB,EAC9B,KAAK,oBAAoB,EAC1B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,KAAK,MAAM,EAAO,MAAM,QAAQ,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,WAAW,EAAE,4BAA4B,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,WAAW,EAAE,4BAA4B,CAAC;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB;AAqBD,wBAAsB,6BAA6B,CAAC,EAAE,SAAS,UAAU,EACvE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,IAAI,EAAE,oBAAoB,GAAG;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/C,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CA+CtC;AAED,wBAAsB,mBAAmB,CAAC,EAAE,SAAS,UAAU,EAC7D,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,IAAI,EAAE;IACJ,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,WAAW,EAAE,4BAA4B,CAAC;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAyE/B;AAED,wBAAsB,iBAAiB,CAAC,EAAE,SAAS,UAAU,EAC3D,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;IACR,wDAAwD;IACxD,YAAY,CAAC,EAAE;QACb,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;QACvD,eAAe,CAAC,CACd,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;KAC/C,CAAC;CACH,GACA,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAkGlC;AAED,wBAAsB,2BAA2B,CAAC,EAAE,SAAS,UAAU,EACrE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,MAAM,SAA2B,GAChC,OAAO,CAAC,MAAM,CAAC,CAOjB"}
|