@syncular/server 0.0.4-26 → 0.0.4-33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobs/index.d.ts +0 -2
- package/dist/blobs/index.d.ts.map +1 -1
- package/dist/blobs/index.js +0 -2
- package/dist/blobs/index.js.map +1 -1
- package/dist/notify.js +2 -2
- package/dist/notify.js.map +1 -1
- package/dist/proxy/oplog.d.ts +1 -1
- package/dist/proxy/oplog.d.ts.map +1 -1
- package/dist/proxy/oplog.js +2 -8
- package/dist/proxy/oplog.js.map +1 -1
- package/dist/pull.d.ts.map +1 -1
- package/dist/pull.js +10 -58
- package/dist/pull.js.map +1 -1
- package/dist/snapshot-chunks/db-metadata.d.ts.map +1 -1
- package/dist/snapshot-chunks/db-metadata.js +6 -9
- package/dist/snapshot-chunks/db-metadata.js.map +1 -1
- package/package.json +2 -2
- package/src/blobs/index.ts +0 -2
- package/src/notify.ts +2 -2
- package/src/proxy/oplog.ts +2 -10
- package/src/pull.ts +15 -81
- package/src/snapshot-chunks/db-metadata.ts +10 -9
- package/dist/blobs/adapters/filesystem.d.ts +0 -31
- package/dist/blobs/adapters/filesystem.d.ts.map +0 -1
- package/dist/blobs/adapters/filesystem.js +0 -140
- package/dist/blobs/adapters/filesystem.js.map +0 -1
- package/dist/blobs/adapters/s3.d.ts +0 -83
- package/dist/blobs/adapters/s3.d.ts.map +0 -1
- package/dist/blobs/adapters/s3.js +0 -219
- package/dist/blobs/adapters/s3.js.map +0 -1
- package/src/blobs/adapters/filesystem.test.ts +0 -132
- package/src/blobs/adapters/filesystem.ts +0 -189
- package/src/blobs/adapters/s3.test.ts +0 -522
- package/src/blobs/adapters/s3.ts +0 -324
package/src/pull.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
-
import { promisify } from 'node:util';
|
|
3
|
-
import { gzip, gzipSync } from 'node:zlib';
|
|
4
1
|
import {
|
|
5
2
|
captureSyncException,
|
|
6
3
|
countSyncMetric,
|
|
7
4
|
distributionSyncMetric,
|
|
8
5
|
encodeSnapshotRowFrames,
|
|
9
6
|
encodeSnapshotRows,
|
|
7
|
+
gzipBytes,
|
|
8
|
+
gzipBytesToStream,
|
|
9
|
+
randomId,
|
|
10
10
|
type ScopeValues,
|
|
11
11
|
SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
12
12
|
SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
type SyncPullResponse,
|
|
18
18
|
type SyncPullSubscriptionResponse,
|
|
19
19
|
type SyncSnapshot,
|
|
20
|
+
sha256Hex,
|
|
20
21
|
startSyncSpan,
|
|
21
22
|
} from '@syncular/core';
|
|
22
23
|
import type { Kysely } from 'kysely';
|
|
@@ -31,9 +32,6 @@ import {
|
|
|
31
32
|
import type { SnapshotChunkStorage } from './snapshot-chunks/types';
|
|
32
33
|
import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve';
|
|
33
34
|
|
|
34
|
-
const gzipAsync = promisify(gzip);
|
|
35
|
-
const ASYNC_GZIP_MIN_BYTES = 64 * 1024;
|
|
36
|
-
|
|
37
35
|
function concatByteChunks(chunks: readonly Uint8Array[]): Uint8Array {
|
|
38
36
|
if (chunks.length === 1) {
|
|
39
37
|
return chunks[0] ?? new Uint8Array();
|
|
@@ -53,62 +51,6 @@ function concatByteChunks(chunks: readonly Uint8Array[]): Uint8Array {
|
|
|
53
51
|
return merged;
|
|
54
52
|
}
|
|
55
53
|
|
|
56
|
-
function bytesToReadableStream(bytes: Uint8Array): ReadableStream<Uint8Array> {
|
|
57
|
-
return new ReadableStream<Uint8Array>({
|
|
58
|
-
start(controller) {
|
|
59
|
-
controller.enqueue(bytes);
|
|
60
|
-
controller.close();
|
|
61
|
-
},
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function chunksToReadableStream(
|
|
66
|
-
chunks: readonly Uint8Array[]
|
|
67
|
-
): ReadableStream<Uint8Array> {
|
|
68
|
-
return new ReadableStream<Uint8Array>({
|
|
69
|
-
start(controller) {
|
|
70
|
-
for (const chunk of chunks) {
|
|
71
|
-
controller.enqueue(chunk);
|
|
72
|
-
}
|
|
73
|
-
controller.close();
|
|
74
|
-
},
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
async function compressSnapshotPayload(
|
|
79
|
-
payload: Uint8Array
|
|
80
|
-
): Promise<Uint8Array> {
|
|
81
|
-
if (payload.byteLength < ASYNC_GZIP_MIN_BYTES) {
|
|
82
|
-
return new Uint8Array(gzipSync(payload));
|
|
83
|
-
}
|
|
84
|
-
const compressed = await gzipAsync(payload);
|
|
85
|
-
return new Uint8Array(compressed);
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
async function compressSnapshotPayloadStream(
|
|
89
|
-
chunks: readonly Uint8Array[]
|
|
90
|
-
): Promise<{
|
|
91
|
-
stream: ReadableStream<Uint8Array>;
|
|
92
|
-
byteLength?: number;
|
|
93
|
-
}> {
|
|
94
|
-
if (typeof CompressionStream !== 'undefined') {
|
|
95
|
-
const source = chunksToReadableStream(chunks);
|
|
96
|
-
const gzipStream = new CompressionStream(
|
|
97
|
-
'gzip'
|
|
98
|
-
) as unknown as TransformStream<Uint8Array, Uint8Array>;
|
|
99
|
-
return {
|
|
100
|
-
stream: source.pipeThrough(gzipStream),
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
const payload = concatByteChunks(chunks);
|
|
105
|
-
const compressed = await compressSnapshotPayload(payload);
|
|
106
|
-
return {
|
|
107
|
-
stream: bytesToReadableStream(compressed),
|
|
108
|
-
byteLength: compressed.length,
|
|
109
|
-
};
|
|
110
|
-
}
|
|
111
|
-
|
|
112
54
|
export interface PullResult {
|
|
113
55
|
response: SyncPullResponse;
|
|
114
56
|
/**
|
|
@@ -123,7 +65,7 @@ export interface PullResult {
|
|
|
123
65
|
/**
|
|
124
66
|
* Generate a stable cache key for snapshot chunks.
|
|
125
67
|
*/
|
|
126
|
-
function scopesToCacheKey(scopes: ScopeValues): string {
|
|
68
|
+
async function scopesToCacheKey(scopes: ScopeValues): Promise<string> {
|
|
127
69
|
const sorted = Object.entries(scopes)
|
|
128
70
|
.sort(([a], [b]) => a.localeCompare(b))
|
|
129
71
|
.map(([k, v]) => {
|
|
@@ -131,7 +73,7 @@ function scopesToCacheKey(scopes: ScopeValues): string {
|
|
|
131
73
|
return `${k}:${arr.join(',')}`;
|
|
132
74
|
})
|
|
133
75
|
.join('|');
|
|
134
|
-
return
|
|
76
|
+
return await sha256Hex(sorted);
|
|
135
77
|
}
|
|
136
78
|
|
|
137
79
|
/**
|
|
@@ -482,7 +424,7 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
482
424
|
|
|
483
425
|
const snapshots: SyncSnapshot[] = [];
|
|
484
426
|
let nextState: SyncBootstrapState | null = effectiveState;
|
|
485
|
-
const cacheKey = `${partitionId}:${scopesToCacheKey(effectiveScopes)}`;
|
|
427
|
+
const cacheKey = `${partitionId}:${await scopesToCacheKey(effectiveScopes)}`;
|
|
486
428
|
|
|
487
429
|
interface SnapshotBundle {
|
|
488
430
|
table: string;
|
|
@@ -491,7 +433,6 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
491
433
|
isLastPage: boolean;
|
|
492
434
|
pageCount: number;
|
|
493
435
|
ttlMs: number;
|
|
494
|
-
hash: ReturnType<typeof createHash>;
|
|
495
436
|
rowFrameParts: Uint8Array[];
|
|
496
437
|
}
|
|
497
438
|
|
|
@@ -518,7 +459,10 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
518
459
|
|
|
519
460
|
let chunkRef = cached;
|
|
520
461
|
if (!chunkRef) {
|
|
521
|
-
const
|
|
462
|
+
const rowFramePayload = concatByteChunks(
|
|
463
|
+
bundle.rowFrameParts
|
|
464
|
+
);
|
|
465
|
+
const sha256 = await sha256Hex(rowFramePayload);
|
|
522
466
|
const expiresAt = new Date(
|
|
523
467
|
Date.now() + Math.max(1000, bundle.ttlMs)
|
|
524
468
|
).toISOString();
|
|
@@ -526,9 +470,7 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
526
470
|
if (args.chunkStorage) {
|
|
527
471
|
if (args.chunkStorage.storeChunkStream) {
|
|
528
472
|
const { stream: bodyStream, byteLength } =
|
|
529
|
-
await
|
|
530
|
-
bundle.rowFrameParts
|
|
531
|
-
);
|
|
473
|
+
await gzipBytesToStream(rowFramePayload);
|
|
532
474
|
chunkRef = await args.chunkStorage.storeChunkStream({
|
|
533
475
|
partitionId,
|
|
534
476
|
scopeKey: cacheKey,
|
|
@@ -544,9 +486,7 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
544
486
|
expiresAt,
|
|
545
487
|
});
|
|
546
488
|
} else {
|
|
547
|
-
const compressedBody = await
|
|
548
|
-
concatByteChunks(bundle.rowFrameParts)
|
|
549
|
-
);
|
|
489
|
+
const compressedBody = await gzipBytes(rowFramePayload);
|
|
550
490
|
chunkRef = await args.chunkStorage.storeChunk({
|
|
551
491
|
partitionId,
|
|
552
492
|
scopeKey: cacheKey,
|
|
@@ -562,10 +502,8 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
562
502
|
});
|
|
563
503
|
}
|
|
564
504
|
} else {
|
|
565
|
-
const compressedBody = await
|
|
566
|
-
|
|
567
|
-
);
|
|
568
|
-
const chunkId = randomUUID();
|
|
505
|
+
const compressedBody = await gzipBytes(rowFramePayload);
|
|
506
|
+
const chunkId = randomId();
|
|
569
507
|
chunkRef = await insertSnapshotChunk(trx, {
|
|
570
508
|
chunkId,
|
|
571
509
|
partitionId,
|
|
@@ -617,9 +555,7 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
617
555
|
if (activeBundle) {
|
|
618
556
|
await flushSnapshotBundle(activeBundle);
|
|
619
557
|
}
|
|
620
|
-
const bundleHash = createHash('sha256');
|
|
621
558
|
const bundleHeader = encodeSnapshotRows([]);
|
|
622
|
-
bundleHash.update(bundleHeader);
|
|
623
559
|
activeBundle = {
|
|
624
560
|
table: nextTableName,
|
|
625
561
|
startCursor: nextState.rowCursor,
|
|
@@ -628,7 +564,6 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
628
564
|
pageCount: 0,
|
|
629
565
|
ttlMs:
|
|
630
566
|
tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000,
|
|
631
|
-
hash: bundleHash,
|
|
632
567
|
rowFrameParts: [bundleHeader],
|
|
633
568
|
};
|
|
634
569
|
}
|
|
@@ -645,7 +580,6 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
645
580
|
);
|
|
646
581
|
|
|
647
582
|
const rowFrames = encodeSnapshotRowFrames(page.rows ?? []);
|
|
648
|
-
activeBundle.hash.update(rowFrames);
|
|
649
583
|
activeBundle.rowFrameParts.push(rowFrames);
|
|
650
584
|
activeBundle.pageCount += 1;
|
|
651
585
|
|
|
@@ -5,14 +5,15 @@
|
|
|
5
5
|
* body content in blob storage adapter.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import { createHash } from 'node:crypto';
|
|
9
8
|
import {
|
|
10
9
|
type BlobStorageAdapter,
|
|
10
|
+
randomId,
|
|
11
11
|
SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
12
12
|
SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
13
13
|
type SyncSnapshotChunkCompression,
|
|
14
14
|
type SyncSnapshotChunkEncoding,
|
|
15
15
|
type SyncSnapshotChunkRef,
|
|
16
|
+
sha256Hex,
|
|
16
17
|
} from '@syncular/core';
|
|
17
18
|
import type { Kysely } from 'kysely';
|
|
18
19
|
import type { SyncCoreDb } from '../schema';
|
|
@@ -65,14 +66,14 @@ export function createDbMetadataChunkStorage(
|
|
|
65
66
|
const { db, blobAdapter, chunkIdPrefix = 'chunk_' } = options;
|
|
66
67
|
|
|
67
68
|
// Generate deterministic blob hash from chunk identity metadata.
|
|
68
|
-
function computeBlobHash(metadata: {
|
|
69
|
+
async function computeBlobHash(metadata: {
|
|
69
70
|
encoding: SyncSnapshotChunkEncoding;
|
|
70
71
|
compression: SyncSnapshotChunkCompression;
|
|
71
72
|
sha256: string;
|
|
72
|
-
}): string {
|
|
73
|
-
const digest =
|
|
74
|
-
|
|
75
|
-
|
|
73
|
+
}): Promise<string> {
|
|
74
|
+
const digest = await sha256Hex(
|
|
75
|
+
`${metadata.encoding}:${metadata.compression}:${metadata.sha256}`
|
|
76
|
+
);
|
|
76
77
|
return `sha256:${digest}`;
|
|
77
78
|
}
|
|
78
79
|
|
|
@@ -133,7 +134,7 @@ export function createDbMetadataChunkStorage(
|
|
|
133
134
|
|
|
134
135
|
// Generate unique chunk ID
|
|
135
136
|
function generateChunkId(): string {
|
|
136
|
-
return `${chunkIdPrefix}${
|
|
137
|
+
return `${chunkIdPrefix}${randomId()}`;
|
|
137
138
|
}
|
|
138
139
|
|
|
139
140
|
async function readStoredRef(args: {
|
|
@@ -280,7 +281,7 @@ export function createDbMetadataChunkStorage(
|
|
|
280
281
|
}
|
|
281
282
|
): Promise<SyncSnapshotChunkRef> {
|
|
282
283
|
const { body, ...metaWithoutBody } = metadata;
|
|
283
|
-
const blobHash = computeBlobHash(metaWithoutBody);
|
|
284
|
+
const blobHash = await computeBlobHash(metaWithoutBody);
|
|
284
285
|
|
|
285
286
|
// Check if blob already exists (content-addressed dedup)
|
|
286
287
|
const blobExists = await blobAdapter.exists(blobHash);
|
|
@@ -340,7 +341,7 @@ export function createDbMetadataChunkStorage(
|
|
|
340
341
|
}
|
|
341
342
|
): Promise<SyncSnapshotChunkRef> {
|
|
342
343
|
const { bodyStream, byteLength, ...metaWithoutBody } = metadata;
|
|
343
|
-
const blobHash = computeBlobHash(metaWithoutBody);
|
|
344
|
+
const blobHash = await computeBlobHash(metaWithoutBody);
|
|
344
345
|
|
|
345
346
|
const blobExists = await blobAdapter.exists(blobHash);
|
|
346
347
|
let observedByteLength: number;
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Filesystem blob storage adapter.
|
|
3
|
-
*
|
|
4
|
-
* Stores blobs as files on disk with 2-level hash-based subdirectories.
|
|
5
|
-
* Uploads/downloads go through the server's blob routes using signed tokens
|
|
6
|
-
* (same pattern as the database adapter).
|
|
7
|
-
*/
|
|
8
|
-
import type { BlobStorageAdapter } from '@syncular/core';
|
|
9
|
-
import type { BlobTokenSigner } from './database';
|
|
10
|
-
export interface FilesystemBlobStorageAdapterOptions {
|
|
11
|
-
/** Directory root for blob files */
|
|
12
|
-
basePath: string;
|
|
13
|
-
/** Server base URL for upload/download routes (e.g. "/api/sync") */
|
|
14
|
-
baseUrl: string;
|
|
15
|
-
/** Token signer for authorization */
|
|
16
|
-
tokenSigner: BlobTokenSigner;
|
|
17
|
-
}
|
|
18
|
-
/**
|
|
19
|
-
* Create a filesystem blob storage adapter.
|
|
20
|
-
*
|
|
21
|
-
* @example
|
|
22
|
-
* ```typescript
|
|
23
|
-
* const adapter = createFilesystemBlobStorageAdapter({
|
|
24
|
-
* basePath: '/data/blobs',
|
|
25
|
-
* baseUrl: 'https://api.example.com/api/sync',
|
|
26
|
-
* tokenSigner: createHmacTokenSigner(process.env.BLOB_SECRET!),
|
|
27
|
-
* });
|
|
28
|
-
* ```
|
|
29
|
-
*/
|
|
30
|
-
export declare function createFilesystemBlobStorageAdapter(options: FilesystemBlobStorageAdapterOptions): BlobStorageAdapter;
|
|
31
|
-
//# sourceMappingURL=filesystem.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"filesystem.d.ts","sourceRoot":"","sources":["../../../src/blobs/adapters/filesystem.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAYH,OAAO,KAAK,EAIV,kBAAkB,EACnB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,MAAM,WAAW,mCAAmC;IAClD,oCAAoC;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,WAAW,EAAE,eAAe,CAAC;CAC9B;AAeD;;;;;;;;;;;GAWG;AACH,wBAAgB,kCAAkC,CAChD,OAAO,EAAE,mCAAmC,GAC3C,kBAAkB,CA8HpB"}
|
|
@@ -1,140 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Filesystem blob storage adapter.
|
|
3
|
-
*
|
|
4
|
-
* Stores blobs as files on disk with 2-level hash-based subdirectories.
|
|
5
|
-
* Uploads/downloads go through the server's blob routes using signed tokens
|
|
6
|
-
* (same pattern as the database adapter).
|
|
7
|
-
*/
|
|
8
|
-
import { mkdir, open, readFile, rename, stat, unlink, writeFile, } from 'node:fs/promises';
|
|
9
|
-
import { dirname, join } from 'node:path';
|
|
10
|
-
/**
|
|
11
|
-
* Resolve hash to a 2-level subdirectory path:
|
|
12
|
-
* `{basePath}/{hex[0..2]}/{hex[2..4]}/{hex}`
|
|
13
|
-
*/
|
|
14
|
-
function hashToFilePath(basePath, hash) {
|
|
15
|
-
const hex = hash.startsWith('sha256:') ? hash.slice(7) : hash;
|
|
16
|
-
return join(basePath, hex.slice(0, 2), hex.slice(2, 4), hex);
|
|
17
|
-
}
|
|
18
|
-
function tmpPath(filePath) {
|
|
19
|
-
return `${filePath}.${Date.now()}.tmp`;
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* Create a filesystem blob storage adapter.
|
|
23
|
-
*
|
|
24
|
-
* @example
|
|
25
|
-
* ```typescript
|
|
26
|
-
* const adapter = createFilesystemBlobStorageAdapter({
|
|
27
|
-
* basePath: '/data/blobs',
|
|
28
|
-
* baseUrl: 'https://api.example.com/api/sync',
|
|
29
|
-
* tokenSigner: createHmacTokenSigner(process.env.BLOB_SECRET!),
|
|
30
|
-
* });
|
|
31
|
-
* ```
|
|
32
|
-
*/
|
|
33
|
-
export function createFilesystemBlobStorageAdapter(options) {
|
|
34
|
-
const { basePath, tokenSigner } = options;
|
|
35
|
-
const normalizedBaseUrl = options.baseUrl.replace(/\/$/, '');
|
|
36
|
-
return {
|
|
37
|
-
name: 'filesystem',
|
|
38
|
-
async signUpload(opts) {
|
|
39
|
-
const expiresAt = Date.now() + opts.expiresIn * 1000;
|
|
40
|
-
const token = await tokenSigner.sign({ hash: opts.hash, action: 'upload', expiresAt }, opts.expiresIn);
|
|
41
|
-
const url = `${normalizedBaseUrl}/blobs/${encodeURIComponent(opts.hash)}/upload?token=${encodeURIComponent(token)}`;
|
|
42
|
-
return {
|
|
43
|
-
url,
|
|
44
|
-
method: 'PUT',
|
|
45
|
-
headers: {
|
|
46
|
-
'Content-Type': opts.mimeType,
|
|
47
|
-
'Content-Length': String(opts.size),
|
|
48
|
-
},
|
|
49
|
-
};
|
|
50
|
-
},
|
|
51
|
-
async signDownload(opts) {
|
|
52
|
-
const expiresAt = Date.now() + opts.expiresIn * 1000;
|
|
53
|
-
const token = await tokenSigner.sign({ hash: opts.hash, action: 'download', expiresAt }, opts.expiresIn);
|
|
54
|
-
return `${normalizedBaseUrl}/blobs/${encodeURIComponent(opts.hash)}/download?token=${encodeURIComponent(token)}`;
|
|
55
|
-
},
|
|
56
|
-
async exists(hash) {
|
|
57
|
-
try {
|
|
58
|
-
await stat(hashToFilePath(basePath, hash));
|
|
59
|
-
return true;
|
|
60
|
-
}
|
|
61
|
-
catch {
|
|
62
|
-
return false;
|
|
63
|
-
}
|
|
64
|
-
},
|
|
65
|
-
async delete(hash) {
|
|
66
|
-
try {
|
|
67
|
-
await unlink(hashToFilePath(basePath, hash));
|
|
68
|
-
}
|
|
69
|
-
catch (err) {
|
|
70
|
-
if (err.code !== 'ENOENT')
|
|
71
|
-
throw err;
|
|
72
|
-
}
|
|
73
|
-
},
|
|
74
|
-
async getMetadata(hash) {
|
|
75
|
-
try {
|
|
76
|
-
const s = await stat(hashToFilePath(basePath, hash));
|
|
77
|
-
return { size: s.size };
|
|
78
|
-
}
|
|
79
|
-
catch {
|
|
80
|
-
return null;
|
|
81
|
-
}
|
|
82
|
-
},
|
|
83
|
-
async put(hash, data) {
|
|
84
|
-
const filePath = hashToFilePath(basePath, hash);
|
|
85
|
-
const tmp = tmpPath(filePath);
|
|
86
|
-
await mkdir(dirname(filePath), { recursive: true });
|
|
87
|
-
await writeFile(tmp, data);
|
|
88
|
-
await rename(tmp, filePath);
|
|
89
|
-
},
|
|
90
|
-
async putStream(hash, stream) {
|
|
91
|
-
const filePath = hashToFilePath(basePath, hash);
|
|
92
|
-
const tmp = tmpPath(filePath);
|
|
93
|
-
await mkdir(dirname(filePath), { recursive: true });
|
|
94
|
-
const fh = await open(tmp, 'w');
|
|
95
|
-
try {
|
|
96
|
-
const reader = stream.getReader();
|
|
97
|
-
while (true) {
|
|
98
|
-
const { done, value } = await reader.read();
|
|
99
|
-
if (done)
|
|
100
|
-
break;
|
|
101
|
-
await fh.write(value);
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
finally {
|
|
105
|
-
await fh.close();
|
|
106
|
-
}
|
|
107
|
-
await rename(tmp, filePath);
|
|
108
|
-
},
|
|
109
|
-
async get(hash) {
|
|
110
|
-
try {
|
|
111
|
-
const buf = await readFile(hashToFilePath(basePath, hash));
|
|
112
|
-
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
113
|
-
}
|
|
114
|
-
catch (err) {
|
|
115
|
-
if (err.code === 'ENOENT')
|
|
116
|
-
return null;
|
|
117
|
-
throw err;
|
|
118
|
-
}
|
|
119
|
-
},
|
|
120
|
-
async getStream(hash) {
|
|
121
|
-
let data;
|
|
122
|
-
try {
|
|
123
|
-
data = await readFile(hashToFilePath(basePath, hash));
|
|
124
|
-
}
|
|
125
|
-
catch (err) {
|
|
126
|
-
if (err.code === 'ENOENT')
|
|
127
|
-
return null;
|
|
128
|
-
throw err;
|
|
129
|
-
}
|
|
130
|
-
const bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
131
|
-
return new ReadableStream({
|
|
132
|
-
start(controller) {
|
|
133
|
-
controller.enqueue(bytes);
|
|
134
|
-
controller.close();
|
|
135
|
-
},
|
|
136
|
-
});
|
|
137
|
-
},
|
|
138
|
-
};
|
|
139
|
-
}
|
|
140
|
-
//# sourceMappingURL=filesystem.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"filesystem.js","sourceRoot":"","sources":["../../../src/blobs/adapters/filesystem.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,KAAK,EACL,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,MAAM,EACN,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAkB1C;;;GAGG;AACH,SAAS,cAAc,CAAC,QAAgB,EAAE,IAAY,EAAU;IAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,OAAO,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAAA,CAC9D;AAED,SAAS,OAAO,CAAC,QAAgB,EAAU;IACzC,OAAO,GAAG,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC;AAAA,CACxC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,kCAAkC,CAChD,OAA4C,EACxB;IACpB,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IAC1C,MAAM,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAE7D,OAAO;QACL,IAAI,EAAE,YAAY;QAElB,KAAK,CAAC,UAAU,CAAC,IAA2B,EAA6B;YACvE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,CAClC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,EAChD,IAAI,CAAC,SAAS,CACf,CAAC;YAEF,MAAM,GAAG,GAAG,GAAG,iBAAiB,UAAU,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;YAEpH,OAAO;gBACL,GAAG;gBACH,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE;oBACP,cAAc,EAAE,IAAI,CAAC,QAAQ;oBAC7B,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;iBACpC;aACF,CAAC;QAAA,CACH;QAED,KAAK,CAAC,YAAY,CAAC,IAA6B,EAAmB;YACjE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,CAClC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,EAClD,IAAI,CAAC,SAAS,CACf,CAAC;YAEF,OAAO,GAAG,iBAAiB,UAAU,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;QAAA,CAClH;QAED,KAAK,CAAC,MAAM,CAAC,IAAY,EAAoB;YAC3C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC3C,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QAAA,CACF;QAED,KAAK,CAAC,MAAM,CAAC,IAAY,EAAiB;YACxC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;oBAAE,MAAM,GAAG,CAAC;YAClE,CAAC;QAAA,CACF;QAED,KAAK,CAAC,WAAW,CACf,IAAY,EACyC;YACrD,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;gBACrD,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QAAA,CACF;QAED,KAAK,CAAC,GAAG,CAAC,IAAY,EAAE,IAAgB,EAAiB;YACvD,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAChD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC9B,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACpD,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAC3B,MAAM,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAAA,CAC7B;QAED,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,MAAkC,EACnB;YACf,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAChD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC9B,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEpD,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;gBAClC,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAChB,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,CAAC;YAED,MAAM,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAAA,CAC7B;QAED,KAAK,CAAC,GAAG,CAAC,IAAY,EAA8B;YAClD,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC3D,OAAO,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;oBAAE,OAAO,IAAI,CAAC;gBAClE,MAAM,GAAG,CAAC;YACZ,CAAC;QAAA,CACF;QAED,KAAK,CAAC,SAAS,CAAC,IAAY,EAA8C;YACxE,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;YACxD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;oBAAE,OAAO,IAAI,CAAC;gBAClE,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,UAAU,CAC1B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,OAAO,IAAI,cAAc,CAAa;gBACpC,KAAK,CAAC,UAAU,EAAE;oBAChB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBAC1B,UAAU,CAAC,KAAK,EAAE,CAAC;gBAAA,CACpB;aACF,CAAC,CAAC;QAAA,CACJ;KACF,CAAC;AAAA,CACH"}
|
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* S3-compatible blob storage adapter.
|
|
3
|
-
*
|
|
4
|
-
* Works with AWS S3, Cloudflare R2, MinIO, and other S3-compatible services.
|
|
5
|
-
* Requires @aws-sdk/client-s3 and @aws-sdk/s3-request-presigner as peer dependencies.
|
|
6
|
-
*/
|
|
7
|
-
import type { BlobStorageAdapter } from '@syncular/core';
|
|
8
|
-
/**
|
|
9
|
-
* S3 client interface (minimal subset of @aws-sdk/client-s3).
|
|
10
|
-
* This allows users to pass in their own configured S3 client.
|
|
11
|
-
*/
|
|
12
|
-
export interface S3ClientLike {
|
|
13
|
-
send(command: unknown): Promise<unknown>;
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* Function to create presigned URLs.
|
|
17
|
-
* This should be getSignedUrl from @aws-sdk/s3-request-presigner.
|
|
18
|
-
*/
|
|
19
|
-
export type GetSignedUrlFn = (client: S3ClientLike, command: unknown, options: {
|
|
20
|
-
expiresIn: number;
|
|
21
|
-
}) => Promise<string>;
|
|
22
|
-
/**
|
|
23
|
-
* S3 command constructors.
|
|
24
|
-
* These should be imported from @aws-sdk/client-s3.
|
|
25
|
-
*/
|
|
26
|
-
export interface S3Commands {
|
|
27
|
-
PutObjectCommand: new (input: {
|
|
28
|
-
Bucket: string;
|
|
29
|
-
Key: string;
|
|
30
|
-
ContentLength?: number;
|
|
31
|
-
ContentType?: string;
|
|
32
|
-
ChecksumSHA256?: string;
|
|
33
|
-
Body?: Uint8Array | ReadableStream<Uint8Array>;
|
|
34
|
-
}) => unknown;
|
|
35
|
-
GetObjectCommand: new (input: {
|
|
36
|
-
Bucket: string;
|
|
37
|
-
Key: string;
|
|
38
|
-
}) => unknown;
|
|
39
|
-
HeadObjectCommand: new (input: {
|
|
40
|
-
Bucket: string;
|
|
41
|
-
Key: string;
|
|
42
|
-
}) => unknown;
|
|
43
|
-
DeleteObjectCommand: new (input: {
|
|
44
|
-
Bucket: string;
|
|
45
|
-
Key: string;
|
|
46
|
-
}) => unknown;
|
|
47
|
-
}
|
|
48
|
-
export interface S3BlobStorageAdapterOptions {
|
|
49
|
-
/** S3 client instance */
|
|
50
|
-
client: S3ClientLike;
|
|
51
|
-
/** S3 bucket name */
|
|
52
|
-
bucket: string;
|
|
53
|
-
/** Optional key prefix for all blobs */
|
|
54
|
-
keyPrefix?: string;
|
|
55
|
-
/** S3 command constructors */
|
|
56
|
-
commands: S3Commands;
|
|
57
|
-
/** getSignedUrl function from @aws-sdk/s3-request-presigner */
|
|
58
|
-
getSignedUrl: GetSignedUrlFn;
|
|
59
|
-
/**
|
|
60
|
-
* Whether to require SHA-256 checksum validation on upload.
|
|
61
|
-
* Supported by S3 and R2. Default: true.
|
|
62
|
-
*/
|
|
63
|
-
requireChecksum?: boolean;
|
|
64
|
-
}
|
|
65
|
-
/**
|
|
66
|
-
* Create an S3-compatible blob storage adapter.
|
|
67
|
-
*
|
|
68
|
-
* @example
|
|
69
|
-
* ```typescript
|
|
70
|
-
* import { S3Client, PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
|
|
71
|
-
* import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
|
72
|
-
*
|
|
73
|
-
* const adapter = createS3BlobStorageAdapter({
|
|
74
|
-
* client: new S3Client({ region: 'us-east-1' }),
|
|
75
|
-
* bucket: 'my-bucket',
|
|
76
|
-
* keyPrefix: 'blobs/',
|
|
77
|
-
* commands: { PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand },
|
|
78
|
-
* getSignedUrl,
|
|
79
|
-
* });
|
|
80
|
-
* ```
|
|
81
|
-
*/
|
|
82
|
-
export declare function createS3BlobStorageAdapter(options: S3BlobStorageAdapterOptions): BlobStorageAdapter;
|
|
83
|
-
//# sourceMappingURL=s3.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../../src/blobs/adapters/s3.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAIV,kBAAkB,EACnB,MAAM,gBAAgB,CAAC;AAExB;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC1C;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,KAC3B,OAAO,CAAC,MAAM,CAAC,CAAC;AAErB;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,gBAAgB,EAAE,KAAK,KAAK,EAAE;QAC5B,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAC;QACZ,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;KAChD,KAAK,OAAO,CAAC;IACd,gBAAgB,EAAE,KAAK,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC;IAC1E,iBAAiB,EAAE,KAAK,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC;IAC3E,mBAAmB,EAAE,KAAK,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC;CAC9E;AAED,MAAM,WAAW,2BAA2B;IAC1C,yBAAyB;IACzB,MAAM,EAAE,YAAY,CAAC;IACrB,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,QAAQ,EAAE,UAAU,CAAC;IACrB,+DAA+D;IAC/D,YAAY,EAAE,cAAc,CAAC;IAC7B;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,2BAA2B,GACnC,kBAAkB,CAqLpB"}
|