@syncular/client-plugin-blob 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +62 -0
- package/src/index.test.ts +223 -0
- package/src/index.ts +481 -0
- package/src/migrate.ts +54 -0
- package/src/types.ts +77 -0
package/package.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@syncular/client-plugin-blob",
|
|
3
|
+
"version": "0.0.0",
|
|
4
|
+
"description": "Blob storage plugin for the Syncular client",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"author": "Benjamin Kniffler",
|
|
7
|
+
"homepage": "https://syncular.dev",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "https://github.com/syncular/syncular.git",
|
|
11
|
+
"directory": "plugins/blob/client"
|
|
12
|
+
},
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/syncular/syncular/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"sync",
|
|
18
|
+
"offline-first",
|
|
19
|
+
"realtime",
|
|
20
|
+
"database",
|
|
21
|
+
"typescript",
|
|
22
|
+
"blob",
|
|
23
|
+
"storage"
|
|
24
|
+
],
|
|
25
|
+
"private": false,
|
|
26
|
+
"publishConfig": {
|
|
27
|
+
"access": "public"
|
|
28
|
+
},
|
|
29
|
+
"type": "module",
|
|
30
|
+
"exports": {
|
|
31
|
+
".": {
|
|
32
|
+
"bun": "./src/index.ts",
|
|
33
|
+
"browser": "./src/index.ts",
|
|
34
|
+
"import": {
|
|
35
|
+
"types": "./dist/index.d.ts",
|
|
36
|
+
"default": "./dist/index.js"
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"scripts": {
|
|
41
|
+
"test": "bun test --pass-with-no-tests",
|
|
42
|
+
"tsgo": "tsgo --noEmit",
|
|
43
|
+
"build": "tsgo",
|
|
44
|
+
"release": "bunx syncular-publish"
|
|
45
|
+
},
|
|
46
|
+
"dependencies": {
|
|
47
|
+
"@syncular/client": "0.0.0",
|
|
48
|
+
"@syncular/core": "0.0.0"
|
|
49
|
+
},
|
|
50
|
+
"devDependencies": {
|
|
51
|
+
"@syncular/config": "0.0.0",
|
|
52
|
+
"kysely": "*",
|
|
53
|
+
"kysely-bun-sqlite": "^0.4.0"
|
|
54
|
+
},
|
|
55
|
+
"peerDependencies": {
|
|
56
|
+
"kysely": "*"
|
|
57
|
+
},
|
|
58
|
+
"files": [
|
|
59
|
+
"dist",
|
|
60
|
+
"src"
|
|
61
|
+
]
|
|
62
|
+
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
|
+
import {
|
|
3
|
+
createDatabase,
|
|
4
|
+
type SyncTransport,
|
|
5
|
+
} from '@syncular/core';
|
|
6
|
+
import type { Kysely } from 'kysely';
|
|
7
|
+
import { sql } from 'kysely';
|
|
8
|
+
import { createBunSqliteDialect } from '../../../../packages/dialect-bun-sqlite/src';
|
|
9
|
+
import {
|
|
10
|
+
Client,
|
|
11
|
+
type SyncClientDb,
|
|
12
|
+
} from '../../../../packages/client/src';
|
|
13
|
+
import type { ClientHandlerCollection } from '../../../../packages/client/src/handlers/collection';
|
|
14
|
+
import { ensureClientSyncSchema } from '../../../../packages/client/src/migrate';
|
|
15
|
+
import { createBlobPlugin, ensureClientBlobSchema, type ClientBlobStorage } from './index';
|
|
16
|
+
|
|
17
|
+
interface TasksTable {
|
|
18
|
+
id: string;
|
|
19
|
+
title: string;
|
|
20
|
+
server_version: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
interface TestDb extends SyncClientDb {
|
|
24
|
+
tasks: TasksTable;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const noopTransport: SyncTransport = {
|
|
28
|
+
async sync() {
|
|
29
|
+
return {};
|
|
30
|
+
},
|
|
31
|
+
async fetchSnapshotChunk() {
|
|
32
|
+
return new Uint8Array();
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
function createMemoryBlobStorage(): ClientBlobStorage {
|
|
37
|
+
const memory = new Map<string, Uint8Array>();
|
|
38
|
+
return {
|
|
39
|
+
async write(hash, data) {
|
|
40
|
+
if (data instanceof ReadableStream) {
|
|
41
|
+
const reader = data.getReader();
|
|
42
|
+
const chunks: Uint8Array[] = [];
|
|
43
|
+
let total = 0;
|
|
44
|
+
while (true) {
|
|
45
|
+
const chunk = await reader.read();
|
|
46
|
+
if (chunk.done) break;
|
|
47
|
+
chunks.push(chunk.value);
|
|
48
|
+
total += chunk.value.length;
|
|
49
|
+
}
|
|
50
|
+
const combined = new Uint8Array(total);
|
|
51
|
+
let offset = 0;
|
|
52
|
+
for (const chunk of chunks) {
|
|
53
|
+
combined.set(chunk, offset);
|
|
54
|
+
offset += chunk.length;
|
|
55
|
+
}
|
|
56
|
+
memory.set(hash, combined);
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
memory.set(hash, new Uint8Array(data));
|
|
60
|
+
},
|
|
61
|
+
async read(hash) {
|
|
62
|
+
const data = memory.get(hash);
|
|
63
|
+
return data ? new Uint8Array(data) : null;
|
|
64
|
+
},
|
|
65
|
+
async delete(hash) {
|
|
66
|
+
memory.delete(hash);
|
|
67
|
+
},
|
|
68
|
+
async exists(hash) {
|
|
69
|
+
return memory.has(hash);
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
describe('blob client plugin', () => {
|
|
75
|
+
let db: Kysely<TestDb>;
|
|
76
|
+
let client: Client<TestDb>;
|
|
77
|
+
let initiateCalls = 0;
|
|
78
|
+
|
|
79
|
+
async function insertBlobOutboxRow(input: {
|
|
80
|
+
hash: string;
|
|
81
|
+
status: string;
|
|
82
|
+
attemptCount: number;
|
|
83
|
+
updatedAt: number;
|
|
84
|
+
}): Promise<void> {
|
|
85
|
+
const now = Date.now();
|
|
86
|
+
await sql`
|
|
87
|
+
insert into ${sql.table('sync_blob_outbox')} (
|
|
88
|
+
${sql.join([
|
|
89
|
+
sql.ref('hash'),
|
|
90
|
+
sql.ref('size'),
|
|
91
|
+
sql.ref('mime_type'),
|
|
92
|
+
sql.ref('body'),
|
|
93
|
+
sql.ref('encrypted'),
|
|
94
|
+
sql.ref('key_id'),
|
|
95
|
+
sql.ref('status'),
|
|
96
|
+
sql.ref('attempt_count'),
|
|
97
|
+
sql.ref('error'),
|
|
98
|
+
sql.ref('created_at'),
|
|
99
|
+
sql.ref('updated_at'),
|
|
100
|
+
])}
|
|
101
|
+
) values (
|
|
102
|
+
${sql.join([
|
|
103
|
+
sql.val(input.hash),
|
|
104
|
+
sql.val(3),
|
|
105
|
+
sql.val('application/octet-stream'),
|
|
106
|
+
sql.val(new Uint8Array([1, 2, 3])),
|
|
107
|
+
sql.val(0),
|
|
108
|
+
sql.val(null),
|
|
109
|
+
sql.val(input.status),
|
|
110
|
+
sql.val(input.attemptCount),
|
|
111
|
+
sql.val(null),
|
|
112
|
+
sql.val(now),
|
|
113
|
+
sql.val(input.updatedAt),
|
|
114
|
+
])}
|
|
115
|
+
)
|
|
116
|
+
`.execute(db);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
beforeEach(async () => {
|
|
120
|
+
db = createDatabase<TestDb>({
|
|
121
|
+
dialect: createBunSqliteDialect({ path: ':memory:' }),
|
|
122
|
+
family: 'sqlite',
|
|
123
|
+
});
|
|
124
|
+
await ensureClientSyncSchema(db);
|
|
125
|
+
await ensureClientBlobSchema(db);
|
|
126
|
+
initiateCalls = 0;
|
|
127
|
+
|
|
128
|
+
const handlers: ClientHandlerCollection<TestDb> = [];
|
|
129
|
+
const transport: SyncTransport = {
|
|
130
|
+
...noopTransport,
|
|
131
|
+
blobs: {
|
|
132
|
+
async initiateUpload() {
|
|
133
|
+
initiateCalls++;
|
|
134
|
+
return { exists: true };
|
|
135
|
+
},
|
|
136
|
+
async completeUpload() {
|
|
137
|
+
return { ok: true };
|
|
138
|
+
},
|
|
139
|
+
async getDownloadUrl() {
|
|
140
|
+
return {
|
|
141
|
+
url: 'https://example.invalid/blob',
|
|
142
|
+
expiresAt: new Date(0).toISOString(),
|
|
143
|
+
};
|
|
144
|
+
},
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
client = new Client<TestDb>({
|
|
149
|
+
db,
|
|
150
|
+
transport,
|
|
151
|
+
tableHandlers: handlers,
|
|
152
|
+
clientId: 'client-1',
|
|
153
|
+
actorId: 'u1',
|
|
154
|
+
subscriptions: [],
|
|
155
|
+
plugins: [createBlobPlugin({ storage: createMemoryBlobStorage() })],
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
afterEach(async () => {
|
|
160
|
+
client.destroy();
|
|
161
|
+
await db.destroy();
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it('attaches client.blobs through plugin setup', () => {
|
|
165
|
+
expect(client.blobs).toBeDefined();
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it('requeues stale uploading rows and uploads them on the next queue run', async () => {
|
|
169
|
+
await insertBlobOutboxRow({
|
|
170
|
+
hash: 'sha256:stale-upload',
|
|
171
|
+
status: 'uploading',
|
|
172
|
+
attemptCount: 0,
|
|
173
|
+
updatedAt: Date.now() - 31_000,
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
const result = await client.blobs.processUploadQueue();
|
|
177
|
+
|
|
178
|
+
expect(result.uploaded).toBe(1);
|
|
179
|
+
expect(result.failed).toBe(0);
|
|
180
|
+
expect(initiateCalls).toBe(1);
|
|
181
|
+
|
|
182
|
+
const remaining = await sql<{ count: number | bigint }>`
|
|
183
|
+
select count(${sql.ref('hash')}) as count
|
|
184
|
+
from ${sql.table('sync_blob_outbox')}
|
|
185
|
+
where ${sql.ref('hash')} = ${'sha256:stale-upload'}
|
|
186
|
+
`.execute(db);
|
|
187
|
+
expect(Number(remaining.rows[0]?.count ?? 0)).toBe(0);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
it('marks stale uploading rows as failed after max retries', async () => {
|
|
191
|
+
await insertBlobOutboxRow({
|
|
192
|
+
hash: 'sha256:stale-failed',
|
|
193
|
+
status: 'uploading',
|
|
194
|
+
attemptCount: 2,
|
|
195
|
+
updatedAt: Date.now() - 31_000,
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
await client.blobs.processUploadQueue();
|
|
199
|
+
|
|
200
|
+
expect(initiateCalls).toBe(0);
|
|
201
|
+
|
|
202
|
+
const rowResult = await sql<{
|
|
203
|
+
status: string;
|
|
204
|
+
attempt_count: number;
|
|
205
|
+
error: string | null;
|
|
206
|
+
}>`
|
|
207
|
+
select
|
|
208
|
+
${sql.ref('status')} as status,
|
|
209
|
+
${sql.ref('attempt_count')} as attempt_count,
|
|
210
|
+
${sql.ref('error')} as error
|
|
211
|
+
from ${sql.table('sync_blob_outbox')}
|
|
212
|
+
where ${sql.ref('hash')} = ${'sha256:stale-failed'}
|
|
213
|
+
limit 1
|
|
214
|
+
`.execute(db);
|
|
215
|
+
const row = rowResult.rows[0];
|
|
216
|
+
if (!row) {
|
|
217
|
+
throw new Error('Expected stale failed row to remain in outbox');
|
|
218
|
+
}
|
|
219
|
+
expect(row.status).toBe('failed');
|
|
220
|
+
expect(row.attempt_count).toBe(3);
|
|
221
|
+
expect(row.error).toContain('Upload timed out while in uploading state');
|
|
222
|
+
});
|
|
223
|
+
});
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,481 @@
|
|
|
1
|
+
import type { SyncTransport } from '@syncular/core';
|
|
2
|
+
import type {
|
|
3
|
+
SyncClientDb,
|
|
4
|
+
SyncClientPlugin,
|
|
5
|
+
} from '@syncular/client';
|
|
6
|
+
import { sql } from 'kysely';
|
|
7
|
+
import { ensureClientBlobSchema } from './migrate';
|
|
8
|
+
import type { BlobClient, ClientBlobStorage } from './types';
|
|
9
|
+
|
|
10
|
+
export * from './migrate';
|
|
11
|
+
export * from './types';
|
|
12
|
+
|
|
13
|
+
export const BLOB_PLUGIN_KIND = 'blob';
|
|
14
|
+
|
|
15
|
+
export interface BlobPluginOptions {
|
|
16
|
+
storage: ClientBlobStorage;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
declare module '@syncular/client' {
|
|
20
|
+
interface SyncClientFeatureRegistry {
|
|
21
|
+
blobs: BlobClient;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
declare module 'syncular/client' {
|
|
26
|
+
interface SyncClientFeatureRegistry {
|
|
27
|
+
blobs: BlobClient;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function createBlobPlugin(options: BlobPluginOptions): SyncClientPlugin {
|
|
32
|
+
return {
|
|
33
|
+
kind: BLOB_PLUGIN_KIND,
|
|
34
|
+
name: BLOB_PLUGIN_KIND,
|
|
35
|
+
setup(ctx) {
|
|
36
|
+
if (!ctx.transport.blobs) {
|
|
37
|
+
throw new Error(
|
|
38
|
+
'Blob plugin requires a transport with blob support enabled'
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
ctx.defineFeature(
|
|
43
|
+
'blobs',
|
|
44
|
+
createBlobClient({
|
|
45
|
+
db: ctx.db,
|
|
46
|
+
transport: ctx.transport,
|
|
47
|
+
storage: options.storage,
|
|
48
|
+
emit: ctx.emit,
|
|
49
|
+
})
|
|
50
|
+
);
|
|
51
|
+
},
|
|
52
|
+
async migrate(ctx) {
|
|
53
|
+
await ensureClientBlobSchema(ctx.db);
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function createBlobClient(args: {
|
|
59
|
+
db: import('kysely').Kysely<SyncClientDb>;
|
|
60
|
+
transport: SyncTransport;
|
|
61
|
+
storage: ClientBlobStorage;
|
|
62
|
+
emit: (event: string, payload: object) => void;
|
|
63
|
+
}): BlobClient {
|
|
64
|
+
const { db, storage, transport, emit } = args;
|
|
65
|
+
const blobs = transport.blobs!;
|
|
66
|
+
const staleUploadingTimeoutMs = 30_000;
|
|
67
|
+
const maxUploadRetries = 3;
|
|
68
|
+
|
|
69
|
+
return {
|
|
70
|
+
async store(data, options) {
|
|
71
|
+
const bytes = await toUint8Array(data);
|
|
72
|
+
const mimeType =
|
|
73
|
+
data instanceof Blob
|
|
74
|
+
? data.type
|
|
75
|
+
: (options?.mimeType ?? 'application/octet-stream');
|
|
76
|
+
|
|
77
|
+
const hashHex = await computeSha256Hex(bytes);
|
|
78
|
+
const hash = `sha256:${hashHex}`;
|
|
79
|
+
|
|
80
|
+
await storage.write(hash, bytes);
|
|
81
|
+
|
|
82
|
+
const now = Date.now();
|
|
83
|
+
await sql`
|
|
84
|
+
insert into ${sql.table('sync_blob_cache')} (
|
|
85
|
+
${sql.join([
|
|
86
|
+
sql.ref('hash'),
|
|
87
|
+
sql.ref('size'),
|
|
88
|
+
sql.ref('mime_type'),
|
|
89
|
+
sql.ref('cached_at'),
|
|
90
|
+
sql.ref('last_accessed_at'),
|
|
91
|
+
sql.ref('encrypted'),
|
|
92
|
+
sql.ref('key_id'),
|
|
93
|
+
sql.ref('body'),
|
|
94
|
+
])}
|
|
95
|
+
) values (
|
|
96
|
+
${sql.join([
|
|
97
|
+
sql.val(hash),
|
|
98
|
+
sql.val(bytes.length),
|
|
99
|
+
sql.val(mimeType),
|
|
100
|
+
sql.val(now),
|
|
101
|
+
sql.val(now),
|
|
102
|
+
sql.val(0),
|
|
103
|
+
sql.val(null),
|
|
104
|
+
sql.val(bytes),
|
|
105
|
+
])}
|
|
106
|
+
)
|
|
107
|
+
on conflict (${sql.ref('hash')}) do nothing
|
|
108
|
+
`.execute(db);
|
|
109
|
+
|
|
110
|
+
if (options?.immediate) {
|
|
111
|
+
const initResult = await blobs.initiateUpload({
|
|
112
|
+
hash,
|
|
113
|
+
size: bytes.length,
|
|
114
|
+
mimeType,
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
if (!initResult.exists && initResult.uploadUrl) {
|
|
118
|
+
const uploadResponse = await fetch(initResult.uploadUrl, {
|
|
119
|
+
method: initResult.uploadMethod ?? 'PUT',
|
|
120
|
+
body: bytes.buffer as ArrayBuffer,
|
|
121
|
+
headers: initResult.uploadHeaders,
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
if (!uploadResponse.ok) {
|
|
125
|
+
throw new Error(`Upload failed: ${uploadResponse.statusText}`);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
await blobs.completeUpload(hash);
|
|
129
|
+
}
|
|
130
|
+
} else {
|
|
131
|
+
await sql`
|
|
132
|
+
insert into ${sql.table('sync_blob_outbox')} (
|
|
133
|
+
${sql.join([
|
|
134
|
+
sql.ref('hash'),
|
|
135
|
+
sql.ref('size'),
|
|
136
|
+
sql.ref('mime_type'),
|
|
137
|
+
sql.ref('status'),
|
|
138
|
+
sql.ref('created_at'),
|
|
139
|
+
sql.ref('updated_at'),
|
|
140
|
+
sql.ref('attempt_count'),
|
|
141
|
+
sql.ref('error'),
|
|
142
|
+
sql.ref('encrypted'),
|
|
143
|
+
sql.ref('key_id'),
|
|
144
|
+
sql.ref('body'),
|
|
145
|
+
])}
|
|
146
|
+
) values (
|
|
147
|
+
${sql.join([
|
|
148
|
+
sql.val(hash),
|
|
149
|
+
sql.val(bytes.length),
|
|
150
|
+
sql.val(mimeType),
|
|
151
|
+
sql.val('pending'),
|
|
152
|
+
sql.val(now),
|
|
153
|
+
sql.val(now),
|
|
154
|
+
sql.val(0),
|
|
155
|
+
sql.val(null),
|
|
156
|
+
sql.val(0),
|
|
157
|
+
sql.val(null),
|
|
158
|
+
sql.val(bytes),
|
|
159
|
+
])}
|
|
160
|
+
)
|
|
161
|
+
on conflict (${sql.ref('hash')}) do nothing
|
|
162
|
+
`.execute(db);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
return {
|
|
166
|
+
hash,
|
|
167
|
+
size: bytes.length,
|
|
168
|
+
mimeType,
|
|
169
|
+
};
|
|
170
|
+
},
|
|
171
|
+
|
|
172
|
+
async retrieve(ref) {
|
|
173
|
+
const local = await storage.read(ref.hash);
|
|
174
|
+
if (local) {
|
|
175
|
+
await sql`
|
|
176
|
+
update ${sql.table('sync_blob_cache')}
|
|
177
|
+
set ${sql.ref('last_accessed_at')} = ${sql.val(Date.now())}
|
|
178
|
+
where ${sql.ref('hash')} = ${sql.val(ref.hash)}
|
|
179
|
+
`.execute(db);
|
|
180
|
+
return local;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const { url } = await blobs.getDownloadUrl(ref.hash);
|
|
184
|
+
const response = await fetch(url);
|
|
185
|
+
if (!response.ok) {
|
|
186
|
+
throw new Error(`Download failed: ${response.statusText}`);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
190
|
+
await storage.write(ref.hash, bytes);
|
|
191
|
+
const now = Date.now();
|
|
192
|
+
await sql`
|
|
193
|
+
insert into ${sql.table('sync_blob_cache')} (
|
|
194
|
+
${sql.join([
|
|
195
|
+
sql.ref('hash'),
|
|
196
|
+
sql.ref('size'),
|
|
197
|
+
sql.ref('mime_type'),
|
|
198
|
+
sql.ref('cached_at'),
|
|
199
|
+
sql.ref('last_accessed_at'),
|
|
200
|
+
sql.ref('encrypted'),
|
|
201
|
+
sql.ref('key_id'),
|
|
202
|
+
sql.ref('body'),
|
|
203
|
+
])}
|
|
204
|
+
) values (
|
|
205
|
+
${sql.join([
|
|
206
|
+
sql.val(ref.hash),
|
|
207
|
+
sql.val(bytes.length),
|
|
208
|
+
sql.val(ref.mimeType),
|
|
209
|
+
sql.val(now),
|
|
210
|
+
sql.val(now),
|
|
211
|
+
sql.val(0),
|
|
212
|
+
sql.val(null),
|
|
213
|
+
sql.val(bytes),
|
|
214
|
+
])}
|
|
215
|
+
)
|
|
216
|
+
on conflict (${sql.ref('hash')}) do nothing
|
|
217
|
+
`.execute(db);
|
|
218
|
+
|
|
219
|
+
return bytes;
|
|
220
|
+
},
|
|
221
|
+
|
|
222
|
+
async isLocal(hash) {
|
|
223
|
+
return storage.exists(hash);
|
|
224
|
+
},
|
|
225
|
+
|
|
226
|
+
async preload(refs) {
|
|
227
|
+
await Promise.all(refs.map((ref) => this.retrieve(ref)));
|
|
228
|
+
},
|
|
229
|
+
|
|
230
|
+
async processUploadQueue() {
|
|
231
|
+
let uploaded = 0;
|
|
232
|
+
let failed = 0;
|
|
233
|
+
const now = Date.now();
|
|
234
|
+
const staleThreshold = now - staleUploadingTimeoutMs;
|
|
235
|
+
|
|
236
|
+
await sql`
|
|
237
|
+
update ${sql.table('sync_blob_outbox')}
|
|
238
|
+
set
|
|
239
|
+
${sql.ref('status')} = ${sql.val('failed')},
|
|
240
|
+
${sql.ref('attempt_count')} = ${sql.ref('attempt_count')} + ${sql.val(
|
|
241
|
+
1
|
|
242
|
+
)},
|
|
243
|
+
${sql.ref('error')} = ${sql.val(
|
|
244
|
+
'Upload timed out while in uploading state'
|
|
245
|
+
)},
|
|
246
|
+
${sql.ref('updated_at')} = ${sql.val(now)}
|
|
247
|
+
where ${sql.ref('status')} = ${sql.val('uploading')}
|
|
248
|
+
and ${sql.ref('updated_at')} < ${sql.val(staleThreshold)}
|
|
249
|
+
and ${sql.ref('attempt_count')} + ${sql.val(1)} >= ${sql.val(
|
|
250
|
+
maxUploadRetries
|
|
251
|
+
)}
|
|
252
|
+
`.execute(db);
|
|
253
|
+
|
|
254
|
+
await sql`
|
|
255
|
+
update ${sql.table('sync_blob_outbox')}
|
|
256
|
+
set
|
|
257
|
+
${sql.ref('status')} = ${sql.val('pending')},
|
|
258
|
+
${sql.ref('attempt_count')} = ${sql.ref('attempt_count')} + ${sql.val(
|
|
259
|
+
1
|
|
260
|
+
)},
|
|
261
|
+
${sql.ref('error')} = ${sql.val(
|
|
262
|
+
'Upload timed out while in uploading state; retrying'
|
|
263
|
+
)},
|
|
264
|
+
${sql.ref('updated_at')} = ${sql.val(now)}
|
|
265
|
+
where ${sql.ref('status')} = ${sql.val('uploading')}
|
|
266
|
+
and ${sql.ref('updated_at')} < ${sql.val(staleThreshold)}
|
|
267
|
+
and ${sql.ref('attempt_count')} + ${sql.val(1)} < ${sql.val(
|
|
268
|
+
maxUploadRetries
|
|
269
|
+
)}
|
|
270
|
+
`.execute(db);
|
|
271
|
+
|
|
272
|
+
const pendingResult = await sql<{
|
|
273
|
+
hash: string;
|
|
274
|
+
size: number;
|
|
275
|
+
mime_type: string;
|
|
276
|
+
body: Uint8Array | null;
|
|
277
|
+
attempt_count: number;
|
|
278
|
+
}>`
|
|
279
|
+
select
|
|
280
|
+
${sql.ref('hash')},
|
|
281
|
+
${sql.ref('size')},
|
|
282
|
+
${sql.ref('mime_type')},
|
|
283
|
+
${sql.ref('body')},
|
|
284
|
+
${sql.ref('attempt_count')}
|
|
285
|
+
from ${sql.table('sync_blob_outbox')}
|
|
286
|
+
where ${sql.ref('status')} = ${sql.val('pending')}
|
|
287
|
+
and ${sql.ref('attempt_count')} < ${sql.val(maxUploadRetries)}
|
|
288
|
+
limit ${sql.val(10)}
|
|
289
|
+
`.execute(db);
|
|
290
|
+
const pending = pendingResult.rows;
|
|
291
|
+
|
|
292
|
+
for (const item of pending) {
|
|
293
|
+
const nextAttemptCount = item.attempt_count + 1;
|
|
294
|
+
try {
|
|
295
|
+
await sql`
|
|
296
|
+
update ${sql.table('sync_blob_outbox')}
|
|
297
|
+
set
|
|
298
|
+
${sql.ref('status')} = ${sql.val('uploading')},
|
|
299
|
+
${sql.ref('attempt_count')} = ${sql.val(nextAttemptCount)},
|
|
300
|
+
${sql.ref('error')} = ${sql.val(null)},
|
|
301
|
+
${sql.ref('updated_at')} = ${sql.val(Date.now())}
|
|
302
|
+
where ${sql.ref('hash')} = ${sql.val(item.hash)}
|
|
303
|
+
and ${sql.ref('status')} = ${sql.val('pending')}
|
|
304
|
+
`.execute(db);
|
|
305
|
+
|
|
306
|
+
const initResult = await blobs.initiateUpload({
|
|
307
|
+
hash: item.hash,
|
|
308
|
+
size: item.size,
|
|
309
|
+
mimeType: item.mime_type,
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
if (!initResult.exists && initResult.uploadUrl && item.body) {
|
|
313
|
+
const uploadBody = new ArrayBuffer(item.body.byteLength);
|
|
314
|
+
new Uint8Array(uploadBody).set(item.body);
|
|
315
|
+
|
|
316
|
+
const uploadResponse = await fetch(initResult.uploadUrl, {
|
|
317
|
+
method: initResult.uploadMethod ?? 'PUT',
|
|
318
|
+
body: uploadBody,
|
|
319
|
+
headers: initResult.uploadHeaders,
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
if (!uploadResponse.ok) {
|
|
323
|
+
throw new Error(`Upload failed: ${uploadResponse.statusText}`);
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
const completeResult = await blobs.completeUpload(item.hash);
|
|
327
|
+
if (!completeResult.ok) {
|
|
328
|
+
throw new Error(
|
|
329
|
+
completeResult.error ?? 'Failed to complete blob upload'
|
|
330
|
+
);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
await sql`
|
|
335
|
+
delete from ${sql.table('sync_blob_outbox')}
|
|
336
|
+
where ${sql.ref('hash')} = ${sql.val(item.hash)}
|
|
337
|
+
`.execute(db);
|
|
338
|
+
|
|
339
|
+
emit('blob:upload:complete', {
|
|
340
|
+
hash: item.hash,
|
|
341
|
+
size: item.size,
|
|
342
|
+
mimeType: item.mime_type,
|
|
343
|
+
});
|
|
344
|
+
uploaded++;
|
|
345
|
+
} catch (err) {
|
|
346
|
+
const nextStatus =
|
|
347
|
+
nextAttemptCount >= maxUploadRetries ? 'failed' : 'pending';
|
|
348
|
+
const errorMessage =
|
|
349
|
+
err instanceof Error ? err.message : 'Unknown error';
|
|
350
|
+
|
|
351
|
+
await sql`
|
|
352
|
+
update ${sql.table('sync_blob_outbox')}
|
|
353
|
+
set
|
|
354
|
+
${sql.ref('status')} = ${sql.val(nextStatus)},
|
|
355
|
+
${sql.ref('error')} = ${sql.val(errorMessage)},
|
|
356
|
+
${sql.ref('updated_at')} = ${sql.val(Date.now())}
|
|
357
|
+
where ${sql.ref('hash')} = ${sql.val(item.hash)}
|
|
358
|
+
`.execute(db);
|
|
359
|
+
|
|
360
|
+
emit('blob:upload:error', {
|
|
361
|
+
hash: item.hash,
|
|
362
|
+
error: errorMessage,
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
if (nextStatus === 'failed') {
|
|
366
|
+
failed++;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
return { uploaded, failed };
|
|
372
|
+
},
|
|
373
|
+
|
|
374
|
+
async getUploadQueueStats() {
|
|
375
|
+
const rowsResult = await sql<{
|
|
376
|
+
status: string;
|
|
377
|
+
count: number | bigint;
|
|
378
|
+
}>`
|
|
379
|
+
select
|
|
380
|
+
${sql.ref('status')} as status,
|
|
381
|
+
count(${sql.ref('hash')}) as count
|
|
382
|
+
from ${sql.table('sync_blob_outbox')}
|
|
383
|
+
group by ${sql.ref('status')}
|
|
384
|
+
`.execute(db);
|
|
385
|
+
|
|
386
|
+
const stats = { pending: 0, uploading: 0, failed: 0 };
|
|
387
|
+
for (const row of rowsResult.rows) {
|
|
388
|
+
if (row.status === 'pending') stats.pending = Number(row.count);
|
|
389
|
+
if (row.status === 'uploading') stats.uploading = Number(row.count);
|
|
390
|
+
if (row.status === 'failed') stats.failed = Number(row.count);
|
|
391
|
+
}
|
|
392
|
+
return stats;
|
|
393
|
+
},
|
|
394
|
+
|
|
395
|
+
async getCacheStats() {
|
|
396
|
+
const result = await sql<{
|
|
397
|
+
count: number | bigint;
|
|
398
|
+
totalBytes: number | bigint | null;
|
|
399
|
+
}>`
|
|
400
|
+
select
|
|
401
|
+
count(${sql.ref('hash')}) as count,
|
|
402
|
+
sum(${sql.ref('size')}) as totalBytes
|
|
403
|
+
from ${sql.table('sync_blob_cache')}
|
|
404
|
+
`.execute(db);
|
|
405
|
+
const row = result.rows[0];
|
|
406
|
+
|
|
407
|
+
return {
|
|
408
|
+
count: Number(row?.count ?? 0),
|
|
409
|
+
totalBytes: Number(row?.totalBytes ?? 0),
|
|
410
|
+
};
|
|
411
|
+
},
|
|
412
|
+
|
|
413
|
+
async pruneCache(maxBytes) {
|
|
414
|
+
if (!maxBytes) return 0;
|
|
415
|
+
|
|
416
|
+
const stats = await this.getCacheStats();
|
|
417
|
+
if (stats.totalBytes <= maxBytes) return 0;
|
|
418
|
+
|
|
419
|
+
const toFree = stats.totalBytes - maxBytes;
|
|
420
|
+
let freed = 0;
|
|
421
|
+
|
|
422
|
+
const oldEntriesResult = await sql<{ hash: string; size: number }>`
|
|
423
|
+
select ${sql.ref('hash')}, ${sql.ref('size')}
|
|
424
|
+
from ${sql.table('sync_blob_cache')}
|
|
425
|
+
order by ${sql.ref('last_accessed_at')} asc
|
|
426
|
+
`.execute(db);
|
|
427
|
+
const oldEntries = oldEntriesResult.rows;
|
|
428
|
+
|
|
429
|
+
for (const entry of oldEntries) {
|
|
430
|
+
if (freed >= toFree) break;
|
|
431
|
+
|
|
432
|
+
await storage.delete(entry.hash);
|
|
433
|
+
await sql`
|
|
434
|
+
delete from ${sql.table('sync_blob_cache')}
|
|
435
|
+
where ${sql.ref('hash')} = ${sql.val(entry.hash)}
|
|
436
|
+
`.execute(db);
|
|
437
|
+
freed += entry.size;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
return freed;
|
|
441
|
+
},
|
|
442
|
+
|
|
443
|
+
async clearCache() {
|
|
444
|
+
if (storage.clear) {
|
|
445
|
+
await storage.clear();
|
|
446
|
+
} else {
|
|
447
|
+
const entriesResult = await sql<{ hash: string }>`
|
|
448
|
+
select ${sql.ref('hash')}
|
|
449
|
+
from ${sql.table('sync_blob_cache')}
|
|
450
|
+
`.execute(db);
|
|
451
|
+
|
|
452
|
+
for (const entry of entriesResult.rows) {
|
|
453
|
+
await storage.delete(entry.hash);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
await sql`delete from ${sql.table('sync_blob_cache')}`.execute(db);
|
|
458
|
+
},
|
|
459
|
+
};
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
async function toUint8Array(
|
|
463
|
+
data: Blob | File | Uint8Array
|
|
464
|
+
): Promise<Uint8Array> {
|
|
465
|
+
if (data instanceof Uint8Array) {
|
|
466
|
+
return data;
|
|
467
|
+
}
|
|
468
|
+
const buffer = await data.arrayBuffer();
|
|
469
|
+
return new Uint8Array(buffer);
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
async function computeSha256Hex(data: Uint8Array): Promise<string> {
|
|
473
|
+
const hashBuffer = await crypto.subtle.digest(
|
|
474
|
+
'SHA-256',
|
|
475
|
+
data.buffer as ArrayBuffer
|
|
476
|
+
);
|
|
477
|
+
const hashArray = new Uint8Array(hashBuffer);
|
|
478
|
+
return Array.from(hashArray)
|
|
479
|
+
.map((b) => b.toString(16).padStart(2, '0'))
|
|
480
|
+
.join('');
|
|
481
|
+
}
|
package/src/migrate.ts
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { Kysely } from 'kysely';
|
|
2
|
+
|
|
3
|
+
export async function ensureClientBlobSchema<DB>(
|
|
4
|
+
db: Kysely<DB>
|
|
5
|
+
): Promise<void> {
|
|
6
|
+
await db.schema
|
|
7
|
+
.createTable('sync_blob_cache')
|
|
8
|
+
.ifNotExists()
|
|
9
|
+
.addColumn('hash', 'text', (col) => col.primaryKey())
|
|
10
|
+
.addColumn('size', 'integer', (col) => col.notNull())
|
|
11
|
+
.addColumn('mime_type', 'text', (col) => col.notNull())
|
|
12
|
+
.addColumn('body', 'blob', (col) => col.notNull())
|
|
13
|
+
.addColumn('encrypted', 'integer', (col) => col.notNull().defaultTo(0))
|
|
14
|
+
.addColumn('key_id', 'text')
|
|
15
|
+
.addColumn('cached_at', 'bigint', (col) => col.notNull())
|
|
16
|
+
.addColumn('last_accessed_at', 'bigint', (col) => col.notNull())
|
|
17
|
+
.execute();
|
|
18
|
+
|
|
19
|
+
await db.schema
|
|
20
|
+
.createIndex('idx_sync_blob_cache_last_accessed')
|
|
21
|
+
.ifNotExists()
|
|
22
|
+
.on('sync_blob_cache')
|
|
23
|
+
.columns(['last_accessed_at'])
|
|
24
|
+
.execute();
|
|
25
|
+
|
|
26
|
+
await db.schema
|
|
27
|
+
.createTable('sync_blob_outbox')
|
|
28
|
+
.ifNotExists()
|
|
29
|
+
.addColumn('id', 'integer', (col) => col.primaryKey().autoIncrement())
|
|
30
|
+
.addColumn('hash', 'text', (col) => col.notNull().unique())
|
|
31
|
+
.addColumn('size', 'integer', (col) => col.notNull())
|
|
32
|
+
.addColumn('mime_type', 'text', (col) => col.notNull())
|
|
33
|
+
.addColumn('body', 'blob', (col) => col.notNull())
|
|
34
|
+
.addColumn('encrypted', 'integer', (col) => col.notNull().defaultTo(0))
|
|
35
|
+
.addColumn('key_id', 'text')
|
|
36
|
+
.addColumn('status', 'text', (col) => col.notNull())
|
|
37
|
+
.addColumn('attempt_count', 'integer', (col) => col.notNull().defaultTo(0))
|
|
38
|
+
.addColumn('error', 'text')
|
|
39
|
+
.addColumn('created_at', 'bigint', (col) => col.notNull())
|
|
40
|
+
.addColumn('updated_at', 'bigint', (col) => col.notNull())
|
|
41
|
+
.execute();
|
|
42
|
+
|
|
43
|
+
await db.schema
|
|
44
|
+
.createIndex('idx_sync_blob_outbox_status')
|
|
45
|
+
.ifNotExists()
|
|
46
|
+
.on('sync_blob_outbox')
|
|
47
|
+
.columns(['status', 'created_at'])
|
|
48
|
+
.execute();
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export async function dropClientBlobSchema<DB>(db: Kysely<DB>): Promise<void> {
|
|
52
|
+
await db.schema.dropTable('sync_blob_outbox').ifExists().execute();
|
|
53
|
+
await db.schema.dropTable('sync_blob_cache').ifExists().execute();
|
|
54
|
+
}
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import type { Generated } from 'kysely';
|
|
2
|
+
|
|
3
|
+
export interface ClientBlobStorage {
|
|
4
|
+
write(
|
|
5
|
+
hash: string,
|
|
6
|
+
data: Uint8Array | ReadableStream<Uint8Array>
|
|
7
|
+
): Promise<void>;
|
|
8
|
+
read(hash: string): Promise<Uint8Array | null>;
|
|
9
|
+
readStream?(hash: string): Promise<ReadableStream<Uint8Array> | null>;
|
|
10
|
+
delete(hash: string): Promise<void>;
|
|
11
|
+
exists(hash: string): Promise<boolean>;
|
|
12
|
+
getUsage?(): Promise<number>;
|
|
13
|
+
clear?(): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface BlobStoreOptions {
|
|
17
|
+
mimeType?: string;
|
|
18
|
+
immediate?: boolean;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface BlobClient {
|
|
22
|
+
store(
|
|
23
|
+
data: Blob | File | Uint8Array,
|
|
24
|
+
options?: BlobStoreOptions
|
|
25
|
+
): Promise<import('@syncular/core').BlobRef>;
|
|
26
|
+
retrieve(ref: import('@syncular/core').BlobRef): Promise<Uint8Array>;
|
|
27
|
+
isLocal(hash: string): Promise<boolean>;
|
|
28
|
+
preload(refs: import('@syncular/core').BlobRef[]): Promise<void>;
|
|
29
|
+
processUploadQueue(): Promise<{ uploaded: number; failed: number }>;
|
|
30
|
+
getUploadQueueStats(): Promise<{
|
|
31
|
+
pending: number;
|
|
32
|
+
uploading: number;
|
|
33
|
+
failed: number;
|
|
34
|
+
}>;
|
|
35
|
+
getCacheStats(): Promise<{ count: number; totalBytes: number }>;
|
|
36
|
+
pruneCache(maxBytes?: number): Promise<number>;
|
|
37
|
+
clearCache(): Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface SyncBlobCacheTable {
|
|
41
|
+
hash: string;
|
|
42
|
+
size: number;
|
|
43
|
+
mime_type: string;
|
|
44
|
+
body: Uint8Array;
|
|
45
|
+
encrypted: number;
|
|
46
|
+
key_id: string | null;
|
|
47
|
+
cached_at: number;
|
|
48
|
+
last_accessed_at: number;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export type BlobUploadStatus =
|
|
52
|
+
| 'pending'
|
|
53
|
+
| 'uploading'
|
|
54
|
+
| 'uploaded'
|
|
55
|
+
| 'confirming'
|
|
56
|
+
| 'complete'
|
|
57
|
+
| 'failed';
|
|
58
|
+
|
|
59
|
+
export interface SyncBlobOutboxTable {
|
|
60
|
+
id: Generated<number>;
|
|
61
|
+
hash: string;
|
|
62
|
+
size: number;
|
|
63
|
+
mime_type: string;
|
|
64
|
+
body: Uint8Array;
|
|
65
|
+
encrypted: number;
|
|
66
|
+
key_id: string | null;
|
|
67
|
+
status: BlobUploadStatus;
|
|
68
|
+
attempt_count: number;
|
|
69
|
+
error: string | null;
|
|
70
|
+
created_at: number;
|
|
71
|
+
updated_at: number;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export interface SyncBlobClientDb {
|
|
75
|
+
sync_blob_cache: SyncBlobCacheTable;
|
|
76
|
+
sync_blob_outbox: SyncBlobOutboxTable;
|
|
77
|
+
}
|