@kubun/store-blob 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/lib/api.d.ts +78 -4
  2. package/lib/api.js +458 -59
  3. package/lib/index.d.ts +3 -3
  4. package/lib/index.js +1 -1
  5. package/lib/migrations.js +29 -3
  6. package/lib/tables.d.ts +31 -0
  7. package/package.json +8 -8
package/lib/api.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { Adapter } from '@kubun/db-adapter';
1
+ import { type Adapter } from '@kubun/db-adapter';
2
2
  import type { Kysely } from 'kysely';
3
3
  import type { BlobEntry, BlobRef, BlobTables } from './tables.js';
4
4
  export type BlobState = 'local' | 'partial' | 'remote-only';
@@ -12,6 +12,12 @@ export type BlobEntryInput = {
12
12
  pinned?: boolean;
13
13
  createdAt: number;
14
14
  };
15
+ export type LoadMode = 'EAGER' | 'ON_DEMAND';
16
+ export type LoadPolicy = {
17
+ modelID: string;
18
+ fieldPath: string;
19
+ mode: LoadMode;
20
+ };
15
21
  export type BlobChunkInput = {
16
22
  index: number;
17
23
  digest: Uint8Array;
@@ -26,9 +32,77 @@ export type BlobStoreAPI = {
26
32
  getRefsForDocument(documentID: string): Promise<Array<BlobRef>>;
27
33
  getRefsForAttachment(attachmentID: string): Promise<Array<BlobRef>>;
28
34
  removeRefsForDocument(documentID: string): Promise<void>;
29
- beginTransfer(attachmentID: string, chunkSize: number, chunks: Array<BlobChunkInput>): Promise<void>;
30
- recordTransferChunk(attachmentID: string, index: number): Promise<void>;
35
+ putSource(attachmentID: string, documentID: string): Promise<void>;
36
+ putSourceIfLocal(attachmentID: string, documentID: string): Promise<boolean>;
37
+ hasSource(attachmentID: string, documentID: string): Promise<boolean>;
38
+ setLoadPolicy(params: LoadPolicy): Promise<void>;
39
+ listLoadPolicies(): Promise<Array<LoadPolicy>>;
40
+ getLoadMode(modelID: string, fieldPath: string): Promise<LoadMode>;
41
+ beginTransfer(attachmentID: string, chunkSize: number, chunks: Array<BlobChunkInput>, documentID: string): Promise<BeginTransferResult>;
42
+ resetTransfer(attachmentID: string): Promise<void>;
43
+ recordTransferChunk(attachmentID: string, index: number, documentID: string): Promise<void>;
31
44
  getPresentChunkIndexes(attachmentID: string): Promise<Array<number>>;
32
- finalizeTransfer(attachmentID: string): Promise<void>;
45
+ finalizeTransfer(attachmentID: string, documentID: string): Promise<void>;
46
+ enqueueByteDeletion(attachmentID: string, queuedAt: number): Promise<boolean>;
47
+ claimByteDeletion(attachmentID: string): Promise<ByteDeletionClaim>;
48
+ clearByteDeletion(attachmentID: string): Promise<void>;
49
+ listByteDeletions(): Promise<Array<string>>;
50
+ queueFetchJob(params: QueueFetchJobParams): Promise<boolean>;
51
+ ensureFetchStub(params: {
52
+ attachmentID: string;
53
+ contentLength: number;
54
+ now: number;
55
+ }): Promise<void>;
56
+ dropFetchJobs(documentID: string): Promise<void>;
57
+ getFetchJob(attachmentID: string, documentID: string): Promise<FetchJob | null>;
58
+ listDueFetchJobs(params: {
59
+ now: number;
60
+ limit: number;
61
+ }): Promise<Array<FetchJob>>;
62
+ nextFetchJobAt(): Promise<number | null>;
63
+ recordFetchJobFailure(params: {
64
+ attachmentID: string;
65
+ documentID: string;
66
+ error: string;
67
+ nextAttemptAt: number;
68
+ }): Promise<void>;
69
+ completeFetchJob(attachmentID: string, documentID: string): Promise<void>;
70
+ sweepDocumentRows(documentID: string): Promise<Array<string>>;
71
+ lockEntryRow(attachmentID: string): Promise<boolean>;
72
+ listTransfers(params: {
73
+ after?: string;
74
+ limit: number;
75
+ }): Promise<Array<StagingTransfer>>;
76
+ deleteOrphanTransfer(attachmentID: string, documentID: string): Promise<boolean>;
77
+ };
78
+ export type StagingTransfer = {
79
+ attachmentID: string;
80
+ documentID: string;
81
+ };
82
+ export type QueueFetchJobParams = {
83
+ attachmentID: string;
84
+ documentID: string;
85
+ now: number;
86
+ };
87
+ export type FetchJob = {
88
+ attachmentID: string;
89
+ documentID: string;
90
+ attempts: number;
91
+ nextAttemptAt: number;
92
+ lastError: string | null;
33
93
  };
94
+ export type ByteDeletionClaim = 'delete' | 'stale' | 'needed';
95
+ export type BeginTransferResult = 'started' | 'resumed' | 'restarted' | 'skipped';
96
+ export declare class BlobEntryLockRequiresTransactionError extends Error {
97
+ get code(): 'BLOB_ENTRY_LOCK_REQUIRES_TRANSACTION';
98
+ constructor();
99
+ }
100
+ /**
101
+ * The transfer is no longer the caller's: another document's fetch restarted
102
+ * it, or it already finished. A local race, not a holder failure.
103
+ */
104
+ export declare class BlobTransferSupersededError extends Error {
105
+ get code(): 'BLOB_TRANSFER_SUPERSEDED';
106
+ constructor(attachmentID: string, documentID: string);
107
+ }
34
108
  export declare function createBlobStore(db: Kysely<BlobTables>, adapter: Adapter): BlobStoreAPI;
package/lib/api.js CHANGED
@@ -1,28 +1,150 @@
1
+ import { AbstractPostgresAdapter } from '@kubun/db-adapter';
2
+ export class BlobEntryLockRequiresTransactionError extends Error {
3
+ get code() {
4
+ return 'BLOB_ENTRY_LOCK_REQUIRES_TRANSACTION';
5
+ }
6
+ constructor(){
7
+ super('A blob entry lock is held only by a transaction');
8
+ this.name = 'BlobEntryLockRequiresTransactionError';
9
+ }
10
+ }
11
+ /**
12
+ * The transfer is no longer the caller's: another document's fetch restarted
13
+ * it, or it already finished. A local race, not a holder failure.
14
+ */ export class BlobTransferSupersededError extends Error {
15
+ get code() {
16
+ return 'BLOB_TRANSFER_SUPERSEDED';
17
+ }
18
+ constructor(attachmentID, documentID){
19
+ super(`The transfer of ${attachmentID} is no longer for document ${documentID}`);
20
+ this.name = 'BlobTransferSupersededError';
21
+ }
22
+ }
23
+ function bytesEqual(a, b) {
24
+ if (a.length !== b.length) return false;
25
+ for(let i = 0; i < a.length; i++){
26
+ if (a[i] !== b[i]) return false;
27
+ }
28
+ return true;
29
+ }
30
+ function fetchJobFromRow(row) {
31
+ return {
32
+ attachmentID: row.attachment_id,
33
+ documentID: row.document_id,
34
+ attempts: Number(row.attempts),
35
+ nextAttemptAt: Number(row.next_attempt_at),
36
+ lastError: row.last_error
37
+ };
38
+ }
39
+ function toLoadMode(mode) {
40
+ return mode === 'EAGER' ? 'EAGER' : 'ON_DEMAND';
41
+ }
1
42
  export function createBlobStore(db, adapter) {
43
+ // SQLite rejects `FOR UPDATE` and serializes write transactions anyway.
44
+ const isPostgres = adapter instanceof AbstractPostgresAdapter;
45
+ // Joins the caller's transaction when there is one.
46
+ const inTransaction = async (fn)=>db.isTransaction ? await fn(db) : await db.transaction().execute(fn);
47
+ // The entry's transfer fields, with its row locked until the transaction
48
+ // ends. Every transfer state change takes this lock, so they serialize per
49
+ // blob, and each decides from what it read under the lock.
50
+ const lockEntry = async (tx, attachmentID)=>{
51
+ const query = tx.selectFrom('kubun_blob_entries').select([
52
+ 'state',
53
+ 'chunk_size',
54
+ 'transfer_document_id',
55
+ 'pinned',
56
+ 'generation'
57
+ ]).where('attachment_id', '=', attachmentID);
58
+ return await (isPostgres ? query.forUpdate() : query).executeTakeFirst() ?? null;
59
+ };
60
+ const lockEntryState = async (tx, attachmentID)=>(await lockEntry(tx, attachmentID))?.state ?? null;
61
+ // Under the entry lock: the transfer must still be staging for this document.
62
+ const requireTransferFor = async (tx, attachmentID, documentID)=>{
63
+ const entry = await lockEntry(tx, attachmentID);
64
+ if (entry?.state !== 'partial' || entry.transfer_document_id !== documentID) {
65
+ throw new BlobTransferSupersededError(attachmentID, documentID);
66
+ }
67
+ };
68
+ const deleteRows = async (tx, attachmentID)=>{
69
+ await clearTransfer(tx, attachmentID);
70
+ await tx.deleteFrom('kubun_blob_sources').where('attachment_id', '=', attachmentID).execute();
71
+ await tx.deleteFrom('kubun_blob_entries').where('attachment_id', '=', attachmentID).execute();
72
+ };
73
+ // The bytes are wanted again: a byte deletion queued before this is stale.
74
+ const bumpGeneration = async (tx, attachmentID)=>{
75
+ await tx.updateTable('kubun_blob_entries').set((eb)=>({
76
+ generation: eb('generation', '+', 1)
77
+ })).where('attachment_id', '=', attachmentID).execute();
78
+ };
79
+ const dropByteDeletion = async (tx, attachmentID)=>{
80
+ await tx.deleteFrom('kubun_blob_byte_deletions').where('attachment_id', '=', attachmentID).execute();
81
+ };
82
+ const isNeeded = async (tx, attachmentID)=>{
83
+ for (const table of [
84
+ 'kubun_blob_refs',
85
+ 'kubun_blob_sources',
86
+ 'kubun_blob_transfers',
87
+ 'kubun_blob_fetch_jobs'
88
+ ]){
89
+ const row = await tx.selectFrom(table).select('attachment_id').where('attachment_id', '=', attachmentID).limit(1).executeTakeFirst();
90
+ if (row != null) return true;
91
+ }
92
+ return false;
93
+ };
94
+ const clearTransfer = async (tx, attachmentID)=>{
95
+ await tx.deleteFrom('kubun_blob_chunks').where('attachment_id', '=', attachmentID).execute();
96
+ await tx.deleteFrom('kubun_blob_transfers').where('attachment_id', '=', attachmentID).execute();
97
+ };
2
98
  return {
3
99
  async insertEntry (entry, chunks) {
4
- await db.insertInto('kubun_blob_entries').values({
5
- attachment_id: entry.attachmentID,
6
- content_length: entry.contentLength,
7
- encrypted: entry.encrypted ? 1 : 0,
8
- key_id: entry.keyID ?? null,
9
- chunk_size: entry.chunkSize,
10
- state: entry.state,
11
- pinned: entry.pinned ? 1 : 0,
12
- created_at: entry.createdAt
13
- })// Content-addressed: identical id means identical bytes, so a second
14
- // write is a no-op rather than a conflict.
15
- .onConflict((oc)=>oc.column('attachment_id').doNothing()).execute();
16
- if (chunks.length > 0) {
17
- await db.insertInto('kubun_blob_chunks').values(chunks.map((chunk)=>({
18
- attachment_id: entry.attachmentID,
19
- index: chunk.index,
20
- digest: adapter.encodeBinary(chunk.digest)
21
- }))).onConflict((oc)=>oc.columns([
22
- 'attachment_id',
23
- 'index'
24
- ]).doNothing()).execute();
25
- }
100
+ await inTransaction(async (tx)=>{
101
+ await tx.insertInto('kubun_blob_entries').values({
102
+ attachment_id: entry.attachmentID,
103
+ content_length: entry.contentLength,
104
+ encrypted: entry.encrypted ? 1 : 0,
105
+ key_id: entry.keyID ?? null,
106
+ chunk_size: entry.chunkSize,
107
+ state: entry.state,
108
+ pinned: entry.pinned ? 1 : 0,
109
+ created_at: entry.createdAt,
110
+ generation: 0
111
+ })// Content-addressed: identical id means identical bytes, so a second
112
+ // write keeps the row. It still bumps the generation, which also takes
113
+ // the row lock: DO NOTHING would not wait for a byte deletion holding
114
+ // it, and a ref recorded beside it would outlive the deleted entry.
115
+ .onConflict((oc)=>oc.column('attachment_id').doUpdateSet((eb)=>({
116
+ generation: eb('kubun_blob_entries.generation', '+', 1)
117
+ }))).execute();
118
+ // Generation 0 only on the row this insert created: a conflict bumps it.
119
+ const current = await lockEntry(tx, entry.attachmentID);
120
+ if (current != null && current.generation !== 0) {
121
+ // A stored manifest stands unless these are the first local bytes:
122
+ // another layout's chunks would mix into it, and local never goes back.
123
+ if (entry.state !== 'local' || current.state === 'local') return;
124
+ // A transfer in flight is superseded: its manifest and present chunks
125
+ // go, so its stage and finalize are refused.
126
+ await clearTransfer(tx, entry.attachmentID);
127
+ await tx.updateTable('kubun_blob_entries').set({
128
+ state: 'local',
129
+ chunk_size: entry.chunkSize,
130
+ encrypted: entry.encrypted ? 1 : 0,
131
+ key_id: entry.keyID ?? null,
132
+ transfer_document_id: null,
133
+ // A pin outlives the stub it was set on.
134
+ pinned: entry.pinned || current.pinned === 1 ? 1 : 0
135
+ }).where('attachment_id', '=', entry.attachmentID).execute();
136
+ }
137
+ if (chunks.length > 0) {
138
+ await tx.insertInto('kubun_blob_chunks').values(chunks.map((chunk)=>({
139
+ attachment_id: entry.attachmentID,
140
+ index: chunk.index,
141
+ digest: adapter.encodeBinary(chunk.digest)
142
+ }))).onConflict((oc)=>oc.columns([
143
+ 'attachment_id',
144
+ 'index'
145
+ ]).doNothing()).execute();
146
+ }
147
+ });
26
148
  },
27
149
  async getEntry (attachmentID) {
28
150
  const row = await db.selectFrom('kubun_blob_entries').selectAll().where('attachment_id', '=', attachmentID).executeTakeFirst();
@@ -36,13 +158,20 @@ export function createBlobStore(db, adapter) {
36
158
  return rows.map((row)=>row.digest);
37
159
  },
38
160
  async setPinned (attachmentID, pinned) {
39
- await db.updateTable('kubun_blob_entries').set({
40
- pinned: pinned ? 1 : 0
41
- }).where('attachment_id', '=', attachmentID).execute();
161
+ await db.updateTable('kubun_blob_entries').set((eb)=>pinned ? {
162
+ pinned: 1,
163
+ generation: eb('generation', '+', 1)
164
+ } : {
165
+ pinned: 0
166
+ }).where('attachment_id', '=', attachmentID).execute();
42
167
  },
43
168
  async deleteEntry (attachmentID) {
44
- await db.deleteFrom('kubun_blob_chunks').where('attachment_id', '=', attachmentID).execute();
45
- await db.deleteFrom('kubun_blob_entries').where('attachment_id', '=', attachmentID).execute();
169
+ await inTransaction(async (tx)=>{
170
+ // Entry lock first, as every transfer change takes it: chunk rows first
171
+ // would deadlock against a concurrent reset.
172
+ await lockEntry(tx, attachmentID);
173
+ await deleteRows(tx, attachmentID);
174
+ });
46
175
  },
47
176
  async putRef (attachmentID, modelID, documentID, refKey) {
48
177
  await db.insertInto('kubun_blob_refs').values({
@@ -67,47 +196,317 @@ export function createBlobStore(db, adapter) {
67
196
  async removeRefsForDocument (documentID) {
68
197
  await db.deleteFrom('kubun_blob_refs').where('document_id', '=', documentID).execute();
69
198
  },
70
- async beginTransfer (attachmentID, chunkSize, chunks) {
71
- await db.updateTable('kubun_blob_entries').set({
72
- chunk_size: chunkSize,
73
- state: 'partial'
74
- }).where('attachment_id', '=', attachmentID).execute();
75
- if (chunks.length > 0) {
76
- await db.insertInto('kubun_blob_chunks').values(chunks.map((chunk)=>({
77
- attachment_id: attachmentID,
78
- index: chunk.index,
79
- digest: adapter.encodeBinary(chunk.digest)
80
- }))).onConflict((oc)=>oc.columns([
199
+ async putSource (attachmentID, documentID) {
200
+ await inTransaction(async (tx)=>{
201
+ await bumpGeneration(tx, attachmentID);
202
+ await tx.insertInto('kubun_blob_sources').values({
203
+ attachment_id: attachmentID,
204
+ document_id: documentID
205
+ }).onConflict((oc)=>oc.columns([
81
206
  'attachment_id',
82
- 'index'
207
+ 'document_id'
83
208
  ]).doNothing()).execute();
84
- }
209
+ });
85
210
  },
86
- async recordTransferChunk (attachmentID, index) {
87
- await db.insertInto('kubun_blob_transfers').values({
88
- attachment_id: attachmentID,
89
- index
211
+ async putSourceIfLocal (attachmentID, documentID) {
212
+ return await inTransaction(async (tx)=>{
213
+ // The bytes may have gone since the caller last looked.
214
+ if (await lockEntryState(tx, attachmentID) !== 'local') return false;
215
+ await bumpGeneration(tx, attachmentID);
216
+ await tx.insertInto('kubun_blob_sources').values({
217
+ attachment_id: attachmentID,
218
+ document_id: documentID
219
+ }).onConflict((oc)=>oc.columns([
220
+ 'attachment_id',
221
+ 'document_id'
222
+ ]).doNothing()).execute();
223
+ return true;
224
+ });
225
+ },
226
+ async hasSource (attachmentID, documentID) {
227
+ const row = await db.selectFrom('kubun_blob_sources').select('document_id').where('attachment_id', '=', attachmentID).where('document_id', '=', documentID).executeTakeFirst();
228
+ return row != null;
229
+ },
230
+ async setLoadPolicy ({ modelID, fieldPath, mode }) {
231
+ await db.insertInto('kubun_blob_load_policies').values({
232
+ model_id: modelID,
233
+ field_path: fieldPath,
234
+ mode
90
235
  }).onConflict((oc)=>oc.columns([
91
- 'attachment_id',
92
- 'index'
93
- ]).doNothing()).execute();
236
+ 'model_id',
237
+ 'field_path'
238
+ ]).doUpdateSet({
239
+ mode
240
+ })).execute();
241
+ },
242
+ async listLoadPolicies () {
243
+ const rows = await db.selectFrom('kubun_blob_load_policies').selectAll().orderBy('model_id', 'asc').orderBy('field_path', 'asc').execute();
244
+ return rows.map((row)=>({
245
+ modelID: row.model_id,
246
+ fieldPath: row.field_path,
247
+ mode: toLoadMode(row.mode)
248
+ }));
249
+ },
250
+ async getLoadMode (modelID, fieldPath) {
251
+ const row = await db.selectFrom('kubun_blob_load_policies').select('mode').where('model_id', '=', modelID).where('field_path', '=', fieldPath).executeTakeFirst();
252
+ return toLoadMode(row?.mode);
253
+ },
254
+ async beginTransfer (attachmentID, chunkSize, chunks, documentID) {
255
+ return await inTransaction(async (tx)=>{
256
+ const entry = await lockEntry(tx, attachmentID);
257
+ // Bytes already local keep their manifest: a download that raced a
258
+ // finished one must not reopen it.
259
+ if (entry == null || entry.state === 'local') return 'skipped';
260
+ let restarted = false;
261
+ if (entry.state === 'partial') {
262
+ const stored = await tx.selectFrom('kubun_blob_chunks').select([
263
+ 'index',
264
+ 'digest'
265
+ ]).where('attachment_id', '=', attachmentID).orderBy('index', 'asc').execute();
266
+ const sameManifest = entry.chunk_size === chunkSize && stored.length === chunks.length && stored.every((row, position)=>{
267
+ const chunk = chunks[position];
268
+ return chunk != null && chunk.index === row.index && bytesEqual(row.digest, chunk.digest);
269
+ });
270
+ // Chunks staged for another document were never served through this
271
+ // one, and the ID alone lets anyone forge a matching manifest.
272
+ if (sameManifest && entry.transfer_document_id === documentID) return 'resumed';
273
+ await clearTransfer(tx, attachmentID);
274
+ restarted = true;
275
+ }
276
+ await tx.updateTable('kubun_blob_entries').set((eb)=>({
277
+ chunk_size: chunkSize,
278
+ state: 'partial',
279
+ transfer_document_id: documentID,
280
+ generation: eb('generation', '+', 1)
281
+ })).where('attachment_id', '=', attachmentID).execute();
282
+ if (chunks.length > 0) {
283
+ await tx.insertInto('kubun_blob_chunks').values(chunks.map((chunk)=>({
284
+ attachment_id: attachmentID,
285
+ index: chunk.index,
286
+ digest: adapter.encodeBinary(chunk.digest)
287
+ }))).onConflict((oc)=>oc.columns([
288
+ 'attachment_id',
289
+ 'index'
290
+ ]).doNothing()).execute();
291
+ }
292
+ return restarted ? 'restarted' : 'started';
293
+ });
294
+ },
295
+ async resetTransfer (attachmentID) {
296
+ await inTransaction(async (tx)=>{
297
+ // Locked first, so a finalize in flight commits before this reads the
298
+ // state, and a local entry is never cleared.
299
+ if (await lockEntryState(tx, attachmentID) !== 'partial') return;
300
+ await clearTransfer(tx, attachmentID);
301
+ await tx.updateTable('kubun_blob_entries').set({
302
+ state: 'remote-only',
303
+ chunk_size: 0,
304
+ transfer_document_id: null
305
+ }).where('attachment_id', '=', attachmentID).execute();
306
+ });
307
+ },
308
+ async recordTransferChunk (attachmentID, index, documentID) {
309
+ await inTransaction(async (tx)=>{
310
+ await requireTransferFor(tx, attachmentID, documentID);
311
+ await tx.insertInto('kubun_blob_transfers').values({
312
+ attachment_id: attachmentID,
313
+ index
314
+ }).onConflict((oc)=>oc.columns([
315
+ 'attachment_id',
316
+ 'index'
317
+ ]).doNothing()).execute();
318
+ });
94
319
  },
95
320
  async getPresentChunkIndexes (attachmentID) {
96
321
  const rows = await db.selectFrom('kubun_blob_transfers').select('index').where('attachment_id', '=', attachmentID).orderBy('index', 'asc').execute();
97
322
  return rows.map((row)=>row.index);
98
323
  },
99
- async finalizeTransfer (attachmentID) {
100
- const manifest = await db.selectFrom('kubun_blob_chunks').select('index').where('attachment_id', '=', attachmentID).execute();
101
- const present = await db.selectFrom('kubun_blob_transfers').select('index').where('attachment_id', '=', attachmentID).execute();
102
- const presentSet = new Set(present.map((row)=>row.index));
103
- const missing = manifest.filter((row)=>!presentSet.has(row.index));
104
- if (missing.length > 0) {
105
- throw new Error(`Cannot finalize transfer ${attachmentID}: ${missing.length} chunk(s) missing`);
106
- }
107
- await db.updateTable('kubun_blob_entries').set({
108
- state: 'local'
109
- }).where('attachment_id', '=', attachmentID).execute();
110
- await db.deleteFrom('kubun_blob_transfers').where('attachment_id', '=', attachmentID).execute();
324
+ async finalizeTransfer (attachmentID, documentID) {
325
+ await inTransaction(async (tx)=>{
326
+ // Locked so a concurrent reset or restart cannot clear the manifest
327
+ // under it, and only the document that staged the transfer finalizes.
328
+ await requireTransferFor(tx, attachmentID, documentID);
329
+ const manifest = await tx.selectFrom('kubun_blob_chunks').select('index').where('attachment_id', '=', attachmentID).execute();
330
+ const present = await tx.selectFrom('kubun_blob_transfers').select('index').where('attachment_id', '=', attachmentID).execute();
331
+ const presentSet = new Set(present.map((row)=>row.index));
332
+ const missing = manifest.filter((row)=>!presentSet.has(row.index));
333
+ if (missing.length > 0) {
334
+ throw new Error(`Cannot finalize transfer ${attachmentID}: ${missing.length} chunk(s) missing`);
335
+ }
336
+ await tx.updateTable('kubun_blob_entries').set((eb)=>({
337
+ state: 'local',
338
+ transfer_document_id: null,
339
+ generation: eb('generation', '+', 1)
340
+ })).where('attachment_id', '=', attachmentID).execute();
341
+ await tx.deleteFrom('kubun_blob_transfers').where('attachment_id', '=', attachmentID).execute();
342
+ });
343
+ },
344
+ async enqueueByteDeletion (attachmentID, queuedAt) {
345
+ return await inTransaction(async (tx)=>{
346
+ const entry = await lockEntry(tx, attachmentID);
347
+ if (entry == null) return false;
348
+ await tx.insertInto('kubun_blob_byte_deletions').values({
349
+ attachment_id: attachmentID,
350
+ generation: entry.generation,
351
+ queued_at: queuedAt
352
+ }).onConflict((oc)=>oc.column('attachment_id').doUpdateSet({
353
+ generation: entry.generation,
354
+ queued_at: queuedAt
355
+ })).execute();
356
+ return true;
357
+ });
358
+ },
359
+ async claimByteDeletion (attachmentID) {
360
+ return await inTransaction(async (tx)=>{
361
+ // Entry lock first: every write, fetch, source and pin takes it too, so
362
+ // none lands between the checks below and the row delete.
363
+ const entry = await lockEntry(tx, attachmentID);
364
+ const job = await tx.selectFrom('kubun_blob_byte_deletions').select('generation').where('attachment_id', '=', attachmentID).executeTakeFirst();
365
+ if (job == null) return 'stale';
366
+ if (job.generation == null) {
367
+ // Rows already deleted: replay the bytes, unless the blob was written
368
+ // again since, in which case the bytes are the new entry's.
369
+ if (entry == null) return 'delete';
370
+ await dropByteDeletion(tx, attachmentID);
371
+ return 'stale';
372
+ }
373
+ if (entry == null || entry.generation !== job.generation) {
374
+ await dropByteDeletion(tx, attachmentID);
375
+ return 'stale';
376
+ }
377
+ if (entry.pinned === 1 || entry.state === 'partial' || await isNeeded(tx, attachmentID)) {
378
+ await dropByteDeletion(tx, attachmentID);
379
+ return 'needed';
380
+ }
381
+ await deleteRows(tx, attachmentID);
382
+ await tx.updateTable('kubun_blob_byte_deletions').set({
383
+ generation: null
384
+ }).where('attachment_id', '=', attachmentID).execute();
385
+ return 'delete';
386
+ });
387
+ },
388
+ async clearByteDeletion (attachmentID) {
389
+ await dropByteDeletion(db, attachmentID);
390
+ },
391
+ async queueFetchJob ({ attachmentID, documentID, now }) {
392
+ return await inTransaction(async (tx)=>{
393
+ await lockEntry(tx, attachmentID);
394
+ const job = await tx.selectFrom('kubun_blob_fetch_jobs').select('document_id').where('attachment_id', '=', attachmentID).where('document_id', '=', documentID).executeTakeFirst();
395
+ if (job != null) return false;
396
+ const source = await tx.selectFrom('kubun_blob_sources').select('document_id').where('attachment_id', '=', attachmentID).where('document_id', '=', documentID).executeTakeFirst();
397
+ if (source != null) return false;
398
+ await tx.insertInto('kubun_blob_fetch_jobs').values({
399
+ attachment_id: attachmentID,
400
+ document_id: documentID,
401
+ attempts: 0,
402
+ next_attempt_at: now,
403
+ last_error: null
404
+ }).onConflict((oc)=>oc.columns([
405
+ 'attachment_id',
406
+ 'document_id'
407
+ ]).doNothing()).execute();
408
+ return true;
409
+ });
410
+ },
411
+ async ensureFetchStub ({ attachmentID, contentLength, now }) {
412
+ // The upsert takes the entry lock: it waits for a byte deletion holding it.
413
+ await db.insertInto('kubun_blob_entries').values({
414
+ attachment_id: attachmentID,
415
+ content_length: contentLength,
416
+ encrypted: 0,
417
+ key_id: null,
418
+ chunk_size: 0,
419
+ state: 'remote-only',
420
+ pinned: 0,
421
+ created_at: now,
422
+ generation: 0
423
+ }).onConflict((oc)=>oc.column('attachment_id').doUpdateSet((eb)=>({
424
+ generation: eb('kubun_blob_entries.generation', '+', 1)
425
+ }))).execute();
426
+ },
427
+ async dropFetchJobs (documentID) {
428
+ await db.deleteFrom('kubun_blob_fetch_jobs').where('document_id', '=', documentID).execute();
429
+ },
430
+ async getFetchJob (attachmentID, documentID) {
431
+ const row = await db.selectFrom('kubun_blob_fetch_jobs').selectAll().where('attachment_id', '=', attachmentID).where('document_id', '=', documentID).executeTakeFirst();
432
+ return row == null ? null : fetchJobFromRow(row);
433
+ },
434
+ async listDueFetchJobs ({ now, limit }) {
435
+ const rows = await db.selectFrom('kubun_blob_fetch_jobs').selectAll().where('next_attempt_at', '<=', now).orderBy('next_attempt_at', 'asc').orderBy('attachment_id', 'asc').orderBy('document_id', 'asc').limit(limit).execute();
436
+ return rows.map(fetchJobFromRow);
437
+ },
438
+ async nextFetchJobAt () {
439
+ const row = await db.selectFrom('kubun_blob_fetch_jobs').select((eb)=>eb.fn.min('next_attempt_at').as('next_attempt_at')).executeTakeFirst();
440
+ return row?.next_attempt_at == null ? null : Number(row.next_attempt_at);
441
+ },
442
+ async recordFetchJobFailure ({ attachmentID, documentID, error, nextAttemptAt }) {
443
+ await db.updateTable('kubun_blob_fetch_jobs').set((eb)=>({
444
+ attempts: eb('attempts', '+', 1),
445
+ next_attempt_at: nextAttemptAt,
446
+ last_error: error
447
+ })).where('attachment_id', '=', attachmentID).where('document_id', '=', documentID).execute();
448
+ },
449
+ async completeFetchJob (attachmentID, documentID) {
450
+ await db.deleteFrom('kubun_blob_fetch_jobs').where('attachment_id', '=', attachmentID).where('document_id', '=', documentID).execute();
451
+ },
452
+ async sweepDocumentRows (documentID) {
453
+ return await inTransaction(async (tx)=>{
454
+ const named = new Set();
455
+ for (const table of [
456
+ 'kubun_blob_refs',
457
+ 'kubun_blob_sources',
458
+ 'kubun_blob_fetch_jobs'
459
+ ]){
460
+ const rows = await tx.selectFrom(table).select('attachment_id').where('document_id', '=', documentID).execute();
461
+ for (const row of rows)named.add(row.attachment_id);
462
+ }
463
+ const attachmentIDs = [
464
+ ...named
465
+ ].sort();
466
+ // Sorted entry locks, as every byte claim and fetch takes them: no row of the document
467
+ // goes while a claim is deciding from it.
468
+ for (const attachmentID of attachmentIDs)await lockEntry(tx, attachmentID);
469
+ for (const table of [
470
+ 'kubun_blob_refs',
471
+ 'kubun_blob_sources',
472
+ 'kubun_blob_fetch_jobs'
473
+ ]){
474
+ await tx.deleteFrom(table).where('document_id', '=', documentID).execute();
475
+ }
476
+ return attachmentIDs;
477
+ });
478
+ },
479
+ async lockEntryRow (attachmentID) {
480
+ if (!db.isTransaction) throw new BlobEntryLockRequiresTransactionError();
481
+ return await lockEntry(db, attachmentID) != null;
482
+ },
483
+ async listTransfers ({ after, limit }) {
484
+ let query = db.selectFrom('kubun_blob_entries').select([
485
+ 'attachment_id',
486
+ 'transfer_document_id'
487
+ ]).where('state', '=', 'partial').where('transfer_document_id', 'is not', null).orderBy('attachment_id', 'asc').limit(limit);
488
+ if (after !== undefined) query = query.where('attachment_id', '>', after);
489
+ return (await query.execute()).map((row)=>({
490
+ attachmentID: row.attachment_id,
491
+ documentID: row.transfer_document_id
492
+ }));
493
+ },
494
+ async deleteOrphanTransfer (attachmentID, documentID) {
495
+ return await inTransaction(async (tx)=>{
496
+ const entry = await lockEntry(tx, attachmentID);
497
+ if (entry?.state !== 'partial' || entry.transfer_document_id !== documentID) return false;
498
+ await clearTransfer(tx, attachmentID);
499
+ await tx.updateTable('kubun_blob_entries').set({
500
+ state: 'remote-only',
501
+ chunk_size: 0,
502
+ transfer_document_id: null
503
+ }).where('attachment_id', '=', attachmentID).execute();
504
+ return true;
505
+ });
506
+ },
507
+ async listByteDeletions () {
508
+ const rows = await db.selectFrom('kubun_blob_byte_deletions').select('attachment_id').orderBy('queued_at', 'asc').orderBy('attachment_id', 'asc').execute();
509
+ return rows.map((row)=>row.attachment_id);
111
510
  }
112
511
  };
113
512
  }
package/lib/index.d.ts CHANGED
@@ -1,9 +1,9 @@
1
1
  import type { StoreProvider } from '@kubun/db';
2
2
  import type { BlobStoreAPI } from './api.js';
3
- export type { BlobChunkInput, BlobEntryInput, BlobState, BlobStoreAPI, } from './api.js';
4
- export { createBlobStore } from './api.js';
3
+ export type { BeginTransferResult, BlobChunkInput, BlobEntryInput, BlobState, BlobStoreAPI, ByteDeletionClaim, FetchJob, LoadMode, LoadPolicy, QueueFetchJobParams, StagingTransfer, } from './api.js';
4
+ export { BlobEntryLockRequiresTransactionError, BlobTransferSupersededError, createBlobStore, } from './api.js';
5
5
  export { blobStoreDefinition } from './definition.js';
6
6
  export declare const BLOB_STORE: "blob";
7
7
  export declare const BLOB_BYTE_ROUTE_PREFIX: "/kubun-blob";
8
8
  export declare function getBlobStore(provider: StoreProvider): Promise<BlobStoreAPI>;
9
- export type { BlobChunk, BlobChunkTable, BlobEntry, BlobEntryTable, BlobRef, BlobRefTable, BlobTables, BlobTransfer, BlobTransferTable, } from './tables.js';
9
+ export type { BlobByteDeletion, BlobByteDeletionTable, BlobChunk, BlobChunkTable, BlobEntry, BlobEntryTable, BlobFetchJob, BlobFetchJobTable, BlobLoadPolicy, BlobLoadPolicyTable, BlobRef, BlobRefTable, BlobSource, BlobSourceTable, BlobTables, BlobTransfer, BlobTransferTable, } from './tables.js';
package/lib/index.js CHANGED
@@ -1,4 +1,4 @@
1
- export { createBlobStore } from './api.js';
1
+ export { BlobEntryLockRequiresTransactionError, BlobTransferSupersededError, createBlobStore } from './api.js';
2
2
  export { blobStoreDefinition } from './definition.js';
3
3
  export const BLOB_STORE = 'blob';
4
4
  // Well-known path for the direct-peer blob byte route (data plane). The holder's
package/lib/migrations.js CHANGED
@@ -8,15 +8,18 @@ export function getBlobMigrations(ctx) {
8
8
  // denormalized (also in the id) for SQL byte-accounting.
9
9
  .addColumn('content_length', t.bigint, (col)=>col.notNull())// Booleans as integer 0/1 — ColumnTypes has no bool.
10
10
  .addColumn('encrypted', 'integer', (col)=>col.notNull().defaultTo(0)).addColumn('key_id', t.text).addColumn('chunk_size', 'integer', (col)=>col.notNull()).addColumn('state', t.text, (col)=>col.notNull()).addColumn('pinned', 'integer', (col)=>col.notNull().defaultTo(0))// ms-epoch (Date.now) overflows int32 on Postgres; bigint holds it.
11
- .addColumn('created_at', t.bigint, (col)=>col.notNull()).execute();
11
+ .addColumn('created_at', t.bigint, (col)=>col.notNull())// The document a partial transfer was started for, so a resume never
12
+ // binds another document to bytes no holder served through it.
13
+ .addColumn('transfer_document_id', t.text)// Bumped whenever the bytes are wanted again; fences byte deletion.
14
+ .addColumn('generation', 'integer', (col)=>col.notNull().defaultTo(0)).execute();
12
15
  // Chunk manifest: persistent per-chunk digest. Survives finalize so this
13
- // node can verify and serve ranges later (Phase 2).
16
+ // node can verify and serve ranges after finalization.
14
17
  await db.schema.createTable('kubun_blob_chunks').ifNotExists().addColumn('attachment_id', t.text, (col)=>col.notNull()).addColumn('index', 'integer', (col)=>col.notNull()).addColumn('digest', t.binary, (col)=>col.notNull()).addPrimaryKeyConstraint('kubun_blob_chunks_pkey', [
15
18
  'attachment_id',
16
19
  'index'
17
20
  ]).execute();
18
21
  // Transfer present-bitmap: which chunks are on disk for an in-flight
19
- // download. Purged on finalize. Written in Phase 2.
22
+ // download. Purged on finalize.
20
23
  await db.schema.createTable('kubun_blob_transfers').ifNotExists().addColumn('attachment_id', t.text, (col)=>col.notNull()).addColumn('index', 'integer', (col)=>col.notNull()).addPrimaryKeyConstraint('kubun_blob_transfers_pkey', [
21
24
  'attachment_id',
22
25
  'index'
@@ -26,8 +29,31 @@ export function getBlobMigrations(ctx) {
26
29
  'document_id',
27
30
  'ref_key'
28
31
  ]).execute();
32
+ // Documents through which this node holds a blob's bytes.
33
+ await db.schema.createTable('kubun_blob_sources').ifNotExists().addColumn('attachment_id', t.text, (col)=>col.notNull()).addColumn('document_id', t.text, (col)=>col.notNull()).addPrimaryKeyConstraint('kubun_blob_sources_pkey', [
34
+ 'attachment_id',
35
+ 'document_id'
36
+ ]).execute();
37
+ // Per-field attachment load modes. A field without a row loads on demand.
38
+ await db.schema.createTable('kubun_blob_load_policies').ifNotExists().addColumn('model_id', t.text, (col)=>col.notNull()).addColumn('field_path', t.text, (col)=>col.notNull()).addColumn('mode', t.text, (col)=>col.notNull()).addPrimaryKeyConstraint('kubun_blob_load_policies_pkey', [
39
+ 'model_id',
40
+ 'field_path'
41
+ ]).execute();
42
+ // Pending byte deletions, replayed until the bytes are gone.
43
+ await db.schema.createTable('kubun_blob_byte_deletions').ifNotExists().addColumn('attachment_id', t.text, (col)=>col.notNull().primaryKey()).addColumn('generation', 'integer').addColumn('queued_at', t.bigint, (col)=>col.notNull()).execute();
44
+ // Attachments a pinned document needs bound, retried until a holder serves them.
45
+ await db.schema.createTable('kubun_blob_fetch_jobs').ifNotExists().addColumn('attachment_id', t.text, (col)=>col.notNull()).addColumn('document_id', t.text, (col)=>col.notNull()).addColumn('attempts', 'integer', (col)=>col.notNull().defaultTo(0)).addColumn('next_attempt_at', t.bigint, (col)=>col.notNull()).addColumn('last_error', t.text).addPrimaryKeyConstraint('kubun_blob_fetch_jobs_pkey', [
46
+ 'attachment_id',
47
+ 'document_id'
48
+ ]).execute();
49
+ await db.schema.createIndex('idx_blob_fetch_jobs_next_attempt_at').ifNotExists().on('kubun_blob_fetch_jobs').column('next_attempt_at').execute();
50
+ await db.schema.createIndex('idx_blob_fetch_jobs_document_id').ifNotExists().on('kubun_blob_fetch_jobs').column('document_id').execute();
29
51
  },
30
52
  async down (db) {
53
+ await db.schema.dropTable('kubun_blob_fetch_jobs').ifExists().execute();
54
+ await db.schema.dropTable('kubun_blob_byte_deletions').ifExists().execute();
55
+ await db.schema.dropTable('kubun_blob_load_policies').ifExists().execute();
56
+ await db.schema.dropTable('kubun_blob_sources').ifExists().execute();
31
57
  await db.schema.dropTable('kubun_blob_refs').ifExists().execute();
32
58
  await db.schema.dropTable('kubun_blob_transfers').ifExists().execute();
33
59
  await db.schema.dropTable('kubun_blob_chunks').ifExists().execute();
package/lib/tables.d.ts CHANGED
@@ -8,6 +8,8 @@ export type BlobEntryTable = {
8
8
  state: string;
9
9
  pinned: number;
10
10
  created_at: number;
11
+ transfer_document_id: string | null;
12
+ generation: number;
11
13
  };
12
14
  export type BlobEntry = Selectable<BlobEntryTable>;
13
15
  export type BlobChunkTable = {
@@ -28,9 +30,38 @@ export type BlobRefTable = {
28
30
  ref_key: string;
29
31
  };
30
32
  export type BlobRef = Selectable<BlobRefTable>;
33
+ export type BlobSourceTable = {
34
+ attachment_id: string;
35
+ document_id: string;
36
+ };
37
+ export type BlobSource = Selectable<BlobSourceTable>;
38
+ export type BlobLoadPolicyTable = {
39
+ model_id: string;
40
+ field_path: string;
41
+ mode: string;
42
+ };
43
+ export type BlobLoadPolicy = Selectable<BlobLoadPolicyTable>;
44
+ export type BlobByteDeletionTable = {
45
+ attachment_id: string;
46
+ generation: number | null;
47
+ queued_at: number;
48
+ };
49
+ export type BlobByteDeletion = Selectable<BlobByteDeletionTable>;
50
+ export type BlobFetchJobTable = {
51
+ attachment_id: string;
52
+ document_id: string;
53
+ attempts: number;
54
+ next_attempt_at: number;
55
+ last_error: string | null;
56
+ };
57
+ export type BlobFetchJob = Selectable<BlobFetchJobTable>;
31
58
  export type BlobTables = {
32
59
  kubun_blob_entries: BlobEntryTable;
33
60
  kubun_blob_chunks: BlobChunkTable;
34
61
  kubun_blob_transfers: BlobTransferTable;
35
62
  kubun_blob_refs: BlobRefTable;
63
+ kubun_blob_sources: BlobSourceTable;
64
+ kubun_blob_load_policies: BlobLoadPolicyTable;
65
+ kubun_blob_byte_deletions: BlobByteDeletionTable;
66
+ kubun_blob_fetch_jobs: BlobFetchJobTable;
36
67
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@kubun/store-blob",
3
- "version": "0.17.0",
3
+ "version": "0.18.0",
4
4
  "keywords": [],
5
5
  "license": "see LICENSE.md",
6
6
  "sideEffects": false,
@@ -15,15 +15,15 @@
15
15
  "LICENSE.md"
16
16
  ],
17
17
  "dependencies": {
18
- "@kubun/db": "^0.17.0",
19
- "@kubun/db-adapter": "^0.17.0",
20
- "@kubun/id": "^0.17.0",
21
- "kysely": "^0.29.5"
18
+ "@kubun/db": "^0.18.0",
19
+ "@kubun/db-adapter": "^0.18.0",
20
+ "@kubun/id": "^0.18.0",
21
+ "kysely": "^0.29.7"
22
22
  },
23
23
  "devDependencies": {
24
- "@kubun/db-better-sqlite": "^0.17.0",
25
- "@kubun/db-postgres": "^0.17.0",
26
- "@testcontainers/postgresql": "^12.1.0"
24
+ "@kubun/db-better-sqlite": "^0.18.0",
25
+ "@kubun/db-postgres": "^0.18.0",
26
+ "@testcontainers/postgresql": "^12.2.1"
27
27
  },
28
28
  "publishConfig": {
29
29
  "access": "public"