@context-use/open-sync 0.2.1 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/LICENSE +3 -0
  2. package/package.json +7 -5
  3. package/src/api.ts +18 -16
  4. package/src/connector/client.ts +61 -28
  5. package/src/connector/failure.ts +49 -0
  6. package/src/connector/file.ts +104 -0
  7. package/src/connector/response.ts +22 -1
  8. package/src/db/client.ts +1 -11
  9. package/src/db/providers.sql +2 -3
  10. package/src/db/providers.ts +1 -11
  11. package/src/db/schema.sql +41 -36
  12. package/src/execution/abortable.ts +12 -0
  13. package/src/execution/diagnostics.ts +1 -2
  14. package/src/execution/provider.ts +1 -0
  15. package/src/execution/worker.ts +162 -39
  16. package/src/http/controller.ts +63 -42
  17. package/src/models/asset.ts +61 -0
  18. package/src/models/definition.ts +30 -15
  19. package/src/models/delivery.ts +25 -31
  20. package/src/models/error.ts +12 -1
  21. package/src/models/limits.ts +8 -3
  22. package/src/models/metadata.ts +42 -0
  23. package/src/models/page.ts +68 -13
  24. package/src/models/provider-path.ts +25 -0
  25. package/src/models/providers.ts +13 -2
  26. package/src/models/record.ts +22 -0
  27. package/src/models/registry.ts +15 -15
  28. package/src/models/source-http-error.ts +37 -0
  29. package/src/models/sync.ts +71 -0
  30. package/src/models/validation.ts +1 -1
  31. package/src/open-sync.ts +26 -7
  32. package/src/repositories/acquisition/contract.ts +18 -10
  33. package/src/repositories/acquisition/lease.ts +80 -94
  34. package/src/repositories/acquisition/outbox.ts +22 -22
  35. package/src/repositories/acquisition/records.ts +44 -13
  36. package/src/repositories/acquisition/sqlite.ts +78 -31
  37. package/src/repositories/assets/capacity.ts +16 -0
  38. package/src/repositories/assets/contract.ts +25 -0
  39. package/src/repositories/assets/filesystem.ts +103 -0
  40. package/src/repositories/assets/sqlite.ts +174 -0
  41. package/src/repositories/catalog/contract.ts +16 -19
  42. package/src/repositories/catalog/sqlite.ts +98 -121
  43. package/src/repositories/delivery/contract.ts +7 -11
  44. package/src/repositories/delivery/lease.ts +16 -0
  45. package/src/repositories/delivery/sqlite.ts +57 -34
  46. package/src/repositories/queue-usage.ts +7 -8
  47. package/src/repositories/rows.ts +10 -29
  48. package/src/runtime.ts +27 -6
  49. package/src/services/acquisition.ts +178 -52
  50. package/src/services/delivery.ts +39 -19
  51. package/src/services/management.ts +88 -51
  52. package/src/services/providers/service.ts +100 -10
  53. package/src/models/installation.ts +0 -37
@@ -1,118 +1,104 @@
1
1
  import type { Database } from 'bun:sqlite';
2
- import { definitionKey } from '../../models/definition';
3
2
  import { fail } from '../../models/error';
4
3
  import { workerScope } from '../../models/identity';
5
- import type { Installation } from '../../models/installation';
6
- import { readInstallation } from '../rows';
7
- import type { RunLease } from './contract';
4
+ import type { Sync } from '../../models/sync';
5
+ import { readSync } from '../rows';
6
+ import type { AcquisitionLease, AcquisitionRepository } from './contract';
8
7
 
9
- export function assertRun(input: { db: Database; lease: RunLease }): Installation {
10
- const { db, lease } = input;
8
+ export function assertAcquisition({ db, lease }: { db: Database; lease: AcquisitionLease }): Sync {
11
9
  const valid = db
12
- .query(
13
- "SELECT 1 FROM runs WHERE owner_id=? AND id=? AND installation_id=? AND binding_epoch=? AND definition_ref=? AND state='running' AND worker_id=? AND generation=? AND expires_at>?",
14
- )
15
- .get(
16
- lease.ownerId,
17
- lease.id,
18
- lease.installation.id,
19
- lease.installation.bindingEpoch,
20
- definitionKey(lease.installation.definition),
21
- lease.workerId,
22
- lease.generation,
23
- Date.now(),
24
- );
25
- const installation = readInstallation({ db, scope: { ...lease, id: lease.installation.id } });
26
- if (
27
- !valid ||
28
- !installation.enabled ||
29
- installation.bindingEpoch !== lease.installation.bindingEpoch ||
30
- definitionKey(installation.definition) !== definitionKey(lease.installation.definition)
31
- ) {
10
+ .query(`SELECT 1 FROM syncs WHERE owner_id=? AND id=? AND enabled=1
11
+ AND status='running' AND generation=? AND expires_at>?`)
12
+ .get(lease.ownerId, lease.sync.id, lease.generation, Date.now());
13
+ if (!valid) {
32
14
  fail('lease_lost');
33
15
  }
34
- if (installation.checkpointRevision !== lease.checkpointRevision) {
35
- fail('checkpoint_conflict');
36
- }
37
- return installation;
16
+ return readSync({ db, scope: { ...lease, id: lease.sync.id } });
38
17
  }
39
- export function claimRun(input: {
18
+
19
+ export function claimAcquisition({
20
+ db,
21
+ leaseMs,
22
+ }: {
40
23
  db: Database;
41
24
  leaseMs: number;
42
- historyLimit: number;
43
- }): RunLease | undefined {
44
- const { db } = input;
25
+ }): AcquisitionLease | undefined {
45
26
  return db
46
27
  .transaction(() => {
47
- recoverRuns(input);
48
- if (db.query("SELECT 1 FROM runs WHERE state='running'").get()) {
49
- return;
50
- }
28
+ const now = Date.now();
29
+ db.query(`UPDATE sync_polls SET state='interrupted',error_code='lease_expired'
30
+ WHERE completed_at IS NULL AND EXISTS (SELECT 1 FROM syncs
31
+ WHERE syncs.owner_id=sync_polls.owner_id AND syncs.id=sync_polls.sync_id
32
+ AND enabled=1 AND expires_at<=?)`).run(now);
33
+ db.query(`UPDATE syncs SET next_due_at=?,status='interrupted',error_code='lease_expired',expires_at=NULL
34
+ WHERE enabled=1 AND expires_at<=?`).run(now, now);
51
35
  const row = db
52
- .query<{ id: string; owner_id: string }, [number]>(
53
- 'SELECT id,owner_id FROM installations WHERE enabled=1 AND next_due_at<=? ORDER BY next_due_at,id LIMIT 1',
54
- )
55
- .get(Date.now());
36
+ .query<
37
+ {
38
+ id: string;
39
+ owner_id: string;
40
+ generation: number;
41
+ resync: number;
42
+ failure_count: number;
43
+ },
44
+ [number]
45
+ >(`SELECT id,owner_id,generation,resync,failure_count FROM syncs
46
+ WHERE enabled=1 AND expires_at IS NULL AND next_due_at<=?
47
+ ORDER BY next_due_at,generation,id LIMIT 1`)
48
+ .get(now);
56
49
  if (!row) {
57
50
  return;
58
51
  }
59
52
  const scope = workerScope(row.owner_id);
60
- const installation = readInstallation({ db, scope: { ...scope, id: row.id } });
61
- const lease: RunLease = {
62
- ...scope,
63
- id: `run_${crypto.randomUUID()}`,
64
- installation,
65
- workerId: crypto.randomUUID(),
66
- generation: 1,
67
- checkpointRevision: installation.checkpointRevision,
68
- };
69
- db.query(`INSERT INTO runs(owner_id,id,installation_id,definition_ref,binding_epoch,worker_id,generation,expires_at,checkpoint_revision,state,started_at)
70
- VALUES (?,?,?,?,?,?,?,?,?,'running',?)`).run(
53
+ // Subsequent pages and retries share one diagnostic row, independent of the lease.
54
+ db.query(`INSERT INTO sync_polls(owner_id,sync_id,started_at,state)
55
+ SELECT ?,?,?,'running' WHERE NOT EXISTS (
56
+ SELECT 1 FROM sync_polls WHERE owner_id=? AND sync_id=? AND completed_at IS NULL)`).run(
71
57
  scope.ownerId,
72
- lease.id,
73
- installation.id,
74
- definitionKey(installation.definition),
75
- installation.bindingEpoch,
76
- lease.workerId,
77
- lease.generation,
78
- Date.now() + input.leaseMs,
79
- lease.checkpointRevision,
80
- Date.now(),
81
- );
82
- db.query("UPDATE installations SET status='running' WHERE owner_id=? AND id=?").run(
58
+ row.id,
59
+ now,
83
60
  scope.ownerId,
84
- installation.id,
61
+ row.id,
85
62
  );
86
- return lease;
63
+ db.query(`UPDATE sync_polls SET state='running',error_code=NULL
64
+ WHERE owner_id=? AND sync_id=? AND completed_at IS NULL`).run(scope.ownerId, row.id);
65
+ db.query(`UPDATE syncs SET status='running',error_code=NULL,generation=generation+1,expires_at=?
66
+ WHERE owner_id=? AND id=?`).run(now + leaseMs, scope.ownerId, row.id);
67
+ return {
68
+ ...scope,
69
+ sync: readSync({ db, scope: { ...scope, id: row.id } }),
70
+ generation: row.generation + 1,
71
+ force: row.resync === 1,
72
+ failureCount: row.failure_count,
73
+ };
87
74
  })
88
75
  .immediate();
89
76
  }
90
- function recoverRuns(input: { db: Database; historyLimit: number }): void {
91
- input.db
92
- .query(`UPDATE installations SET next_due_at=?,status='lease_expired' WHERE enabled=1 AND EXISTS (
93
- SELECT 1 FROM runs WHERE owner_id=installations.owner_id AND installation_id=installations.id AND state='running' AND expires_at<=?)`)
94
- .run(Date.now(), Date.now());
95
- input.db
96
- .query(
97
- "UPDATE runs SET state='lease_expired',completed_at=? WHERE state='running' AND expires_at<=?",
98
- )
99
- .run(Date.now(), Date.now());
100
- input.db
101
- .query(
102
- "DELETE FROM runs WHERE state!='running' AND rowid NOT IN (SELECT rowid FROM runs ORDER BY started_at DESC,rowid DESC LIMIT ?)",
103
- )
104
- .run(input.historyLimit);
105
- }
106
- export function finishRun(input: {
107
- db: Database;
108
- lease: RunLease;
109
- state: string;
110
- delay: number;
111
- }): void {
112
- input.db
113
- .query('UPDATE runs SET state=?,completed_at=? WHERE owner_id=? AND id=?')
114
- .run(input.state, Date.now(), input.lease.ownerId, input.lease.id);
115
- input.db
116
- .query('UPDATE installations SET status=?,next_due_at=? WHERE owner_id=? AND id=?')
117
- .run(input.state, Date.now() + input.delay, input.lease.ownerId, input.lease.installation.id);
77
+
78
+ export function finishAcquisition(
79
+ input: Parameters<AcquisitionRepository['finish']>[0] & { db: Database },
80
+ ): void {
81
+ const { db, lease } = input;
82
+ const now = Date.now();
83
+ db.query(`UPDATE sync_polls SET state=?,error_code=?,completed_at=?
84
+ WHERE owner_id=? AND sync_id=? AND completed_at IS NULL`).run(
85
+ input.pause ? 'disabled' : input.state,
86
+ input.errorCode ?? null,
87
+ input.state === 'succeeded' ? now : null,
88
+ lease.ownerId,
89
+ lease.sync.id,
90
+ );
91
+ db.query(`UPDATE syncs SET status=?,error_code=?,next_due_at=?,expires_at=NULL,
92
+ enabled=CASE WHEN ? THEN 0 ELSE enabled END,
93
+ failure_count=COALESCE(?,failure_count),resync=CASE WHEN ? THEN 0 ELSE resync END
94
+ WHERE owner_id=? AND id=?`).run(
95
+ input.pause ? 'disabled' : input.state,
96
+ input.errorCode ?? null,
97
+ now + input.delay,
98
+ Number(input.pause ?? false),
99
+ input.failureCount ?? null,
100
+ Number(input.state === 'succeeded'),
101
+ lease.ownerId,
102
+ lease.sync.id,
103
+ );
118
104
  }
@@ -1,29 +1,33 @@
1
1
  import type { Database } from 'bun:sqlite';
2
- import type { DeliveredRecord, Delivery } from '../../models/delivery';
2
+ import type { DeliveryAsset } from '../../models/asset';
3
+ import type { Deliverable, DeliveredRecord } from '../../models/delivery';
3
4
  import { fail } from '../../models/error';
4
- import type { Installation } from '../../models/installation';
5
5
  import { canonicalJson } from '../../models/json';
6
6
  import type { QueueLimits } from '../../models/limits';
7
+ import type { Sync } from '../../models/sync';
7
8
  import { queueUsage } from '../queue-usage';
9
+ import type { AcquisitionLease } from './contract';
8
10
 
9
11
  export function enqueue(input: {
10
12
  db: Database;
11
- installation: Installation;
13
+ sync: Sync;
12
14
  records: DeliveredRecord[];
13
15
  limits: QueueLimits;
16
+ assets: DeliveryAsset[];
17
+ lease: AcquisitionLease;
14
18
  }): void {
15
19
  if (!input.records.length) {
16
20
  return;
17
21
  }
18
- const { installation, db, limits } = input;
19
- const delivery: Delivery = {
20
- version: 1,
22
+ const { sync, db, limits } = input;
23
+ const { assets } = input;
24
+ const delivery: Omit<Deliverable, 'openAsset'> = {
21
25
  id: `delivery_${crypto.randomUUID()}`,
22
- ownerId: installation.ownerId,
23
- sourceId: installation.sourceId,
24
- installationId: installation.id,
25
- definition: installation.definition,
26
- deliverable: { records: input.records },
26
+ ownerId: sync.ownerId,
27
+ syncId: sync.id,
28
+ definition: sync.definition,
29
+ records: input.records,
30
+ assets,
27
31
  };
28
32
  const body = canonicalJson(delivery).json;
29
33
  const bytes = Buffer.byteLength(body);
@@ -38,15 +42,11 @@ export function enqueue(input: {
38
42
  fail('waiting_for_capacity');
39
43
  }
40
44
  db.query(
41
- 'INSERT INTO deliveries(owner_id,id,installation_id,destination_id,body,bytes,record_count,due_at) VALUES (?,?,?,?,?,?,?,?)',
42
- ).run(
43
- installation.ownerId,
44
- delivery.id,
45
- installation.id,
46
- installation.destinationId,
47
- body,
48
- bytes,
49
- input.records.length,
50
- Date.now(),
51
- );
45
+ 'INSERT INTO deliveries(owner_id,id,sync_id,body,bytes,record_count,due_at) VALUES (?,?,?,?,?,?,?)',
46
+ ).run(sync.ownerId, delivery.id, sync.id, body, bytes, input.records.length, Date.now());
47
+ for (const asset of assets) {
48
+ db.query(
49
+ `UPDATE delivery_assets SET delivery_id=? WHERE owner_id=? AND sync_id=? AND generation=? AND asset_id=? AND asset_version=?`,
50
+ ).run(delivery.id, sync.ownerId, sync.id, input.lease.generation, asset.id, asset.version);
51
+ }
52
52
  }
@@ -1,37 +1,68 @@
1
1
  import type { Database } from 'bun:sqlite';
2
- import type { DeliveredRecord, SyncRecord } from '../../models/delivery';
3
- import type { Installation } from '../../models/installation';
2
+ import type { DeliveryAsset } from '../../models/asset';
3
+ import type { DeliveredRecord } from '../../models/delivery';
4
4
  import { canonicalJson } from '../../models/json';
5
+ import type { SyncRecord } from '../../models/record';
6
+ import type { Sync } from '../../models/sync';
5
7
 
6
8
  export function writeRecord(input: {
7
9
  db: Database;
8
- installation: Installation;
10
+ sync: Sync;
9
11
  record: SyncRecord;
12
+ assets: DeliveryAsset[];
13
+ force: boolean;
10
14
  }): DeliveredRecord | undefined {
11
- const { db, installation, record } = input;
15
+ const { db, sync, record } = input;
12
16
  const previous = db
13
17
  .query<{ hash: string; revision: number; deleted: number }, [string, string, string, string]>(
14
- 'SELECT hash,revision,deleted FROM records WHERE owner_id=? AND installation_id=? AND kind=? AND id=?',
18
+ 'SELECT hash,revision,deleted FROM record_state WHERE owner_id=? AND sync_id=? AND kind=? AND id=?',
15
19
  )
16
- .get(installation.ownerId, installation.id, record.kind, record.id);
17
- if (record.operation === 'delete' && (!previous || previous.deleted === 1)) {
20
+ .get(sync.ownerId, sync.id, record.kind, record.id);
21
+ if (!input.force && record.operation === 'delete' && (!previous || previous.deleted === 1)) {
18
22
  return;
19
23
  }
20
24
  const contentHash =
21
- record.operation === 'upsert' ? canonicalJson(record.data).sha256 : previous!.hash;
22
- if (record.operation === 'upsert' && previous?.deleted === 0 && previous.hash === contentHash) {
25
+ record.operation === 'upsert'
26
+ ? hashRecord({ record, assets: input.assets })
27
+ : (previous?.hash ?? '');
28
+ if (
29
+ !input.force &&
30
+ record.operation === 'upsert' &&
31
+ previous?.deleted === 0 &&
32
+ previous.hash === contentHash
33
+ ) {
23
34
  return;
24
35
  }
25
36
  const revision = (previous?.revision ?? 0) + 1;
26
- db.query(`INSERT INTO records VALUES (?,?,?,?,?,?,?) ON CONFLICT(owner_id,installation_id,kind,id)
37
+ db.query(`INSERT INTO record_state VALUES (?,?,?,?,?,?,?) ON CONFLICT(owner_id,sync_id,kind,id)
27
38
  DO UPDATE SET hash=excluded.hash,revision=excluded.revision,deleted=excluded.deleted`).run(
28
- installation.ownerId,
29
- installation.id,
39
+ sync.ownerId,
40
+ sync.id,
30
41
  record.kind,
31
42
  record.id,
32
43
  contentHash,
33
44
  revision,
34
45
  Number(record.operation === 'delete'),
35
46
  );
36
- return { ...record, eventId: `event_${crypto.randomUUID()}`, revision, contentHash };
47
+ return { ...record, revision };
48
+ }
49
+
50
+ function hashRecord({
51
+ record,
52
+ assets,
53
+ }: {
54
+ record: Extract<SyncRecord, { operation: 'upsert' }>;
55
+ assets: DeliveryAsset[];
56
+ }) {
57
+ return canonicalJson({
58
+ id: record.id,
59
+ kind: record.kind,
60
+ data: record.data,
61
+ content: record.content ?? null,
62
+ assetRefs: record.assetRefs ?? {},
63
+ assets,
64
+ preview: record.preview ?? null,
65
+ createdAt: record.createdAt ?? null,
66
+ updatedAt: record.updatedAt ?? null,
67
+ }).sha256;
37
68
  }
@@ -1,60 +1,107 @@
1
1
  import type { Database } from 'bun:sqlite';
2
- import { definitionKey, type SyncDefinition, type SyncPage } from '../../models/definition';
2
+ import { assetKey, type DeliveryAsset } from '../../models/asset';
3
+ import type { SyncDefinition, SyncStep } from '../../models/definition';
3
4
  import type { DeliveredRecord } from '../../models/delivery';
4
5
  import { fail } from '../../models/error';
5
6
  import { canonicalJson } from '../../models/json';
6
7
  import type { QueueLimits } from '../../models/limits';
7
8
  import { preparePage } from '../../models/page';
9
+ import { capturedAssets } from '../assets/sqlite';
8
10
  import { hasQueueCapacity } from '../queue-usage';
9
- import type { AcquisitionRepository, RunLease } from './contract';
10
- import { assertRun, claimRun, finishRun } from './lease';
11
+ import type { AcquisitionLease, AcquisitionRepository } from './contract';
12
+ import { assertAcquisition, claimAcquisition, finishAcquisition } from './lease';
11
13
  import { enqueue } from './outbox';
12
14
  import { writeRecord } from './records';
13
15
 
14
16
  export class SqliteAcquisition implements AcquisitionRepository {
15
- constructor(
16
- private readonly input: { db: Database; limits: QueueLimits; historyLimit: number },
17
- ) {}
17
+ constructor(private readonly input: { db: Database; limits: QueueLimits }) {}
18
+ capacityReleased(): void {
19
+ this.input.db
20
+ .query("UPDATE syncs SET next_due_at=? WHERE enabled=1 AND status='waiting_for_capacity'")
21
+ .run(Date.now());
22
+ }
23
+ nextDue(): number | undefined {
24
+ return (
25
+ this.input.db
26
+ .query<{ due: number | null }, []>(`
27
+ SELECT MIN(COALESCE(expires_at,next_due_at)) AS due FROM syncs WHERE enabled=1`)
28
+ .get()?.due ?? undefined
29
+ );
30
+ }
18
31
  claim(leaseMs: number) {
19
- return claimRun({ ...this.input, leaseMs });
32
+ return claimAcquisition({ ...this.input, leaseMs });
20
33
  }
21
34
  hasCapacity() {
22
35
  return hasQueueCapacity(this.input);
23
36
  }
24
- commit(input: { lease: RunLease; page: SyncPage; definition: SyncDefinition }): void {
37
+ commit(input: { lease: AcquisitionLease; page: SyncStep; definition: SyncDefinition }): void {
25
38
  const { db, limits } = this.input;
26
39
  const page = preparePage({ ...input, limits });
27
40
  db.transaction(() => {
28
- const installation = assertRun({ db, lease: input.lease });
29
- if (definitionKey(input.definition) !== definitionKey(installation.definition)) {
41
+ const sync = assertAcquisition({ db, lease: input.lease });
42
+ if (input.definition.id !== sync.definition) {
30
43
  fail('definition_conflict');
31
44
  }
32
- const records: DeliveredRecord[] = [];
33
- for (const record of page.deliverable.records) {
34
- const changed = writeRecord({ db, installation, record });
35
- if (changed) {
36
- records.push(changed);
37
- }
38
- }
39
- enqueue({ db, installation, records, limits });
40
- db.query(
41
- 'UPDATE installations SET checkpoint=?,checkpoint_revision=checkpoint_revision+1 WHERE owner_id=? AND id=?',
42
- ).run(canonicalJson(page.checkpoint).json, installation.ownerId, installation.id);
43
- db.query(
44
- 'UPDATE runs SET checkpoint_revision=checkpoint_revision+1,pages=pages+1 WHERE owner_id=? AND id=?',
45
- ).run(input.lease.ownerId, input.lease.id);
46
- if (page.complete) {
47
- finishRun({ db, lease: input.lease, state: 'succeeded', delay: installation.intervalMs });
48
- }
45
+ const { records, assets } = changedRecords({ db, lease: input.lease, sync, page });
46
+ enqueue({ db, sync, records, limits, assets, lease: input.lease });
47
+ db.query(`UPDATE sync_polls SET records_processed=records_processed+?,records_queued=records_queued+?
48
+ WHERE owner_id=? AND sync_id=? AND completed_at IS NULL`).run(
49
+ page.records.length,
50
+ records.length,
51
+ sync.ownerId,
52
+ sync.id,
53
+ );
54
+ db.query('UPDATE syncs SET checkpoint=? WHERE owner_id=? AND id=?').run(
55
+ canonicalJson(page.checkpoint).json,
56
+ sync.ownerId,
57
+ sync.id,
58
+ );
59
+ finishAcquisition({
60
+ db,
61
+ lease: input.lease,
62
+ state: page.complete ? 'succeeded' : 'ready',
63
+ delay: page.complete ? sync.intervalMs : 0,
64
+ failureCount: page.complete ? 0 : undefined,
65
+ });
49
66
  }).immediate();
50
- input.lease.checkpointRevision++;
51
67
  }
52
- finish(input: { lease: RunLease; state: string; delay: number }): void {
68
+ finish(input: Parameters<AcquisitionRepository['finish']>[0]): void {
53
69
  this.input.db
54
70
  .transaction(() => {
55
- assertRun({ db: this.input.db, lease: input.lease });
56
- finishRun({ db: this.input.db, ...input });
71
+ assertAcquisition({ db: this.input.db, lease: input.lease });
72
+ finishAcquisition({ db: this.input.db, ...input });
57
73
  })
58
74
  .immediate();
59
75
  }
60
76
  }
77
+
78
+ function changedRecords(input: {
79
+ db: Database;
80
+ lease: AcquisitionLease;
81
+ sync: import('../../models/sync').Sync;
82
+ page: SyncStep;
83
+ }) {
84
+ const records: DeliveredRecord[] = [];
85
+ const assets = new Map<string, DeliveryAsset>();
86
+ for (const record of input.page.records) {
87
+ const referenced = capturedAssets({
88
+ db: input.db,
89
+ lease: input.lease,
90
+ refs: record.operation === 'upsert' ? Object.values(record.assetRefs ?? {}) : [],
91
+ });
92
+ const changed = writeRecord({
93
+ db: input.db,
94
+ sync: input.sync,
95
+ record,
96
+ assets: referenced,
97
+ force: input.lease.force,
98
+ });
99
+ if (changed) {
100
+ records.push(changed);
101
+ for (const asset of referenced) {
102
+ assets.set(assetKey(asset), asset);
103
+ }
104
+ }
105
+ }
106
+ return { records, assets: [...assets.values()] };
107
+ }
@@ -0,0 +1,16 @@
1
+ import type { Database } from 'bun:sqlite';
2
+ import type { AcquisitionLease } from '../acquisition/contract';
3
+
4
+ export function assetBytes(input: { db: Database; lease?: AcquisitionLease }): number {
5
+ return input.db
6
+ .query<{ bytes: number }, (string | number | null)[]>(
7
+ `SELECT COALESCE(SUM(bytes),0) AS bytes FROM delivery_assets
8
+ WHERE (? IS NULL OR (owner_id=? AND sync_id=? AND generation=?))`,
9
+ )
10
+ .get(
11
+ input.lease?.ownerId ?? null,
12
+ input.lease?.ownerId ?? null,
13
+ input.lease?.sync.id ?? null,
14
+ input.lease?.generation ?? null,
15
+ )!.bytes;
16
+ }
@@ -0,0 +1,25 @@
1
+ import type { AssetMetadata, AssetRef } from '../../models/asset';
2
+ import type { AcquisitionLease } from '../acquisition/contract';
3
+ import type { DeliveryLease } from '../delivery/contract';
4
+
5
+ export interface AssetRepository {
6
+ stage(input: { lease: AcquisitionLease; asset: AssetMetadata; unavailable?: string }): string;
7
+ captured(input: { lease: AcquisitionLease; id: string; size: number; sha256: string }): void;
8
+ reserve(input: { lease: AcquisitionLease; id: string; bytes: number }): void;
9
+ discarded(id: string): void;
10
+ garbage(): string[];
11
+ retained(id: string): boolean;
12
+ read(input: { lease: DeliveryLease; asset: AssetRef }): string;
13
+ }
14
+ export interface AssetFiles {
15
+ write(input: {
16
+ id: string;
17
+ body: ReadableStream<Uint8Array>;
18
+ reserve(file: { id: string; bytes: number }): void;
19
+ maxBytes: number;
20
+ signal: AbortSignal;
21
+ }): Promise<{ id: string; size: number; sha256: string }>;
22
+ open(id: string): Promise<ReadableStream<Uint8Array>>;
23
+ remove(id: string): Promise<void>;
24
+ sweep(input: { retain(id: string): boolean }): Promise<void>;
25
+ }
@@ -0,0 +1,103 @@
1
+ const privateFileMode = 0o600;
2
+
3
+ import { createHash } from 'node:crypto';
4
+ import { mkdir, open, readdir, unlink } from 'node:fs/promises';
5
+ import { join } from 'node:path';
6
+ import { fail } from '../../models/error';
7
+ import type { AssetFiles } from './contract';
8
+
9
+ /** A directory dedicated to this queue. Names come only from generated UUIDs, never provider filenames. */
10
+ export class DirectoryAssets implements AssetFiles {
11
+ constructor(private readonly directory: string) {}
12
+ async write(input: Parameters<AssetFiles['write']>[0]) {
13
+ const reader = input.body.getReader();
14
+ const { id } = input;
15
+ let file: Awaited<ReturnType<typeof open>> | undefined;
16
+ const hash = createHash('sha256');
17
+ let size = 0;
18
+ const abort = () => {
19
+ void reader.cancel().catch(() => undefined);
20
+ };
21
+ input.signal.addEventListener('abort', abort, { once: true });
22
+ try {
23
+ input.signal.throwIfAborted();
24
+ input.reserve({ id, bytes: 0 });
25
+ await mkdir(this.directory, { recursive: true, mode: 0o700 });
26
+ file = await open(this.path(id), 'wx', privateFileMode);
27
+ while (true) {
28
+ input.signal.throwIfAborted();
29
+ const chunk = await reader.read();
30
+ input.signal.throwIfAborted();
31
+ if (chunk.done) {
32
+ break;
33
+ }
34
+ size += chunk.value.byteLength;
35
+ if (size > input.maxBytes) {
36
+ fail('asset_too_large');
37
+ }
38
+ input.reserve({ id, bytes: size });
39
+ hash.update(chunk.value);
40
+ let offset = 0;
41
+ while (offset < chunk.value.byteLength) {
42
+ const written = await file.write(chunk.value.subarray(offset));
43
+ offset += written.bytesWritten;
44
+ }
45
+ }
46
+ await file.sync();
47
+ await file.close();
48
+ file = undefined;
49
+ const directory = await open(this.directory, 'r');
50
+ try {
51
+ await directory.sync();
52
+ } finally {
53
+ await directory.close();
54
+ }
55
+ return { id, size, sha256: hash.digest('hex') };
56
+ } catch (error) {
57
+ try {
58
+ await file?.close();
59
+ } finally {
60
+ await this.remove(id);
61
+ }
62
+ throw error;
63
+ } finally {
64
+ input.signal.removeEventListener('abort', abort);
65
+ await reader.cancel().catch(() => undefined);
66
+ reader.releaseLock();
67
+ }
68
+ }
69
+ async open(id: string) {
70
+ const file = Bun.file(this.path(id));
71
+ if (!(await file.exists())) {
72
+ fail('asset_content_missing');
73
+ }
74
+ return file.stream();
75
+ }
76
+ async remove(id: string) {
77
+ await unlink(this.path(id)).catch((error: NodeJS.ErrnoException) => {
78
+ if (error.code !== 'ENOENT') {
79
+ throw error;
80
+ }
81
+ });
82
+ }
83
+ async sweep(input: Parameters<AssetFiles['sweep']>[0]) {
84
+ const names = await readdir(this.directory).catch((error: NodeJS.ErrnoException) => {
85
+ if (error.code !== 'ENOENT') {
86
+ throw error;
87
+ }
88
+ return [] as string[];
89
+ });
90
+ for (const name of names) {
91
+ if (!/^[0-9a-f-]{36}$/.test(name) || input.retain(name)) {
92
+ continue;
93
+ }
94
+ await this.remove(name);
95
+ }
96
+ }
97
+ private path(id: string) {
98
+ if (!/^[0-9a-f-]{36}$/.test(id)) {
99
+ fail('invalid_asset_file');
100
+ }
101
+ return join(this.directory, id);
102
+ }
103
+ }