@context-use/open-sync 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +3 -0
- package/package.json +7 -5
- package/src/api.ts +18 -16
- package/src/connector/client.ts +61 -28
- package/src/connector/failure.ts +49 -0
- package/src/connector/file.ts +104 -0
- package/src/connector/response.ts +22 -1
- package/src/db/client.ts +1 -11
- package/src/db/providers.sql +2 -3
- package/src/db/providers.ts +1 -11
- package/src/db/schema.sql +41 -36
- package/src/execution/abortable.ts +12 -0
- package/src/execution/diagnostics.ts +1 -2
- package/src/execution/provider.ts +1 -0
- package/src/execution/worker.ts +162 -39
- package/src/http/controller.ts +63 -42
- package/src/models/asset.ts +61 -0
- package/src/models/definition.ts +30 -15
- package/src/models/delivery.ts +25 -31
- package/src/models/error.ts +12 -1
- package/src/models/limits.ts +8 -3
- package/src/models/metadata.ts +42 -0
- package/src/models/page.ts +68 -13
- package/src/models/provider-path.ts +25 -0
- package/src/models/providers.ts +13 -2
- package/src/models/record.ts +22 -0
- package/src/models/registry.ts +15 -15
- package/src/models/source-http-error.ts +37 -0
- package/src/models/sync.ts +71 -0
- package/src/models/validation.ts +1 -1
- package/src/open-sync.ts +26 -7
- package/src/repositories/acquisition/contract.ts +18 -10
- package/src/repositories/acquisition/lease.ts +80 -94
- package/src/repositories/acquisition/outbox.ts +22 -22
- package/src/repositories/acquisition/records.ts +44 -13
- package/src/repositories/acquisition/sqlite.ts +78 -31
- package/src/repositories/assets/capacity.ts +16 -0
- package/src/repositories/assets/contract.ts +25 -0
- package/src/repositories/assets/filesystem.ts +103 -0
- package/src/repositories/assets/sqlite.ts +174 -0
- package/src/repositories/catalog/contract.ts +16 -19
- package/src/repositories/catalog/sqlite.ts +98 -121
- package/src/repositories/delivery/contract.ts +7 -11
- package/src/repositories/delivery/lease.ts +16 -0
- package/src/repositories/delivery/sqlite.ts +57 -34
- package/src/repositories/queue-usage.ts +7 -8
- package/src/repositories/rows.ts +10 -29
- package/src/runtime.ts +27 -6
- package/src/services/acquisition.ts +178 -52
- package/src/services/delivery.ts +39 -19
- package/src/services/management.ts +88 -51
- package/src/services/providers/service.ts +100 -10
- package/src/models/installation.ts +0 -37
|
@@ -1,118 +1,104 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
|
-
import { definitionKey } from '../../models/definition';
|
|
3
2
|
import { fail } from '../../models/error';
|
|
4
3
|
import { workerScope } from '../../models/identity';
|
|
5
|
-
import type {
|
|
6
|
-
import {
|
|
7
|
-
import type {
|
|
4
|
+
import type { Sync } from '../../models/sync';
|
|
5
|
+
import { readSync } from '../rows';
|
|
6
|
+
import type { AcquisitionLease, AcquisitionRepository } from './contract';
|
|
8
7
|
|
|
9
|
-
export function
|
|
10
|
-
const { db, lease } = input;
|
|
8
|
+
export function assertAcquisition({ db, lease }: { db: Database; lease: AcquisitionLease }): Sync {
|
|
11
9
|
const valid = db
|
|
12
|
-
.query(
|
|
13
|
-
|
|
14
|
-
)
|
|
15
|
-
|
|
16
|
-
lease.ownerId,
|
|
17
|
-
lease.id,
|
|
18
|
-
lease.installation.id,
|
|
19
|
-
lease.installation.bindingEpoch,
|
|
20
|
-
definitionKey(lease.installation.definition),
|
|
21
|
-
lease.workerId,
|
|
22
|
-
lease.generation,
|
|
23
|
-
Date.now(),
|
|
24
|
-
);
|
|
25
|
-
const installation = readInstallation({ db, scope: { ...lease, id: lease.installation.id } });
|
|
26
|
-
if (
|
|
27
|
-
!valid ||
|
|
28
|
-
!installation.enabled ||
|
|
29
|
-
installation.bindingEpoch !== lease.installation.bindingEpoch ||
|
|
30
|
-
definitionKey(installation.definition) !== definitionKey(lease.installation.definition)
|
|
31
|
-
) {
|
|
10
|
+
.query(`SELECT 1 FROM syncs WHERE owner_id=? AND id=? AND enabled=1
|
|
11
|
+
AND status='running' AND generation=? AND expires_at>?`)
|
|
12
|
+
.get(lease.ownerId, lease.sync.id, lease.generation, Date.now());
|
|
13
|
+
if (!valid) {
|
|
32
14
|
fail('lease_lost');
|
|
33
15
|
}
|
|
34
|
-
|
|
35
|
-
fail('checkpoint_conflict');
|
|
36
|
-
}
|
|
37
|
-
return installation;
|
|
16
|
+
return readSync({ db, scope: { ...lease, id: lease.sync.id } });
|
|
38
17
|
}
|
|
39
|
-
|
|
18
|
+
|
|
19
|
+
export function claimAcquisition({
|
|
20
|
+
db,
|
|
21
|
+
leaseMs,
|
|
22
|
+
}: {
|
|
40
23
|
db: Database;
|
|
41
24
|
leaseMs: number;
|
|
42
|
-
|
|
43
|
-
}): RunLease | undefined {
|
|
44
|
-
const { db } = input;
|
|
25
|
+
}): AcquisitionLease | undefined {
|
|
45
26
|
return db
|
|
46
27
|
.transaction(() => {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
28
|
+
const now = Date.now();
|
|
29
|
+
db.query(`UPDATE sync_polls SET state='interrupted',error_code='lease_expired'
|
|
30
|
+
WHERE completed_at IS NULL AND EXISTS (SELECT 1 FROM syncs
|
|
31
|
+
WHERE syncs.owner_id=sync_polls.owner_id AND syncs.id=sync_polls.sync_id
|
|
32
|
+
AND enabled=1 AND expires_at<=?)`).run(now);
|
|
33
|
+
db.query(`UPDATE syncs SET next_due_at=?,status='interrupted',error_code='lease_expired',expires_at=NULL
|
|
34
|
+
WHERE enabled=1 AND expires_at<=?`).run(now, now);
|
|
51
35
|
const row = db
|
|
52
|
-
.query<
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
36
|
+
.query<
|
|
37
|
+
{
|
|
38
|
+
id: string;
|
|
39
|
+
owner_id: string;
|
|
40
|
+
generation: number;
|
|
41
|
+
resync: number;
|
|
42
|
+
failure_count: number;
|
|
43
|
+
},
|
|
44
|
+
[number]
|
|
45
|
+
>(`SELECT id,owner_id,generation,resync,failure_count FROM syncs
|
|
46
|
+
WHERE enabled=1 AND expires_at IS NULL AND next_due_at<=?
|
|
47
|
+
ORDER BY next_due_at,generation,id LIMIT 1`)
|
|
48
|
+
.get(now);
|
|
56
49
|
if (!row) {
|
|
57
50
|
return;
|
|
58
51
|
}
|
|
59
52
|
const scope = workerScope(row.owner_id);
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
installation,
|
|
65
|
-
workerId: crypto.randomUUID(),
|
|
66
|
-
generation: 1,
|
|
67
|
-
checkpointRevision: installation.checkpointRevision,
|
|
68
|
-
};
|
|
69
|
-
db.query(`INSERT INTO runs(owner_id,id,installation_id,definition_ref,binding_epoch,worker_id,generation,expires_at,checkpoint_revision,state,started_at)
|
|
70
|
-
VALUES (?,?,?,?,?,?,?,?,?,'running',?)`).run(
|
|
53
|
+
// Subsequent pages and retries share one diagnostic row, independent of the lease.
|
|
54
|
+
db.query(`INSERT INTO sync_polls(owner_id,sync_id,started_at,state)
|
|
55
|
+
SELECT ?,?,?,'running' WHERE NOT EXISTS (
|
|
56
|
+
SELECT 1 FROM sync_polls WHERE owner_id=? AND sync_id=? AND completed_at IS NULL)`).run(
|
|
71
57
|
scope.ownerId,
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
definitionKey(installation.definition),
|
|
75
|
-
installation.bindingEpoch,
|
|
76
|
-
lease.workerId,
|
|
77
|
-
lease.generation,
|
|
78
|
-
Date.now() + input.leaseMs,
|
|
79
|
-
lease.checkpointRevision,
|
|
80
|
-
Date.now(),
|
|
81
|
-
);
|
|
82
|
-
db.query("UPDATE installations SET status='running' WHERE owner_id=? AND id=?").run(
|
|
58
|
+
row.id,
|
|
59
|
+
now,
|
|
83
60
|
scope.ownerId,
|
|
84
|
-
|
|
61
|
+
row.id,
|
|
85
62
|
);
|
|
86
|
-
|
|
63
|
+
db.query(`UPDATE sync_polls SET state='running',error_code=NULL
|
|
64
|
+
WHERE owner_id=? AND sync_id=? AND completed_at IS NULL`).run(scope.ownerId, row.id);
|
|
65
|
+
db.query(`UPDATE syncs SET status='running',error_code=NULL,generation=generation+1,expires_at=?
|
|
66
|
+
WHERE owner_id=? AND id=?`).run(now + leaseMs, scope.ownerId, row.id);
|
|
67
|
+
return {
|
|
68
|
+
...scope,
|
|
69
|
+
sync: readSync({ db, scope: { ...scope, id: row.id } }),
|
|
70
|
+
generation: row.generation + 1,
|
|
71
|
+
force: row.resync === 1,
|
|
72
|
+
failureCount: row.failure_count,
|
|
73
|
+
};
|
|
87
74
|
})
|
|
88
75
|
.immediate();
|
|
89
76
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
.
|
|
100
|
-
|
|
101
|
-
.
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
.
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
.run(input.state, Date.now() + input.delay, input.lease.ownerId, input.lease.installation.id);
|
|
77
|
+
|
|
78
|
+
export function finishAcquisition(
|
|
79
|
+
input: Parameters<AcquisitionRepository['finish']>[0] & { db: Database },
|
|
80
|
+
): void {
|
|
81
|
+
const { db, lease } = input;
|
|
82
|
+
const now = Date.now();
|
|
83
|
+
db.query(`UPDATE sync_polls SET state=?,error_code=?,completed_at=?
|
|
84
|
+
WHERE owner_id=? AND sync_id=? AND completed_at IS NULL`).run(
|
|
85
|
+
input.pause ? 'disabled' : input.state,
|
|
86
|
+
input.errorCode ?? null,
|
|
87
|
+
input.state === 'succeeded' ? now : null,
|
|
88
|
+
lease.ownerId,
|
|
89
|
+
lease.sync.id,
|
|
90
|
+
);
|
|
91
|
+
db.query(`UPDATE syncs SET status=?,error_code=?,next_due_at=?,expires_at=NULL,
|
|
92
|
+
enabled=CASE WHEN ? THEN 0 ELSE enabled END,
|
|
93
|
+
failure_count=COALESCE(?,failure_count),resync=CASE WHEN ? THEN 0 ELSE resync END
|
|
94
|
+
WHERE owner_id=? AND id=?`).run(
|
|
95
|
+
input.pause ? 'disabled' : input.state,
|
|
96
|
+
input.errorCode ?? null,
|
|
97
|
+
now + input.delay,
|
|
98
|
+
Number(input.pause ?? false),
|
|
99
|
+
input.failureCount ?? null,
|
|
100
|
+
Number(input.state === 'succeeded'),
|
|
101
|
+
lease.ownerId,
|
|
102
|
+
lease.sync.id,
|
|
103
|
+
);
|
|
118
104
|
}
|
|
@@ -1,29 +1,33 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
|
-
import type {
|
|
2
|
+
import type { DeliveryAsset } from '../../models/asset';
|
|
3
|
+
import type { Deliverable, DeliveredRecord } from '../../models/delivery';
|
|
3
4
|
import { fail } from '../../models/error';
|
|
4
|
-
import type { Installation } from '../../models/installation';
|
|
5
5
|
import { canonicalJson } from '../../models/json';
|
|
6
6
|
import type { QueueLimits } from '../../models/limits';
|
|
7
|
+
import type { Sync } from '../../models/sync';
|
|
7
8
|
import { queueUsage } from '../queue-usage';
|
|
9
|
+
import type { AcquisitionLease } from './contract';
|
|
8
10
|
|
|
9
11
|
export function enqueue(input: {
|
|
10
12
|
db: Database;
|
|
11
|
-
|
|
13
|
+
sync: Sync;
|
|
12
14
|
records: DeliveredRecord[];
|
|
13
15
|
limits: QueueLimits;
|
|
16
|
+
assets: DeliveryAsset[];
|
|
17
|
+
lease: AcquisitionLease;
|
|
14
18
|
}): void {
|
|
15
19
|
if (!input.records.length) {
|
|
16
20
|
return;
|
|
17
21
|
}
|
|
18
|
-
const {
|
|
19
|
-
const
|
|
20
|
-
|
|
22
|
+
const { sync, db, limits } = input;
|
|
23
|
+
const { assets } = input;
|
|
24
|
+
const delivery: Omit<Deliverable, 'openAsset'> = {
|
|
21
25
|
id: `delivery_${crypto.randomUUID()}`,
|
|
22
|
-
ownerId:
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
26
|
+
ownerId: sync.ownerId,
|
|
27
|
+
syncId: sync.id,
|
|
28
|
+
definition: sync.definition,
|
|
29
|
+
records: input.records,
|
|
30
|
+
assets,
|
|
27
31
|
};
|
|
28
32
|
const body = canonicalJson(delivery).json;
|
|
29
33
|
const bytes = Buffer.byteLength(body);
|
|
@@ -38,15 +42,11 @@ export function enqueue(input: {
|
|
|
38
42
|
fail('waiting_for_capacity');
|
|
39
43
|
}
|
|
40
44
|
db.query(
|
|
41
|
-
'INSERT INTO deliveries(owner_id,id,
|
|
42
|
-
).run(
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
bytes,
|
|
49
|
-
input.records.length,
|
|
50
|
-
Date.now(),
|
|
51
|
-
);
|
|
45
|
+
'INSERT INTO deliveries(owner_id,id,sync_id,body,bytes,record_count,due_at) VALUES (?,?,?,?,?,?,?)',
|
|
46
|
+
).run(sync.ownerId, delivery.id, sync.id, body, bytes, input.records.length, Date.now());
|
|
47
|
+
for (const asset of assets) {
|
|
48
|
+
db.query(
|
|
49
|
+
`UPDATE delivery_assets SET delivery_id=? WHERE owner_id=? AND sync_id=? AND generation=? AND asset_id=? AND asset_version=?`,
|
|
50
|
+
).run(delivery.id, sync.ownerId, sync.id, input.lease.generation, asset.id, asset.version);
|
|
51
|
+
}
|
|
52
52
|
}
|
|
@@ -1,37 +1,68 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
|
-
import type {
|
|
3
|
-
import type {
|
|
2
|
+
import type { DeliveryAsset } from '../../models/asset';
|
|
3
|
+
import type { DeliveredRecord } from '../../models/delivery';
|
|
4
4
|
import { canonicalJson } from '../../models/json';
|
|
5
|
+
import type { SyncRecord } from '../../models/record';
|
|
6
|
+
import type { Sync } from '../../models/sync';
|
|
5
7
|
|
|
6
8
|
export function writeRecord(input: {
|
|
7
9
|
db: Database;
|
|
8
|
-
|
|
10
|
+
sync: Sync;
|
|
9
11
|
record: SyncRecord;
|
|
12
|
+
assets: DeliveryAsset[];
|
|
13
|
+
force: boolean;
|
|
10
14
|
}): DeliveredRecord | undefined {
|
|
11
|
-
const { db,
|
|
15
|
+
const { db, sync, record } = input;
|
|
12
16
|
const previous = db
|
|
13
17
|
.query<{ hash: string; revision: number; deleted: number }, [string, string, string, string]>(
|
|
14
|
-
'SELECT hash,revision,deleted FROM
|
|
18
|
+
'SELECT hash,revision,deleted FROM record_state WHERE owner_id=? AND sync_id=? AND kind=? AND id=?',
|
|
15
19
|
)
|
|
16
|
-
.get(
|
|
17
|
-
if (record.operation === 'delete' && (!previous || previous.deleted === 1)) {
|
|
20
|
+
.get(sync.ownerId, sync.id, record.kind, record.id);
|
|
21
|
+
if (!input.force && record.operation === 'delete' && (!previous || previous.deleted === 1)) {
|
|
18
22
|
return;
|
|
19
23
|
}
|
|
20
24
|
const contentHash =
|
|
21
|
-
record.operation === 'upsert'
|
|
22
|
-
|
|
25
|
+
record.operation === 'upsert'
|
|
26
|
+
? hashRecord({ record, assets: input.assets })
|
|
27
|
+
: (previous?.hash ?? '');
|
|
28
|
+
if (
|
|
29
|
+
!input.force &&
|
|
30
|
+
record.operation === 'upsert' &&
|
|
31
|
+
previous?.deleted === 0 &&
|
|
32
|
+
previous.hash === contentHash
|
|
33
|
+
) {
|
|
23
34
|
return;
|
|
24
35
|
}
|
|
25
36
|
const revision = (previous?.revision ?? 0) + 1;
|
|
26
|
-
db.query(`INSERT INTO
|
|
37
|
+
db.query(`INSERT INTO record_state VALUES (?,?,?,?,?,?,?) ON CONFLICT(owner_id,sync_id,kind,id)
|
|
27
38
|
DO UPDATE SET hash=excluded.hash,revision=excluded.revision,deleted=excluded.deleted`).run(
|
|
28
|
-
|
|
29
|
-
|
|
39
|
+
sync.ownerId,
|
|
40
|
+
sync.id,
|
|
30
41
|
record.kind,
|
|
31
42
|
record.id,
|
|
32
43
|
contentHash,
|
|
33
44
|
revision,
|
|
34
45
|
Number(record.operation === 'delete'),
|
|
35
46
|
);
|
|
36
|
-
return { ...record,
|
|
47
|
+
return { ...record, revision };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function hashRecord({
|
|
51
|
+
record,
|
|
52
|
+
assets,
|
|
53
|
+
}: {
|
|
54
|
+
record: Extract<SyncRecord, { operation: 'upsert' }>;
|
|
55
|
+
assets: DeliveryAsset[];
|
|
56
|
+
}) {
|
|
57
|
+
return canonicalJson({
|
|
58
|
+
id: record.id,
|
|
59
|
+
kind: record.kind,
|
|
60
|
+
data: record.data,
|
|
61
|
+
content: record.content ?? null,
|
|
62
|
+
assetRefs: record.assetRefs ?? {},
|
|
63
|
+
assets,
|
|
64
|
+
preview: record.preview ?? null,
|
|
65
|
+
createdAt: record.createdAt ?? null,
|
|
66
|
+
updatedAt: record.updatedAt ?? null,
|
|
67
|
+
}).sha256;
|
|
37
68
|
}
|
|
@@ -1,60 +1,107 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
|
-
import {
|
|
2
|
+
import { assetKey, type DeliveryAsset } from '../../models/asset';
|
|
3
|
+
import type { SyncDefinition, SyncStep } from '../../models/definition';
|
|
3
4
|
import type { DeliveredRecord } from '../../models/delivery';
|
|
4
5
|
import { fail } from '../../models/error';
|
|
5
6
|
import { canonicalJson } from '../../models/json';
|
|
6
7
|
import type { QueueLimits } from '../../models/limits';
|
|
7
8
|
import { preparePage } from '../../models/page';
|
|
9
|
+
import { capturedAssets } from '../assets/sqlite';
|
|
8
10
|
import { hasQueueCapacity } from '../queue-usage';
|
|
9
|
-
import type {
|
|
10
|
-
import {
|
|
11
|
+
import type { AcquisitionLease, AcquisitionRepository } from './contract';
|
|
12
|
+
import { assertAcquisition, claimAcquisition, finishAcquisition } from './lease';
|
|
11
13
|
import { enqueue } from './outbox';
|
|
12
14
|
import { writeRecord } from './records';
|
|
13
15
|
|
|
14
16
|
export class SqliteAcquisition implements AcquisitionRepository {
|
|
15
|
-
constructor(
|
|
16
|
-
|
|
17
|
-
|
|
17
|
+
constructor(private readonly input: { db: Database; limits: QueueLimits }) {}
|
|
18
|
+
capacityReleased(): void {
|
|
19
|
+
this.input.db
|
|
20
|
+
.query("UPDATE syncs SET next_due_at=? WHERE enabled=1 AND status='waiting_for_capacity'")
|
|
21
|
+
.run(Date.now());
|
|
22
|
+
}
|
|
23
|
+
nextDue(): number | undefined {
|
|
24
|
+
return (
|
|
25
|
+
this.input.db
|
|
26
|
+
.query<{ due: number | null }, []>(`
|
|
27
|
+
SELECT MIN(COALESCE(expires_at,next_due_at)) AS due FROM syncs WHERE enabled=1`)
|
|
28
|
+
.get()?.due ?? undefined
|
|
29
|
+
);
|
|
30
|
+
}
|
|
18
31
|
claim(leaseMs: number) {
|
|
19
|
-
return
|
|
32
|
+
return claimAcquisition({ ...this.input, leaseMs });
|
|
20
33
|
}
|
|
21
34
|
hasCapacity() {
|
|
22
35
|
return hasQueueCapacity(this.input);
|
|
23
36
|
}
|
|
24
|
-
commit(input: { lease:
|
|
37
|
+
commit(input: { lease: AcquisitionLease; page: SyncStep; definition: SyncDefinition }): void {
|
|
25
38
|
const { db, limits } = this.input;
|
|
26
39
|
const page = preparePage({ ...input, limits });
|
|
27
40
|
db.transaction(() => {
|
|
28
|
-
const
|
|
29
|
-
if (
|
|
41
|
+
const sync = assertAcquisition({ db, lease: input.lease });
|
|
42
|
+
if (input.definition.id !== sync.definition) {
|
|
30
43
|
fail('definition_conflict');
|
|
31
44
|
}
|
|
32
|
-
const records
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
45
|
+
const { records, assets } = changedRecords({ db, lease: input.lease, sync, page });
|
|
46
|
+
enqueue({ db, sync, records, limits, assets, lease: input.lease });
|
|
47
|
+
db.query(`UPDATE sync_polls SET records_processed=records_processed+?,records_queued=records_queued+?
|
|
48
|
+
WHERE owner_id=? AND sync_id=? AND completed_at IS NULL`).run(
|
|
49
|
+
page.records.length,
|
|
50
|
+
records.length,
|
|
51
|
+
sync.ownerId,
|
|
52
|
+
sync.id,
|
|
53
|
+
);
|
|
54
|
+
db.query('UPDATE syncs SET checkpoint=? WHERE owner_id=? AND id=?').run(
|
|
55
|
+
canonicalJson(page.checkpoint).json,
|
|
56
|
+
sync.ownerId,
|
|
57
|
+
sync.id,
|
|
58
|
+
);
|
|
59
|
+
finishAcquisition({
|
|
60
|
+
db,
|
|
61
|
+
lease: input.lease,
|
|
62
|
+
state: page.complete ? 'succeeded' : 'ready',
|
|
63
|
+
delay: page.complete ? sync.intervalMs : 0,
|
|
64
|
+
failureCount: page.complete ? 0 : undefined,
|
|
65
|
+
});
|
|
49
66
|
}).immediate();
|
|
50
|
-
input.lease.checkpointRevision++;
|
|
51
67
|
}
|
|
52
|
-
finish(input:
|
|
68
|
+
finish(input: Parameters<AcquisitionRepository['finish']>[0]): void {
|
|
53
69
|
this.input.db
|
|
54
70
|
.transaction(() => {
|
|
55
|
-
|
|
56
|
-
|
|
71
|
+
assertAcquisition({ db: this.input.db, lease: input.lease });
|
|
72
|
+
finishAcquisition({ db: this.input.db, ...input });
|
|
57
73
|
})
|
|
58
74
|
.immediate();
|
|
59
75
|
}
|
|
60
76
|
}
|
|
77
|
+
|
|
78
|
+
function changedRecords(input: {
|
|
79
|
+
db: Database;
|
|
80
|
+
lease: AcquisitionLease;
|
|
81
|
+
sync: import('../../models/sync').Sync;
|
|
82
|
+
page: SyncStep;
|
|
83
|
+
}) {
|
|
84
|
+
const records: DeliveredRecord[] = [];
|
|
85
|
+
const assets = new Map<string, DeliveryAsset>();
|
|
86
|
+
for (const record of input.page.records) {
|
|
87
|
+
const referenced = capturedAssets({
|
|
88
|
+
db: input.db,
|
|
89
|
+
lease: input.lease,
|
|
90
|
+
refs: record.operation === 'upsert' ? Object.values(record.assetRefs ?? {}) : [],
|
|
91
|
+
});
|
|
92
|
+
const changed = writeRecord({
|
|
93
|
+
db: input.db,
|
|
94
|
+
sync: input.sync,
|
|
95
|
+
record,
|
|
96
|
+
assets: referenced,
|
|
97
|
+
force: input.lease.force,
|
|
98
|
+
});
|
|
99
|
+
if (changed) {
|
|
100
|
+
records.push(changed);
|
|
101
|
+
for (const asset of referenced) {
|
|
102
|
+
assets.set(assetKey(asset), asset);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return { records, assets: [...assets.values()] };
|
|
107
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import type { AcquisitionLease } from '../acquisition/contract';
|
|
3
|
+
|
|
4
|
+
export function assetBytes(input: { db: Database; lease?: AcquisitionLease }): number {
|
|
5
|
+
return input.db
|
|
6
|
+
.query<{ bytes: number }, (string | number | null)[]>(
|
|
7
|
+
`SELECT COALESCE(SUM(bytes),0) AS bytes FROM delivery_assets
|
|
8
|
+
WHERE (? IS NULL OR (owner_id=? AND sync_id=? AND generation=?))`,
|
|
9
|
+
)
|
|
10
|
+
.get(
|
|
11
|
+
input.lease?.ownerId ?? null,
|
|
12
|
+
input.lease?.ownerId ?? null,
|
|
13
|
+
input.lease?.sync.id ?? null,
|
|
14
|
+
input.lease?.generation ?? null,
|
|
15
|
+
)!.bytes;
|
|
16
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { AssetMetadata, AssetRef } from '../../models/asset';
|
|
2
|
+
import type { AcquisitionLease } from '../acquisition/contract';
|
|
3
|
+
import type { DeliveryLease } from '../delivery/contract';
|
|
4
|
+
|
|
5
|
+
export interface AssetRepository {
|
|
6
|
+
stage(input: { lease: AcquisitionLease; asset: AssetMetadata; unavailable?: string }): string;
|
|
7
|
+
captured(input: { lease: AcquisitionLease; id: string; size: number; sha256: string }): void;
|
|
8
|
+
reserve(input: { lease: AcquisitionLease; id: string; bytes: number }): void;
|
|
9
|
+
discarded(id: string): void;
|
|
10
|
+
garbage(): string[];
|
|
11
|
+
retained(id: string): boolean;
|
|
12
|
+
read(input: { lease: DeliveryLease; asset: AssetRef }): string;
|
|
13
|
+
}
|
|
14
|
+
export interface AssetFiles {
|
|
15
|
+
write(input: {
|
|
16
|
+
id: string;
|
|
17
|
+
body: ReadableStream<Uint8Array>;
|
|
18
|
+
reserve(file: { id: string; bytes: number }): void;
|
|
19
|
+
maxBytes: number;
|
|
20
|
+
signal: AbortSignal;
|
|
21
|
+
}): Promise<{ id: string; size: number; sha256: string }>;
|
|
22
|
+
open(id: string): Promise<ReadableStream<Uint8Array>>;
|
|
23
|
+
remove(id: string): Promise<void>;
|
|
24
|
+
sweep(input: { retain(id: string): boolean }): Promise<void>;
|
|
25
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
const privateFileMode = 0o600;
|
|
2
|
+
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { mkdir, open, readdir, unlink } from 'node:fs/promises';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { fail } from '../../models/error';
|
|
7
|
+
import type { AssetFiles } from './contract';
|
|
8
|
+
|
|
9
|
+
/** A directory dedicated to this queue. Names come only from generated UUIDs, never provider filenames. */
|
|
10
|
+
export class DirectoryAssets implements AssetFiles {
|
|
11
|
+
constructor(private readonly directory: string) {}
|
|
12
|
+
async write(input: Parameters<AssetFiles['write']>[0]) {
|
|
13
|
+
const reader = input.body.getReader();
|
|
14
|
+
const { id } = input;
|
|
15
|
+
let file: Awaited<ReturnType<typeof open>> | undefined;
|
|
16
|
+
const hash = createHash('sha256');
|
|
17
|
+
let size = 0;
|
|
18
|
+
const abort = () => {
|
|
19
|
+
void reader.cancel().catch(() => undefined);
|
|
20
|
+
};
|
|
21
|
+
input.signal.addEventListener('abort', abort, { once: true });
|
|
22
|
+
try {
|
|
23
|
+
input.signal.throwIfAborted();
|
|
24
|
+
input.reserve({ id, bytes: 0 });
|
|
25
|
+
await mkdir(this.directory, { recursive: true, mode: 0o700 });
|
|
26
|
+
file = await open(this.path(id), 'wx', privateFileMode);
|
|
27
|
+
while (true) {
|
|
28
|
+
input.signal.throwIfAborted();
|
|
29
|
+
const chunk = await reader.read();
|
|
30
|
+
input.signal.throwIfAborted();
|
|
31
|
+
if (chunk.done) {
|
|
32
|
+
break;
|
|
33
|
+
}
|
|
34
|
+
size += chunk.value.byteLength;
|
|
35
|
+
if (size > input.maxBytes) {
|
|
36
|
+
fail('asset_too_large');
|
|
37
|
+
}
|
|
38
|
+
input.reserve({ id, bytes: size });
|
|
39
|
+
hash.update(chunk.value);
|
|
40
|
+
let offset = 0;
|
|
41
|
+
while (offset < chunk.value.byteLength) {
|
|
42
|
+
const written = await file.write(chunk.value.subarray(offset));
|
|
43
|
+
offset += written.bytesWritten;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
await file.sync();
|
|
47
|
+
await file.close();
|
|
48
|
+
file = undefined;
|
|
49
|
+
const directory = await open(this.directory, 'r');
|
|
50
|
+
try {
|
|
51
|
+
await directory.sync();
|
|
52
|
+
} finally {
|
|
53
|
+
await directory.close();
|
|
54
|
+
}
|
|
55
|
+
return { id, size, sha256: hash.digest('hex') };
|
|
56
|
+
} catch (error) {
|
|
57
|
+
try {
|
|
58
|
+
await file?.close();
|
|
59
|
+
} finally {
|
|
60
|
+
await this.remove(id);
|
|
61
|
+
}
|
|
62
|
+
throw error;
|
|
63
|
+
} finally {
|
|
64
|
+
input.signal.removeEventListener('abort', abort);
|
|
65
|
+
await reader.cancel().catch(() => undefined);
|
|
66
|
+
reader.releaseLock();
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
async open(id: string) {
|
|
70
|
+
const file = Bun.file(this.path(id));
|
|
71
|
+
if (!(await file.exists())) {
|
|
72
|
+
fail('asset_content_missing');
|
|
73
|
+
}
|
|
74
|
+
return file.stream();
|
|
75
|
+
}
|
|
76
|
+
async remove(id: string) {
|
|
77
|
+
await unlink(this.path(id)).catch((error: NodeJS.ErrnoException) => {
|
|
78
|
+
if (error.code !== 'ENOENT') {
|
|
79
|
+
throw error;
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
async sweep(input: Parameters<AssetFiles['sweep']>[0]) {
|
|
84
|
+
const names = await readdir(this.directory).catch((error: NodeJS.ErrnoException) => {
|
|
85
|
+
if (error.code !== 'ENOENT') {
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
return [] as string[];
|
|
89
|
+
});
|
|
90
|
+
for (const name of names) {
|
|
91
|
+
if (!/^[0-9a-f-]{36}$/.test(name) || input.retain(name)) {
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
await this.remove(name);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
private path(id: string) {
|
|
98
|
+
if (!/^[0-9a-f-]{36}$/.test(id)) {
|
|
99
|
+
fail('invalid_asset_file');
|
|
100
|
+
}
|
|
101
|
+
return join(this.directory, id);
|
|
102
|
+
}
|
|
103
|
+
}
|