@context-use/open-sync 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +3 -0
- package/package.json +7 -5
- package/src/api.ts +18 -16
- package/src/connector/client.ts +61 -28
- package/src/connector/failure.ts +49 -0
- package/src/connector/file.ts +104 -0
- package/src/connector/response.ts +22 -1
- package/src/db/client.ts +1 -11
- package/src/db/providers.sql +2 -3
- package/src/db/providers.ts +1 -11
- package/src/db/schema.sql +41 -36
- package/src/execution/abortable.ts +12 -0
- package/src/execution/diagnostics.ts +1 -2
- package/src/execution/provider.ts +1 -0
- package/src/execution/worker.ts +162 -39
- package/src/http/controller.ts +63 -42
- package/src/models/asset.ts +61 -0
- package/src/models/definition.ts +24 -15
- package/src/models/delivery.ts +25 -31
- package/src/models/error.ts +12 -1
- package/src/models/limits.ts +8 -3
- package/src/models/metadata.ts +42 -0
- package/src/models/page.ts +68 -13
- package/src/models/provider-path.ts +25 -0
- package/src/models/providers.ts +13 -2
- package/src/models/record.ts +22 -0
- package/src/models/registry.ts +15 -15
- package/src/models/source-http-error.ts +37 -0
- package/src/models/sync.ts +71 -0
- package/src/models/validation.ts +1 -1
- package/src/open-sync.ts +26 -7
- package/src/repositories/acquisition/contract.ts +18 -10
- package/src/repositories/acquisition/lease.ts +80 -94
- package/src/repositories/acquisition/outbox.ts +22 -22
- package/src/repositories/acquisition/records.ts +44 -13
- package/src/repositories/acquisition/sqlite.ts +78 -31
- package/src/repositories/assets/capacity.ts +16 -0
- package/src/repositories/assets/contract.ts +25 -0
- package/src/repositories/assets/filesystem.ts +103 -0
- package/src/repositories/assets/sqlite.ts +174 -0
- package/src/repositories/catalog/contract.ts +16 -19
- package/src/repositories/catalog/sqlite.ts +98 -121
- package/src/repositories/delivery/contract.ts +7 -11
- package/src/repositories/delivery/lease.ts +16 -0
- package/src/repositories/delivery/sqlite.ts +57 -34
- package/src/repositories/queue-usage.ts +7 -8
- package/src/repositories/rows.ts +10 -29
- package/src/runtime.ts +27 -6
- package/src/services/acquisition.ts +178 -52
- package/src/services/delivery.ts +39 -19
- package/src/services/management.ts +88 -51
- package/src/services/providers/service.ts +100 -10
- package/src/models/installation.ts +0 -37
|
@@ -1,44 +1,55 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
|
-
import type { DeliveryResult, PendingDelivery } from '../../models/delivery';
|
|
2
|
+
import type { Deliverable, DeliveryResult, PendingDelivery } from '../../models/delivery';
|
|
3
3
|
import { fail } from '../../models/error';
|
|
4
4
|
import { type Resource, type Scope, workerScope } from '../../models/identity';
|
|
5
5
|
import { queueUsage } from '../queue-usage';
|
|
6
|
-
import { type Row,
|
|
6
|
+
import { type Row, readSync } from '../rows';
|
|
7
7
|
import type { DeliveryLease, DeliveryRepository } from './contract';
|
|
8
|
+
import { assertDelivery } from './lease';
|
|
9
|
+
|
|
10
|
+
const headOfQueue = `NOT EXISTS (SELECT 1 FROM deliveries prior
|
|
11
|
+
WHERE prior.owner_id=d.owner_id AND prior.sync_id=d.sync_id
|
|
12
|
+
AND prior.sequence<d.sequence)`;
|
|
8
13
|
|
|
9
14
|
export class SqliteDeliveries implements DeliveryRepository {
|
|
10
15
|
constructor(private readonly db: Database) {}
|
|
16
|
+
nextDue(): number | undefined {
|
|
17
|
+
return (
|
|
18
|
+
this.db
|
|
19
|
+
.query<{ due: number | null }, []>(`SELECT MIN(
|
|
20
|
+
CASE WHEN d.state='leased' THEN MAX(d.due_at,d.expires_at) ELSE d.due_at END) AS due
|
|
21
|
+
FROM deliveries d WHERE d.state!='blocked' AND ${headOfQueue}`)
|
|
22
|
+
.get()?.due ?? undefined
|
|
23
|
+
);
|
|
24
|
+
}
|
|
11
25
|
claim(leaseMs: number): DeliveryLease | undefined {
|
|
12
26
|
return this.db
|
|
13
27
|
.transaction(() => {
|
|
14
|
-
// A blocked delivery stops only its
|
|
28
|
+
// A blocked delivery stops only its sync; ordering survives retries and restarts.
|
|
15
29
|
const row = this.db
|
|
16
30
|
.query<
|
|
17
31
|
Row,
|
|
18
32
|
[number, number]
|
|
19
33
|
>(`SELECT d.* FROM deliveries d WHERE d.state!='blocked' AND d.due_at<=?
|
|
20
|
-
AND (d.state='pending' OR d.expires_at<=?) AND
|
|
21
|
-
SELECT 1 FROM deliveries prior WHERE prior.owner_id=d.owner_id AND prior.destination_id=d.destination_id AND prior.sequence<d.sequence)
|
|
34
|
+
AND (d.state='pending' OR d.expires_at<=?) AND ${headOfQueue}
|
|
22
35
|
ORDER BY d.sequence LIMIT 1`)
|
|
23
36
|
.get(Date.now(), Date.now());
|
|
24
37
|
if (!row) {
|
|
25
38
|
return;
|
|
26
39
|
}
|
|
27
|
-
const workerId = crypto.randomUUID();
|
|
28
40
|
const scope = workerScope(String(row.owner_id));
|
|
29
41
|
this.db
|
|
30
42
|
.query(
|
|
31
|
-
"UPDATE deliveries SET state='leased',
|
|
43
|
+
"UPDATE deliveries SET state='leased',generation=generation+1,attempt=attempt+1,expires_at=? WHERE owner_id=? AND id=?",
|
|
32
44
|
)
|
|
33
|
-
.run(
|
|
45
|
+
.run(Date.now() + leaseMs, scope.ownerId, row.id!);
|
|
34
46
|
return {
|
|
35
47
|
...scope,
|
|
36
48
|
delivery: JSON.parse(String(row.body)),
|
|
37
|
-
destination:
|
|
49
|
+
destination: readSync({
|
|
38
50
|
db: this.db,
|
|
39
|
-
scope: { ...scope, id: String(row.
|
|
40
|
-
}),
|
|
41
|
-
workerId,
|
|
51
|
+
scope: { ...scope, id: String(row.sync_id) },
|
|
52
|
+
}).destination,
|
|
42
53
|
generation: Number(row.generation) + 1,
|
|
43
54
|
attempt: Number(row.attempt) + 1,
|
|
44
55
|
};
|
|
@@ -49,15 +60,7 @@ export class SqliteDeliveries implements DeliveryRepository {
|
|
|
49
60
|
const { lease, result } = input;
|
|
50
61
|
this.db
|
|
51
62
|
.transaction(() => {
|
|
52
|
-
|
|
53
|
-
!this.db
|
|
54
|
-
.query(
|
|
55
|
-
"SELECT 1 FROM deliveries WHERE owner_id=? AND id=? AND state='leased' AND worker_id=? AND generation=? AND expires_at>?",
|
|
56
|
-
)
|
|
57
|
-
.get(lease.ownerId, lease.delivery.id, lease.workerId, lease.generation, Date.now())
|
|
58
|
-
) {
|
|
59
|
-
fail('lease_lost');
|
|
60
|
-
}
|
|
63
|
+
assertDelivery({ db: this.db, lease });
|
|
61
64
|
if (result.status === 'accepted') {
|
|
62
65
|
this.db
|
|
63
66
|
.query('DELETE FROM deliveries WHERE owner_id=? AND id=?')
|
|
@@ -65,13 +68,13 @@ export class SqliteDeliveries implements DeliveryRepository {
|
|
|
65
68
|
// Capacity is global; all paused acquisitions can compete for the released budget.
|
|
66
69
|
this.db
|
|
67
70
|
.query(
|
|
68
|
-
"UPDATE
|
|
71
|
+
"UPDATE syncs SET next_due_at=? WHERE enabled=1 AND status='waiting_for_capacity'",
|
|
69
72
|
)
|
|
70
73
|
.run(Date.now());
|
|
71
74
|
} else {
|
|
72
75
|
this.db
|
|
73
76
|
.query(
|
|
74
|
-
'UPDATE deliveries SET state=?,due_at=?,
|
|
77
|
+
'UPDATE deliveries SET state=?,due_at=?,expires_at=NULL,error_code=? WHERE owner_id=? AND id=?',
|
|
75
78
|
)
|
|
76
79
|
.run(
|
|
77
80
|
result.status === 'rejected' ? 'blocked' : 'pending',
|
|
@@ -87,25 +90,45 @@ export class SqliteDeliveries implements DeliveryRepository {
|
|
|
87
90
|
status(scope: Scope) {
|
|
88
91
|
return queueUsage({ db: this.db, ownerId: scope.ownerId });
|
|
89
92
|
}
|
|
90
|
-
pending(input: Scope & {
|
|
91
|
-
const
|
|
93
|
+
pending(input: Scope & { syncId: string; before?: number }) {
|
|
94
|
+
const pageSize = 20;
|
|
92
95
|
const rows = this.db
|
|
93
96
|
.query<
|
|
94
|
-
PendingDelivery,
|
|
95
|
-
[string, number, number]
|
|
96
|
-
>(`SELECT id,
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
97
|
+
PendingDelivery & { sequence: number },
|
|
98
|
+
[string, string, number, number]
|
|
99
|
+
>(`SELECT sequence,id,sync_id AS syncId,state,bytes,
|
|
100
|
+
record_count AS recordCount,json_array_length(body,'$.assets') AS assetCount,
|
|
101
|
+
attempt,due_at AS nextAttemptAt,error_code AS errorCode FROM deliveries
|
|
102
|
+
WHERE owner_id=? AND sync_id=? AND sequence<? ORDER BY sequence DESC LIMIT ?`)
|
|
103
|
+
.all(input.ownerId, input.syncId, input.before ?? Number.MAX_SAFE_INTEGER, pageSize + 1);
|
|
104
|
+
const page = rows.slice(0, pageSize);
|
|
105
|
+
return {
|
|
106
|
+
deliveries: page.map(({ sequence: _, ...delivery }) => delivery),
|
|
107
|
+
nextCursor: rows.length > pageSize ? page.at(-1)!.sequence : null,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
deliverable(input: Resource & { syncId: string }): Omit<Deliverable, 'openAsset'> {
|
|
111
|
+
const row = this.db
|
|
112
|
+
.query<{ body: string }, [string, string, string]>(
|
|
113
|
+
'SELECT body FROM deliveries WHERE owner_id=? AND sync_id=? AND id=?',
|
|
114
|
+
)
|
|
115
|
+
.get(input.ownerId, input.syncId, input.id);
|
|
116
|
+
if (!row) {
|
|
117
|
+
fail('not_found');
|
|
118
|
+
}
|
|
119
|
+
return JSON.parse(row.body);
|
|
100
120
|
}
|
|
101
121
|
retry(input: Resource): void {
|
|
102
122
|
const updated = this.db
|
|
103
123
|
.query(
|
|
104
|
-
"UPDATE deliveries SET state='pending',due_at=?,generation=generation+1,
|
|
124
|
+
"UPDATE deliveries SET state='pending',due_at=?,generation=generation+1,expires_at=NULL,error_code=NULL WHERE owner_id=? AND id=? AND (state!='leased' OR expires_at<=?)",
|
|
105
125
|
)
|
|
106
|
-
.run(Date.now(), input.ownerId, input.id);
|
|
126
|
+
.run(Date.now(), input.ownerId, input.id, Date.now());
|
|
107
127
|
if (!updated.changes) {
|
|
108
|
-
|
|
128
|
+
const exists = this.db
|
|
129
|
+
.query('SELECT 1 FROM deliveries WHERE owner_id=? AND id=?')
|
|
130
|
+
.get(input.ownerId, input.id);
|
|
131
|
+
fail(exists ? 'busy' : 'not_found');
|
|
109
132
|
}
|
|
110
133
|
}
|
|
111
134
|
}
|
|
@@ -1,21 +1,20 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
2
|
import type { QueueStatus } from '../models/delivery';
|
|
3
3
|
import type { QueueLimits } from '../models/limits';
|
|
4
|
+
import { assetBytes } from './assets/capacity';
|
|
4
5
|
export function queueUsage(input: { db: Database; ownerId?: string }): QueueStatus {
|
|
5
6
|
const row = input.db
|
|
6
|
-
.query<
|
|
7
|
-
QueueStatus,
|
|
8
|
-
[string | null, string | null]
|
|
9
|
-
>(`SELECT coalesce(sum(bytes),0) AS pendingBytes,
|
|
7
|
+
.query<QueueStatus, (string | null)[]>(`SELECT coalesce(sum(bytes),0) AS pendingBytes,
|
|
10
8
|
coalesce(sum(record_count),0) AS pendingRecords,count(*) AS pendingDeliveries,
|
|
11
9
|
coalesce(sum(state='blocked'),0) AS blockedDeliveries FROM deliveries WHERE (? IS NULL OR owner_id=?)`)
|
|
12
10
|
.get(input.ownerId ?? null, input.ownerId ?? null)!;
|
|
13
11
|
return row;
|
|
14
12
|
}
|
|
15
|
-
export function hasQueueCapacity(
|
|
16
|
-
const usage = queueUsage({ db
|
|
13
|
+
export function hasQueueCapacity({ db, limits }: { db: Database; limits: QueueLimits }): boolean {
|
|
14
|
+
const usage = queueUsage({ db });
|
|
17
15
|
return (
|
|
18
|
-
usage.pendingBytes <
|
|
19
|
-
usage.pendingRecords <
|
|
16
|
+
usage.pendingBytes < limits.maxPendingBytes &&
|
|
17
|
+
usage.pendingRecords < limits.maxPendingRecords &&
|
|
18
|
+
assetBytes({ db }) < limits.maxPendingAssetBytes
|
|
20
19
|
);
|
|
21
20
|
}
|
package/src/repositories/rows.ts
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
|
-
import type { Destination } from '../models/delivery';
|
|
3
2
|
import { fail } from '../models/error';
|
|
4
3
|
import type { Resource } from '../models/identity';
|
|
5
|
-
import type {
|
|
4
|
+
import type { Sync, SyncStatus } from '../models/sync';
|
|
6
5
|
|
|
7
6
|
export type Row = Record<string, string | number | null>;
|
|
8
|
-
export function
|
|
7
|
+
export function readSync(input: { db: Database; scope: Resource }): Sync {
|
|
9
8
|
const row = input.db
|
|
10
|
-
.query<Row, [string, string]>('SELECT * FROM
|
|
9
|
+
.query<Row, [string, string]>('SELECT * FROM syncs WHERE owner_id=? AND id=?')
|
|
11
10
|
.get(input.scope.ownerId, input.scope.id);
|
|
12
11
|
if (!row) {
|
|
13
12
|
fail('not_found');
|
|
@@ -15,36 +14,18 @@ export function readInstallation(input: { db: Database; scope: Resource }): Inst
|
|
|
15
14
|
return {
|
|
16
15
|
id: String(row.id),
|
|
17
16
|
ownerId: String(row.owner_id),
|
|
18
|
-
|
|
19
|
-
definition: {
|
|
20
|
-
id: String(row.definition_id),
|
|
21
|
-
version: String(row.definition_version),
|
|
22
|
-
artifactId: String(row.artifact_id),
|
|
23
|
-
},
|
|
17
|
+
definition: String(row.definition_id),
|
|
24
18
|
connection: row.connection === null ? undefined : JSON.parse(String(row.connection)),
|
|
25
19
|
config: JSON.parse(String(row.config)),
|
|
26
|
-
|
|
20
|
+
destination: {
|
|
21
|
+
type: String(row.destination_type),
|
|
22
|
+
config: JSON.parse(String(row.destination_config)),
|
|
23
|
+
},
|
|
27
24
|
enabled: row.enabled === 1,
|
|
28
|
-
bindingEpoch: Number(row.binding_epoch),
|
|
29
25
|
checkpoint: JSON.parse(String(row.checkpoint)),
|
|
30
|
-
checkpointRevision: Number(row.checkpoint_revision),
|
|
31
26
|
intervalMs: Number(row.interval_ms),
|
|
32
27
|
nextDueAt: Number(row.next_due_at),
|
|
33
|
-
status:
|
|
34
|
-
|
|
35
|
-
}
|
|
36
|
-
export function readDestination(input: { db: Database; scope: Resource }): Destination {
|
|
37
|
-
const row = input.db
|
|
38
|
-
.query<Row, [string, string]>('SELECT * FROM destinations WHERE owner_id=? AND id=?')
|
|
39
|
-
.get(input.scope.ownerId, input.scope.id);
|
|
40
|
-
if (!row) {
|
|
41
|
-
fail('not_found');
|
|
42
|
-
}
|
|
43
|
-
return {
|
|
44
|
-
id: String(row.id),
|
|
45
|
-
ownerId: String(row.owner_id),
|
|
46
|
-
type: String(row.type),
|
|
47
|
-
version: String(row.version),
|
|
48
|
-
config: JSON.parse(String(row.config)),
|
|
28
|
+
status: row.status as SyncStatus,
|
|
29
|
+
errorCode: row.error_code === null ? null : String(row.error_code),
|
|
49
30
|
};
|
|
50
31
|
}
|
package/src/runtime.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { syncApi } from './api';
|
|
1
2
|
import { openDatabase } from './db/client';
|
|
2
3
|
import { type Logger, safeLogger } from './execution/diagnostics';
|
|
3
4
|
import type { ProviderGateway } from './execution/provider';
|
|
@@ -14,6 +15,8 @@ import {
|
|
|
14
15
|
} from './models/limits';
|
|
15
16
|
import { Registry } from './models/registry';
|
|
16
17
|
import { SqliteAcquisition } from './repositories/acquisition/sqlite';
|
|
18
|
+
import { DirectoryAssets } from './repositories/assets/filesystem';
|
|
19
|
+
import { SqliteAssets } from './repositories/assets/sqlite';
|
|
17
20
|
import { SqliteCatalog } from './repositories/catalog/sqlite';
|
|
18
21
|
import { SqliteDeliveries } from './repositories/delivery/sqlite';
|
|
19
22
|
import { AcquisitionService } from './services/acquisition';
|
|
@@ -22,6 +25,8 @@ import { SyncManagement } from './services/management';
|
|
|
22
25
|
|
|
23
26
|
export interface SyncRuntimeOptions {
|
|
24
27
|
databasePath: string;
|
|
28
|
+
/** Private directory dedicated to queued asset bytes. */
|
|
29
|
+
assetDirectory?: string;
|
|
25
30
|
definitions: readonly SyncRegistration[];
|
|
26
31
|
destinationTypes: Readonly<Record<string, DestinationType>>;
|
|
27
32
|
connector?: ProviderGateway;
|
|
@@ -46,22 +51,38 @@ export function createSyncRuntime(options: SyncRuntimeOptions) {
|
|
|
46
51
|
const db = openDatabase(options.databasePath);
|
|
47
52
|
try {
|
|
48
53
|
const catalog = new SqliteCatalog(db);
|
|
49
|
-
for (const definition of registry.definitions()) {
|
|
50
|
-
catalog.register(definition);
|
|
51
|
-
}
|
|
52
54
|
const deliveries = new SqliteDeliveries(db);
|
|
55
|
+
const assets = new SqliteAssets({
|
|
56
|
+
db,
|
|
57
|
+
maxBytes: limits.maxPendingAssetBytes,
|
|
58
|
+
});
|
|
59
|
+
const files = new DirectoryAssets(options.assetDirectory ?? `${options.databasePath}.assets`);
|
|
53
60
|
const log = safeLogger(options.onEvent);
|
|
61
|
+
let recoverFiles = true;
|
|
54
62
|
const worker = new Worker({
|
|
55
63
|
acquisition: new AcquisitionService({
|
|
56
|
-
repository: new SqliteAcquisition({ db, limits
|
|
64
|
+
repository: new SqliteAcquisition({ db, limits }),
|
|
57
65
|
registry,
|
|
66
|
+
assets,
|
|
67
|
+
files,
|
|
68
|
+
maxAssetBytes: limits.maxAssetBytes,
|
|
58
69
|
gateway: options.connector,
|
|
59
70
|
timing,
|
|
60
71
|
log,
|
|
61
72
|
}),
|
|
62
|
-
delivery: new DeliveryService({ repository: deliveries, registry, timing }),
|
|
73
|
+
delivery: new DeliveryService({ repository: deliveries, registry, timing, assets, files }),
|
|
63
74
|
timing,
|
|
64
75
|
log,
|
|
76
|
+
async cleanup() {
|
|
77
|
+
for (const id of assets.garbage()) {
|
|
78
|
+
await files.remove(id);
|
|
79
|
+
assets.discarded(id);
|
|
80
|
+
}
|
|
81
|
+
if (recoverFiles) {
|
|
82
|
+
await files.sweep({ retain: (id) => assets.retained(id) });
|
|
83
|
+
recoverFiles = false;
|
|
84
|
+
}
|
|
85
|
+
},
|
|
65
86
|
});
|
|
66
87
|
const api = new SyncManagement({
|
|
67
88
|
catalog,
|
|
@@ -74,7 +95,7 @@ export function createSyncRuntime(options: SyncRuntimeOptions) {
|
|
|
74
95
|
});
|
|
75
96
|
let closing: Promise<void> | undefined;
|
|
76
97
|
return {
|
|
77
|
-
api,
|
|
98
|
+
api: syncApi(api),
|
|
78
99
|
start: () => worker.start(),
|
|
79
100
|
tick: () => worker.tick(),
|
|
80
101
|
close: () => {
|
|
@@ -1,100 +1,226 @@
|
|
|
1
|
+
import { abortable } from '../execution/abortable';
|
|
1
2
|
import type { Logger } from '../execution/diagnostics';
|
|
2
3
|
import { bindProvider, type ProviderGateway } from '../execution/provider';
|
|
4
|
+
import { assetKey, type SourceAssets } from '../models/asset';
|
|
3
5
|
import { SyncError } from '../models/error';
|
|
4
|
-
import
|
|
6
|
+
import { canonicalJson } from '../models/json';
|
|
7
|
+
import { retryDelay, type Timing } from '../models/limits';
|
|
5
8
|
import type { Registry } from '../models/registry';
|
|
6
|
-
import
|
|
9
|
+
import { retryableStatus } from '../models/source-http-error';
|
|
10
|
+
import { identifier } from '../models/validation';
|
|
11
|
+
import type { AcquisitionLease, AcquisitionRepository } from '../repositories/acquisition/contract';
|
|
12
|
+
import type { AssetFiles, AssetRepository } from '../repositories/assets/contract';
|
|
7
13
|
|
|
8
14
|
export class AcquisitionService {
|
|
9
15
|
constructor(
|
|
10
16
|
private readonly input: {
|
|
11
17
|
repository: AcquisitionRepository;
|
|
18
|
+
assets: AssetRepository;
|
|
19
|
+
files: AssetFiles;
|
|
20
|
+
maxAssetBytes: number;
|
|
12
21
|
registry: Registry;
|
|
13
22
|
gateway?: ProviderGateway;
|
|
14
23
|
timing: Timing;
|
|
15
24
|
log: Logger;
|
|
16
25
|
},
|
|
17
26
|
) {}
|
|
27
|
+
capacityReleased() {
|
|
28
|
+
this.input.repository.capacityReleased();
|
|
29
|
+
}
|
|
30
|
+
nextDue() {
|
|
31
|
+
return this.input.repository.nextDue();
|
|
32
|
+
}
|
|
18
33
|
claim() {
|
|
19
34
|
return this.input.repository.claim(this.input.timing.leaseMs);
|
|
20
35
|
}
|
|
21
|
-
async execute(input: { lease:
|
|
22
|
-
const { repository, timing
|
|
23
|
-
const { lease
|
|
36
|
+
async execute(input: { lease: AcquisitionLease; signal: AbortSignal }): Promise<void> {
|
|
37
|
+
const { repository, timing } = this.input;
|
|
38
|
+
const { lease } = input;
|
|
24
39
|
try {
|
|
25
40
|
if (!repository.hasCapacity()) {
|
|
26
41
|
this.finish({ lease, state: 'waiting_for_capacity', delay: timing.retryMs });
|
|
27
42
|
return;
|
|
28
43
|
}
|
|
29
|
-
await this.consume(input);
|
|
44
|
+
await abortable({ signal: input.signal, run: () => this.consume(input) });
|
|
30
45
|
} catch (error) {
|
|
31
|
-
|
|
32
|
-
? 'cancelled'
|
|
33
|
-
: error instanceof SyncError
|
|
34
|
-
? error.code
|
|
35
|
-
: 'execution_failed';
|
|
36
|
-
log({ code, ownerId: lease.ownerId, installationId: lease.installation.id });
|
|
37
|
-
this.finish({ lease, state: code, delay: timing.retryMs });
|
|
46
|
+
this.failed({ ...input, error });
|
|
38
47
|
}
|
|
39
48
|
}
|
|
40
|
-
private
|
|
41
|
-
const {
|
|
49
|
+
private failed(input: { lease: AcquisitionLease; signal: AbortSignal; error: unknown }): void {
|
|
50
|
+
const { timing, log } = this.input;
|
|
51
|
+
const { lease, signal, error } = input;
|
|
52
|
+
const code = signal.aborted
|
|
53
|
+
? abortCode(signal)
|
|
54
|
+
: error instanceof SyncError
|
|
55
|
+
? error.code
|
|
56
|
+
: 'execution_failed';
|
|
57
|
+
const failed = !['paused', 'interrupted', 'waiting_for_capacity'].includes(code);
|
|
58
|
+
const failureCount = failed ? lease.failureCount + 1 : lease.failureCount;
|
|
59
|
+
const status = error instanceof SyncError ? error.status : undefined;
|
|
60
|
+
const pause = !signal.aborted && status !== undefined && !retryableStatus(status);
|
|
61
|
+
const delay = failed
|
|
62
|
+
? retryDelay({ attempt: failureCount, retryMs: timing.retryMs })
|
|
63
|
+
: timing.retryMs;
|
|
64
|
+
log({
|
|
65
|
+
code,
|
|
66
|
+
ownerId: lease.ownerId,
|
|
67
|
+
syncId: lease.sync.id,
|
|
68
|
+
fields: {
|
|
69
|
+
...(error instanceof SyncError ? error.diagnostics : {}),
|
|
70
|
+
...(status === undefined ? {} : { httpStatus: status }),
|
|
71
|
+
failureCount,
|
|
72
|
+
...(pause ? { paused: true } : { retryAfterMs: delay }),
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
this.finish({
|
|
76
|
+
lease,
|
|
77
|
+
state:
|
|
78
|
+
code === 'waiting_for_capacity'
|
|
79
|
+
? 'waiting_for_capacity'
|
|
80
|
+
: signal.aborted && code !== 'timed_out'
|
|
81
|
+
? 'interrupted'
|
|
82
|
+
: 'retrying',
|
|
83
|
+
errorCode: code === 'waiting_for_capacity' ? undefined : code,
|
|
84
|
+
delay,
|
|
85
|
+
failureCount,
|
|
86
|
+
pause,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
private async consume(input: { lease: AcquisitionLease; signal: AbortSignal }): Promise<void> {
|
|
90
|
+
const { repository, registry } = this.input;
|
|
42
91
|
const { lease, signal } = input;
|
|
43
|
-
const entry = registry.definition(lease.
|
|
92
|
+
const entry = registry.definition(lease.sync.definition);
|
|
44
93
|
const provider = await bindProvider({
|
|
45
94
|
actorId: lease.actorId,
|
|
46
95
|
ownerId: lease.ownerId,
|
|
47
96
|
gateway: this.input.gateway,
|
|
48
|
-
connection: lease.
|
|
97
|
+
connection: lease.sync.connection,
|
|
49
98
|
requirements: entry.definition.provider,
|
|
50
99
|
signal,
|
|
51
100
|
});
|
|
52
101
|
const executable = await entry.load();
|
|
53
102
|
signal.throwIfAborted();
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
sourceId: lease.installation.sourceId,
|
|
103
|
+
const page = await executable.step({
|
|
104
|
+
config: lease.sync.config,
|
|
105
|
+
checkpoint: lease.sync.checkpoint,
|
|
106
|
+
syncId: lease.sync.id,
|
|
59
107
|
signal,
|
|
60
108
|
provider,
|
|
61
|
-
|
|
62
|
-
this.input.
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
})) {
|
|
69
|
-
signal.throwIfAborted();
|
|
70
|
-
repository.commit({ lease, page, definition: entry.definition });
|
|
71
|
-
pages++;
|
|
72
|
-
if (page.complete) {
|
|
73
|
-
return;
|
|
74
|
-
}
|
|
75
|
-
if (!repository.hasCapacity()) {
|
|
76
|
-
this.finish({ lease, state: 'waiting_for_capacity', delay: timing.retryMs });
|
|
77
|
-
return;
|
|
78
|
-
}
|
|
79
|
-
if (pages >= timing.maxPages) {
|
|
80
|
-
this.finish({ lease, state: 'yielded', delay: 0 });
|
|
81
|
-
return;
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
throw new SyncError({
|
|
85
|
-
code: 'incomplete_run',
|
|
86
|
-
message: 'Definition ended without a complete page.',
|
|
109
|
+
assets: sourceAssets({
|
|
110
|
+
repository: this.input.assets,
|
|
111
|
+
files: this.input.files,
|
|
112
|
+
lease,
|
|
113
|
+
signal,
|
|
114
|
+
maxBytes: this.input.maxAssetBytes,
|
|
115
|
+
}),
|
|
87
116
|
});
|
|
117
|
+
signal.throwIfAborted();
|
|
118
|
+
repository.commit({ lease, page, definition: entry.definition });
|
|
88
119
|
}
|
|
89
|
-
|
|
120
|
+
|
|
121
|
+
private finish(input: Parameters<AcquisitionRepository['finish']>[0]): void {
|
|
90
122
|
try {
|
|
91
123
|
this.input.repository.finish(input);
|
|
92
124
|
} catch (error) {
|
|
93
|
-
if (
|
|
94
|
-
!(error instanceof SyncError && ['lease_lost', 'checkpoint_conflict'].includes(error.code))
|
|
95
|
-
) {
|
|
125
|
+
if (!(error instanceof SyncError && error.code === 'lease_lost')) {
|
|
96
126
|
throw error;
|
|
97
127
|
}
|
|
98
128
|
}
|
|
99
129
|
}
|
|
100
130
|
}
|
|
131
|
+
|
|
132
|
+
function abortCode(signal: AbortSignal): string {
|
|
133
|
+
if (signal.reason instanceof DOMException && signal.reason.name === 'TimeoutError') {
|
|
134
|
+
return 'timed_out';
|
|
135
|
+
}
|
|
136
|
+
return signal.reason === 'paused' ? 'paused' : 'interrupted';
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function sourceAssets(input: {
|
|
140
|
+
repository: AssetRepository;
|
|
141
|
+
files: AssetFiles;
|
|
142
|
+
lease: AcquisitionLease;
|
|
143
|
+
signal: AbortSignal;
|
|
144
|
+
maxBytes: number;
|
|
145
|
+
}): SourceAssets {
|
|
146
|
+
const captures = new Map<
|
|
147
|
+
string,
|
|
148
|
+
{
|
|
149
|
+
metadata: string;
|
|
150
|
+
ref: import('../models/asset').AssetRef;
|
|
151
|
+
pending?: Promise<import('../models/asset').AssetRef>;
|
|
152
|
+
}
|
|
153
|
+
>();
|
|
154
|
+
const stage = ({
|
|
155
|
+
capture,
|
|
156
|
+
unavailable,
|
|
157
|
+
}: {
|
|
158
|
+
capture: import('../models/asset').AssetMetadata;
|
|
159
|
+
unavailable?: string;
|
|
160
|
+
}) => {
|
|
161
|
+
input.signal.throwIfAborted();
|
|
162
|
+
const asset = assetMetadata(capture);
|
|
163
|
+
const metadata = canonicalJson({ ...asset, unavailable: unavailable ?? null }).json;
|
|
164
|
+
const previous = captures.get(assetKey(asset));
|
|
165
|
+
if (previous) {
|
|
166
|
+
if (previous.metadata !== metadata) {
|
|
167
|
+
throw new SyncError({
|
|
168
|
+
code: 'asset_version_conflict',
|
|
169
|
+
message: 'Conflicting asset within one step.',
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
return { previous, asset, metadata };
|
|
173
|
+
}
|
|
174
|
+
const id = input.repository.stage({ lease: input.lease, asset, unavailable });
|
|
175
|
+
return { id, asset, metadata };
|
|
176
|
+
};
|
|
177
|
+
return {
|
|
178
|
+
unavailable(capture) {
|
|
179
|
+
identifier(capture.code);
|
|
180
|
+
const result = stage({ capture, unavailable: capture.code });
|
|
181
|
+
if (result.previous) {
|
|
182
|
+
return result.previous.ref;
|
|
183
|
+
}
|
|
184
|
+
const ref = { id: capture.id, version: capture.version };
|
|
185
|
+
captures.set(assetKey(ref), { ref, metadata: result.metadata });
|
|
186
|
+
return ref;
|
|
187
|
+
},
|
|
188
|
+
capture(capture) {
|
|
189
|
+
const result = stage({ capture });
|
|
190
|
+
if (result.previous) {
|
|
191
|
+
return result.previous.pending!;
|
|
192
|
+
}
|
|
193
|
+
const ref = { id: capture.id, version: capture.version };
|
|
194
|
+
const pending = (async () => {
|
|
195
|
+
const body = await capture.read();
|
|
196
|
+
input.signal.throwIfAborted();
|
|
197
|
+
const file = await input.files.write({
|
|
198
|
+
id: result.id!,
|
|
199
|
+
body,
|
|
200
|
+
maxBytes: input.maxBytes,
|
|
201
|
+
signal: input.signal,
|
|
202
|
+
reserve: (reservation) => {
|
|
203
|
+
input.signal.throwIfAborted();
|
|
204
|
+
input.repository.reserve({ lease: input.lease, ...reservation });
|
|
205
|
+
},
|
|
206
|
+
});
|
|
207
|
+
input.signal.throwIfAborted();
|
|
208
|
+
input.repository.captured({ lease: input.lease, ...file });
|
|
209
|
+
return ref;
|
|
210
|
+
})();
|
|
211
|
+
captures.set(assetKey(ref), { ref, metadata: result.metadata, pending });
|
|
212
|
+
return pending;
|
|
213
|
+
},
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function assetMetadata(input: import('../models/asset').AssetMetadata) {
|
|
218
|
+
return {
|
|
219
|
+
id: input.id,
|
|
220
|
+
version: input.version,
|
|
221
|
+
name: input.name,
|
|
222
|
+
mediaType: input.mediaType,
|
|
223
|
+
...(input.createdAt !== undefined ? { createdAt: input.createdAt } : {}),
|
|
224
|
+
...(input.updatedAt !== undefined ? { updatedAt: input.updatedAt } : {}),
|
|
225
|
+
};
|
|
226
|
+
}
|