@context-use/open-sync 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/LICENSE +3 -0
  2. package/package.json +7 -5
  3. package/src/api.ts +18 -16
  4. package/src/connector/client.ts +61 -28
  5. package/src/connector/failure.ts +49 -0
  6. package/src/connector/file.ts +104 -0
  7. package/src/connector/response.ts +22 -1
  8. package/src/db/client.ts +1 -11
  9. package/src/db/providers.sql +2 -3
  10. package/src/db/providers.ts +1 -11
  11. package/src/db/schema.sql +41 -36
  12. package/src/execution/abortable.ts +12 -0
  13. package/src/execution/diagnostics.ts +1 -2
  14. package/src/execution/provider.ts +1 -0
  15. package/src/execution/worker.ts +162 -39
  16. package/src/http/controller.ts +63 -42
  17. package/src/models/asset.ts +61 -0
  18. package/src/models/definition.ts +24 -15
  19. package/src/models/delivery.ts +25 -31
  20. package/src/models/error.ts +12 -1
  21. package/src/models/limits.ts +8 -3
  22. package/src/models/metadata.ts +42 -0
  23. package/src/models/page.ts +68 -13
  24. package/src/models/provider-path.ts +25 -0
  25. package/src/models/providers.ts +13 -2
  26. package/src/models/record.ts +22 -0
  27. package/src/models/registry.ts +15 -15
  28. package/src/models/source-http-error.ts +37 -0
  29. package/src/models/sync.ts +71 -0
  30. package/src/models/validation.ts +1 -1
  31. package/src/open-sync.ts +26 -7
  32. package/src/repositories/acquisition/contract.ts +18 -10
  33. package/src/repositories/acquisition/lease.ts +80 -94
  34. package/src/repositories/acquisition/outbox.ts +22 -22
  35. package/src/repositories/acquisition/records.ts +44 -13
  36. package/src/repositories/acquisition/sqlite.ts +78 -31
  37. package/src/repositories/assets/capacity.ts +16 -0
  38. package/src/repositories/assets/contract.ts +25 -0
  39. package/src/repositories/assets/filesystem.ts +103 -0
  40. package/src/repositories/assets/sqlite.ts +174 -0
  41. package/src/repositories/catalog/contract.ts +16 -19
  42. package/src/repositories/catalog/sqlite.ts +98 -121
  43. package/src/repositories/delivery/contract.ts +7 -11
  44. package/src/repositories/delivery/lease.ts +16 -0
  45. package/src/repositories/delivery/sqlite.ts +57 -34
  46. package/src/repositories/queue-usage.ts +7 -8
  47. package/src/repositories/rows.ts +10 -29
  48. package/src/runtime.ts +27 -6
  49. package/src/services/acquisition.ts +178 -52
  50. package/src/services/delivery.ts +39 -19
  51. package/src/services/management.ts +88 -51
  52. package/src/services/providers/service.ts +100 -10
  53. package/src/models/installation.ts +0 -37
@@ -1,44 +1,55 @@
1
1
  import type { Database } from 'bun:sqlite';
2
- import type { DeliveryResult, PendingDelivery } from '../../models/delivery';
2
+ import type { Deliverable, DeliveryResult, PendingDelivery } from '../../models/delivery';
3
3
  import { fail } from '../../models/error';
4
4
  import { type Resource, type Scope, workerScope } from '../../models/identity';
5
5
  import { queueUsage } from '../queue-usage';
6
- import { type Row, readDestination } from '../rows';
6
+ import { type Row, readSync } from '../rows';
7
7
  import type { DeliveryLease, DeliveryRepository } from './contract';
8
+ import { assertDelivery } from './lease';
9
+
10
+ const headOfQueue = `NOT EXISTS (SELECT 1 FROM deliveries prior
11
+ WHERE prior.owner_id=d.owner_id AND prior.sync_id=d.sync_id
12
+ AND prior.sequence<d.sequence)`;
8
13
 
9
14
  export class SqliteDeliveries implements DeliveryRepository {
10
15
  constructor(private readonly db: Database) {}
16
+ nextDue(): number | undefined {
17
+ return (
18
+ this.db
19
+ .query<{ due: number | null }, []>(`SELECT MIN(
20
+ CASE WHEN d.state='leased' THEN MAX(d.due_at,d.expires_at) ELSE d.due_at END) AS due
21
+ FROM deliveries d WHERE d.state!='blocked' AND ${headOfQueue}`)
22
+ .get()?.due ?? undefined
23
+ );
24
+ }
11
25
  claim(leaseMs: number): DeliveryLease | undefined {
12
26
  return this.db
13
27
  .transaction(() => {
14
- // A blocked delivery stops only its destination; ordering survives retries and restarts.
28
+ // A blocked delivery stops only its sync; ordering survives retries and restarts.
15
29
  const row = this.db
16
30
  .query<
17
31
  Row,
18
32
  [number, number]
19
33
  >(`SELECT d.* FROM deliveries d WHERE d.state!='blocked' AND d.due_at<=?
20
- AND (d.state='pending' OR d.expires_at<=?) AND NOT EXISTS (
21
- SELECT 1 FROM deliveries prior WHERE prior.owner_id=d.owner_id AND prior.destination_id=d.destination_id AND prior.sequence<d.sequence)
34
+ AND (d.state='pending' OR d.expires_at<=?) AND ${headOfQueue}
22
35
  ORDER BY d.sequence LIMIT 1`)
23
36
  .get(Date.now(), Date.now());
24
37
  if (!row) {
25
38
  return;
26
39
  }
27
- const workerId = crypto.randomUUID();
28
40
  const scope = workerScope(String(row.owner_id));
29
41
  this.db
30
42
  .query(
31
- "UPDATE deliveries SET state='leased',worker_id=?,generation=generation+1,attempt=attempt+1,expires_at=? WHERE owner_id=? AND id=?",
43
+ "UPDATE deliveries SET state='leased',generation=generation+1,attempt=attempt+1,expires_at=? WHERE owner_id=? AND id=?",
32
44
  )
33
- .run(workerId, Date.now() + leaseMs, scope.ownerId, row.id!);
45
+ .run(Date.now() + leaseMs, scope.ownerId, row.id!);
34
46
  return {
35
47
  ...scope,
36
48
  delivery: JSON.parse(String(row.body)),
37
- destination: readDestination({
49
+ destination: readSync({
38
50
  db: this.db,
39
- scope: { ...scope, id: String(row.destination_id) },
40
- }),
41
- workerId,
51
+ scope: { ...scope, id: String(row.sync_id) },
52
+ }).destination,
42
53
  generation: Number(row.generation) + 1,
43
54
  attempt: Number(row.attempt) + 1,
44
55
  };
@@ -49,15 +60,7 @@ export class SqliteDeliveries implements DeliveryRepository {
49
60
  const { lease, result } = input;
50
61
  this.db
51
62
  .transaction(() => {
52
- if (
53
- !this.db
54
- .query(
55
- "SELECT 1 FROM deliveries WHERE owner_id=? AND id=? AND state='leased' AND worker_id=? AND generation=? AND expires_at>?",
56
- )
57
- .get(lease.ownerId, lease.delivery.id, lease.workerId, lease.generation, Date.now())
58
- ) {
59
- fail('lease_lost');
60
- }
63
+ assertDelivery({ db: this.db, lease });
61
64
  if (result.status === 'accepted') {
62
65
  this.db
63
66
  .query('DELETE FROM deliveries WHERE owner_id=? AND id=?')
@@ -65,13 +68,13 @@ export class SqliteDeliveries implements DeliveryRepository {
65
68
  // Capacity is global; all paused acquisitions can compete for the released budget.
66
69
  this.db
67
70
  .query(
68
- "UPDATE installations SET next_due_at=? WHERE enabled=1 AND status='waiting_for_capacity'",
71
+ "UPDATE syncs SET next_due_at=? WHERE enabled=1 AND status='waiting_for_capacity'",
69
72
  )
70
73
  .run(Date.now());
71
74
  } else {
72
75
  this.db
73
76
  .query(
74
- 'UPDATE deliveries SET state=?,due_at=?,worker_id=NULL,expires_at=NULL,error_code=? WHERE owner_id=? AND id=?',
77
+ 'UPDATE deliveries SET state=?,due_at=?,expires_at=NULL,error_code=? WHERE owner_id=? AND id=?',
75
78
  )
76
79
  .run(
77
80
  result.status === 'rejected' ? 'blocked' : 'pending',
@@ -87,25 +90,45 @@ export class SqliteDeliveries implements DeliveryRepository {
87
90
  status(scope: Scope) {
88
91
  return queueUsage({ db: this.db, ownerId: scope.ownerId });
89
92
  }
90
- pending(input: Scope & { offset: number }) {
91
- const limit = 50;
93
+ pending(input: Scope & { syncId: string; before?: number }) {
94
+ const pageSize = 20;
92
95
  const rows = this.db
93
96
  .query<
94
- PendingDelivery,
95
- [string, number, number]
96
- >(`SELECT id,installation_id AS installationId,destination_id AS destinationId,state,bytes,
97
- record_count AS recordCount,attempt,due_at AS nextAttemptAt,error_code AS errorCode FROM deliveries WHERE owner_id=? ORDER BY sequence LIMIT ? OFFSET ?`)
98
- .all(input.ownerId, limit + 1, input.offset);
99
- return { deliveries: rows.slice(0, limit), hasMore: rows.length > limit, pageSize: limit };
97
+ PendingDelivery & { sequence: number },
98
+ [string, string, number, number]
99
+ >(`SELECT sequence,id,sync_id AS syncId,state,bytes,
100
+ record_count AS recordCount,json_array_length(body,'$.assets') AS assetCount,
101
+ attempt,due_at AS nextAttemptAt,error_code AS errorCode FROM deliveries
102
+ WHERE owner_id=? AND sync_id=? AND sequence<? ORDER BY sequence DESC LIMIT ?`)
103
+ .all(input.ownerId, input.syncId, input.before ?? Number.MAX_SAFE_INTEGER, pageSize + 1);
104
+ const page = rows.slice(0, pageSize);
105
+ return {
106
+ deliveries: page.map(({ sequence: _, ...delivery }) => delivery),
107
+ nextCursor: rows.length > pageSize ? page.at(-1)!.sequence : null,
108
+ };
109
+ }
110
+ deliverable(input: Resource & { syncId: string }): Omit<Deliverable, 'openAsset'> {
111
+ const row = this.db
112
+ .query<{ body: string }, [string, string, string]>(
113
+ 'SELECT body FROM deliveries WHERE owner_id=? AND sync_id=? AND id=?',
114
+ )
115
+ .get(input.ownerId, input.syncId, input.id);
116
+ if (!row) {
117
+ fail('not_found');
118
+ }
119
+ return JSON.parse(row.body);
100
120
  }
101
121
  retry(input: Resource): void {
102
122
  const updated = this.db
103
123
  .query(
104
- "UPDATE deliveries SET state='pending',due_at=?,generation=generation+1,worker_id=NULL,expires_at=NULL,error_code=NULL WHERE owner_id=? AND id=?",
124
+ "UPDATE deliveries SET state='pending',due_at=?,generation=generation+1,expires_at=NULL,error_code=NULL WHERE owner_id=? AND id=? AND (state!='leased' OR expires_at<=?)",
105
125
  )
106
- .run(Date.now(), input.ownerId, input.id);
126
+ .run(Date.now(), input.ownerId, input.id, Date.now());
107
127
  if (!updated.changes) {
108
- fail('not_found');
128
+ const exists = this.db
129
+ .query('SELECT 1 FROM deliveries WHERE owner_id=? AND id=?')
130
+ .get(input.ownerId, input.id);
131
+ fail(exists ? 'busy' : 'not_found');
109
132
  }
110
133
  }
111
134
  }
@@ -1,21 +1,20 @@
1
1
  import type { Database } from 'bun:sqlite';
2
2
  import type { QueueStatus } from '../models/delivery';
3
3
  import type { QueueLimits } from '../models/limits';
4
+ import { assetBytes } from './assets/capacity';
4
5
  export function queueUsage(input: { db: Database; ownerId?: string }): QueueStatus {
5
6
  const row = input.db
6
- .query<
7
- QueueStatus,
8
- [string | null, string | null]
9
- >(`SELECT coalesce(sum(bytes),0) AS pendingBytes,
7
+ .query<QueueStatus, (string | null)[]>(`SELECT coalesce(sum(bytes),0) AS pendingBytes,
10
8
  coalesce(sum(record_count),0) AS pendingRecords,count(*) AS pendingDeliveries,
11
9
  coalesce(sum(state='blocked'),0) AS blockedDeliveries FROM deliveries WHERE (? IS NULL OR owner_id=?)`)
12
10
  .get(input.ownerId ?? null, input.ownerId ?? null)!;
13
11
  return row;
14
12
  }
15
- export function hasQueueCapacity(input: { db: Database; limits: QueueLimits }): boolean {
16
- const usage = queueUsage({ db: input.db });
13
+ export function hasQueueCapacity({ db, limits }: { db: Database; limits: QueueLimits }): boolean {
14
+ const usage = queueUsage({ db });
17
15
  return (
18
- usage.pendingBytes < input.limits.maxPendingBytes &&
19
- usage.pendingRecords < input.limits.maxPendingRecords
16
+ usage.pendingBytes < limits.maxPendingBytes &&
17
+ usage.pendingRecords < limits.maxPendingRecords &&
18
+ assetBytes({ db }) < limits.maxPendingAssetBytes
20
19
  );
21
20
  }
@@ -1,13 +1,12 @@
1
1
  import type { Database } from 'bun:sqlite';
2
- import type { Destination } from '../models/delivery';
3
2
  import { fail } from '../models/error';
4
3
  import type { Resource } from '../models/identity';
5
- import type { Installation } from '../models/installation';
4
+ import type { Sync, SyncStatus } from '../models/sync';
6
5
 
7
6
  export type Row = Record<string, string | number | null>;
8
- export function readInstallation(input: { db: Database; scope: Resource }): Installation {
7
+ export function readSync(input: { db: Database; scope: Resource }): Sync {
9
8
  const row = input.db
10
- .query<Row, [string, string]>('SELECT * FROM installations WHERE owner_id=? AND id=?')
9
+ .query<Row, [string, string]>('SELECT * FROM syncs WHERE owner_id=? AND id=?')
11
10
  .get(input.scope.ownerId, input.scope.id);
12
11
  if (!row) {
13
12
  fail('not_found');
@@ -15,36 +14,18 @@ export function readInstallation(input: { db: Database; scope: Resource }): Inst
15
14
  return {
16
15
  id: String(row.id),
17
16
  ownerId: String(row.owner_id),
18
- sourceId: String(row.source_id),
19
- definition: {
20
- id: String(row.definition_id),
21
- version: String(row.definition_version),
22
- artifactId: String(row.artifact_id),
23
- },
17
+ definition: String(row.definition_id),
24
18
  connection: row.connection === null ? undefined : JSON.parse(String(row.connection)),
25
19
  config: JSON.parse(String(row.config)),
26
- destinationId: String(row.destination_id),
20
+ destination: {
21
+ type: String(row.destination_type),
22
+ config: JSON.parse(String(row.destination_config)),
23
+ },
27
24
  enabled: row.enabled === 1,
28
- bindingEpoch: Number(row.binding_epoch),
29
25
  checkpoint: JSON.parse(String(row.checkpoint)),
30
- checkpointRevision: Number(row.checkpoint_revision),
31
26
  intervalMs: Number(row.interval_ms),
32
27
  nextDueAt: Number(row.next_due_at),
33
- status: String(row.status),
34
- };
35
- }
36
- export function readDestination(input: { db: Database; scope: Resource }): Destination {
37
- const row = input.db
38
- .query<Row, [string, string]>('SELECT * FROM destinations WHERE owner_id=? AND id=?')
39
- .get(input.scope.ownerId, input.scope.id);
40
- if (!row) {
41
- fail('not_found');
42
- }
43
- return {
44
- id: String(row.id),
45
- ownerId: String(row.owner_id),
46
- type: String(row.type),
47
- version: String(row.version),
48
- config: JSON.parse(String(row.config)),
28
+ status: row.status as SyncStatus,
29
+ errorCode: row.error_code === null ? null : String(row.error_code),
49
30
  };
50
31
  }
package/src/runtime.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { syncApi } from './api';
1
2
  import { openDatabase } from './db/client';
2
3
  import { type Logger, safeLogger } from './execution/diagnostics';
3
4
  import type { ProviderGateway } from './execution/provider';
@@ -14,6 +15,8 @@ import {
14
15
  } from './models/limits';
15
16
  import { Registry } from './models/registry';
16
17
  import { SqliteAcquisition } from './repositories/acquisition/sqlite';
18
+ import { DirectoryAssets } from './repositories/assets/filesystem';
19
+ import { SqliteAssets } from './repositories/assets/sqlite';
17
20
  import { SqliteCatalog } from './repositories/catalog/sqlite';
18
21
  import { SqliteDeliveries } from './repositories/delivery/sqlite';
19
22
  import { AcquisitionService } from './services/acquisition';
@@ -22,6 +25,8 @@ import { SyncManagement } from './services/management';
22
25
 
23
26
  export interface SyncRuntimeOptions {
24
27
  databasePath: string;
28
+ /** Private directory dedicated to queued asset bytes. */
29
+ assetDirectory?: string;
25
30
  definitions: readonly SyncRegistration[];
26
31
  destinationTypes: Readonly<Record<string, DestinationType>>;
27
32
  connector?: ProviderGateway;
@@ -46,22 +51,38 @@ export function createSyncRuntime(options: SyncRuntimeOptions) {
46
51
  const db = openDatabase(options.databasePath);
47
52
  try {
48
53
  const catalog = new SqliteCatalog(db);
49
- for (const definition of registry.definitions()) {
50
- catalog.register(definition);
51
- }
52
54
  const deliveries = new SqliteDeliveries(db);
55
+ const assets = new SqliteAssets({
56
+ db,
57
+ maxBytes: limits.maxPendingAssetBytes,
58
+ });
59
+ const files = new DirectoryAssets(options.assetDirectory ?? `${options.databasePath}.assets`);
53
60
  const log = safeLogger(options.onEvent);
61
+ let recoverFiles = true;
54
62
  const worker = new Worker({
55
63
  acquisition: new AcquisitionService({
56
- repository: new SqliteAcquisition({ db, limits, historyLimit: timing.historyLimit }),
64
+ repository: new SqliteAcquisition({ db, limits }),
57
65
  registry,
66
+ assets,
67
+ files,
68
+ maxAssetBytes: limits.maxAssetBytes,
58
69
  gateway: options.connector,
59
70
  timing,
60
71
  log,
61
72
  }),
62
- delivery: new DeliveryService({ repository: deliveries, registry, timing }),
73
+ delivery: new DeliveryService({ repository: deliveries, registry, timing, assets, files }),
63
74
  timing,
64
75
  log,
76
+ async cleanup() {
77
+ for (const id of assets.garbage()) {
78
+ await files.remove(id);
79
+ assets.discarded(id);
80
+ }
81
+ if (recoverFiles) {
82
+ await files.sweep({ retain: (id) => assets.retained(id) });
83
+ recoverFiles = false;
84
+ }
85
+ },
65
86
  });
66
87
  const api = new SyncManagement({
67
88
  catalog,
@@ -74,7 +95,7 @@ export function createSyncRuntime(options: SyncRuntimeOptions) {
74
95
  });
75
96
  let closing: Promise<void> | undefined;
76
97
  return {
77
- api,
98
+ api: syncApi(api),
78
99
  start: () => worker.start(),
79
100
  tick: () => worker.tick(),
80
101
  close: () => {
@@ -1,100 +1,226 @@
1
+ import { abortable } from '../execution/abortable';
1
2
  import type { Logger } from '../execution/diagnostics';
2
3
  import { bindProvider, type ProviderGateway } from '../execution/provider';
4
+ import { assetKey, type SourceAssets } from '../models/asset';
3
5
  import { SyncError } from '../models/error';
4
- import type { Timing } from '../models/limits';
6
+ import { canonicalJson } from '../models/json';
7
+ import { retryDelay, type Timing } from '../models/limits';
5
8
  import type { Registry } from '../models/registry';
6
- import type { AcquisitionRepository, RunLease } from '../repositories/acquisition/contract';
9
+ import { retryableStatus } from '../models/source-http-error';
10
+ import { identifier } from '../models/validation';
11
+ import type { AcquisitionLease, AcquisitionRepository } from '../repositories/acquisition/contract';
12
+ import type { AssetFiles, AssetRepository } from '../repositories/assets/contract';
7
13
 
8
14
  export class AcquisitionService {
9
15
  constructor(
10
16
  private readonly input: {
11
17
  repository: AcquisitionRepository;
18
+ assets: AssetRepository;
19
+ files: AssetFiles;
20
+ maxAssetBytes: number;
12
21
  registry: Registry;
13
22
  gateway?: ProviderGateway;
14
23
  timing: Timing;
15
24
  log: Logger;
16
25
  },
17
26
  ) {}
27
+ capacityReleased() {
28
+ this.input.repository.capacityReleased();
29
+ }
30
+ nextDue() {
31
+ return this.input.repository.nextDue();
32
+ }
18
33
  claim() {
19
34
  return this.input.repository.claim(this.input.timing.leaseMs);
20
35
  }
21
- async execute(input: { lease: RunLease; signal: AbortSignal }): Promise<void> {
22
- const { repository, timing, log } = this.input;
23
- const { lease, signal } = input;
36
+ async execute(input: { lease: AcquisitionLease; signal: AbortSignal }): Promise<void> {
37
+ const { repository, timing } = this.input;
38
+ const { lease } = input;
24
39
  try {
25
40
  if (!repository.hasCapacity()) {
26
41
  this.finish({ lease, state: 'waiting_for_capacity', delay: timing.retryMs });
27
42
  return;
28
43
  }
29
- await this.consume(input);
44
+ await abortable({ signal: input.signal, run: () => this.consume(input) });
30
45
  } catch (error) {
31
- const code = signal.aborted
32
- ? 'cancelled'
33
- : error instanceof SyncError
34
- ? error.code
35
- : 'execution_failed';
36
- log({ code, ownerId: lease.ownerId, installationId: lease.installation.id });
37
- this.finish({ lease, state: code, delay: timing.retryMs });
46
+ this.failed({ ...input, error });
38
47
  }
39
48
  }
40
- private async consume(input: { lease: RunLease; signal: AbortSignal }): Promise<void> {
41
- const { repository, registry, timing } = this.input;
49
+ private failed(input: { lease: AcquisitionLease; signal: AbortSignal; error: unknown }): void {
50
+ const { timing, log } = this.input;
51
+ const { lease, signal, error } = input;
52
+ const code = signal.aborted
53
+ ? abortCode(signal)
54
+ : error instanceof SyncError
55
+ ? error.code
56
+ : 'execution_failed';
57
+ const failed = !['paused', 'interrupted', 'waiting_for_capacity'].includes(code);
58
+ const failureCount = failed ? lease.failureCount + 1 : lease.failureCount;
59
+ const status = error instanceof SyncError ? error.status : undefined;
60
+ const pause = !signal.aborted && status !== undefined && !retryableStatus(status);
61
+ const delay = failed
62
+ ? retryDelay({ attempt: failureCount, retryMs: timing.retryMs })
63
+ : timing.retryMs;
64
+ log({
65
+ code,
66
+ ownerId: lease.ownerId,
67
+ syncId: lease.sync.id,
68
+ fields: {
69
+ ...(error instanceof SyncError ? error.diagnostics : {}),
70
+ ...(status === undefined ? {} : { httpStatus: status }),
71
+ failureCount,
72
+ ...(pause ? { paused: true } : { retryAfterMs: delay }),
73
+ },
74
+ });
75
+ this.finish({
76
+ lease,
77
+ state:
78
+ code === 'waiting_for_capacity'
79
+ ? 'waiting_for_capacity'
80
+ : signal.aborted && code !== 'timed_out'
81
+ ? 'interrupted'
82
+ : 'retrying',
83
+ errorCode: code === 'waiting_for_capacity' ? undefined : code,
84
+ delay,
85
+ failureCount,
86
+ pause,
87
+ });
88
+ }
89
+ private async consume(input: { lease: AcquisitionLease; signal: AbortSignal }): Promise<void> {
90
+ const { repository, registry } = this.input;
42
91
  const { lease, signal } = input;
43
- const entry = registry.definition(lease.installation.definition);
92
+ const entry = registry.definition(lease.sync.definition);
44
93
  const provider = await bindProvider({
45
94
  actorId: lease.actorId,
46
95
  ownerId: lease.ownerId,
47
96
  gateway: this.input.gateway,
48
- connection: lease.installation.connection,
97
+ connection: lease.sync.connection,
49
98
  requirements: entry.definition.provider,
50
99
  signal,
51
100
  });
52
101
  const executable = await entry.load();
53
102
  signal.throwIfAborted();
54
- let pages = 0;
55
- for await (const page of executable.run({
56
- config: lease.installation.config,
57
- checkpoint: lease.installation.checkpoint,
58
- sourceId: lease.installation.sourceId,
103
+ const page = await executable.step({
104
+ config: lease.sync.config,
105
+ checkpoint: lease.sync.checkpoint,
106
+ syncId: lease.sync.id,
59
107
  signal,
60
108
  provider,
61
- log: (event) =>
62
- this.input.log({
63
- ...event,
64
- code: 'definition_log',
65
- ownerId: lease.ownerId,
66
- installationId: lease.installation.id,
67
- }),
68
- })) {
69
- signal.throwIfAborted();
70
- repository.commit({ lease, page, definition: entry.definition });
71
- pages++;
72
- if (page.complete) {
73
- return;
74
- }
75
- if (!repository.hasCapacity()) {
76
- this.finish({ lease, state: 'waiting_for_capacity', delay: timing.retryMs });
77
- return;
78
- }
79
- if (pages >= timing.maxPages) {
80
- this.finish({ lease, state: 'yielded', delay: 0 });
81
- return;
82
- }
83
- }
84
- throw new SyncError({
85
- code: 'incomplete_run',
86
- message: 'Definition ended without a complete page.',
109
+ assets: sourceAssets({
110
+ repository: this.input.assets,
111
+ files: this.input.files,
112
+ lease,
113
+ signal,
114
+ maxBytes: this.input.maxAssetBytes,
115
+ }),
87
116
  });
117
+ signal.throwIfAborted();
118
+ repository.commit({ lease, page, definition: entry.definition });
88
119
  }
89
- private finish(input: { lease: RunLease; state: string; delay: number }): void {
120
+
121
+ private finish(input: Parameters<AcquisitionRepository['finish']>[0]): void {
90
122
  try {
91
123
  this.input.repository.finish(input);
92
124
  } catch (error) {
93
- if (
94
- !(error instanceof SyncError && ['lease_lost', 'checkpoint_conflict'].includes(error.code))
95
- ) {
125
+ if (!(error instanceof SyncError && error.code === 'lease_lost')) {
96
126
  throw error;
97
127
  }
98
128
  }
99
129
  }
100
130
  }
131
+
132
+ function abortCode(signal: AbortSignal): string {
133
+ if (signal.reason instanceof DOMException && signal.reason.name === 'TimeoutError') {
134
+ return 'timed_out';
135
+ }
136
+ return signal.reason === 'paused' ? 'paused' : 'interrupted';
137
+ }
138
+
139
+ function sourceAssets(input: {
140
+ repository: AssetRepository;
141
+ files: AssetFiles;
142
+ lease: AcquisitionLease;
143
+ signal: AbortSignal;
144
+ maxBytes: number;
145
+ }): SourceAssets {
146
+ const captures = new Map<
147
+ string,
148
+ {
149
+ metadata: string;
150
+ ref: import('../models/asset').AssetRef;
151
+ pending?: Promise<import('../models/asset').AssetRef>;
152
+ }
153
+ >();
154
+ const stage = ({
155
+ capture,
156
+ unavailable,
157
+ }: {
158
+ capture: import('../models/asset').AssetMetadata;
159
+ unavailable?: string;
160
+ }) => {
161
+ input.signal.throwIfAborted();
162
+ const asset = assetMetadata(capture);
163
+ const metadata = canonicalJson({ ...asset, unavailable: unavailable ?? null }).json;
164
+ const previous = captures.get(assetKey(asset));
165
+ if (previous) {
166
+ if (previous.metadata !== metadata) {
167
+ throw new SyncError({
168
+ code: 'asset_version_conflict',
169
+ message: 'Conflicting asset within one step.',
170
+ });
171
+ }
172
+ return { previous, asset, metadata };
173
+ }
174
+ const id = input.repository.stage({ lease: input.lease, asset, unavailable });
175
+ return { id, asset, metadata };
176
+ };
177
+ return {
178
+ unavailable(capture) {
179
+ identifier(capture.code);
180
+ const result = stage({ capture, unavailable: capture.code });
181
+ if (result.previous) {
182
+ return result.previous.ref;
183
+ }
184
+ const ref = { id: capture.id, version: capture.version };
185
+ captures.set(assetKey(ref), { ref, metadata: result.metadata });
186
+ return ref;
187
+ },
188
+ capture(capture) {
189
+ const result = stage({ capture });
190
+ if (result.previous) {
191
+ return result.previous.pending!;
192
+ }
193
+ const ref = { id: capture.id, version: capture.version };
194
+ const pending = (async () => {
195
+ const body = await capture.read();
196
+ input.signal.throwIfAborted();
197
+ const file = await input.files.write({
198
+ id: result.id!,
199
+ body,
200
+ maxBytes: input.maxBytes,
201
+ signal: input.signal,
202
+ reserve: (reservation) => {
203
+ input.signal.throwIfAborted();
204
+ input.repository.reserve({ lease: input.lease, ...reservation });
205
+ },
206
+ });
207
+ input.signal.throwIfAborted();
208
+ input.repository.captured({ lease: input.lease, ...file });
209
+ return ref;
210
+ })();
211
+ captures.set(assetKey(ref), { ref, metadata: result.metadata, pending });
212
+ return pending;
213
+ },
214
+ };
215
+ }
216
+
217
+ function assetMetadata(input: import('../models/asset').AssetMetadata) {
218
+ return {
219
+ id: input.id,
220
+ version: input.version,
221
+ name: input.name,
222
+ mediaType: input.mediaType,
223
+ ...(input.createdAt !== undefined ? { createdAt: input.createdAt } : {}),
224
+ ...(input.updatedAt !== undefined ? { updatedAt: input.updatedAt } : {}),
225
+ };
226
+ }