@context-use/open-sync 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/LICENSE +3 -0
  2. package/package.json +7 -4
  3. package/src/api.ts +18 -14
  4. package/src/build.ts +3 -21
  5. package/src/connector/client.ts +61 -28
  6. package/src/connector/failure.ts +49 -0
  7. package/src/connector/file.ts +104 -0
  8. package/src/connector/response.ts +22 -1
  9. package/src/db/client.ts +1 -11
  10. package/src/db/providers.sql +2 -3
  11. package/src/db/providers.ts +1 -11
  12. package/src/db/schema.sql +41 -36
  13. package/src/execution/abortable.ts +12 -0
  14. package/src/execution/diagnostics.ts +1 -2
  15. package/src/execution/provider.ts +1 -0
  16. package/src/execution/worker.ts +162 -39
  17. package/src/http/controller.ts +63 -42
  18. package/src/http/providers.ts +13 -0
  19. package/src/models/asset.ts +61 -0
  20. package/src/models/definition.ts +24 -15
  21. package/src/models/delivery.ts +25 -31
  22. package/src/models/error.ts +12 -1
  23. package/src/models/limits.ts +8 -3
  24. package/src/models/metadata.ts +42 -0
  25. package/src/models/page.ts +68 -13
  26. package/src/models/provider-catalog.ts +57 -0
  27. package/src/models/provider-path.ts +25 -0
  28. package/src/models/providers.ts +13 -2
  29. package/src/models/record.ts +22 -0
  30. package/src/models/registry.ts +15 -15
  31. package/src/models/source-http-error.ts +37 -0
  32. package/src/models/sync.ts +71 -0
  33. package/src/models/validation.ts +1 -1
  34. package/src/open-sync.ts +31 -7
  35. package/src/repositories/acquisition/contract.ts +18 -10
  36. package/src/repositories/acquisition/lease.ts +80 -94
  37. package/src/repositories/acquisition/outbox.ts +22 -22
  38. package/src/repositories/acquisition/records.ts +44 -13
  39. package/src/repositories/acquisition/sqlite.ts +78 -31
  40. package/src/repositories/assets/capacity.ts +16 -0
  41. package/src/repositories/assets/contract.ts +25 -0
  42. package/src/repositories/assets/filesystem.ts +103 -0
  43. package/src/repositories/assets/sqlite.ts +174 -0
  44. package/src/repositories/catalog/contract.ts +16 -18
  45. package/src/repositories/catalog/sqlite.ts +107 -115
  46. package/src/repositories/delivery/contract.ts +7 -11
  47. package/src/repositories/delivery/lease.ts +16 -0
  48. package/src/repositories/delivery/sqlite.ts +57 -34
  49. package/src/repositories/providers/contract.ts +2 -1
  50. package/src/repositories/providers/sqlite.ts +22 -6
  51. package/src/repositories/queue-usage.ts +7 -8
  52. package/src/repositories/rows.ts +10 -29
  53. package/src/runtime.ts +27 -6
  54. package/src/services/acquisition.ts +178 -52
  55. package/src/services/delivery.ts +39 -19
  56. package/src/services/management.ts +108 -44
  57. package/src/services/providers/service.ts +187 -31
  58. package/src/build/connector.ts +0 -143
  59. package/src/models/installation.ts +0 -37
@@ -7,14 +7,26 @@ import type { Logger } from './diagnostics';
7
7
 
8
8
  export interface WorkerControl {
9
9
  ensureOpen(): void;
10
- cancel(input: Resource): Promise<void>;
10
+ wake(): void;
11
+ cancelAcquisition(input: Resource): Promise<void>;
12
+ cancelSync(input: Resource): Promise<void>;
11
13
  }
12
14
  export class Worker implements WorkerControl {
13
15
  readonly #lifetime = new AbortController();
14
- #timer?: ReturnType<typeof setInterval>;
15
- #acquisition?: Promise<void>;
16
- #delivery?: Promise<void>;
17
- #active?: { ownerId: string; id: string; abort: AbortController };
16
+ #timer?: ReturnType<typeof setTimeout>;
17
+ #started = false;
18
+ readonly #acquisitions = new Map<
19
+ Promise<void>,
20
+ { ownerId: string; syncId: string; abort: AbortController }
21
+ >();
22
+ readonly #deliveries = new Map<
23
+ Promise<void>,
24
+ { ownerId: string; syncId: string; abort: AbortController }
25
+ >();
26
+ #cleaning?: Promise<void>;
27
+ #cleanupRequested = false;
28
+ #capacityReleased = false;
29
+ #cleanupTimer?: ReturnType<typeof setTimeout>;
18
30
  #closing?: Promise<void>;
19
31
  constructor(
20
32
  private readonly input: {
@@ -22,6 +34,7 @@ export class Worker implements WorkerControl {
22
34
  delivery: DeliveryService;
23
35
  timing: Timing;
24
36
  log: Logger;
37
+ cleanup(): Promise<void>;
25
38
  },
26
39
  ) {}
27
40
  ensureOpen(): void {
@@ -31,24 +44,124 @@ export class Worker implements WorkerControl {
31
44
  }
32
45
  start(): void {
33
46
  this.ensureOpen();
34
- if (this.#timer) {
47
+ if (this.#started) {
35
48
  return;
36
49
  }
37
- const pump = () => {
38
- void this.tick().catch(() => this.input.log({ code: 'runtime_failed' }));
39
- };
40
- this.#timer = setInterval(pump, this.input.timing.pollMs);
41
- pump();
50
+ this.#started = true;
51
+ this.completed();
42
52
  }
53
+ wake(): void {
54
+ this.schedule(0);
55
+ }
56
+ private schedule(delay: number): void {
57
+ if (!this.#started || this.#lifetime.signal.aborted) {
58
+ return;
59
+ }
60
+ clearTimeout(this.#timer);
61
+ // Yield to the event loop even when another step is immediately runnable.
62
+ const maxTimerDelay = 2_147_483_647;
63
+ this.#timer = setTimeout(
64
+ () => {
65
+ this.#timer = undefined;
66
+ try {
67
+ this.dispatch();
68
+ } catch {
69
+ this.input.log({ code: 'runtime_failed' });
70
+ this.schedule(this.input.timing.retryMs);
71
+ }
72
+ },
73
+ Math.min(delay, maxTimerDelay),
74
+ );
75
+ }
76
+ /** A deterministic dispatch round for hosts/tests that do not start the background worker. */
43
77
  tick(): Promise<void> {
44
78
  this.ensureOpen();
45
- this.#acquisition ??= this.acquire().finally(() => {
46
- this.#acquisition = undefined;
47
- });
48
- this.#delivery ??= this.input.delivery.execute(this.signal()).finally(() => {
49
- this.#delivery = undefined;
50
- });
51
- return Promise.all([this.#acquisition, this.#delivery]).then(() => undefined);
79
+ this.dispatch();
80
+ return Promise.all([...this.#acquisitions.keys(), ...this.#deliveries.keys()]).then(() =>
81
+ this.cleanup(),
82
+ );
83
+ }
84
+ private dispatch(): void {
85
+ this.ensureOpen();
86
+ const { acquisition, delivery, timing } = this.input;
87
+ while (this.#acquisitions.size < timing.sourceConcurrency) {
88
+ const lease = acquisition.claim();
89
+ if (!lease) {
90
+ break;
91
+ }
92
+ const abort = new AbortController();
93
+ const task = acquisition
94
+ .execute({ lease, signal: this.signal(abort.signal) })
95
+ .catch(() => this.input.log({ code: 'acquisition_failed' }))
96
+ .finally(() => {
97
+ this.#acquisitions.delete(task);
98
+ this.completed();
99
+ });
100
+ this.#acquisitions.set(task, { ownerId: lease.ownerId, syncId: lease.sync.id, abort });
101
+ }
102
+ while (this.#deliveries.size < timing.deliveryConcurrency) {
103
+ const lease = delivery.claim();
104
+ if (!lease) {
105
+ break;
106
+ }
107
+ const abort = new AbortController();
108
+ const task = delivery
109
+ .execute({ lease, signal: this.signal(abort.signal) })
110
+ .then((accepted) => {
111
+ this.#capacityReleased ||= accepted;
112
+ })
113
+ .catch(() => this.input.log({ code: 'delivery_failed' }))
114
+ .finally(() => {
115
+ this.#deliveries.delete(task);
116
+ this.completed();
117
+ });
118
+ this.#deliveries.set(task, { ownerId: lease.ownerId, syncId: lease.delivery.syncId, abort });
119
+ }
120
+ const due = [
121
+ this.#acquisitions.size < timing.sourceConcurrency ? acquisition.nextDue() : undefined,
122
+ this.#deliveries.size < timing.deliveryConcurrency ? delivery.nextDue() : undefined,
123
+ ].filter((value): value is number => value !== undefined);
124
+ if (due.length) {
125
+ this.schedule(Math.max(0, Math.min(...due) - Date.now()));
126
+ }
127
+ }
128
+ private completed(): void {
129
+ this.wake();
130
+ if (this.#started) {
131
+ void this.cleanup().then(
132
+ () => this.wake(),
133
+ () => {
134
+ this.input.log({ code: 'cleanup_failed' });
135
+ if (!this.#lifetime.signal.aborted && !this.#cleanupTimer) {
136
+ this.#cleanupTimer = setTimeout(() => {
137
+ this.#cleanupTimer = undefined;
138
+ this.completed();
139
+ }, this.input.timing.retryMs);
140
+ }
141
+ },
142
+ );
143
+ }
144
+ }
145
+ private cleanup(): Promise<void> {
146
+ this.#cleanupRequested = true;
147
+ this.#cleaning ??= Promise.resolve().then(() => this.drainCleanup());
148
+ return this.#cleaning;
149
+ }
150
+ private async drainCleanup(): Promise<void> {
151
+ try {
152
+ do {
153
+ this.#cleanupRequested = false;
154
+ await this.input.cleanup();
155
+ } while (this.#cleanupRequested);
156
+ if (this.#capacityReleased) {
157
+ this.#capacityReleased = false;
158
+ this.input.acquisition.capacityReleased();
159
+ }
160
+ clearTimeout(this.#cleanupTimer);
161
+ this.#cleanupTimer = undefined;
162
+ } finally {
163
+ this.#cleaning = undefined;
164
+ }
52
165
  }
53
166
  private signal(extra?: AbortSignal): AbortSignal {
54
167
  return AbortSignal.any([
@@ -57,34 +170,44 @@ export class Worker implements WorkerControl {
57
170
  ...(extra ? [extra] : []),
58
171
  ]);
59
172
  }
60
- private async acquire(): Promise<void> {
61
- const lease = this.input.acquisition.claim();
62
- if (!lease) {
63
- return;
64
- }
65
- const abort = new AbortController();
66
- this.#active = { ownerId: lease.ownerId, id: lease.installation.id, abort };
67
- try {
68
- await this.input.acquisition.execute({ lease, signal: this.signal(abort.signal) });
69
- } finally {
70
- this.#active = undefined;
71
- }
173
+ async cancelAcquisition(input: Resource): Promise<void> {
174
+ await this.cancelTasks({ scope: input, tasks: this.#acquisitions });
175
+ this.completed();
72
176
  }
73
- async cancel(input: Resource): Promise<void> {
74
- if (this.#active?.ownerId === input.ownerId && this.#active.id === input.id) {
75
- this.#active.abort.abort();
76
- await this.#acquisition;
177
+ async cancelSync(input: Resource): Promise<void> {
178
+ await Promise.all([
179
+ this.cancelTasks({ scope: input, tasks: this.#acquisitions }),
180
+ this.cancelTasks({ scope: input, tasks: this.#deliveries }),
181
+ ]);
182
+ this.#capacityReleased = true;
183
+ this.completed();
184
+ await this.cleanup().catch(() => this.input.log({ code: 'cleanup_failed' }));
185
+ }
186
+ private async cancelTasks({
187
+ scope,
188
+ tasks,
189
+ }: {
190
+ scope: Resource;
191
+ tasks: Map<Promise<void>, { ownerId: string; syncId: string; abort: AbortController }>;
192
+ }): Promise<void> {
193
+ const cancelled = [];
194
+ for (const [task, entry] of tasks) {
195
+ if (entry.ownerId === scope.ownerId && entry.syncId === scope.id) {
196
+ entry.abort.abort('interrupted');
197
+ cancelled.push(task);
198
+ }
77
199
  }
200
+ await Promise.all(cancelled);
78
201
  }
79
202
  close(): Promise<void> {
80
203
  this.#closing ??= this.stop();
81
204
  return this.#closing;
82
205
  }
83
206
  private async stop(): Promise<void> {
84
- if (this.#timer) {
85
- clearInterval(this.#timer);
86
- }
87
- this.#lifetime.abort();
88
- await Promise.allSettled([this.#acquisition, this.#delivery]);
207
+ clearTimeout(this.#timer);
208
+ clearTimeout(this.#cleanupTimer);
209
+ this.#lifetime.abort('interrupted');
210
+ await Promise.allSettled([...this.#acquisitions.keys(), ...this.#deliveries.keys()]);
211
+ await this.cleanup();
89
212
  }
90
213
  }
@@ -5,12 +5,15 @@ import type { JsonObject } from '../models/json';
5
5
  import { syncErrorResponse } from './index';
6
6
 
7
7
  const identifier = t.String({ minLength: 1, maxLength: 1024 });
8
+ const pageQuery = t.Object({
9
+ before: t.Optional(t.Integer({ minimum: 1, maximum: Number.MAX_SAFE_INTEGER })),
10
+ });
11
+ const deliveryParams = t.Object({ id: identifier, deliveryId: identifier });
8
12
  const config = t.Record(t.String(), t.Unknown());
9
- const definition = t.Object({ id: identifier, version: identifier, artifactId: identifier });
10
- const installationBody = t.Object({
11
- definition,
13
+ const syncBody = t.Object({
14
+ definition: identifier,
12
15
  config,
13
- destinationId: identifier,
16
+ destination: t.Object({ type: identifier, input: config }),
14
17
  connection: t.Optional(t.Object({ id: identifier, service: identifier })),
15
18
  intervalMs: t.Optional(t.Integer({ minimum: 1 })),
16
19
  enabled: t.Optional(t.Boolean()),
@@ -22,7 +25,11 @@ export function createSyncController(input: {
22
25
  authorize(request: Request): Scope | null | Promise<Scope | null>;
23
26
  }) {
24
27
  return new Elysia({ prefix: '/sync' })
25
- .onError(({ error }) => syncErrorResponse(error))
28
+ .onError(({ error, code, status }) =>
29
+ code === 'VALIDATION'
30
+ ? status('Unprocessable Content', { error: 'invalid_input' })
31
+ : syncErrorResponse(error),
32
+ )
26
33
  .resolve(async ({ request, status }) => {
27
34
  const scope = await input.authorize(request);
28
35
  if (!scope) {
@@ -32,64 +39,78 @@ export function createSyncController(input: {
32
39
  })
33
40
  .get('/definitions', ({ scope }) => ({ definitions: input.api.definitions(scope) }))
34
41
  .get('/destination-types', ({ scope }) => ({ types: input.api.destinationTypes(scope) }))
35
- .get('/destinations', ({ scope }) => ({ destinations: input.api.destinations(scope) }))
42
+ .get('/syncs', ({ scope }) => ({ syncs: input.api.syncs(scope) }))
36
43
  .post(
37
- '/destinations',
44
+ '/syncs',
38
45
  ({ scope, body }) =>
39
- input.api.createDestination({ ...body, config: body.config as JsonObject, ...scope }),
40
- {
41
- body: t.Object({ type: identifier, config }),
42
- },
43
- )
44
- .get('/installations', ({ scope }) => ({ installations: input.api.installations(scope) }))
45
- .post(
46
- '/installations',
47
- ({ scope, body }) =>
48
- input.api.createInstallation({ ...body, config: body.config as JsonObject, ...scope }),
49
- { body: installationBody },
46
+ input.api.createSync({
47
+ ...body,
48
+ config: body.config as JsonObject,
49
+ destination: { type: body.destination.type, input: body.destination.input as JsonObject },
50
+ ...scope,
51
+ }),
52
+ { body: syncBody },
50
53
  )
51
54
  .get(
52
- '/installations/:id',
53
- ({ scope, params }) => input.api.installation({ ...scope, id: params.id }),
55
+ '/syncs/:id',
56
+ ({ scope, params }) => input.api.sync({ ...scope, id: params.id }),
54
57
  resourceParams,
55
58
  )
56
- .get(
57
- '/installations/:id/runs',
58
- ({ scope, params, query }) =>
59
- input.api.runs({ ...scope, id: params.id, offset: query.offset }),
60
- {
61
- ...resourceParams,
62
- query: t.Object({ offset: t.Optional(t.Integer({ minimum: 0, maximum: 1000000 })) }),
63
- },
64
- )
65
59
  .patch(
66
- '/installations/:id',
60
+ '/syncs/:id',
67
61
  ({ scope, params, body }) =>
68
62
  input.api.setEnabled({ ...scope, id: params.id, enabled: body.enabled }),
69
63
  { ...resourceParams, body: t.Object({ enabled: t.Boolean() }) },
70
64
  )
65
+ .get(
66
+ '/syncs/:id/polls',
67
+ ({ scope, params, query }) =>
68
+ input.api.polls({ ...scope, id: params.id, before: query.before }),
69
+ { ...resourceParams, query: pageQuery },
70
+ )
71
+ .post(
72
+ '/syncs/:id/run',
73
+ ({ scope, params }) => {
74
+ input.api.runNow({ ...scope, id: params.id });
75
+ return { queued: true };
76
+ },
77
+ resourceParams,
78
+ )
71
79
  .post(
72
- '/installations/:id/run',
73
- ({ scope, params, body }) => {
74
- input.api.queueRun({ ...scope, id: params.id, backfill: body.backfill });
80
+ '/syncs/:id/resync',
81
+ async ({ scope, params }) => {
82
+ await input.api.resync({ ...scope, id: params.id });
75
83
  return { queued: true };
76
84
  },
77
- { ...resourceParams, body: t.Object({ backfill: t.Optional(t.Boolean()) }) },
85
+ resourceParams,
86
+ )
87
+ .delete(
88
+ '/syncs/:id',
89
+ async ({ scope, params }) => {
90
+ await input.api.removeSync({ ...scope, id: params.id });
91
+ return { removed: true };
92
+ },
93
+ resourceParams,
78
94
  )
79
95
  .get('/status', ({ scope }) => input.api.status(scope))
80
96
  .get(
81
- '/deliveries',
82
- ({ scope, query }) => input.api.deliveries({ ...scope, offset: query.offset }),
83
- {
84
- query: t.Object({ offset: t.Optional(t.Integer({ minimum: 0, maximum: 1000000 })) }),
85
- },
97
+ '/syncs/:id/deliverables',
98
+ ({ scope, params, query }) =>
99
+ input.api.deliveries({ ...scope, syncId: params.id, before: query.before }),
100
+ { ...resourceParams, query: pageQuery },
101
+ )
102
+ .get(
103
+ '/syncs/:id/deliverables/:deliveryId',
104
+ ({ scope, params }) =>
105
+ input.api.deliverable({ ...scope, syncId: params.id, id: params.deliveryId }),
106
+ { params: deliveryParams },
86
107
  )
87
108
  .post(
88
- '/deliveries/:id/retry',
109
+ '/syncs/:id/deliverables/:deliveryId/retry',
89
110
  ({ scope, params }) => {
90
- input.api.retryDelivery({ ...scope, id: params.id });
111
+ input.api.retryDelivery({ ...scope, syncId: params.id, id: params.deliveryId });
91
112
  return { queued: true };
92
113
  },
93
- resourceParams,
114
+ { params: deliveryParams },
94
115
  );
95
116
  }
@@ -28,6 +28,17 @@ export function createProviderController(input: {
28
28
  async ({ request }) => await input.providers.connections(await scope(request)),
29
29
  )
30
30
  .get('/', async ({ request }) => await input.providers.catalog(await scope(request)))
31
+ .get(
32
+ '/catalog',
33
+ async ({ request, query }) =>
34
+ input.providers.catalogPage({ ...(await scope(request)), ...query }),
35
+ {
36
+ query: t.Object({
37
+ q: t.Optional(t.String({ maxLength: 200 })),
38
+ offset: t.Optional(t.Integer({ minimum: 0, maximum: 1000000 })),
39
+ }),
40
+ },
41
+ )
31
42
  .get(
32
43
  '/:service',
33
44
  async ({ request, params }) =>
@@ -50,6 +61,7 @@ export function createProviderController(input: {
50
61
  {
51
62
  params: t.Object({ service }),
52
63
  body: t.Object({
64
+ connectionId: t.Optional(id),
53
65
  authorizationOptionIds: t.Optional(
54
66
  t.Array(t.String({ maxLength: 256 }), { maxItems: 128 }),
55
67
  ),
@@ -63,6 +75,7 @@ export function createProviderController(input: {
63
75
  {
64
76
  params: t.Object({ service }),
65
77
  body: t.Object({
78
+ connectionId: t.Optional(id),
66
79
  authType: t.Union([t.Literal('api_key'), t.Literal('custom_credential')]),
67
80
  values,
68
81
  }),
@@ -0,0 +1,61 @@
1
+ import { fail } from './error';
2
+ import type { SourceTimestamps } from './metadata';
3
+
4
+ /** Identity is scoped to one owner and configured sync. A version must identify immutable content. */
5
+ export interface AssetRef {
6
+ id: string;
7
+ version: string;
8
+ }
9
+ export interface AssetMetadata extends AssetRef, SourceTimestamps {
10
+ name: string;
11
+ mediaType: string;
12
+ }
13
+ export type DeliveryAsset = AssetMetadata &
14
+ ({ size: number; sha256: string } | { unavailable: string });
15
+ export interface AssetCapture extends AssetMetadata {
16
+ /** Called only during acquisition. Credentials and temporary URLs never enter the queue. */
17
+ read(): Promise<ReadableStream<Uint8Array>>;
18
+ }
19
+ export interface SourceAssets {
20
+ /** Capture for this step. A read failure fails the step; later steps capture again. */
21
+ capture(input: AssetCapture): Promise<AssetRef>;
22
+ /** Declare a known permanent source limitation without downloading or discarding the attachment. */
23
+ unavailable(input: AssetMetadata & { code: string }): AssetRef;
24
+ }
25
+ export function assetKey(asset: AssetRef): string {
26
+ return JSON.stringify([asset.id, asset.version]);
27
+ }
28
+
29
+ const prefix = 'open-sync-asset:';
30
+ const keyPattern = /^[A-Za-z0-9_-]+$/;
31
+ /** Create a record-local asset reference for an exact JSON value or a content link target. */
32
+ export function assetPlaceholder(key: string): string {
33
+ if (!keyPattern.test(key)) {
34
+ fail('invalid_asset_placeholder');
35
+ }
36
+ return `${prefix}${key}`;
37
+ }
38
+ /** Parse an exact placeholder; ordinary strings return undefined, malformed placeholders throw. */
39
+ export function assetPlaceholderKey(value: string): string | undefined {
40
+ if (!value.startsWith(prefix)) {
41
+ return;
42
+ }
43
+ const key = value.slice(prefix.length);
44
+ assetPlaceholder(key);
45
+ return key;
46
+ }
47
+
48
+ /** Resolve an exact placeholder through the record's declared references. Does not inspect content. */
49
+ export function resolveAssetReference(input: {
50
+ value: string;
51
+ assetRefs: Readonly<Record<string, AssetRef>>;
52
+ }): AssetRef | undefined {
53
+ const key = assetPlaceholderKey(input.value);
54
+ if (key === undefined) {
55
+ return;
56
+ }
57
+ if (!Object.hasOwn(input.assetRefs, key)) {
58
+ return fail('unknown_asset_reference');
59
+ }
60
+ return input.assetRefs[key];
61
+ }
@@ -1,12 +1,11 @@
1
1
  import type { Schema } from '@cfworker/json-schema';
2
- import type { Deliverable } from './delivery';
2
+ import type { SourceAssets } from './asset';
3
3
  import type { JsonObject, JsonValue } from './json';
4
+ import type { SyncRecord } from './record';
5
+
6
+ // biome-ignore lint/performance/noBarrelFile: Public source-authoring entry point exposes its failure contract.
7
+ export { SourceHttpError } from './source-http-error';
4
8
 
5
- export interface DefinitionRef {
6
- id: string;
7
- version: string;
8
- artifactId: string;
9
- }
10
9
  export interface ConnectionRef {
11
10
  id: string;
12
11
  service: string;
@@ -25,10 +24,19 @@ export interface ProviderResponse {
25
24
  }
26
25
  export interface ProviderOperations {
27
26
  action(input: { id: string; input: JsonObject }): Promise<JsonValue>;
27
+ /** Execute a declared file action and consume its temporary file through the owned Connector. */
28
+ download?(input: {
29
+ id: string;
30
+ input: JsonObject;
31
+ /** Top-level result field containing the file; omit when the action returns it directly. */
32
+ fileField?: string;
33
+ }): Promise<ReadableStream<Uint8Array>>;
28
34
  get(input: { path: string; query?: JsonObject }): Promise<ProviderResponse>;
29
35
  post(input: { path: string; body: JsonObject }): Promise<ProviderResponse>;
30
36
  }
31
- export interface SyncDefinition extends DefinitionRef {
37
+ export interface SyncDefinition {
38
+ /** Host-defined registration name, shared by configured syncs using this implementation. */
39
+ id: string;
32
40
  name?: string;
33
41
  description?: string;
34
42
  configSchema: Schema;
@@ -37,27 +45,28 @@ export interface SyncDefinition extends DefinitionRef {
37
45
  kinds: Readonly<Record<string, Schema>>;
38
46
  provider?: ProviderRequirements;
39
47
  }
40
- export interface SyncPage {
41
- deliverable: Deliverable;
48
+ /** One complete source-defined unit: fetch its records and assets before returning.
49
+ * The engine commits the output with its resume position; checkpoints do not carry unfinished records.
50
+ */
51
+ export interface SyncStep {
52
+ records: readonly SyncRecord[];
42
53
  checkpoint: JsonValue;
43
54
  complete: boolean;
44
55
  }
45
56
  export interface SyncContext {
46
57
  config: JsonObject;
47
58
  checkpoint: JsonValue;
48
- sourceId: string;
59
+ /** Identity of this configured sync, also supplied to the destination. */
60
+ syncId: string;
49
61
  signal: AbortSignal;
50
62
  provider: ProviderOperations;
51
- log(input: { message: string; fields?: JsonObject }): void;
63
+ assets: SourceAssets;
52
64
  }
53
65
  export interface SyncExecutable {
54
- run(context: SyncContext): AsyncIterable<SyncPage>;
66
+ step(context: SyncContext): Promise<SyncStep>;
55
67
  }
56
68
  /** Trusted host code only. Loading uploaded code requires a separate isolated execution layer. */
57
69
  export interface SyncRegistration {
58
70
  definition: SyncDefinition;
59
71
  load(): SyncExecutable | Promise<SyncExecutable>;
60
72
  }
61
- export function definitionKey(ref: DefinitionRef): string {
62
- return JSON.stringify([ref.id, ref.version, ref.artifactId]);
63
- }
@@ -1,27 +1,23 @@
1
1
  import type { Schema } from '@cfworker/json-schema';
2
- import type { DefinitionRef } from './definition';
2
+ import type { AssetRef, DeliveryAsset } from './asset';
3
3
  import type { Scope } from './identity';
4
4
  import type { JsonObject } from './json';
5
+ import type { SyncRecord } from './record';
5
6
 
6
- export type SyncRecord =
7
- | { operation: 'upsert'; kind: string; id: string; data: JsonObject }
8
- | { operation: 'delete'; kind: string; id: string };
9
- export interface Deliverable {
10
- records: readonly SyncRecord[];
11
- }
7
+ /** Source-authored record plus the monotonic revision used to reject stale replays. */
12
8
  export type DeliveredRecord = SyncRecord & {
13
- eventId: string;
14
9
  revision: number;
15
- contentHash: string;
16
10
  };
17
- export interface Delivery {
18
- version: 1;
11
+ /** One stable batch, replayed until the destination durably accepts all records and assets. */
12
+ export interface Deliverable {
19
13
  id: string;
20
14
  ownerId: string;
21
- sourceId: string;
22
- installationId: string;
23
- definition: DefinitionRef;
24
- deliverable: { records: DeliveredRecord[] };
15
+ syncId: string;
16
+ definition: string;
17
+ records: DeliveredRecord[];
18
+ assets: DeliveryAsset[];
19
+ /** Open a fresh stream during this attempt. Files may be deleted after acceptance. */
20
+ openAsset(asset: AssetRef): Promise<ReadableStream<Uint8Array>>;
25
21
  }
26
22
  export type DeliveryResult =
27
23
  | { status: 'accepted' }
@@ -30,24 +26,23 @@ export type DeliveryResult =
30
26
  export interface DestinationType {
31
27
  name?: string;
32
28
  description?: string;
33
- /** Pin endpoint/interpretation changes to a new version. Existing work is never rerouted. */
34
- version: string;
35
29
  configSchema: Schema;
36
- /** Accepted means durable acceptance of the whole delivery. Receivers must tolerate retries. */
30
+ /** User input is validated before preparation; only the resulting config is persisted. */
31
+ setup?: {
32
+ schema: Schema;
33
+ prepare(input: { scope: Scope; input: JsonObject }): JsonObject | Promise<JsonObject>;
34
+ };
35
+ /** At-least-once delivery: accepted means durable acceptance of all records and assets.
36
+ * Receive the original source representation and use deliverable.openAsset() for captured bytes.
37
+ * The destination chooses whether to send the bundle or upload assets and resolve references first.
38
+ */
37
39
  deliver(input: {
38
40
  scope: Scope;
39
41
  config: JsonObject;
40
- delivery: Delivery;
42
+ deliverable: Deliverable;
41
43
  signal: AbortSignal;
42
44
  }): Promise<DeliveryResult>;
43
45
  }
44
- export interface Destination {
45
- id: string;
46
- ownerId: string;
47
- type: string;
48
- version: string;
49
- config: JsonObject;
50
- }
51
46
  export interface QueueStatus {
52
47
  pendingBytes: number;
53
48
  pendingRecords: number;
@@ -56,11 +51,11 @@ export interface QueueStatus {
56
51
  }
57
52
  export interface PendingDelivery {
58
53
  id: string;
59
- installationId: string;
60
- destinationId: string;
61
- state: string;
54
+ syncId: string;
55
+ state: 'pending' | 'leased' | 'blocked';
62
56
  bytes: number;
63
57
  recordCount: number;
58
+ assetCount: number;
64
59
  attempt: number;
65
60
  nextAttemptAt: number;
66
61
  errorCode: string | null;
@@ -68,6 +63,5 @@ export interface PendingDelivery {
68
63
 
69
64
  export interface DeliveryPage {
70
65
  deliveries: PendingDelivery[];
71
- hasMore: boolean;
72
- pageSize: number;
66
+ nextCursor: number | null;
73
67
  }
@@ -1,9 +1,20 @@
1
+ import type { JsonObject } from './json';
2
+
1
3
  export class SyncError extends Error {
2
4
  readonly code: string;
3
- constructor(input: { code: string; message: string }) {
5
+ readonly diagnostics?: JsonObject;
6
+ readonly status?: number;
7
+ constructor(input: {
8
+ code: string;
9
+ message: string;
10
+ diagnostics?: JsonObject;
11
+ status?: number;
12
+ }) {
4
13
  super(input.message);
5
14
  this.name = 'SyncError';
6
15
  this.code = input.code;
16
+ this.diagnostics = input.diagnostics;
17
+ this.status = input.status;
7
18
  }
8
19
  }
9
20
  export function fail(code: string): never {