@context-use/open-sync 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/LICENSE +3 -0
  2. package/package.json +7 -5
  3. package/src/api.ts +18 -16
  4. package/src/connector/client.ts +61 -28
  5. package/src/connector/failure.ts +49 -0
  6. package/src/connector/file.ts +104 -0
  7. package/src/connector/response.ts +22 -1
  8. package/src/db/client.ts +1 -11
  9. package/src/db/providers.sql +2 -3
  10. package/src/db/providers.ts +1 -11
  11. package/src/db/schema.sql +41 -36
  12. package/src/execution/abortable.ts +12 -0
  13. package/src/execution/diagnostics.ts +1 -2
  14. package/src/execution/provider.ts +1 -0
  15. package/src/execution/worker.ts +162 -39
  16. package/src/http/controller.ts +63 -42
  17. package/src/models/asset.ts +61 -0
  18. package/src/models/definition.ts +24 -15
  19. package/src/models/delivery.ts +25 -31
  20. package/src/models/error.ts +12 -1
  21. package/src/models/limits.ts +8 -3
  22. package/src/models/metadata.ts +42 -0
  23. package/src/models/page.ts +68 -13
  24. package/src/models/provider-path.ts +25 -0
  25. package/src/models/providers.ts +13 -2
  26. package/src/models/record.ts +22 -0
  27. package/src/models/registry.ts +15 -15
  28. package/src/models/source-http-error.ts +37 -0
  29. package/src/models/sync.ts +71 -0
  30. package/src/models/validation.ts +1 -1
  31. package/src/open-sync.ts +26 -7
  32. package/src/repositories/acquisition/contract.ts +18 -10
  33. package/src/repositories/acquisition/lease.ts +80 -94
  34. package/src/repositories/acquisition/outbox.ts +22 -22
  35. package/src/repositories/acquisition/records.ts +44 -13
  36. package/src/repositories/acquisition/sqlite.ts +78 -31
  37. package/src/repositories/assets/capacity.ts +16 -0
  38. package/src/repositories/assets/contract.ts +25 -0
  39. package/src/repositories/assets/filesystem.ts +103 -0
  40. package/src/repositories/assets/sqlite.ts +174 -0
  41. package/src/repositories/catalog/contract.ts +16 -19
  42. package/src/repositories/catalog/sqlite.ts +98 -121
  43. package/src/repositories/delivery/contract.ts +7 -11
  44. package/src/repositories/delivery/lease.ts +16 -0
  45. package/src/repositories/delivery/sqlite.ts +57 -34
  46. package/src/repositories/queue-usage.ts +7 -8
  47. package/src/repositories/rows.ts +10 -29
  48. package/src/runtime.ts +27 -6
  49. package/src/services/acquisition.ts +178 -52
  50. package/src/services/delivery.ts +39 -19
  51. package/src/services/management.ts +88 -51
  52. package/src/services/providers/service.ts +100 -10
  53. package/src/models/installation.ts +0 -37
@@ -1,35 +1,53 @@
1
+ import { abortable } from '../execution/abortable';
1
2
  import type { DeliveryResult } from '../models/delivery';
2
3
  import { validateResult } from '../models/delivery-result';
3
4
  import { SyncError } from '../models/error';
4
5
  import { retryDelay, type Timing } from '../models/limits';
5
6
  import type { Registry } from '../models/registry';
6
- import type { DeliveryRepository } from '../repositories/delivery/contract';
7
+ import type { AssetFiles, AssetRepository } from '../repositories/assets/contract';
8
+ import type { DeliveryLease, DeliveryRepository } from '../repositories/delivery/contract';
7
9
 
8
10
  export class DeliveryService {
9
11
  constructor(
10
- private readonly input: { repository: DeliveryRepository; registry: Registry; timing: Timing },
12
+ private readonly input: {
13
+ repository: DeliveryRepository;
14
+ registry: Registry;
15
+ timing: Timing;
16
+ assets: AssetRepository;
17
+ files: AssetFiles;
18
+ },
11
19
  ) {}
12
- async execute(signal: AbortSignal): Promise<void> {
20
+ claim() {
21
+ return this.input.repository.claim(this.input.timing.leaseMs);
22
+ }
23
+ nextDue() {
24
+ return this.input.repository.nextDue();
25
+ }
26
+ async execute(input: { lease: DeliveryLease; signal: AbortSignal }): Promise<boolean> {
13
27
  const { repository, registry, timing } = this.input;
14
- const lease = repository.claim(timing.leaseMs);
15
- if (!lease) {
16
- return;
17
- }
28
+ const { lease, signal } = input;
18
29
  let result: DeliveryResult;
19
30
  try {
20
31
  const type = registry.destination(lease.destination.type);
21
- if (type.version !== lease.destination.version) {
22
- result = { status: 'rejected', code: 'destination_unavailable' };
23
- } else {
24
- result = validateResult(
25
- await type.deliver({
26
- scope: { actorId: lease.actorId, ownerId: lease.ownerId },
27
- config: structuredClone(lease.destination.config),
28
- delivery: structuredClone(lease.delivery),
29
- signal,
30
- }),
31
- );
32
- }
32
+ result = validateResult(
33
+ await abortable({
34
+ signal,
35
+ run: () =>
36
+ type.deliver({
37
+ scope: { actorId: lease.actorId, ownerId: lease.ownerId },
38
+ config: structuredClone(lease.destination.config),
39
+ deliverable: {
40
+ ...structuredClone(lease.delivery),
41
+ openAsset: async (asset) => {
42
+ signal.throwIfAborted();
43
+ const id = this.input.assets.read({ lease, asset });
44
+ return await this.input.files.open(id);
45
+ },
46
+ },
47
+ signal,
48
+ }),
49
+ }),
50
+ );
33
51
  signal.throwIfAborted();
34
52
  } catch (error) {
35
53
  result =
@@ -44,10 +62,12 @@ export class DeliveryService {
44
62
  });
45
63
  try {
46
64
  repository.complete({ lease, result, delay });
65
+ return result.status === 'accepted';
47
66
  } catch (error) {
48
67
  if (!(error instanceof SyncError && error.code === 'lease_lost')) {
49
68
  throw error;
50
69
  }
70
+ return false;
51
71
  }
52
72
  }
53
73
  }
@@ -3,10 +3,10 @@ import type { WorkerControl } from '../execution/worker';
3
3
  import type { ConnectionRef } from '../models/definition';
4
4
  import { fail } from '../models/error';
5
5
  import type { Resource, Scope } from '../models/identity';
6
- import type { CreateInstallation } from '../models/installation';
7
6
  import type { JsonObject } from '../models/json';
8
7
  import { positive, type QueueLimits } from '../models/limits';
9
8
  import type { Registry } from '../models/registry';
9
+ import { type CreateSync, summarizeSync } from '../models/sync';
10
10
  import { identifier, validate } from '../models/validation';
11
11
  import type { CatalogRepository } from '../repositories/catalog/contract';
12
12
  import type { DeliveryRepository } from '../repositories/delivery/contract';
@@ -30,36 +30,40 @@ export class SyncManagement {
30
30
  }
31
31
  definitions(scope: Scope) {
32
32
  this.guard(scope);
33
- return this.input.registry.definitions();
33
+ return this.input.registry
34
+ .definitions()
35
+ .map(({ id, name, description, configSchema, provider }) => ({
36
+ id,
37
+ name,
38
+ description,
39
+ configSchema,
40
+ provider,
41
+ }));
34
42
  }
35
43
  destinationTypes(scope: Scope) {
36
44
  this.guard(scope);
37
45
  return this.input.registry.destinationTypes();
38
46
  }
39
- runs(input: Resource & { offset?: number }) {
40
- this.guard(input);
41
- const offset = input.offset ?? 0;
42
- positive(offset + 1);
43
- return this.input.catalog.runs({ ...input, offset });
44
- }
45
- destinations(scope: Scope) {
47
+ private async prepareDestination(input: Scope & { destination: CreateSync['destination'] }) {
48
+ const scope = { actorId: input.actorId, ownerId: input.ownerId };
49
+ const type = this.input.registry.destination(input.destination.type);
50
+ const values = validate({
51
+ value: input.destination.input,
52
+ schema: type.setup?.schema ?? type.configSchema,
53
+ }) as JsonObject;
54
+ let prepared = values;
55
+ if (type.setup) {
56
+ try {
57
+ prepared = await type.setup.prepare({ scope: { ...scope }, input: values });
58
+ } catch {
59
+ fail('destination_setup_failed');
60
+ }
61
+ }
46
62
  this.guard(scope);
47
- return this.input.catalog
48
- .destinations(scope)
49
- .map(({ id, type, version }) => ({ id, type, version }));
63
+ const config = validate({ value: prepared, schema: type.configSchema }) as JsonObject;
64
+ return { type: input.destination.type, config };
50
65
  }
51
- createDestination(input: Scope & { type: string; config: JsonObject }) {
52
- this.guard(input);
53
- const type = this.input.registry.destination(input.type);
54
- const config = validate({ value: input.config, schema: type.configSchema }) as JsonObject;
55
- const { id, version } = this.input.catalog.createDestination({
56
- ...input,
57
- config,
58
- version: type.version,
59
- });
60
- return { id, type: input.type, version };
61
- }
62
- async createInstallation(input: CreateInstallation) {
66
+ async createSync(input: CreateSync) {
63
67
  this.guard(input);
64
68
  const { definition } = this.input.registry.definition(input.definition);
65
69
  const config = validate({ value: input.config, schema: definition.configSchema }) as JsonObject;
@@ -79,27 +83,37 @@ export class SyncManagement {
79
83
  fail('unexpected_connection');
80
84
  }
81
85
  this.guard(input);
82
- return this.input.catalog.createInstallation({
86
+ const destination = await this.prepareDestination(input);
87
+ const sync = this.input.catalog.createSync({
83
88
  ...input,
84
89
  config,
90
+ destination,
85
91
  initialCheckpoint: definition.initialCheckpoint,
86
92
  });
93
+ this.input.worker.wake();
94
+ return summarizeSync(sync);
87
95
  }
88
- installations(scope: Scope) {
96
+ syncs(scope: Scope) {
89
97
  this.guard(scope);
90
- return this.input.catalog.installations(scope);
98
+ return this.input.catalog.syncs(scope).map(summarizeSync);
99
+ }
100
+ sync(input: Resource) {
101
+ this.guard(input);
102
+ return summarizeSync(this.input.catalog.sync(input));
91
103
  }
92
- installation(input: Resource) {
104
+ /** Polling iterations, newest first, including the current scan. */
105
+ polls(input: Resource & { before?: number }) {
93
106
  this.guard(input);
94
- return this.input.catalog.installation(input);
107
+ validateCursor(input.before);
108
+ return this.input.catalog.polls(input);
95
109
  }
96
- async connectInstallation(input: Resource & { connection: ConnectionRef }) {
110
+ async connectSync(input: Resource & { connection: ConnectionRef }) {
97
111
  this.guard(input);
98
- const installation = this.input.catalog.installation(input);
99
- if (installation.connection || installation.enabled) {
112
+ const sync = this.input.catalog.sync(input);
113
+ if (sync.connection || sync.enabled) {
100
114
  fail('already_connected');
101
115
  }
102
- const { definition } = this.input.registry.definition(installation.definition);
116
+ const { definition } = this.input.registry.definition(sync.definition);
103
117
  if (!definition.provider) {
104
118
  fail('unexpected_connection');
105
119
  }
@@ -110,42 +124,65 @@ export class SyncManagement {
110
124
  signal: AbortSignal.timeout(this.input.timeoutMs),
111
125
  });
112
126
  this.guard(input);
113
- return this.input.catalog.connectInstallation(input);
127
+ const connected = this.input.catalog.connectSync(input);
128
+ this.input.worker.wake();
129
+ return summarizeSync(connected);
114
130
  }
115
131
  async setEnabled(input: Resource & { enabled: boolean }) {
116
132
  this.guard(input);
117
- const installation = this.input.catalog.installation(input);
118
- if (input.enabled && !installation.connection) {
119
- const { definition } = this.input.registry.definition(installation.definition);
133
+ const sync = this.input.catalog.sync(input);
134
+ if (sync.enabled === input.enabled) {
135
+ return summarizeSync(sync);
136
+ }
137
+ if (input.enabled && !sync.connection) {
138
+ const { definition } = this.input.registry.definition(sync.definition);
120
139
  if (definition.provider) {
121
140
  fail('connection_required');
122
141
  }
123
142
  }
124
143
  const result = this.input.catalog.setEnabled(input);
125
- await this.input.worker.cancel(input);
126
- return result;
144
+ await this.input.worker.cancelAcquisition(input);
145
+ return summarizeSync(result);
127
146
  }
128
- queueRun(input: Resource & { backfill?: boolean }): void {
147
+ runNow(input: Resource): void {
129
148
  this.guard(input);
130
- const installation = this.input.catalog.installation(input);
131
- const { definition } = this.input.registry.definition(installation.definition);
132
- this.input.catalog.queue({
133
- ...input,
134
- checkpoint: input.backfill ? definition.initialCheckpoint : undefined,
135
- });
149
+ this.input.catalog.runNow(input);
150
+ this.input.worker.wake();
151
+ }
152
+ async resync(input: Resource): Promise<void> {
153
+ this.guard(input);
154
+ const sync = this.input.catalog.sync(input);
155
+ const { definition } = this.input.registry.definition(sync.definition);
156
+ this.input.catalog.resync({ ...input, checkpoint: definition.initialCheckpoint });
157
+ await this.input.worker.cancelAcquisition(input);
158
+ }
159
+ async removeSync(input: Resource): Promise<void> {
160
+ this.guard(input);
161
+ this.input.catalog.removeSync(input);
162
+ await this.input.worker.cancelSync(input);
136
163
  }
137
164
  status(scope: Scope) {
138
165
  this.guard(scope);
139
166
  return { queue: this.input.deliveries.status(scope), limits: { ...this.input.limits } };
140
167
  }
141
- deliveries(input: Scope & { offset?: number }) {
168
+ deliveries(input: Scope & { syncId: string; before?: number }) {
142
169
  this.guard(input);
143
- const offset = input.offset ?? 0;
144
- positive(offset + 1);
145
- return this.input.deliveries.pending({ ...input, offset });
170
+ validateCursor(input.before);
171
+ this.input.catalog.sync({ ...input, id: input.syncId });
172
+ return this.input.deliveries.pending(input);
146
173
  }
147
- retryDelivery(input: Resource): void {
174
+ deliverable(input: Resource & { syncId: string }) {
148
175
  this.guard(input);
176
+ return this.input.deliveries.deliverable(input);
177
+ }
178
+ retryDelivery(input: Resource & { syncId: string }): void {
179
+ this.deliverable(input);
149
180
  this.input.deliveries.retry(input);
181
+ this.input.worker.wake();
182
+ }
183
+ }
184
+ function validateCursor(before?: number): void {
185
+ if (before !== undefined && (!Number.isSafeInteger(before) || before < 1)) {
186
+ fail('invalid_input');
150
187
  }
151
188
  }
@@ -3,6 +3,7 @@ import { ProviderCatalog } from '../../models/provider-catalog';
3
3
  import { oauthClientValues, publicConnection } from '../../models/provider-values';
4
4
  import type {
5
5
  ConnectorManagement,
6
+ OAuthClientRegistration,
6
7
  ProviderScope,
7
8
  ProviderSetup as RuntimeProviderSetup,
8
9
  } from '../../models/providers';
@@ -11,12 +12,14 @@ import type { ProviderRepository } from '../../repositories/providers/contract';
11
12
 
12
13
  export class ProviderService {
13
14
  private providerCatalog?: Promise<ProviderCatalog>;
15
+ private readonly clientConfigurations = new Map<string, Promise<void>>();
14
16
  constructor(
15
17
  private readonly input: {
16
18
  repository: ProviderRepository;
17
19
  connector: ConnectorManagement;
18
20
  returnUrl(input: { service: string; id: string }): string;
19
21
  canConfigure(scope: ProviderScope): Promise<boolean>;
22
+ registrations?: Readonly<Record<string, OAuthClientRegistration>>;
20
23
  signal: AbortSignal;
21
24
  onConnected?(
22
25
  input: ProviderScope & { connection: { id: string; service: string } },
@@ -64,11 +67,23 @@ export class ProviderService {
64
67
  this.guard(input);
65
68
  return (await this.loadCatalog()).page(input);
66
69
  }
70
+ private registration(service: string) {
71
+ const registrations = this.input.registrations;
72
+ return registrations && Object.hasOwn(registrations, service)
73
+ ? registrations[service]
74
+ : undefined;
75
+ }
67
76
  private async setup(service: string): Promise<RuntimeProviderSetup> {
68
77
  const setup = await this.input.connector.call({
69
78
  path: `/v1/providers/${encodeURIComponent(service)}/setup`,
70
79
  });
71
- return setup as unknown as RuntimeProviderSetup;
80
+ const result = setup as unknown as RuntimeProviderSetup;
81
+ return {
82
+ ...result,
83
+ ...(result.oauthClient && this.registration(service)
84
+ ? { oauthClient: { ...result.oauthClient, automaticRegistration: true } }
85
+ : {}),
86
+ };
72
87
  }
73
88
  async status(input: ProviderScope & { service: string }) {
74
89
  this.guard(input);
@@ -114,21 +129,95 @@ export class ProviderService {
114
129
  if (!(await this.input.canConfigure(input))) {
115
130
  fail('forbidden');
116
131
  }
117
- const setup = await this.setup(input.service);
132
+ await this.configureClient({
133
+ service: input.service,
134
+ configure: async () => {
135
+ const setup = await this.setup(input.service);
136
+ const auth = setup.auth.find((method) => method.type === 'oauth2');
137
+ if (!auth) {
138
+ throw new SyncError({
139
+ code: 'provider_request_failed',
140
+ message: 'This provider does not support OAuth.',
141
+ });
142
+ }
143
+ await this.input.connector.call({
144
+ path: `/api/oauth/configs/${encodeURIComponent(input.service)}`,
145
+ method: 'PUT',
146
+ body: oauthClientValues({ auth, values: input.values }),
147
+ });
148
+ },
149
+ });
150
+ return { configured: true };
151
+ }
152
+ private async configureClient(input: { service: string; configure: () => Promise<void> }) {
153
+ const { service, configure } = input;
154
+ const pending = (this.clientConfigurations.get(service) ?? Promise.resolve())
155
+ .catch(() => undefined)
156
+ .then(configure);
157
+ this.clientConfigurations.set(service, pending);
158
+ try {
159
+ await pending;
160
+ } finally {
161
+ if (this.clientConfigurations.get(service) === pending) {
162
+ this.clientConfigurations.delete(service);
163
+ }
164
+ }
165
+ }
166
+ private async ensureClient(input: ProviderScope & { service: string }) {
167
+ const register = this.registration(input.service);
168
+ if (!register) {
169
+ return;
170
+ }
171
+ await this.configureClient({
172
+ service: input.service,
173
+ configure: async () => {
174
+ const setup = await this.setup(input.service);
175
+ if (setup.oauthClient?.configured) {
176
+ return;
177
+ }
178
+ if (!(await this.input.canConfigure(input))) {
179
+ fail('forbidden');
180
+ }
181
+ await this.registerClient({ service: input.service, register, setup });
182
+ },
183
+ });
184
+ }
185
+
186
+ private async registerClient(input: {
187
+ service: string;
188
+ register: OAuthClientRegistration;
189
+ setup: RuntimeProviderSetup;
190
+ }) {
191
+ const { setup } = input;
118
192
  const auth = setup.auth.find((method) => method.type === 'oauth2');
119
- if (!auth) {
193
+ if (!auth || !setup.oauthClient) {
194
+ fail('provider_request_failed');
195
+ }
196
+ try {
197
+ const timeoutMs = 30_000;
198
+ const values = await input.register({
199
+ redirectUri: setup.oauthClient.expectedRedirectUri,
200
+ scopes: auth.scopes,
201
+ signal: AbortSignal.any([this.input.signal, AbortSignal.timeout(timeoutMs)]),
202
+ });
203
+ this.input.signal.throwIfAborted();
204
+ await this.input.connector.call({
205
+ path: `/api/oauth/configs/${encodeURIComponent(input.service)}`,
206
+ method: 'PUT',
207
+ body: oauthClientValues({
208
+ auth,
209
+ values: { clientId: values.clientId, clientSecret: values.clientSecret ?? '' },
210
+ }),
211
+ });
212
+ } catch {
213
+ // Registration errors may contain credentials or upstream bodies.
120
214
  throw new SyncError({
121
215
  code: 'provider_request_failed',
122
- message: 'This provider does not support OAuth.',
216
+ message: 'Could not register the OAuth client. Try connecting again.',
123
217
  });
124
218
  }
125
- await this.input.connector.call({
126
- path: `/api/oauth/configs/${encodeURIComponent(input.service)}`,
127
- method: 'PUT',
128
- body: oauthClientValues({ auth, values: input.values }),
129
- });
130
- return { configured: true };
131
219
  }
220
+
132
221
  private async reconnectTarget(input: ProviderScope & { service: string; connectionId?: string }) {
133
222
  if (!input.connectionId) {
134
223
  return undefined;
@@ -148,6 +237,7 @@ export class ProviderService {
148
237
  ) {
149
238
  this.guard(input);
150
239
  const target = await this.reconnectTarget(input);
240
+ await this.ensureClient(input);
151
241
  const id = target?.id ?? `connection_${crypto.randomUUID()}`;
152
242
  const result = await this.input.connector.call({
153
243
  path: target
@@ -1,37 +0,0 @@
1
- import type { ConnectionRef, DefinitionRef } from './definition';
2
- import type { Scope } from './identity';
3
- import type { JsonObject, JsonValue } from './json';
4
-
5
- export interface Installation {
6
- id: string;
7
- ownerId: string;
8
- sourceId: string;
9
- definition: DefinitionRef;
10
- connection?: ConnectionRef;
11
- config: JsonObject;
12
- destinationId: string;
13
- enabled: boolean;
14
- bindingEpoch: number;
15
- checkpoint: JsonValue;
16
- checkpointRevision: number;
17
- intervalMs: number;
18
- nextDueAt: number;
19
- status: string;
20
- }
21
- export interface CreateInstallation extends Scope {
22
- definition: DefinitionRef;
23
- connection?: ConnectionRef;
24
- config: JsonObject;
25
- destinationId: string;
26
- intervalMs?: number;
27
- enabled?: boolean;
28
- }
29
-
30
- export interface SyncRun {
31
- id: string;
32
- state: string;
33
- startedAt: number;
34
- completedAt: number | null;
35
- pages: number;
36
- checkpointRevision: number;
37
- }