@context-use/open-sync 0.1.0

package/src/services/acquisition.ts

@@ -0,0 +1,100 @@
+import type { Logger } from '../execution/diagnostics';
+import { bindProvider, type ProviderGateway } from '../execution/provider';
+import { SyncError } from '../models/error';
+import type { Timing } from '../models/limits';
+import type { Registry } from '../models/registry';
+import type { AcquisitionRepository, RunLease } from '../repositories/acquisition/contract';
+
+export class AcquisitionService {
+  constructor(
+    private readonly input: {
+      repository: AcquisitionRepository;
+      registry: Registry;
+      gateway?: ProviderGateway;
+      timing: Timing;
+      log: Logger;
+    },
+  ) {}
+  claim() {
+    return this.input.repository.claim(this.input.timing.leaseMs);
+  }
+  async execute(input: { lease: RunLease; signal: AbortSignal }): Promise<void> {
+    const { repository, timing, log } = this.input;
+    const { lease, signal } = input;
+    try {
+      if (!repository.hasCapacity()) {
+        this.finish({ lease, state: 'waiting_for_capacity', delay: timing.retryMs });
+        return;
+      }
+      await this.consume(input);
+    } catch (error) {
+      const code = signal.aborted
+        ? 'cancelled'
+        : error instanceof SyncError
+          ? error.code
+          : 'execution_failed';
+      log({ code, ownerId: lease.ownerId, installationId: lease.installation.id });
+      this.finish({ lease, state: code, delay: timing.retryMs });
+    }
+  }
+  private async consume(input: { lease: RunLease; signal: AbortSignal }): Promise<void> {
+    const { repository, registry, timing } = this.input;
+    const { lease, signal } = input;
+    const entry = registry.definition(lease.installation.definition);
+    const provider = await bindProvider({
+      actorId: lease.actorId,
+      ownerId: lease.ownerId,
+      gateway: this.input.gateway,
+      connection: lease.installation.connection,
+      requirements: entry.definition.provider,
+      signal,
+    });
+    const executable = await entry.load();
+    signal.throwIfAborted();
+    let pages = 0;
+    for await (const page of executable.run({
+      config: lease.installation.config,
+      checkpoint: lease.installation.checkpoint,
+      sourceId: lease.installation.sourceId,
+      signal,
+      provider,
+      log: (event) =>
+        this.input.log({
+          ...event,
+          code: 'definition_log',
+          ownerId: lease.ownerId,
+          installationId: lease.installation.id,
+        }),
+    })) {
+      signal.throwIfAborted();
+      repository.commit({ lease, page, definition: entry.definition });
+      pages++;
+      if (page.complete) {
+        return;
+      }
+      if (!repository.hasCapacity()) {
+        this.finish({ lease, state: 'waiting_for_capacity', delay: timing.retryMs });
+        return;
+      }
+      if (pages >= timing.maxPages) {
+        this.finish({ lease, state: 'yielded', delay: 0 });
+        return;
+      }
+    }
+    throw new SyncError({
+      code: 'incomplete_run',
+      message: 'Definition ended without a complete page.',
+    });
+  }
+  private finish(input: { lease: RunLease; state: string; delay: number }): void {
+    try {
+      this.input.repository.finish(input);
+    } catch (error) {
+      if (
+        !(error instanceof SyncError && ['lease_lost', 'checkpoint_conflict'].includes(error.code))
+      ) {
+        throw error;
+      }
+    }
+  }
+}

package/src/services/delivery.ts

@@ -0,0 +1,53 @@
+import type { DeliveryResult } from '../models/delivery';
+import { validateResult } from '../models/delivery-result';
+import { SyncError } from '../models/error';
+import { retryDelay, type Timing } from '../models/limits';
+import type { Registry } from '../models/registry';
+import type { DeliveryRepository } from '../repositories/delivery/contract';
+
+export class DeliveryService {
+  constructor(
+    private readonly input: { repository: DeliveryRepository; registry: Registry; timing: Timing },
+  ) {}
+  async execute(signal: AbortSignal): Promise<void> {
+    const { repository, registry, timing } = this.input;
+    const lease = repository.claim(timing.leaseMs);
+    if (!lease) {
+      return;
+    }
+    let result: DeliveryResult;
+    try {
+      const type = registry.destination(lease.destination.type);
+      if (type.version !== lease.destination.version) {
+        result = { status: 'rejected', code: 'destination_unavailable' };
+      } else {
+        result = validateResult(
+          await type.deliver({
+            scope: { actorId: lease.actorId, ownerId: lease.ownerId },
+            config: structuredClone(lease.destination.config),
+            delivery: structuredClone(lease.delivery),
+            signal,
+          }),
+        );
+      }
+      signal.throwIfAborted();
+    } catch (error) {
+      result =
+        error instanceof SyncError && error.code === 'destination_unavailable'
+          ? { status: 'rejected', code: 'destination_unavailable' }
+          : { status: 'retry', code: 'delivery_failed' };
+    }
+    const delay = retryDelay({
+      attempt: lease.attempt,
+      retryMs: timing.retryMs,
+      resultDelay: result.status === 'retry' ? result.retryAfterMs : undefined,
+    });
+    try {
+      repository.complete({ lease, result, delay });
+    } catch (error) {
+      if (!(error instanceof SyncError && error.code === 'lease_lost')) {
+        throw error;
+      }
+    }
+  }
+}

package/src/services/management.ts

@@ -0,0 +1,124 @@
+import { bindProvider, type ProviderGateway } from '../execution/provider';
+import type { WorkerControl } from '../execution/worker';
+import { fail } from '../models/error';
+import type { Resource, Scope } from '../models/identity';
+import type { CreateInstallation } from '../models/installation';
+import type { JsonObject } from '../models/json';
+import { positive, type QueueLimits } from '../models/limits';
+import type { Registry } from '../models/registry';
+import { identifier, validate } from '../models/validation';
+import type { CatalogRepository } from '../repositories/catalog/contract';
+import type { DeliveryRepository } from '../repositories/delivery/contract';
+
+export class SyncManagement {
+  constructor(
+    private readonly input: {
+      catalog: CatalogRepository;
+      deliveries: DeliveryRepository;
+      registry: Registry;
+      worker: WorkerControl;
+      gateway?: ProviderGateway;
+      limits: QueueLimits;
+      timeoutMs: number;
+    },
+  ) {}
+  private guard(scope: Scope): void {
+    this.input.worker.ensureOpen();
+    identifier(scope.actorId);
+    identifier(scope.ownerId);
+  }
+  definitions(scope: Scope) {
+    this.guard(scope);
+    return this.input.registry.definitions();
+  }
+  destinationTypes(scope: Scope) {
+    this.guard(scope);
+    return this.input.registry.destinationTypes();
+  }
+  runs(input: Resource & { offset?: number }) {
+    this.guard(input);
+    const offset = input.offset ?? 0;
+    positive(offset + 1);
+    return this.input.catalog.runs({ ...input, offset });
+  }
+  destinations(scope: Scope) {
+    this.guard(scope);
+    return this.input.catalog
+      .destinations(scope)
+      .map(({ id, type, version }) => ({ id, type, version }));
+  }
+  createDestination(input: Scope & { type: string; config: JsonObject }) {
+    this.guard(input);
+    const type = this.input.registry.destination(input.type);
+    const config = validate({ value: input.config, schema: type.configSchema }) as JsonObject;
+    const { id, version } = this.input.catalog.createDestination({
+      ...input,
+      config,
+      version: type.version,
+    });
+    return { id, type: input.type, version };
+  }
+  async createInstallation(input: CreateInstallation) {
+    this.guard(input);
+    const { definition } = this.input.registry.definition(input.definition);
+    const config = validate({ value: input.config, schema: definition.configSchema }) as JsonObject;
+    if (input.intervalMs !== undefined) {
+      positive(input.intervalMs);
+    }
+    if (definition.provider) {
+      await bindProvider({
+        actorId: input.actorId,
+        ownerId: input.ownerId,
+        connection: input.connection,
+        requirements: definition.provider,
+        gateway: this.input.gateway,
+        signal: AbortSignal.timeout(this.input.timeoutMs),
+      });
+    } else if (input.connection) {
+      fail('unexpected_connection');
+    }
+    this.guard(input);
+    return this.input.catalog.createInstallation({
+      ...input,
+      config,
+      initialCheckpoint: definition.initialCheckpoint,
+    });
+  }
+  installations(scope: Scope) {
+    this.guard(scope);
+    return this.input.catalog.installations(scope);
+  }
+  installation(input: Resource) {
+    this.guard(input);
+    return this.input.catalog.installation(input);
+  }
+  async setEnabled(input: Resource & { enabled: boolean }) {
+    this.guard(input);
+    const result = this.input.catalog.setEnabled(input);
+    await this.input.worker.cancel(input);
+    return result;
+  }
+  queueRun(input: Resource & { backfill?: boolean }): void {
+    this.guard(input);
+    const installation = this.input.catalog.installation(input);
+    const { definition } = this.input.registry.definition(installation.definition);
+    this.input.catalog.queue({
+      ...input,
+      checkpoint: input.backfill ? definition.initialCheckpoint : undefined,
+    });
+  }
+  status(scope: Scope) {
+    this.guard(scope);
+    return { queue: this.input.deliveries.status(scope), limits: { ...this.input.limits } };
+  }
+  deliveries(input: Scope & { offset?: number }) {
+    this.guard(input);
+    const offset = input.offset ?? 0;
+    positive(offset + 1);
+    return this.input.deliveries.pending({ ...input, offset });
+  }
+  retryDelivery(input: Resource): void {
+    this.guard(input);
+    this.input.deliveries.retry(input);
+  }
+}

package/src/services/providers/service.ts

@@ -0,0 +1,190 @@
+import { fail, SyncError } from '../../models/error';
+import { oauthClientValues, publicConnection } from '../../models/provider-values';
+import type {
+  ConnectorManagement,
+  ProviderScope,
+  ProviderSetup as RuntimeProviderSetup,
+} from '../../models/providers';
+import { identifier } from '../../models/validation';
+import type { ProviderRepository } from '../../repositories/providers/contract';
+
+export class ProviderService {
+  constructor(
+    private readonly input: {
+      repository: ProviderRepository;
+      connector: ConnectorManagement;
+      returnUrl(input: { service: string; id: string }): string;
+      canConfigure(scope: ProviderScope): Promise<boolean>;
+      signal: AbortSignal;
+    },
+  ) {}
+
+  private guard(scope: ProviderScope) {
+    this.input.signal.throwIfAborted();
+    identifier(scope.ownerId);
+    identifier(scope.actorId);
+  }
+  async connections(scope: ProviderScope) {
+    this.guard(scope);
+    return (await this.input.repository.list(scope)).map(({ id, service, account }) => ({
+      id,
+      service,
+      account,
+    }));
+  }
+  async connection(input: ProviderScope & { id: string }) {
+    this.guard(input);
+    const connection = await this.input.repository.connection(input);
+    if (!connection) {
+      fail('not_found');
+    }
+    return { id: connection.id, service: connection.service, account: connection.account };
+  }
+  async catalog(scope: ProviderScope) {
+    this.guard(scope);
+    const providers = await this.input.connector.catalog();
+    return providers.map(({ service, displayName, iconUrl, authTypes, categories, scenario }) => ({
+      service,
+      displayName,
+      iconUrl,
+      authTypes,
+      categories,
+      scenario,
+    }));
+  }
+  private async setup(service: string): Promise<RuntimeProviderSetup> {
+    const setup = await this.input.connector.call({
+      path: `/v1/providers/${encodeURIComponent(service)}/setup`,
+    });
+    return setup as unknown as RuntimeProviderSetup;
+  }
+  async status(input: ProviderScope & { service: string }) {
+    this.guard(input);
+    const provider = (await this.catalog(input)).find((entry) => entry.service === input.service);
+    if (!provider) {
+      fail('not_found');
+    }
+    const setup = await this.setup(input.service);
+    const connections = await this.input.repository.list(input);
+    return {
+      provider,
+      setup,
+      connections: await Promise.all(
+        connections
+          .filter((connection) => connection.service === input.service)
+          .map(async ({ id, account, connectorId }) => {
+            const metadata = await this.input.connector
+              .call({ path: `/v1/connections/by-id/${encodeURIComponent(connectorId)}` })
+              .catch(() => null);
+            const status =
+              metadata?.id === connectorId &&
+              metadata.service === input.service &&
+              (metadata.status === 'active' || metadata.status === 'reauth_required')
+                ? metadata.status
+                : 'unknown';
+            return { id, account, status };
+          }),
+      ),
+    };
+  }
+  async configure(input: ProviderScope & { service: string; values: Record<string, string> }) {
+    this.guard(input);
+    if (!(await this.input.canConfigure(input))) {
+      fail('forbidden');
+    }
+    const setup = await this.setup(input.service);
+    const auth = setup.auth.find((method) => method.type === 'oauth2');
+    if (!auth) {
+      throw new SyncError({
+        code: 'provider_request_failed',
+        message: 'This provider does not support OAuth.',
+      });
+    }
+    await this.input.connector.call({
+      path: `/api/oauth/configs/${encodeURIComponent(input.service)}`,
+      method: 'PUT',
+      body: oauthClientValues({ auth, values: input.values }),
+    });
+    return { configured: true };
+  }
+  async start(input: ProviderScope & { service: string; authorizationOptionIds?: string[] }) {
+    this.guard(input);
+    const id = `connection_${crypto.randomUUID()}`;
+    const result = await this.input.connector.call({
+      path: `/v1/connections/${encodeURIComponent(input.service)}/connect`,
+      method: 'POST',
+      body: {
+        returnUri: this.input.returnUrl({ service: input.service, id }),
+        authorizationOptionIds: input.authorizationOptionIds,
+      },
+    });
+    if (
+      typeof result.authorizationUrl !== 'string' ||
+      typeof result.connectionRequestId !== 'string'
+    ) {
+      throw new SyncError({
+        code: 'provider_request_failed',
+        message: 'Incomplete authorization response.',
+      });
+    }
+    this.input.signal.throwIfAborted();
+    await this.input.repository.start({ ...input, id, requestId: result.connectionRequestId });
+    return { authorizationUrl: result.authorizationUrl };
+  }
+  async credentials(
+    input: ProviderScope & {
+      service: string;
+      authType: 'api_key' | 'custom_credential';
+      values: Record<string, string>;
+    },
+  ) {
+    this.guard(input);
+    const { apiKey, ...extra } = input.values;
+    const result = await this.input.connector.call({
+      path: `/v1/connections/${encodeURIComponent(input.service)}/connect/${input.authType === 'api_key' ? 'api-key' : 'custom-credential'}`,
+      method: 'POST',
+      body: input.authType === 'api_key' ? { apiKey, extra } : { values: input.values },
+    });
+    const connection = publicConnection({ metadata: result, service: input.service });
+    this.input.signal.throwIfAborted();
+    await this.input.repository.add({
+      actorId: input.actorId,
+      ownerId: input.ownerId,
+      id: `connection_${crypto.randomUUID()}`,
+      ...connection,
+    });
+    return { connected: true };
+  }
+  async complete(input: ProviderScope & { service: string; id: string }): Promise<void> {
+    this.guard(input);
+    const existing = await this.input.repository.connection(input);
+    if (existing?.service === input.service) {
+      return;
+    }
+    const pending = await this.input.repository.pending(input);
+    if (!pending || pending.service !== input.service) {
+      fail('not_found');
+    }
+    const result = await this.input.connector.call({
+      path: `/v1/connection-requests/${encodeURIComponent(pending.requestId)}`,
+    });
+    if (result.status !== 'connected' || typeof result.appId !== 'string') {
+      throw new SyncError({
+        code: 'provider_request_failed',
+        message: 'Authorization did not complete. Connect again to retry.',
+      });
+    }
+    const metadata = await this.input.connector.call({
+      path: `/v1/connections/by-id/${encodeURIComponent(result.appId)}`,
+    });
+    const connection = publicConnection({ metadata, service: input.service });
+    if (connection.connectorId !== result.appId) {
+      throw new SyncError({ code: 'provider_request_failed', message: 'Invalid connection.' });
+    }
+    this.input.signal.throwIfAborted();
+    await this.input.repository.complete({ ...input, ...connection });
+    if (!(await this.input.repository.connection(input))) {
+      fail('not_found');
+    }
+  }
+}