@context-use/open-sync 0.1.0

package/src/execution/worker.ts

@@ -0,0 +1,90 @@
+import { fail } from '../models/error';
+import type { Resource } from '../models/identity';
+import type { Timing } from '../models/limits';
+import type { AcquisitionService } from '../services/acquisition';
+import type { DeliveryService } from '../services/delivery';
+import type { Logger } from './diagnostics';
+
+export interface WorkerControl {
+  ensureOpen(): void;
+  cancel(input: Resource): Promise<void>;
+}
+export class Worker implements WorkerControl {
+  readonly #lifetime = new AbortController();
+  #timer?: ReturnType<typeof setInterval>;
+  #acquisition?: Promise<void>;
+  #delivery?: Promise<void>;
+  #active?: { ownerId: string; id: string; abort: AbortController };
+  #closing?: Promise<void>;
+  constructor(
+    private readonly input: {
+      acquisition: AcquisitionService;
+      delivery: DeliveryService;
+      timing: Timing;
+      log: Logger;
+    },
+  ) {}
+  ensureOpen(): void {
+    if (this.#lifetime.signal.aborted) {
+      fail('closed');
+    }
+  }
+  start(): void {
+    this.ensureOpen();
+    if (this.#timer) {
+      return;
+    }
+    const pump = () => {
+      void this.tick().catch(() => this.input.log({ code: 'runtime_failed' }));
+    };
+    this.#timer = setInterval(pump, this.input.timing.pollMs);
+    pump();
+  }
+  tick(): Promise<void> {
+    this.ensureOpen();
+    this.#acquisition ??= this.acquire().finally(() => {
+      this.#acquisition = undefined;
+    });
+    this.#delivery ??= this.input.delivery.execute(this.signal()).finally(() => {
+      this.#delivery = undefined;
+    });
+    return Promise.all([this.#acquisition, this.#delivery]).then(() => undefined);
+  }
+  private signal(extra?: AbortSignal): AbortSignal {
+    return AbortSignal.any([
+      this.#lifetime.signal,
+      AbortSignal.timeout(this.input.timing.timeoutMs),
+      ...(extra ? [extra] : []),
+    ]);
+  }
+  private async acquire(): Promise<void> {
+    const lease = this.input.acquisition.claim();
+    if (!lease) {
+      return;
+    }
+    const abort = new AbortController();
+    this.#active = { ownerId: lease.ownerId, id: lease.installation.id, abort };
+    try {
+      await this.input.acquisition.execute({ lease, signal: this.signal(abort.signal) });
+    } finally {
+      this.#active = undefined;
+    }
+  }
+  async cancel(input: Resource): Promise<void> {
+    if (this.#active?.ownerId === input.ownerId && this.#active.id === input.id) {
+      this.#active.abort.abort();
+      await this.#acquisition;
+    }
+  }
+  close(): Promise<void> {
+    this.#closing ??= this.stop();
+    return this.#closing;
+  }
+  private async stop(): Promise<void> {
+    if (this.#timer) {
+      clearInterval(this.#timer);
+    }
+    this.#lifetime.abort();
+    await Promise.allSettled([this.#acquisition, this.#delivery]);
+  }
+}

package/src/http/app.ts

@@ -0,0 +1,21 @@
+import { Elysia } from 'elysia';
+import type { SyncApi } from '../api';
+import type { Scope } from '../models/identity';
+import type { ProviderService } from '../services/providers/service';
+import { createSyncController } from './controller';
+import { syncErrorResponse } from './index';
+import { createProviderController } from './providers';
+
+export function createHttpApp<Prefix extends string>(input: {
+  prefix: Prefix;
+  api: SyncApi;
+  providers: ProviderService;
+  authorize(request: Request): Scope | null | Promise<Scope | null>;
+  authorizationRedirect?(input: { service: string; outcome: 'connected' | 'failed' }): string;
+}) {
+  return new Elysia({ prefix: input.prefix })
+    .onError(({ error }) => syncErrorResponse(error))
+    .use(createSyncController(input))
+    .use(createProviderController(input));
+}
+export type OpenSyncHttp<Prefix extends string = ''> = ReturnType<typeof createHttpApp<Prefix>>;

package/src/http/controller.ts

@@ -0,0 +1,95 @@
+import { Elysia, t } from 'elysia';
+import type { SyncApi } from '../api';
+import type { Scope } from '../models/identity';
+import type { JsonObject } from '../models/json';
+import { syncErrorResponse } from './index';
+
+const identifier = t.String({ minLength: 1, maxLength: 1024 });
+const config = t.Record(t.String(), t.Unknown());
+const definition = t.Object({ id: identifier, version: identifier, artifactId: identifier });
+const installationBody = t.Object({
+  definition,
+  config,
+  destinationId: identifier,
+  connection: t.Optional(t.Object({ id: identifier, service: identifier })),
+  intervalMs: t.Optional(t.Integer({ minimum: 1 })),
+  enabled: t.Optional(t.Boolean()),
+});
+const resourceParams = { params: t.Object({ id: identifier }) };
+/** Optional management transport. The host owns authentication, owner authorization and CSRF policy. */
+export function createSyncController(input: {
+  api: SyncApi;
+  authorize(request: Request): Scope | null | Promise<Scope | null>;
+}) {
+  return new Elysia({ prefix: '/sync' })
+    .onError(({ error }) => syncErrorResponse(error))
+    .resolve(async ({ request, status }) => {
+      const scope = await input.authorize(request);
+      if (!scope) {
+        return status('Unauthorized', { error: 'Unauthorized' });
+      }
+      return { scope };
+    })
+    .get('/definitions', ({ scope }) => ({ definitions: input.api.definitions(scope) }))
+    .get('/destination-types', ({ scope }) => ({ types: input.api.destinationTypes(scope) }))
+    .get('/destinations', ({ scope }) => ({ destinations: input.api.destinations(scope) }))
+    .post(
+      '/destinations',
+      ({ scope, body }) =>
+        input.api.createDestination({ ...body, config: body.config as JsonObject, ...scope }),
+      {
+        body: t.Object({ type: identifier, config }),
+      },
+    )
+    .get('/installations', ({ scope }) => ({ installations: input.api.installations(scope) }))
+    .post(
+      '/installations',
+      ({ scope, body }) =>
+        input.api.createInstallation({ ...body, config: body.config as JsonObject, ...scope }),
+      { body: installationBody },
+    )
+    .get(
+      '/installations/:id',
+      ({ scope, params }) => input.api.installation({ ...scope, id: params.id }),
+      resourceParams,
+    )
+    .get(
+      '/installations/:id/runs',
+      ({ scope, params, query }) =>
+        input.api.runs({ ...scope, id: params.id, offset: query.offset }),
+      {
+        ...resourceParams,
+        query: t.Object({ offset: t.Optional(t.Integer({ minimum: 0, maximum: 1000000 })) }),
+      },
+    )
+    .patch(
+      '/installations/:id',
+      ({ scope, params, body }) =>
+        input.api.setEnabled({ ...scope, id: params.id, enabled: body.enabled }),
+      { ...resourceParams, body: t.Object({ enabled: t.Boolean() }) },
+    )
+    .post(
+      '/installations/:id/run',
+      ({ scope, params, body }) => {
+        input.api.queueRun({ ...scope, id: params.id, backfill: body.backfill });
+        return { queued: true };
+      },
+      { ...resourceParams, body: t.Object({ backfill: t.Optional(t.Boolean()) }) },
+    )
+    .get('/status', ({ scope }) => input.api.status(scope))
+    .get(
+      '/deliveries',
+      ({ scope, query }) => input.api.deliveries({ ...scope, offset: query.offset }),
+      {
+        query: t.Object({ offset: t.Optional(t.Integer({ minimum: 0, maximum: 1000000 })) }),
+      },
+    )
+    .post(
+      '/deliveries/:id/retry',
+      ({ scope, params }) => {
+        input.api.retryDelivery({ ...scope, id: params.id });
+        return { queued: true };
+      },
+      resourceParams,
+    );
+}

package/src/http/index.ts

@@ -0,0 +1,20 @@
+import { status } from 'elysia';
+import { SyncError } from '../models/error';
+
+export type { OpenSyncHttp } from './app';
+
+const notFound = 404;
+const conflict = 409;
+const unauthorized = 401;
+const forbidden = 403;
+const invalidInput = 400;
+export function syncErrorResponse(error: unknown) {
+  if (error instanceof SyncError) {
+    return status(
+      ({ not_found: notFound, busy: conflict, unauthorized, forbidden } as Record<string, number>)[
+        error.code
+      ] ?? invalidInput,
+      { error: error.code },
+    );
+  }
+}

package/src/http/providers.ts

@@ -0,0 +1,94 @@
+import { Elysia, t } from 'elysia';
+import { fail, SyncError } from '../models/error';
+import type { Scope } from '../models/identity';
+import type { ProviderService } from '../services/providers/service';
+import { syncErrorResponse } from './index';
+
+export function createProviderController(input: {
+  providers: ProviderService;
+  authorizationRedirect?(input: { service: string; outcome: 'connected' | 'failed' }): string;
+  authorize(request: Request): Scope | null | Promise<Scope | null>;
+}) {
+  async function scope(request: Request) {
+    const principal = await input.authorize(request);
+    if (!principal) {
+      fail('unauthorized');
+    }
+    return principal;
+  }
+  const service = t.String({ pattern: '^[a-zA-Z0-9_-]+$', maxLength: 128 });
+  const id = t.String({ pattern: '^connection_[a-f0-9-]{36}$' });
+  const values = t.Record(t.String({ maxLength: 128 }), t.String({ maxLength: 65536 }), {
+    maxProperties: 64,
+  });
+  return new Elysia({ prefix: '/providers' })
+    .onError(({ error }) => syncErrorResponse(error))
+    .get(
+      '/connections',
+      async ({ request }) => await input.providers.connections(await scope(request)),
+    )
+    .get('/', async ({ request }) => await input.providers.catalog(await scope(request)))
+    .get(
+      '/:service',
+      async ({ request, params }) =>
+        await input.providers.status({ ...(await scope(request)), ...params }),
+      { params: t.Object({ service }) },
+    )
+    .put(
+      '/:service/oauth-client',
+      async ({ request, params, body }) =>
+        await input.providers.configure({ ...(await scope(request)), ...params, ...body }),
+      {
+        params: t.Object({ service }),
+        body: t.Object({ values }),
+      },
+    )
+    .post(
+      '/:service/connect',
+      async ({ request, params, body }) =>
+        await input.providers.start({ ...(await scope(request)), ...params, ...body }),
+      {
+        params: t.Object({ service }),
+        body: t.Object({
+          authorizationOptionIds: t.Optional(
+            t.Array(t.String({ maxLength: 256 }), { maxItems: 128 }),
+          ),
+        }),
+      },
+    )
+    .post(
+      '/:service/credentials',
+      async ({ request, params, body }) =>
+        await input.providers.credentials({ ...(await scope(request)), ...params, ...body }),
+      {
+        params: t.Object({ service }),
+        body: t.Object({
+          authType: t.Union([t.Literal('api_key'), t.Literal('custom_credential')]),
+          values,
+        }),
+      },
+    )
+    .get(
+      '/:service/return/:id',
+      async ({ request, params, redirect }) => {
+        try {
+          await input.providers.complete({ ...(await scope(request)), ...params });
+        } catch (error) {
+          if (
+            !(error instanceof SyncError) ||
+            error.code !== 'provider_request_failed' ||
+            !input.authorizationRedirect
+          ) {
+            throw error;
+          }
+          return redirect(
+            input.authorizationRedirect({ service: params.service, outcome: 'failed' }),
+          );
+        }
+        return input.authorizationRedirect
+          ? redirect(input.authorizationRedirect({ service: params.service, outcome: 'connected' }))
+          : { connected: true };
+      },
+      { params: t.Object({ service, id }) },
+    );
+}

package/src/models/definition.ts

@@ -0,0 +1,63 @@
+import type { Schema } from '@cfworker/json-schema';
+import type { Deliverable } from './delivery';
+import type { JsonObject, JsonValue } from './json';
+
+export interface DefinitionRef {
+  id: string;
+  version: string;
+  artifactId: string;
+}
+export interface ConnectionRef {
+  id: string;
+  service: string;
+}
+export interface ProviderRequirements {
+  service: string;
+  actions: readonly string[];
+  proxyPaths?: readonly string[];
+  proxyPostPaths?: readonly string[];
+}
+/** JSON or text response from the provider, including non-success HTTP statuses. */
+export interface ProviderResponse {
+  status: number;
+  headers: Readonly<Record<string, string>>;
+  body: JsonValue;
+}
+export interface ProviderOperations {
+  action(input: { id: string; input: JsonObject }): Promise<JsonValue>;
+  get(input: { path: string; query?: JsonObject }): Promise<ProviderResponse>;
+  post(input: { path: string; body: JsonObject }): Promise<ProviderResponse>;
+}
+export interface SyncDefinition extends DefinitionRef {
+  name?: string;
+  description?: string;
+  configSchema: Schema;
+  checkpointSchema: Schema;
+  initialCheckpoint: JsonValue;
+  kinds: Readonly<Record<string, Schema>>;
+  provider?: ProviderRequirements;
+}
+export interface SyncPage {
+  deliverable: Deliverable;
+  checkpoint: JsonValue;
+  complete: boolean;
+}
+export interface SyncContext {
+  config: JsonObject;
+  checkpoint: JsonValue;
+  sourceId: string;
+  signal: AbortSignal;
+  provider: ProviderOperations;
+  log(input: { message: string; fields?: JsonObject }): void;
+}
+export interface SyncExecutable {
+  run(context: SyncContext): AsyncIterable<SyncPage>;
+}
+/** Trusted host code only. Loading uploaded code requires a separate isolated execution layer. */
+export interface SyncRegistration {
+  definition: SyncDefinition;
+  load(): SyncExecutable | Promise<SyncExecutable>;
+}
+export function definitionKey(ref: DefinitionRef): string {
+  return JSON.stringify([ref.id, ref.version, ref.artifactId]);
+}

package/src/models/delivery-result.ts

@@ -0,0 +1,22 @@
+import type { DeliveryResult } from './delivery';
+import { fail } from './error';
+import { identifier } from './validation';
+export function validateResult(result: DeliveryResult): DeliveryResult {
+  if (!['accepted', 'retry', 'rejected'].includes(result?.status)) {
+    fail('invalid_delivery_result');
+  }
+  if (result.status !== 'accepted' && result.code !== undefined) {
+    identifier(result.code);
+  }
+  if (result.status === 'rejected' && !result.code) {
+    fail('invalid_delivery_result');
+  }
+  if (
+    result.status === 'retry' &&
+    result.retryAfterMs !== undefined &&
+    (!Number.isSafeInteger(result.retryAfterMs) || result.retryAfterMs < 0)
+  ) {
+    fail('invalid_delivery_result');
+  }
+  return result;
+}

package/src/models/delivery.ts

@@ -0,0 +1,73 @@
+import type { Schema } from '@cfworker/json-schema';
+import type { DefinitionRef } from './definition';
+import type { Scope } from './identity';
+import type { JsonObject } from './json';
+
+export type SyncRecord =
+  | { operation: 'upsert'; kind: string; id: string; data: JsonObject }
+  | { operation: 'delete'; kind: string; id: string };
+export interface Deliverable {
+  records: readonly SyncRecord[];
+}
+export type DeliveredRecord = SyncRecord & {
+  eventId: string;
+  revision: number;
+  contentHash: string;
+};
+export interface Delivery {
+  version: 1;
+  id: string;
+  ownerId: string;
+  sourceId: string;
+  installationId: string;
+  definition: DefinitionRef;
+  deliverable: { records: DeliveredRecord[] };
+}
+export type DeliveryResult =
+  | { status: 'accepted' }
+  | { status: 'retry'; retryAfterMs?: number; code?: string }
+  | { status: 'rejected'; code: string };
+export interface DestinationType {
+  name?: string;
+  description?: string;
+  /** Pin endpoint/interpretation changes to a new version. Existing work is never rerouted. */
+  version: string;
+  configSchema: Schema;
+  /** Accepted means durable acceptance of the whole delivery. Receivers must tolerate retries. */
+  deliver(input: {
+    scope: Scope;
+    config: JsonObject;
+    delivery: Delivery;
+    signal: AbortSignal;
+  }): Promise<DeliveryResult>;
+}
+export interface Destination {
+  id: string;
+  ownerId: string;
+  type: string;
+  version: string;
+  config: JsonObject;
+}
+export interface QueueStatus {
+  pendingBytes: number;
+  pendingRecords: number;
+  pendingDeliveries: number;
+  blockedDeliveries: number;
+}
+export interface PendingDelivery {
+  id: string;
+  installationId: string;
+  destinationId: string;
+  state: string;
+  bytes: number;
+  recordCount: number;
+  attempt: number;
+  nextAttemptAt: number;
+  errorCode: string | null;
+}
+
+export interface DeliveryPage {
+  deliveries: PendingDelivery[];
+  hasMore: boolean;
+  pageSize: number;
+}

package/src/models/error.ts

@@ -0,0 +1,11 @@
+export class SyncError extends Error {
+  readonly code: string;
+  constructor(input: { code: string; message: string }) {
+    super(input.message);
+    this.name = 'SyncError';
+    this.code = input.code;
+  }
+}
+export function fail(code: string): never {
+  throw new SyncError({ code, message: code.replaceAll('_', ' ') });
+}

package/src/models/identity.ts

@@ -0,0 +1,11 @@
+/** The host authenticates the actor and authorizes access to this owner. */
+export interface Scope {
+  actorId: string;
+  ownerId: string;
+}
+export interface Resource extends Scope {
+  id: string;
+}
+export function workerScope(ownerId: string): Scope {
+  return { actorId: 'open-sync:worker', ownerId };
+}

package/src/models/installation.ts

@@ -0,0 +1,37 @@
+import type { ConnectionRef, DefinitionRef } from './definition';
+import type { Scope } from './identity';
+import type { JsonObject, JsonValue } from './json';
+
+export interface Installation {
+  id: string;
+  ownerId: string;
+  sourceId: string;
+  definition: DefinitionRef;
+  connection?: ConnectionRef;
+  config: JsonObject;
+  destinationId: string;
+  enabled: boolean;
+  bindingEpoch: number;
+  checkpoint: JsonValue;
+  checkpointRevision: number;
+  intervalMs: number;
+  nextDueAt: number;
+  status: string;
+}
+export interface CreateInstallation extends Scope {
+  definition: DefinitionRef;
+  connection?: ConnectionRef;
+  config: JsonObject;
+  destinationId: string;
+  intervalMs?: number;
+  enabled?: boolean;
+}
+
+export interface SyncRun {
+  id: string;
+  state: string;
+  startedAt: number;
+  completedAt: number | null;
+  pages: number;
+  checkpointRevision: number;
+}

package/src/models/json.ts

@@ -0,0 +1,82 @@
+import { createHash } from 'node:crypto';
+
+export type JsonValue = null | boolean | number | string | JsonValue[] | JsonObject;
+export interface JsonObject {
+  [key: string]: JsonValue;
+}
+export interface CanonicalJson {
+  value: JsonValue;
+  json: string;
+  sha256: string;
+}
+interface Visit {
+  value: unknown;
+  ancestors: Set<object>;
+}
+
+// Adapted from massimoalbarello/open-connector's src/sync/record-hash.ts.
+export function canonicalJson(value: unknown): CanonicalJson {
+  const normalized = normalize({ value, ancestors: new Set() });
+  const json = JSON.stringify(normalized);
+  return { value: normalized, json, sha256: createHash('sha256').update(json).digest('hex') };
+}
+function normalize(input: Visit): JsonValue {
+  const { value, ancestors } = input;
+  if (value === null || typeof value === 'boolean' || typeof value === 'string') {
+    return value;
+  }
+  if (typeof value === 'number' && Number.isFinite(value)) {
+    return Object.is(value, -0) ? 0 : value;
+  }
+  if (typeof value !== 'object' || ancestors.has(value)) {
+    throw new TypeError('Invalid JSON value');
+  }
+  ancestors.add(value);
+  try {
+    return Array.isArray(value)
+      ? normalizeArray({ value, ancestors })
+      : normalizeObject({ value, ancestors });
+  } finally {
+    ancestors.delete(value);
+  }
+}
+function normalizeArray(input: { value: unknown[]; ancestors: Set<object> }): JsonValue[] {
+  const descriptors = Object.getOwnPropertyDescriptors(input.value);
+  const keys = Reflect.ownKeys(descriptors);
+  if (keys.length !== input.value.length + 1) {
+    throw new TypeError('Invalid JSON array');
+  }
+  const result: JsonValue[] = [];
+  for (let index = 0; index < input.value.length; index++) {
+    const descriptor = descriptors[index];
+    if (!descriptor || !('value' in descriptor)) {
+      throw new TypeError('Sparse or accessor array');
+    }
+    result.push(normalize({ value: descriptor.value, ancestors: input.ancestors }));
+  }
+  return result;
+}
+function normalizeObject(input: { value: object; ancestors: Set<object> }): JsonObject {
+  const prototype = Object.getPrototypeOf(input.value);
+  if (prototype !== Object.prototype && prototype !== null) {
+    throw new TypeError('Expected plain JSON');
+  }
+  if (Object.getOwnPropertySymbols(input.value).length) {
+    throw new TypeError('Symbol JSON property');
+  }
+  const descriptors = Object.getOwnPropertyDescriptors(input.value);
+  const result: JsonObject = {};
+  for (const key of Object.keys(descriptors).sort()) {
+    const descriptor = descriptors[key]!;
+    if (!descriptor.enumerable || !('value' in descriptor)) {
+      throw new TypeError('Expected data property');
+    }
+    Object.defineProperty(result, key, {
+      value: normalize({ value: descriptor.value, ancestors: input.ancestors }),
+      enumerable: true,
+      writable: true,
+      configurable: true,
+    });
+  }
+  return result;
+}

package/src/models/limits.ts

@@ -0,0 +1,42 @@
+import { fail } from './error';
+export interface QueueLimits {
+  maxPendingBytes: number;
+  maxPendingRecords: number;
+  maxPageBytes: number;
+  maxPageRecords: number;
+}
+export const defaultLimits: QueueLimits = {
+  maxPendingBytes: 67_108_864,
+  maxPendingRecords: 100_000,
+  maxPageBytes: 1_048_576,
+  maxPageRecords: 1000,
+};
+export const defaultTiming = {
+  pollMs: 1000,
+  leaseMs: 60_000,
+  timeoutMs: 30_000,
+  retryMs: 30_000,
+  maxPages: 100,
+  historyLimit: 1000,
+};
+export type Timing = typeof defaultTiming;
+export function positive(value: number): number {
+  if (!Number.isSafeInteger(value) || value < 1) {
+    fail('invalid_positive_integer');
+  }
+  return value;
+}
+export function retryDelay(input: {
+  attempt: number;
+  retryMs: number;
+  resultDelay?: number;
+}): number {
+  const maxBackoff = 3_600_000;
+  const maxRequestedDelay = 86_400_000;
+  const maxExponent = 10;
+  const factor = 2;
+  if (input.resultDelay !== undefined) {
+    return Math.min(maxRequestedDelay, input.resultDelay);
+  }
+  return Math.min(maxBackoff, input.retryMs * factor ** Math.min(input.attempt - 1, maxExponent));
+}