@context-use/open-sync 0.1.0

package/src/models/page.ts

@@ -0,0 +1,69 @@
+import type { SyncDefinition, SyncPage } from './definition';
+import type { SyncRecord } from './delivery';
+import { fail } from './error';
+import { canonicalJson } from './json';
+import type { QueueLimits } from './limits';
+import { identifier, validate } from './validation';
+
+export function preparePage(input: {
+  page: SyncPage;
+  definition: SyncDefinition;
+  limits: QueueLimits;
+}): SyncPage {
+  try {
+    const json = canonicalJson(input.page);
+    if (Buffer.byteLength(json.json) > input.limits.maxPageBytes) {
+      fail('invalid_page');
+    }
+    const page = json.value as unknown as SyncPage;
+    if (
+      Object.keys(page).some((key) => !['deliverable', 'checkpoint', 'complete'].includes(key)) ||
+      typeof page.complete !== 'boolean'
+    ) {
+      fail('invalid_page');
+    }
+    if (
+      Object.keys(page.deliverable).some((key) => key !== 'records') ||
+      !Array.isArray(page.deliverable.records) ||
+      page.deliverable.records.length > input.limits.maxPageRecords
+    ) {
+      fail('invalid_page');
+    }
+    validate({ value: page.checkpoint, schema: input.definition.checkpointSchema });
+    const identities = new Set<string>();
+    for (const record of page.deliverable.records) {
+      validateRecord({ record, definition: input.definition });
+      const key = JSON.stringify([record.kind, record.id]);
+      if (identities.has(key)) {
+        fail('duplicate_record');
+      }
+      identities.add(key);
+    }
+    return page;
+  } catch {
+    return fail('invalid_page');
+  }
+}
+function validateRecord(input: { record: SyncRecord; definition: SyncDefinition }): void {
+  const { record, definition } = input;
+  identifier(record.kind);
+  identifier(record.id);
+  if (!Object.hasOwn(definition.kinds, record.kind)) {
+    fail('unknown_kind');
+  }
+  const fields =
+    record.operation === 'upsert'
+      ? ['operation', 'kind', 'id', 'data']
+      : ['operation', 'kind', 'id'];
+  if (Object.keys(record).some((key) => !fields.includes(key))) {
+    fail('invalid_record');
+  }
+  if (record.operation === 'upsert') {
+    if (!record.data || Array.isArray(record.data) || typeof record.data !== 'object') {
+      fail('invalid_record');
+    }
+    validate({ value: record.data, schema: definition.kinds[record.kind]! });
+  } else if (record.operation !== 'delete') {
+    fail('invalid_record');
+  }
+}

package/src/models/provider-values.ts

@@ -0,0 +1,42 @@
+import { SyncError } from './error';
+import type { ProviderSetup as RuntimeProviderSetup } from './providers';
+
+type OAuthSetup = Extract<RuntimeProviderSetup['auth'][number], { type: 'oauth2' }>;
+
+export function oauthClientValues(input: { auth: OAuthSetup; values: Record<string, string> }) {
+  const extra: Record<string, string> = {};
+  const secretExtra: Record<string, string> = {};
+  for (const field of input.auth.clientFields) {
+    if (field.key === 'clientId' || field.key === 'clientSecret') {
+      continue;
+    }
+    const value = input.values[field.key] ?? field.defaultValue;
+    if (value !== undefined) {
+      (field.location === 'secretExtra' ? secretExtra : extra)[field.key] = value;
+    }
+  }
+  return {
+    clientId: input.values.clientId,
+    clientSecret: input.values.clientSecret,
+    extra,
+    secretExtra,
+  };
+}
+
+export function publicConnection(input: { metadata: Record<string, unknown>; service: string }) {
+  const { metadata } = input;
+  if (
+    typeof metadata.id !== 'string' ||
+    metadata.service !== input.service ||
+    metadata.status !== 'active'
+  ) {
+    throw new SyncError({
+      code: 'connection_unavailable',
+      message: 'Provider connection is unavailable.',
+    });
+  }
+  const account = [metadata.accountLabel, metadata.displayName, metadata.providerAccountId].find(
+    (value): value is string => typeof value === 'string' && value.length > 0,
+  );
+  return { connectorId: metadata.id, service: input.service, account: account ?? input.service };
+}

package/src/models/providers.ts

@@ -0,0 +1,34 @@
+import type { RuntimeProviderSetup } from '@oomol-lab/open-connector';
+export type ProviderSetup = RuntimeProviderSetup;
+
+import type { Scope } from './identity';
+export type ProviderScope = Scope;
+export interface ProviderConnection {
+  id: string;
+  connectorId: string;
+  service: string;
+  account: string;
+}
+export interface ProviderAuthorization {
+  id: string;
+  requestId: string;
+  service: string;
+}
+
+export interface ProviderCatalogEntry {
+  service: string;
+  displayName: string;
+  iconUrl: string | null;
+  categories: { id: string; displayName: string }[];
+  scenario: string;
+  authTypes: RuntimeProviderSetup['auth'][number]['type'][];
+}
+
+export interface ConnectorManagement {
+  catalog(): Promise<ProviderCatalogEntry[]>;
+  call(input: {
+    path: string;
+    method?: 'POST' | 'PUT';
+    body?: Record<string, unknown>;
+  }): Promise<Record<string, unknown>>;
+}

package/src/models/registry.ts

@@ -0,0 +1,74 @@
+import {
+  type DefinitionRef,
+  definitionKey,
+  type SyncDefinition,
+  type SyncRegistration,
+} from './definition';
+import type { DestinationType } from './delivery';
+import { fail } from './error';
+import { canonicalJson } from './json';
+import { identifier, validate } from './validation';
+
+export class Registry {
+  readonly #definitions = new Map<string, SyncRegistration>();
+  readonly #destinations: ReadonlyMap<string, DestinationType>;
+  constructor(input: {
+    definitions: readonly SyncRegistration[];
+    destinations: Readonly<Record<string, DestinationType>>;
+  }) {
+    this.#destinations = new Map(
+      Object.entries(input.destinations).map(([name, type]) => {
+        identifier(name);
+        identifier(type.version);
+        return [name, { ...type, configSchema: structuredClone(type.configSchema) }];
+      }),
+    );
+    for (const registration of input.definitions) {
+      const definition = canonicalJson(registration.definition).value as unknown as SyncDefinition;
+      identifier(definition.id);
+      identifier(definition.version);
+      identifier(definition.artifactId);
+      validate({ value: definition.initialCheckpoint, schema: definition.checkpointSchema });
+      if (!Object.keys(definition.kinds).length) {
+        fail('missing_record_kinds');
+      }
+      for (const kind of Object.keys(definition.kinds)) {
+        identifier(kind);
+      }
+      const key = definitionKey(definition);
+      if (this.#definitions.has(key)) {
+        fail('definition_conflict');
+      }
+      this.#definitions.set(key, { definition, load: () => registration.load() });
+    }
+  }
+  definitions(): SyncDefinition[] {
+    return [...this.#definitions.values()].map((entry) => structuredClone(entry.definition));
+  }
+  definition(ref: DefinitionRef): SyncRegistration {
+    const registration = this.#definitions.get(definitionKey(ref));
+    if (!registration) {
+      fail('definition_unavailable');
+    }
+    return {
+      definition: structuredClone(registration.definition),
+      load: () => registration.load(),
+    };
+  }
+  destinationTypes() {
+    return [...this.#destinations].map(([type, entry]) => ({
+      type,
+      version: entry.version,
+      name: entry.name,
+      description: entry.description,
+      configSchema: structuredClone(entry.configSchema),
+    }));
+  }
+  destination(name: string): DestinationType {
+    const type = this.#destinations.get(name);
+    if (!type) {
+      fail('destination_unavailable');
+    }
+    return type;
+  }
+}

package/src/models/validation.ts

@@ -0,0 +1,24 @@
+import { type Schema, Validator } from '@cfworker/json-schema';
+import { fail } from './error';
+import { canonicalJson, type JsonValue } from './json';
+
+const maxIdentifierLength = 1024;
+export function identifier(value: unknown): asserts value is string {
+  if (typeof value !== 'string' || !value.trim() || value.length > maxIdentifierLength) {
+    fail('invalid_identifier');
+  }
+}
+export function validate(input: { value: unknown; schema: Schema }): JsonValue {
+  try {
+    const json = canonicalJson(input.value);
+    // The validator mutates schemas while compiling; persisted manifests remain immutable.
+    if (
+      !new Validator(structuredClone(input.schema), '2020-12', false).validate(json.value).valid
+    ) {
+      fail('invalid_input');
+    }
+    return json.value;
+  } catch {
+    return fail('invalid_input');
+  }
+}

package/src/open-sync.ts

@@ -0,0 +1,182 @@
+import { join } from 'node:path';
+import { type ProviderApi, providerApi, type SyncApi, syncApi } from './api';
+import { createConnectorClient } from './connector/client';
+import { loadProviderKey } from './connector/encryption-key';
+import { connectorManagement } from './connector/management';
+import { openProviderDatabase } from './db/providers';
+import type { Logger } from './execution/diagnostics';
+import { createHttpApp } from './http/app';
+import type { SyncRegistration } from './models/definition';
+import type { DestinationType } from './models/delivery';
+import { fail } from './models/error';
+import type { Scope } from './models/identity';
+import { defaultTiming, type QueueLimits } from './models/limits';
+import { SqliteProviders } from './repositories/providers/sqlite';
+import { createSyncRuntime } from './runtime';
+import { ProviderService } from './services/providers/service';
+
+export type { Scope } from './models/identity';
+export type { ProviderCatalogEntry, ProviderSetup } from './models/providers';
+
+export interface OpenSyncOptions {
+  definitions: readonly SyncRegistration[];
+  destinationTypes: Readonly<Record<string, DestinationType>>;
+  limits?: Partial<QueueLimits>;
+  executionTimeoutMs?: number;
+  onEvent?: Logger;
+  dataDirectory: string;
+  /** Absolute URL where the host mounts fetch(), including its path prefix. */
+  publicUrl: string;
+  /** Host authentication, ownership and CSRF policy for management HTTP requests. */
+  authorize(request: Request): Scope | null | Promise<Scope | null>;
+  /** OAuth application settings are instance-wide, so require the host's administrator policy. */
+  canConfigureProviders(scope: Scope): Promise<boolean>;
+  /** Final host UI location after Open Sync completes authorization. */
+  authorizationRedirect?(input: { service: string; outcome: 'connected' | 'failed' }): string;
+}
+
+/** Headless application boundary. The host owns its listener; Open Sync owns provider and worker lifecycles. */
+export async function createOpenSync(options: OpenSyncOptions): Promise<OpenSyncRuntime> {
+  const base = new URL(options.publicUrl);
+  if (
+    !['http:', 'https:'].includes(base.protocol) ||
+    base.search ||
+    base.hash ||
+    base.username ||
+    base.password
+  ) {
+    fail('invalid_public_url');
+  }
+  const prefix = base.pathname.replace(/\/$/, '');
+  const publicUrl = `${base.origin}${prefix}`;
+  const encryptionKey = await loadProviderKey(options.dataDirectory);
+  const adminToken = crypto.randomUUID();
+  const runtimeToken = crypto.randomUUID();
+  // Loading the engine alone does not initialize the provider platform.
+  const { createConnectorRuntime } = await import('@oomol-lab/open-connector');
+  const connector = await createConnectorRuntime({
+    dataDir: join(options.dataDirectory, 'connector'),
+    publicOrigin: publicUrl,
+    encryptionKey,
+    adminToken,
+    runtimeToken,
+  });
+  const lifetime = new AbortController();
+  let references: ReturnType<typeof openProviderDatabase> | undefined;
+  let engine: ReturnType<typeof createSyncRuntime> | undefined;
+  try {
+    references = openProviderDatabase(join(options.dataDirectory, 'providers.db'));
+    const repository = new SqliteProviders(references);
+    const transport = (request: Request) => connector.fetch(request);
+    const management = connectorManagement({
+      fetch: transport,
+      baseUrl: publicUrl,
+      adminToken,
+      runtimeToken,
+      signal: lifetime.signal,
+    });
+    const client = createConnectorClient({
+      fetch: transport,
+      baseUrl: publicUrl,
+      adminToken,
+      runtimeToken,
+      authorizeConnection: (input) =>
+        repository.owns({ ...input, connectorId: input.connection.id }),
+    });
+    engine = createSyncRuntime({
+      definitions: options.definitions,
+      destinationTypes: options.destinationTypes,
+      limits: options.limits,
+      onEvent: options.onEvent,
+      timing: {
+        timeoutMs: options.executionTimeoutMs ?? defaultTiming.timeoutMs,
+        leaseMs: (options.executionTimeoutMs ?? defaultTiming.timeoutMs) + defaultTiming.timeoutMs,
+      },
+      databasePath: join(options.dataDirectory, 'sync.db'),
+      connector: {
+        async bind(input) {
+          const owned = await repository.connection({ ...input, id: input.connection.id });
+          if (!owned || owned.service !== input.connection.service) {
+            fail('not_found');
+          }
+          return await client.bind({
+            ...input,
+            connection: { id: owned.connectorId, service: input.connection.service },
+          });
+        },
+      },
+    });
+    const providers = new ProviderService({
+      repository,
+      connector: management,
+      signal: lifetime.signal,
+      canConfigure: options.canConfigureProviders,
+      returnUrl: (input) =>
+        `${publicUrl}/providers/${encodeURIComponent(input.service)}/return/${input.id}`,
+    });
+    const api = syncApi(engine.api);
+    const http = createHttpApp({
+      prefix,
+      api,
+      providers,
+      authorize: options.authorize,
+      authorizationRedirect: options.authorizationRedirect,
+    });
+    let closing: Promise<void> | undefined;
+    const runtime = engine;
+    const database = references;
+    return {
+      api,
+      providers: providerApi(providers),
+      start: () => runtime.start(),
+      async fetch(request: Request): Promise<Response> {
+        if (lifetime.signal.aborted) {
+          return new Response(null, { status: 503 });
+        }
+        const url = new URL(request.url);
+        if (url.pathname === `${prefix}/oauth/callback` && request.method === 'GET') {
+          return await connector.fetch(request);
+        }
+        if (!url.pathname.startsWith(`${prefix}/`)) {
+          return new Response(null, { status: 404 });
+        }
+        return await http.handle(request);
+      },
+      close() {
+        closing ??= (async () => {
+          lifetime.abort();
+          try {
+            await runtime.close();
+          } finally {
+            try {
+              await connector.close();
+            } finally {
+              database.close();
+            }
+          }
+        })();
+        return closing;
+      },
+    };
+  } catch (error) {
+    lifetime.abort();
+    try {
+      await engine?.close();
+    } finally {
+      try {
+        await connector.close();
+      } finally {
+        references?.close();
+      }
+    }
+    throw error;
+  }
+}
+export interface OpenSyncRuntime {
+  api: SyncApi;
+  providers: ProviderApi;
+  fetch(request: Request): Promise<Response>;
+  start(): void;
+  close(): Promise<void>;
+}
+export type { ProviderApi, SyncApi } from './api';

package/src/repositories/acquisition/contract.ts

@@ -0,0 +1,17 @@
+import type { SyncDefinition, SyncPage } from '../../models/definition';
+import type { Scope } from '../../models/identity';
+import type { Installation } from '../../models/installation';
+
+export interface RunLease extends Scope {
+  id: string;
+  installation: Installation;
+  workerId: string;
+  generation: number;
+  checkpointRevision: number;
+}
+export interface AcquisitionRepository {
+  claim(leaseMs: number): RunLease | undefined;
+  hasCapacity(): boolean;
+  commit(input: { lease: RunLease; page: SyncPage; definition: SyncDefinition }): void;
+  finish(input: { lease: RunLease; state: string; delay: number }): void;
+}

package/src/repositories/acquisition/lease.ts

@@ -0,0 +1,118 @@
+import type { Database } from 'bun:sqlite';
+import { definitionKey } from '../../models/definition';
+import { fail } from '../../models/error';
+import { workerScope } from '../../models/identity';
+import type { Installation } from '../../models/installation';
+import { readInstallation } from '../rows';
+import type { RunLease } from './contract';
+
+export function assertRun(input: { db: Database; lease: RunLease }): Installation {
+  const { db, lease } = input;
+  const valid = db
+    .query(
+      "SELECT 1 FROM runs WHERE owner_id=? AND id=? AND installation_id=? AND binding_epoch=? AND definition_ref=? AND state='running' AND worker_id=? AND generation=? AND expires_at>?",
+    )
+    .get(
+      lease.ownerId,
+      lease.id,
+      lease.installation.id,
+      lease.installation.bindingEpoch,
+      definitionKey(lease.installation.definition),
+      lease.workerId,
+      lease.generation,
+      Date.now(),
+    );
+  const installation = readInstallation({ db, scope: { ...lease, id: lease.installation.id } });
+  if (
+    !valid ||
+    !installation.enabled ||
+    installation.bindingEpoch !== lease.installation.bindingEpoch ||
+    definitionKey(installation.definition) !== definitionKey(lease.installation.definition)
+  ) {
+    fail('lease_lost');
+  }
+  if (installation.checkpointRevision !== lease.checkpointRevision) {
+    fail('checkpoint_conflict');
+  }
+  return installation;
+}
+export function claimRun(input: {
+  db: Database;
+  leaseMs: number;
+  historyLimit: number;
+}): RunLease | undefined {
+  const { db } = input;
+  return db
+    .transaction(() => {
+      recoverRuns(input);
+      if (db.query("SELECT 1 FROM runs WHERE state='running'").get()) {
+        return;
+      }
+      const row = db
+        .query<{ id: string; owner_id: string }, [number]>(
+          'SELECT id,owner_id FROM installations WHERE enabled=1 AND next_due_at<=? ORDER BY next_due_at,id LIMIT 1',
+        )
+        .get(Date.now());
+      if (!row) {
+        return;
+      }
+      const scope = workerScope(row.owner_id);
+      const installation = readInstallation({ db, scope: { ...scope, id: row.id } });
+      const lease: RunLease = {
+        ...scope,
+        id: `run_${crypto.randomUUID()}`,
+        installation,
+        workerId: crypto.randomUUID(),
+        generation: 1,
+        checkpointRevision: installation.checkpointRevision,
+      };
+      db.query(`INSERT INTO runs(owner_id,id,installation_id,definition_ref,binding_epoch,worker_id,generation,expires_at,checkpoint_revision,state,started_at)
+      VALUES (?,?,?,?,?,?,?,?,?,'running',?)`).run(
+        scope.ownerId,
+        lease.id,
+        installation.id,
+        definitionKey(installation.definition),
+        installation.bindingEpoch,
+        lease.workerId,
+        lease.generation,
+        Date.now() + input.leaseMs,
+        lease.checkpointRevision,
+        Date.now(),
+      );
+      db.query("UPDATE installations SET status='running' WHERE owner_id=? AND id=?").run(
+        scope.ownerId,
+        installation.id,
+      );
+      return lease;
+    })
+    .immediate();
+}
+function recoverRuns(input: { db: Database; historyLimit: number }): void {
+  input.db
+    .query(`UPDATE installations SET next_due_at=?,status='lease_expired' WHERE enabled=1 AND EXISTS (
+    SELECT 1 FROM runs WHERE owner_id=installations.owner_id AND installation_id=installations.id AND state='running' AND expires_at<=?)`)
+    .run(Date.now(), Date.now());
+  input.db
+    .query(
+      "UPDATE runs SET state='lease_expired',completed_at=? WHERE state='running' AND expires_at<=?",
+    )
+    .run(Date.now(), Date.now());
+  input.db
+    .query(
+      "DELETE FROM runs WHERE state!='running' AND rowid NOT IN (SELECT rowid FROM runs ORDER BY started_at DESC,rowid DESC LIMIT ?)",
+    )
+    .run(input.historyLimit);
+}
+export function finishRun(input: {
+  db: Database;
+  lease: RunLease;
+  state: string;
+  delay: number;
+}): void {
+  input.db
+    .query('UPDATE runs SET state=?,completed_at=? WHERE owner_id=? AND id=?')
+    .run(input.state, Date.now(), input.lease.ownerId, input.lease.id);
+  input.db
+    .query('UPDATE installations SET status=?,next_due_at=? WHERE owner_id=? AND id=?')
+    .run(input.state, Date.now() + input.delay, input.lease.ownerId, input.lease.installation.id);
+}

package/src/repositories/acquisition/outbox.ts

@@ -0,0 +1,52 @@
+import type { Database } from 'bun:sqlite';
+import type { DeliveredRecord, Delivery } from '../../models/delivery';
+import { fail } from '../../models/error';
+import type { Installation } from '../../models/installation';
+import { canonicalJson } from '../../models/json';
+import type { QueueLimits } from '../../models/limits';
+import { queueUsage } from '../queue-usage';
+
+export function enqueue(input: {
+  db: Database;
+  installation: Installation;
+  records: DeliveredRecord[];
+  limits: QueueLimits;
+}): void {
+  if (!input.records.length) {
+    return;
+  }
+  const { installation, db, limits } = input;
+  const delivery: Delivery = {
+    version: 1,
+    id: `delivery_${crypto.randomUUID()}`,
+    ownerId: installation.ownerId,
+    sourceId: installation.sourceId,
+    installationId: installation.id,
+    definition: installation.definition,
+    deliverable: { records: input.records },
+  };
+  const body = canonicalJson(delivery).json;
+  const bytes = Buffer.byteLength(body);
+  if (bytes > limits.maxPendingBytes || input.records.length > limits.maxPendingRecords) {
+    fail('page_exceeds_queue_capacity');
+  }
+  const usage = queueUsage({ db });
+  if (
+    bytes + usage.pendingBytes > limits.maxPendingBytes ||
+    input.records.length + usage.pendingRecords > limits.maxPendingRecords
+  ) {
+    fail('waiting_for_capacity');
+  }
+  db.query(
+    'INSERT INTO deliveries(owner_id,id,installation_id,destination_id,body,bytes,record_count,due_at) VALUES (?,?,?,?,?,?,?,?)',
+  ).run(
+    installation.ownerId,
+    delivery.id,
+    installation.id,
+    installation.destinationId,
+    body,
+    bytes,
+    input.records.length,
+    Date.now(),
+  );
+}

package/src/repositories/acquisition/records.ts

@@ -0,0 +1,37 @@
+import type { Database } from 'bun:sqlite';
+import type { DeliveredRecord, SyncRecord } from '../../models/delivery';
+import type { Installation } from '../../models/installation';
+import { canonicalJson } from '../../models/json';
+
+export function writeRecord(input: {
+  db: Database;
+  installation: Installation;
+  record: SyncRecord;
+}): DeliveredRecord | undefined {
+  const { db, installation, record } = input;
+  const previous = db
+    .query<{ hash: string; revision: number; deleted: number }, [string, string, string, string]>(
+      'SELECT hash,revision,deleted FROM records WHERE owner_id=? AND installation_id=? AND kind=? AND id=?',
+    )
+    .get(installation.ownerId, installation.id, record.kind, record.id);
+  if (record.operation === 'delete' && (!previous || previous.deleted === 1)) {
+    return;
+  }
+  const contentHash =
+    record.operation === 'upsert' ? canonicalJson(record.data).sha256 : previous!.hash;
+  if (record.operation === 'upsert' && previous?.deleted === 0 && previous.hash === contentHash) {
+    return;
+  }
+  const revision = (previous?.revision ?? 0) + 1;
+  db.query(`INSERT INTO records VALUES (?,?,?,?,?,?,?) ON CONFLICT(owner_id,installation_id,kind,id)
+    DO UPDATE SET hash=excluded.hash,revision=excluded.revision,deleted=excluded.deleted`).run(
+    installation.ownerId,
+    installation.id,
+    record.kind,
+    record.id,
+    contentHash,
+    revision,
+    Number(record.operation === 'delete'),
+  );
+  return { ...record, eventId: `event_${crypto.randomUUID()}`, revision, contentHash };
+}