@context-use/open-sync 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@context-use/open-sync",
+  "version": "0.1.0",
+  "type": "module",
+  "exports": {
+    ".": "./src/open-sync.ts",
+    "./definition": "./src/models/definition.ts",
+    "./delivery": "./src/models/delivery.ts",
+    "./json": "./src/models/json.ts",
+    "./http": "./src/http/index.ts",
+    "./engine": "./src/runtime.ts",
+    "./build": "./src/build.ts"
+  },
+  "files": [
+    "src"
+  ],
+  "scripts": {
+    "check:types": "tsc --noEmit",
+    "test": "bun test test",
+    "prepack": "cp ../../LICENSE ./LICENSE",
+    "postpack": "rm ./LICENSE"
+  },
+  "dependencies": {
+    "@cfworker/json-schema": "4.1.1",
+    "elysia": "^1.4.30",
+    "@oomol-lab/open-connector": "1.6.0"
+  },
+  "devDependencies": {
+    "typescript": "^7.0.2"
+  },
+  "engines": {
+    "bun": ">=1.4.0"
+  },
+  "description": "Headless sync runtime for Bun with provider authorization, durable checkpoints, and record delivery.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/massimoalbarello/open-sync.git",
+    "directory": "packages/sync"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/api.ts

@@ -0,0 +1,53 @@
+import type { SyncManagement } from './services/management';
+import type { ProviderService } from './services/providers/service';
+
+/** Deliberate public selections: internal service methods are not automatically host APIs. */
+export type SyncApi = Pick<
+  SyncManagement,
+  | 'definitions'
+  | 'destinationTypes'
+  | 'runs'
+  | 'destinations'
+  | 'createDestination'
+  | 'createInstallation'
+  | 'installations'
+  | 'installation'
+  | 'setEnabled'
+  | 'queueRun'
+  | 'status'
+  | 'deliveries'
+  | 'retryDelivery'
+>;
+export type ProviderApi = Pick<
+  ProviderService,
+  'connections' | 'connection' | 'catalog' | 'status' | 'configure' | 'start' | 'credentials'
+>;
+
+export function syncApi(service: SyncManagement): SyncApi {
+  return {
+    definitions: service.definitions.bind(service),
+    destinations: service.destinations.bind(service),
+    destinationTypes: service.destinationTypes.bind(service),
+    runs: service.runs.bind(service),
+    createDestination: service.createDestination.bind(service),
+    createInstallation: service.createInstallation.bind(service),
+    installations: service.installations.bind(service),
+    installation: service.installation.bind(service),
+    setEnabled: service.setEnabled.bind(service),
+    queueRun: service.queueRun.bind(service),
+    status: service.status.bind(service),
+    deliveries: service.deliveries.bind(service),
+    retryDelivery: service.retryDelivery.bind(service),
+  };
+}
+export function providerApi(service: ProviderService): ProviderApi {
+  return {
+    connections: service.connections.bind(service),
+    connection: service.connection.bind(service),
+    catalog: service.catalog.bind(service),
+    status: service.status.bind(service),
+    configure: service.configure.bind(service),
+    start: service.start.bind(service),
+    credentials: service.credentials.bind(service),
+  };
+}

package/src/build.ts

@@ -0,0 +1,6 @@
+import { getConnectorAssetDirectory } from '@oomol-lab/open-connector';
+
+/** Options needed when a host embeds Open Sync in a Bun executable. */
+export function getOpenSyncBuildOptions() {
+  return { assets: [getConnectorAssetDirectory()], external: ['proxy-agent'] };
+}

package/src/connector/client.ts

@@ -0,0 +1,156 @@
+import type { ProviderGateway } from '../execution/provider';
+import type { ConnectionRef, ProviderOperations, ProviderRequirements } from '../models/definition';
+import { fail } from '../models/error';
+import type { Scope } from '../models/identity';
+import { canonicalJson, type JsonObject, type JsonValue } from '../models/json';
+import { providerResponse } from './response';
+
+interface RequestInput {
+  path: string;
+  admin?: boolean;
+  body?: JsonObject;
+  alias?: string;
+  signal: AbortSignal;
+}
+interface ConnectorClientOptions {
+  /** Borrowed embedded Connector transport; the Open Sync facade owns its lifetime. */
+  fetch(request: Request): Promise<Response>;
+  baseUrl: string;
+  adminToken: string;
+  runtimeToken: string;
+  /** The facade checks ownership before any privileged metadata read. */
+  authorizeConnection(input: Scope & { connection: ConnectionRef }): Promise<boolean>;
+}
+
+/** Public APIs only. Connector currently exposes no atomic credential-generation commit guard. */
+export function createConnectorClient(options: ConnectorClientOptions): ProviderGateway {
+  if (!options.adminToken || !options.runtimeToken) {
+    fail('connector_credentials_required');
+  }
+  const base = options.baseUrl.replace(/\/$/, '');
+  async function request(input: RequestInput): Promise<JsonValue> {
+    input.signal.throwIfAborted();
+    const headers = new Headers({
+      authorization: `Bearer ${input.admin ? options.adminToken : options.runtimeToken}`,
+    });
+    if (input.alias) {
+      headers.set('x-oo-connector-alias', input.alias);
+    }
+    if (input.body) {
+      headers.set('content-type', 'application/json');
+    }
+    const response = await options.fetch(
+      new Request(`${base}${input.path}`, {
+        method: input.body ? 'POST' : 'GET',
+        headers,
+        signal: input.signal,
+        body: input.body ? JSON.stringify(input.body) : undefined,
+      }),
+    );
+    const result = (await response.json()) as { success?: boolean; data?: JsonValue };
+    if (!response.ok || result.success !== true || result.data === undefined) {
+      fail('connector_request_failed');
+    }
+    input.signal.throwIfAborted();
+    return canonicalJson(result.data).value;
+  }
+  return {
+    async bind(input) {
+      if (
+        !(await options.authorizeConnection({
+          actorId: input.actorId,
+          ownerId: input.ownerId,
+          connection: input.connection,
+        }))
+      ) {
+        fail('not_found');
+      }
+      if (input.connection.service !== input.requirements.service) {
+        fail('connection_mismatch');
+      }
+      const metadata = (await request({
+        path: `/v1/connections/by-id/${encodeURIComponent(input.connection.id)}`,
+        admin: true,
+        signal: input.signal,
+      })) as JsonObject;
+      checkConnection({ metadata, connection: input.connection, requirements: input.requirements });
+      return operations({
+        request,
+        requirements: input.requirements,
+        alias: metadata.alias as string,
+        signal: input.signal,
+      });
+    },
+  };
+}
+function checkConnection(input: {
+  metadata: JsonObject;
+  connection: ConnectionRef;
+  requirements: ProviderRequirements;
+}): void {
+  const { metadata, connection, requirements } = input;
+  if (
+    metadata.id !== connection.id ||
+    metadata.service !== requirements.service ||
+    metadata.status !== 'active' ||
+    typeof metadata.alias !== 'string' ||
+    !metadata.alias
+  ) {
+    fail('connection_unavailable');
+  }
+}
+function operations(input: {
+  request(input: RequestInput): Promise<JsonValue>;
+  requirements: ProviderRequirements;
+  alias: string;
+  signal: AbortSignal;
+}): ProviderOperations {
+  const { request, requirements, alias, signal } = input;
+  return {
+    async action(operation) {
+      if (!requirements.actions.includes(operation.id)) {
+        fail('operation_denied');
+      }
+      const path = `/v1/actions/${encodeURIComponent(operation.id)}`;
+      const metadata = (await request({ path, signal })) as JsonObject;
+      if (metadata.service !== requirements.service) {
+        fail('operation_denied');
+      }
+      return await request({ path, body: { input: operation.input }, alias, signal });
+    },
+    get(operation) {
+      const { path } = operation;
+      if (
+        !path.startsWith('/') ||
+        path.startsWith('//') ||
+        /[\\?#]/.test(path) ||
+        !requirements.proxyPaths?.includes(path)
+      ) {
+        fail('operation_denied');
+      }
+      return request({
+        path: `/v1/proxy/${encodeURIComponent(requirements.service)}`,
+        body: { endpoint: path, method: 'GET', query: operation.query ?? {} },
+        alias,
+        signal,
+      }).then(providerResponse);
+    },
+    post(operation) {
+      const { path } = operation;
+      if (
+        !path.startsWith('/') ||
+        path.startsWith('//') ||
+        /[\\?#]/.test(path) ||
+        !requirements.proxyPostPaths?.includes(path)
+      ) {
+        fail('operation_denied');
+      }
+      return request({
+        path: `/v1/proxy/${encodeURIComponent(requirements.service)}`,
+        body: { endpoint: path, method: 'POST', body: operation.body },
+        alias,
+        signal,
+      }).then(providerResponse);
+    },
+  };
+}

package/src/connector/encryption-key.ts

@@ -0,0 +1,23 @@
+import { randomBytes } from 'node:crypto';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+/** Independent from login secrets; retain this key alongside the opaque Connector data. */
+export async function loadProviderKey(dataFolder: string): Promise<string> {
+  await mkdir(dataFolder, { recursive: true });
+  const path = join(dataFolder, '.connector-key');
+  const bytes = 32;
+  const mode = 0o600;
+  try {
+    await writeFile(path, randomBytes(bytes).toString('hex'), { flag: 'wx', mode });
+  } catch (error) {
+    if (!(error instanceof Error && 'code' in error && error.code === 'EEXIST')) {
+      throw error;
+    }
+  }
+  const value = await readFile(path, 'utf8');
+  if (!/^[a-f0-9]{64}$/.test(value)) {
+    throw new Error('Invalid provider encryption key.');
+  }
+  return value;
+}

package/src/connector/management.ts

@@ -0,0 +1,76 @@
+import { SyncError } from '../models/error';
+import type { ConnectorManagement, ProviderCatalogEntry } from '../models/providers';
+
+/** Only this internal adapter uses administrator credentials. No raw Connector routes are exposed to users. */
+export function connectorManagement(input: {
+  fetch(request: Request): Promise<Response>;
+  baseUrl: string;
+  adminToken: string;
+  runtimeToken: string;
+  signal: AbortSignal;
+}): ConnectorManagement {
+  const timeoutMs = 30_000;
+  async function requestData({
+    request,
+    token,
+  }: {
+    request: { path: string; method?: 'POST' | 'PUT'; body?: Record<string, unknown> };
+    token: string;
+  }): Promise<unknown> {
+    const response = await input.fetch(
+      new Request(`${input.baseUrl}${request.path}`, {
+        method: request.method ?? 'GET',
+        headers: {
+          authorization: `Bearer ${token}`,
+          'content-type': 'application/json',
+        },
+        body: request.body ? JSON.stringify(request.body) : undefined,
+        signal: AbortSignal.any([input.signal, AbortSignal.timeout(timeoutMs)]),
+      }),
+    );
+    const result = (await response.json()) as Record<string, unknown>;
+    input.signal.throwIfAborted();
+    // The documented OAuth configuration API returns a bare summary; /v1 uses an envelope.
+    const versioned = request.path.startsWith('/v1/');
+    const data = versioned ? result.data : result;
+    if (
+      !response.ok ||
+      (versioned && result.success !== true) ||
+      !data ||
+      typeof data !== 'object'
+    ) {
+      // Do not echo provider errors or request bodies, which can contain credentials.
+      throw new SyncError({
+        code: 'provider_request_failed',
+        message:
+          'Provider could not complete this request. Check provider configuration and authorization.',
+      });
+    }
+    return data;
+  }
+  return {
+    async catalog() {
+      const data = await requestData({
+        request: { path: '/v1/providers' },
+        token: input.runtimeToken,
+      });
+      if (!Array.isArray(data)) {
+        throw new SyncError({
+          code: 'provider_request_failed',
+          message: 'Invalid provider catalog.',
+        });
+      }
+      return data as ProviderCatalogEntry[];
+    },
+    async call(request) {
+      const data = await requestData({ request, token: input.adminToken });
+      if (Array.isArray(data)) {
+        throw new SyncError({
+          code: 'provider_request_failed',
+          message: 'Invalid provider response.',
+        });
+      }
+      return data as Record<string, unknown>;
+    },
+  };
+}

package/src/connector/response.ts

@@ -0,0 +1,35 @@
+import type { ProviderResponse } from '../models/definition';
+import { fail } from '../models/error';
+import type { JsonValue } from '../models/json';
+import { validate } from '../models/validation';
+
+/** Connector's wire envelope ends here; definitions only consume the Open Sync contract. */
+export function providerResponse(value: unknown): ProviderResponse {
+  let response: {
+    status: number;
+    headers: Record<string, string>;
+    data: JsonValue;
+    bodyEncoding?: unknown;
+  };
+  try {
+    response = validate({
+      value,
+      schema: {
+        type: 'object',
+        required: ['status', 'headers', 'data'],
+        properties: {
+          status: { type: 'integer', minimum: 100, maximum: 599 },
+          headers: { type: 'object', additionalProperties: { type: 'string' } },
+          data: {},
+        },
+      },
+    }) as unknown as typeof response;
+  } catch {
+    return fail('invalid_provider_response');
+  }
+  // Assets are deferred. Never pass an encoded binary body off as provider text.
+  if (response.bodyEncoding !== undefined) {
+    fail('unsupported_provider_response');
+  }
+  return { status: response.status, headers: response.headers, body: response.data };
+}

package/src/db/client.ts

@@ -0,0 +1,24 @@
+import { Database } from 'bun:sqlite';
+import { fail } from '../models/error';
+import schema from './schema.sql' with { type: 'text' };
+
+export function openDatabase(path: string): Database {
+  const db = new Database(path, { create: true, strict: true });
+  try {
+    db.exec('PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;');
+    db.transaction(() => {
+      const { user_version: version } = db
+        .query<{ user_version: number }, []>('PRAGMA user_version')
+        .get()!;
+      if (version === 0) {
+        db.exec(schema);
+      } else if (version !== 1) {
+        fail('schema_version');
+      }
+    }).immediate();
+    return db;
+  } catch (error) {
+    db.close();
+    throw error;
+  }
+}

package/src/db/providers.sql

@@ -0,0 +1,15 @@
+CREATE TABLE provider_authorizations (
+  owner_id TEXT NOT NULL PRIMARY KEY,
+  id TEXT NOT NULL UNIQUE,
+  request_id TEXT NOT NULL UNIQUE,
+  service TEXT NOT NULL
+);
+CREATE TABLE provider_connections (
+  owner_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  connector_id TEXT NOT NULL UNIQUE,
+  account TEXT NOT NULL,
+  service TEXT NOT NULL,
+  PRIMARY KEY (owner_id, id)
+);
+PRAGMA user_version=1;

package/src/db/providers.sql.d.ts

@@ -0,0 +1,2 @@
+declare const sql: string;
+export default sql;

package/src/db/providers.ts

@@ -0,0 +1,24 @@
+import { Database } from 'bun:sqlite';
+import { fail } from '../models/error';
+import schema from './providers.sql' with { type: 'text' };
+
+export function openProviderDatabase(path: string) {
+  const db = new Database(path, { create: true, strict: true });
+  try {
+    db.exec('PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000');
+    db.transaction(() => {
+      const version = db
+        .query<{ user_version: number }, []>('PRAGMA user_version')
+        .get()!.user_version;
+      if (version === 0) {
+        db.exec(schema);
+      } else if (version !== 1) {
+        fail('provider_schema_version');
+      }
+    }).immediate();
+    return db;
+  } catch (error) {
+    db.close();
+    throw error;
+  }
+}

package/src/db/schema.sql

@@ -0,0 +1,45 @@
+CREATE TABLE definitions (
+  id TEXT NOT NULL, version TEXT NOT NULL, artifact_id TEXT NOT NULL, manifest TEXT NOT NULL,
+  PRIMARY KEY(id, version)
+);
+CREATE TABLE destinations (
+  owner_id TEXT NOT NULL, id TEXT NOT NULL, type TEXT NOT NULL, version TEXT NOT NULL, config TEXT NOT NULL,
+  PRIMARY KEY(owner_id, id)
+);
+CREATE TABLE installations (
+  owner_id TEXT NOT NULL, id TEXT NOT NULL, source_id TEXT NOT NULL,
+  definition_id TEXT NOT NULL, definition_version TEXT NOT NULL, artifact_id TEXT NOT NULL,
+  connection TEXT, config TEXT NOT NULL, destination_id TEXT NOT NULL,
+  enabled INTEGER NOT NULL, binding_epoch INTEGER NOT NULL DEFAULT 1,
+  checkpoint TEXT NOT NULL, checkpoint_revision INTEGER NOT NULL DEFAULT 0,
+  interval_ms INTEGER NOT NULL, next_due_at INTEGER NOT NULL, status TEXT NOT NULL,
+  PRIMARY KEY(owner_id, id), UNIQUE(owner_id, source_id),
+  FOREIGN KEY(owner_id, destination_id) REFERENCES destinations(owner_id, id)
+);
+CREATE TABLE runs (
+  owner_id TEXT NOT NULL, id TEXT NOT NULL, installation_id TEXT NOT NULL,
+  definition_ref TEXT NOT NULL, binding_epoch INTEGER NOT NULL,
+  worker_id TEXT NOT NULL, generation INTEGER NOT NULL, expires_at INTEGER NOT NULL,
+  checkpoint_revision INTEGER NOT NULL, state TEXT NOT NULL, started_at INTEGER NOT NULL,
+  completed_at INTEGER, pages INTEGER NOT NULL DEFAULT 0,
+  PRIMARY KEY(owner_id, id), FOREIGN KEY(owner_id, installation_id) REFERENCES installations(owner_id, id)
+);
+CREATE UNIQUE INDEX one_acquisition ON runs((1)) WHERE state='running';
+CREATE TABLE records (
+  owner_id TEXT NOT NULL, installation_id TEXT NOT NULL, kind TEXT NOT NULL, id TEXT NOT NULL,
+  hash TEXT NOT NULL, revision INTEGER NOT NULL, deleted INTEGER NOT NULL,
+  PRIMARY KEY(owner_id, installation_id, kind, id),
+  FOREIGN KEY(owner_id, installation_id) REFERENCES installations(owner_id, id)
+);
+CREATE TABLE deliveries (
+  sequence INTEGER PRIMARY KEY AUTOINCREMENT, owner_id TEXT NOT NULL, id TEXT NOT NULL,
+  installation_id TEXT NOT NULL, destination_id TEXT NOT NULL, body TEXT NOT NULL,
+  bytes INTEGER NOT NULL, record_count INTEGER NOT NULL, state TEXT NOT NULL DEFAULT 'pending',
+  due_at INTEGER NOT NULL, worker_id TEXT, generation INTEGER NOT NULL DEFAULT 0,
+  expires_at INTEGER, attempt INTEGER NOT NULL DEFAULT 0, error_code TEXT,
+  UNIQUE(owner_id, id),
+  FOREIGN KEY(owner_id, installation_id) REFERENCES installations(owner_id, id),
+  FOREIGN KEY(owner_id, destination_id) REFERENCES destinations(owner_id, id)
+);
+CREATE INDEX deliveries_due ON deliveries(state, due_at);
+PRAGMA user_version = 1;

package/src/db/schema.sql.d.ts

@@ -0,0 +1,2 @@
+declare const sql: string;
+export default sql;

package/src/execution/diagnostics.ts

@@ -0,0 +1,18 @@
+import type { JsonObject } from '../models/json';
+export interface SyncEvent {
+  code: string;
+  ownerId?: string;
+  installationId?: string;
+  message?: string;
+  fields?: JsonObject;
+}
+export type Logger = (event: SyncEvent) => void;
+export function safeLogger(logger?: Logger): Logger {
+  return (event) => {
+    try {
+      logger?.(event);
+    } catch {
+      /* Diagnostics cannot change persistence. */
+    }
+  };
+}

package/src/execution/provider.ts

@@ -0,0 +1,37 @@
+import type { ConnectionRef, ProviderOperations, ProviderRequirements } from '../models/definition';
+import { fail } from '../models/error';
+import type { Scope } from '../models/identity';
+export interface ProviderGateway {
+  bind(
+    input: Scope & {
+      connection: ConnectionRef;
+      requirements: ProviderRequirements;
+      signal: AbortSignal;
+    },
+  ): Promise<ProviderOperations>;
+}
+export const noProvider: ProviderOperations = {
+  action: () => Promise.reject(new Error('operation_denied')),
+  get: () => Promise.reject(new Error('operation_denied')),
+  post: () => Promise.reject(new Error('operation_denied')),
+};
+export async function bindProvider(
+  input: Scope & {
+    gateway?: ProviderGateway;
+    connection?: ConnectionRef;
+    requirements?: ProviderRequirements;
+    signal: AbortSignal;
+  },
+): Promise<ProviderOperations> {
+  if (!input.requirements) {
+    return noProvider;
+  }
+  if (!input.gateway || !input.connection) {
+    return fail('connection_required');
+  }
+  return await input.gateway.bind({
+    ...input,
+    connection: input.connection,
+    requirements: input.requirements,
+  });
+}