@context-use/open-sync 0.1.0 → 0.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@context-use/open-sync",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "type": "module",
   "exports": {
     ".": "./src/open-sync.ts",
@@ -23,7 +23,8 @@
   "dependencies": {
     "@cfworker/json-schema": "4.1.1",
     "elysia": "^1.4.30",
-    "@oomol-lab/open-connector": "1.6.0"
+    "minisearch": "^7.2.0",
+    "@oomol-lab/open-connector": "1.6.3"
   },
   "devDependencies": {
     "typescript": "^7.0.2"

package/src/api.ts

@@ -10,6 +10,7 @@ export type SyncApi = Pick<
   | 'destinations'
   | 'createDestination'
   | 'createInstallation'
+  | 'connectInstallation'
   | 'installations'
   | 'installation'
   | 'setEnabled'
@@ -31,6 +32,7 @@ export function syncApi(service: SyncManagement): SyncApi {
     runs: service.runs.bind(service),
     createDestination: service.createDestination.bind(service),
     createInstallation: service.createInstallation.bind(service),
+    connectInstallation: service.connectInstallation.bind(service),
     installations: service.installations.bind(service),
     installation: service.installation.bind(service),
     setEnabled: service.setEnabled.bind(service),

package/src/build.ts

@@ -1,6 +1,22 @@
-import { getConnectorAssetDirectory } from '@oomol-lab/open-connector';
+import { getConnectorBuildOptions } from '@oomol-lab/open-connector/build';
+import type { SyncRegistration } from './models/definition';
 
-/** Options needed when a host embeds Open Sync in a Bun executable. */
-export function getOpenSyncBuildOptions() {
-  return { assets: [getConnectorAssetDirectory()], external: ['proxy-agent'] };
+/** Provider dependencies declared by registrations, without loading their sync executables. */
+export function providersFromDefinitions(definitions: readonly SyncRegistration[]): string[] {
+  return [
+    ...new Set(
+      definitions.flatMap(({ definition }) =>
+        definition.provider ? [definition.provider.service] : [],
+      ),
+    ),
+  ].sort();
+}
+
+/**
+ * Prepare selected providers for a Bun executable. Pass plugins/external to Bun.build and assets
+ * to compile.assets. Await dispose() after the build (also on failure); it removes staged files.
+ * An empty provider list packages no providers. The installed dependency is never modified.
+ */
+export function getOpenSyncBuildOptions(options: { providers: readonly string[] }) {
+  return getConnectorBuildOptions(options);
 }

package/src/http/providers.ts

@@ -28,6 +28,17 @@ export function createProviderController(input: {
       async ({ request }) => await input.providers.connections(await scope(request)),
     )
     .get('/', async ({ request }) => await input.providers.catalog(await scope(request)))
+    .get(
+      '/catalog',
+      async ({ request, query }) =>
+        input.providers.catalogPage({ ...(await scope(request)), ...query }),
+      {
+        query: t.Object({
+          q: t.Optional(t.String({ maxLength: 200 })),
+          offset: t.Optional(t.Integer({ minimum: 0, maximum: 1000000 })),
+        }),
+      },
+    )
     .get(
       '/:service',
       async ({ request, params }) =>
@@ -50,6 +61,7 @@ export function createProviderController(input: {
       {
         params: t.Object({ service }),
         body: t.Object({
+          connectionId: t.Optional(id),
           authorizationOptionIds: t.Optional(
             t.Array(t.String({ maxLength: 256 }), { maxItems: 128 }),
           ),
@@ -63,6 +75,7 @@ export function createProviderController(input: {
       {
         params: t.Object({ service }),
         body: t.Object({
+          connectionId: t.Optional(id),
           authType: t.Union([t.Literal('api_key'), t.Literal('custom_credential')]),
           values,
         }),

package/src/models/provider-catalog.ts

@@ -0,0 +1,57 @@
+import MiniSearch from 'minisearch';
+import type { ProviderCatalogEntry } from './providers';
+
+/** Provider metadata is fixed for the lifetime of a connector runtime. */
+export class ProviderCatalog {
+  private index?: MiniSearch;
+  private byService?: Map<string, ProviderCatalogEntry>;
+  readonly entries: ProviderCatalogEntry[];
+  constructor(entries: ProviderCatalogEntry[]) {
+    this.entries = entries.map(
+      ({ service, displayName, iconUrl, categories, scenario, authTypes }) => ({
+        service,
+        displayName,
+        iconUrl,
+        categories,
+        scenario,
+        authTypes,
+      }),
+    );
+  }
+
+  page(input: { q?: string; offset?: number }) {
+    const providers = this.entries;
+    const query = input.q?.trim();
+    let matches = providers;
+    if (query) {
+      if (!this.index) {
+        this.index = new MiniSearch({
+          idField: 'service',
+          fields: ['service', 'displayName', 'categoryNames', 'scenario', 'authentication'],
+          searchOptions: {
+            prefix: true,
+            combineWith: 'AND',
+            boost: { displayName: 3, service: 2 },
+          },
+        });
+        this.index.addAll(
+          providers.map((provider) => ({
+            ...provider,
+            categoryNames: provider.categories.map((category) => category.displayName).join(' '),
+            authentication: provider.authTypes.map((type) => type.replaceAll('_', ' ')).join(' '),
+          })),
+        );
+      }
+      this.byService ??= new Map(providers.map((provider) => [provider.service, provider]));
+      matches = this.index.search(query).map((result) => this.byService!.get(result.id)!);
+    }
+    const pageSize = 30;
+    const offset = input.offset ?? 0;
+    return {
+      providers: matches.slice(offset, offset + pageSize),
+      total: matches.length,
+      pageSize,
+      hasMore: offset + pageSize < matches.length,
+    };
+  }
+}

package/src/open-sync.ts

@@ -31,6 +31,10 @@ export interface OpenSyncOptions {
   authorize(request: Request): Scope | null | Promise<Scope | null>;
   /** OAuth application settings are instance-wide, so require the host's administrator policy. */
   canConfigureProviders(scope: Scope): Promise<boolean>;
+  /** Called after an owned connection is saved, including a retried OAuth completion. */
+  onProviderConnected?(
+    input: Scope & { connection: { id: string; service: string } },
+  ): Promise<void>;
   /** Final host UI location after Open Sync completes authorization. */
   authorizationRedirect?(input: { service: string; outcome: 'connected' | 'failed' }): string;
 }
@@ -75,6 +79,21 @@ export async function createOpenSync(options: OpenSyncOptions): Promise<OpenSync
       runtimeToken,
       signal: lifetime.signal,
     });
+    const requiredProviders = new Set(
+      options.definitions.flatMap(({ definition }) =>
+        definition.provider ? [definition.provider.service] : [],
+      ),
+    );
+    if (requiredProviders.size) {
+      const available = new Set((await management.catalog()).map((entry) => entry.service));
+      for (const service of requiredProviders) {
+        if (!available.has(service)) {
+          throw new Error(
+            `Open Sync provider ${JSON.stringify(service)} is unavailable. Include it in getOpenSyncBuildOptions({ providers }) and rebuild.`,
+          );
+        }
+      }
+    }
     const client = createConnectorClient({
       fetch: transport,
       baseUrl: publicUrl,
@@ -111,6 +130,7 @@ export async function createOpenSync(options: OpenSyncOptions): Promise<OpenSync
       connector: management,
       signal: lifetime.signal,
       canConfigure: options.canConfigureProviders,
+      onConnected: options.onProviderConnected,
       returnUrl: (input) =>
         `${publicUrl}/providers/${encodeURIComponent(input.service)}/return/${input.id}`,
     });

package/src/repositories/catalog/contract.ts

@@ -1,4 +1,4 @@
-import type { SyncDefinition } from '../../models/definition';
+import type { ConnectionRef, SyncDefinition } from '../../models/definition';
 import type { Destination } from '../../models/delivery';
 import type { Resource, Scope } from '../../models/identity';
 import type { CreateInstallation, Installation, SyncRun } from '../../models/installation';
@@ -18,6 +18,7 @@ export interface CatalogRepository {
     hasMore: boolean;
     pageSize: number;
   };
+  connectInstallation(input: Resource & { connection: ConnectionRef }): Installation;
   setEnabled(input: Resource & { enabled: boolean }): Installation;
   queue(input: Resource & { checkpoint?: JsonValue }): void;
 }

package/src/repositories/catalog/sqlite.ts

@@ -1,5 +1,5 @@
 import type { Database } from 'bun:sqlite';
-import type { SyncDefinition } from '../../models/definition';
+import type { ConnectionRef, SyncDefinition } from '../../models/definition';
 import { fail } from '../../models/error';
 import type { Resource, Scope } from '../../models/identity';
 import type { CreateInstallation } from '../../models/installation';
@@ -113,6 +113,21 @@ export class SqliteCatalog implements CatalogRepository {
       pageSize: limit,
     };
   }
+  connectInstallation(input: Resource & { connection: ConnectionRef }) {
+    return this.db
+      .transaction(() => {
+        const installation = this.installation(input);
+        if (installation.connection || installation.enabled) {
+          fail('already_connected');
+        }
+        this.db
+          .query(`UPDATE installations SET connection=?,enabled=1,
+        binding_epoch=binding_epoch+1,status='ready',next_due_at=? WHERE owner_id=? AND id=?`)
+          .run(canonicalJson(input.connection).json, Date.now(), input.ownerId, input.id);
+        return this.installation(input);
+      })
+      .immediate();
+  }
   setEnabled(input: Resource & { enabled: boolean }) {
     return this.db
       .transaction(() => {

package/src/repositories/providers/contract.ts

@@ -10,6 +10,7 @@ export interface ProviderRepository {
   owns(input: ProviderScope & { connectorId: string }): Promise<boolean>;
   start(input: ProviderScope & ProviderAuthorization): Promise<void>;
   pending(input: ProviderScope & { id: string }): Promise<ProviderAuthorization | null>;
+  updateAccount(input: ProviderScope & ProviderConnection): Promise<void>;
   add(input: ProviderScope & ProviderConnection): Promise<void>;
-  complete(input: ProviderScope & ProviderConnection): Promise<void>;
+  complete(input: ProviderScope & ProviderConnection & { requestId: string }): Promise<void>;
 }

package/src/repositories/providers/sqlite.ts

@@ -52,6 +52,14 @@ export class SqliteProviders implements ProviderRepository {
         .get(input.ownerId, input.id),
     );
   }
+  updateAccount(input: ProviderScope & ProviderConnection) {
+    this.db
+      .query(
+        'UPDATE provider_connections SET account=? WHERE owner_id=? AND id=? AND connector_id=?',
+      )
+      .run(input.account, input.ownerId, input.id, input.connectorId);
+    return Promise.resolve();
+  }
   add(input: ProviderScope & ProviderConnection) {
     this.db
       .query(
@@ -60,18 +68,26 @@ export class SqliteProviders implements ProviderRepository {
       .run(input.ownerId, input.id, input.connectorId, input.account, input.service);
     return Promise.resolve();
   }
-  complete(input: ProviderScope & ProviderConnection) {
+  complete(input: ProviderScope & ProviderConnection & { requestId: string }) {
     this.db
       .transaction(() => {
         // A superseded authorization must never claim a connection.
         this.db
           .query(`INSERT INTO provider_connections(owner_id,id,connector_id,account,service)
-        SELECT owner_id,id,?,?,? FROM provider_authorizations WHERE owner_id=? AND id=?
-        ON CONFLICT(owner_id,id) DO NOTHING`)
-          .run(input.connectorId, input.account, input.service, input.ownerId, input.id);
+        SELECT owner_id,id,?,?,? FROM provider_authorizations WHERE owner_id=? AND id=? AND request_id=?
+        ON CONFLICT(owner_id,id) DO UPDATE SET account=excluded.account
+        WHERE provider_connections.connector_id=excluded.connector_id`)
+          .run(
+            input.connectorId,
+            input.account,
+            input.service,
+            input.ownerId,
+            input.id,
+            input.requestId,
+          );
         this.db
-          .query('DELETE FROM provider_authorizations WHERE owner_id=? AND id=?')
-          .run(input.ownerId, input.id);
+          .query('DELETE FROM provider_authorizations WHERE owner_id=? AND id=? AND request_id=?')
+          .run(input.ownerId, input.id, input.requestId);
       })
       .immediate();
     return Promise.resolve();

package/src/services/management.ts

@@ -1,5 +1,6 @@
 import { bindProvider, type ProviderGateway } from '../execution/provider';
 import type { WorkerControl } from '../execution/worker';
+import type { ConnectionRef } from '../models/definition';
 import { fail } from '../models/error';
 import type { Resource, Scope } from '../models/identity';
 import type { CreateInstallation } from '../models/installation';
@@ -65,7 +66,7 @@ export class SyncManagement {
     if (input.intervalMs !== undefined) {
       positive(input.intervalMs);
     }
-    if (definition.provider) {
+    if (definition.provider && (input.connection || input.enabled !== false)) {
       await bindProvider({
         actorId: input.actorId,
         ownerId: input.ownerId,
@@ -92,8 +93,34 @@ export class SyncManagement {
     this.guard(input);
     return this.input.catalog.installation(input);
   }
+  async connectInstallation(input: Resource & { connection: ConnectionRef }) {
+    this.guard(input);
+    const installation = this.input.catalog.installation(input);
+    if (installation.connection || installation.enabled) {
+      fail('already_connected');
+    }
+    const { definition } = this.input.registry.definition(installation.definition);
+    if (!definition.provider) {
+      fail('unexpected_connection');
+    }
+    await bindProvider({
+      ...input,
+      requirements: definition.provider,
+      gateway: this.input.gateway,
+      signal: AbortSignal.timeout(this.input.timeoutMs),
+    });
+    this.guard(input);
+    return this.input.catalog.connectInstallation(input);
+  }
   async setEnabled(input: Resource & { enabled: boolean }) {
     this.guard(input);
+    const installation = this.input.catalog.installation(input);
+    if (input.enabled && !installation.connection) {
+      const { definition } = this.input.registry.definition(installation.definition);
+      if (definition.provider) {
+        fail('connection_required');
+      }
+    }
     const result = this.input.catalog.setEnabled(input);
     await this.input.worker.cancel(input);
     return result;

package/src/services/providers/service.ts

@@ -1,4 +1,5 @@
 import { fail, SyncError } from '../../models/error';
+import { ProviderCatalog } from '../../models/provider-catalog';
 import { oauthClientValues, publicConnection } from '../../models/provider-values';
 import type {
   ConnectorManagement,
@@ -9,6 +10,7 @@ import { identifier } from '../../models/validation';
 import type { ProviderRepository } from '../../repositories/providers/contract';
 
 export class ProviderService {
+  private providerCatalog?: Promise<ProviderCatalog>;
   constructor(
     private readonly input: {
       repository: ProviderRepository;
@@ -16,9 +18,23 @@ export class ProviderService {
       returnUrl(input: { service: string; id: string }): string;
       canConfigure(scope: ProviderScope): Promise<boolean>;
       signal: AbortSignal;
+      onConnected?(
+        input: ProviderScope & { connection: { id: string; service: string } },
+      ): Promise<void>;
     },
   ) {}
 
+  private loadCatalog() {
+    this.providerCatalog ??= this.input.connector
+      .catalog()
+      .then((entries) => new ProviderCatalog(entries))
+      .catch((error) => {
+        this.providerCatalog = undefined;
+        throw error;
+      });
+    return this.providerCatalog;
+  }
+
   private guard(scope: ProviderScope) {
     this.input.signal.throwIfAborted();
     identifier(scope.ownerId);
@@ -42,15 +58,11 @@ export class ProviderService {
   }
   async catalog(scope: ProviderScope) {
     this.guard(scope);
-    const providers = await this.input.connector.catalog();
-    return providers.map(({ service, displayName, iconUrl, authTypes, categories, scenario }) => ({
-      service,
-      displayName,
-      iconUrl,
-      authTypes,
-      categories,
-      scenario,
-    }));
+    return (await this.loadCatalog()).entries;
+  }
+  async catalogPage(input: ProviderScope & { q?: string; offset?: number }) {
+    this.guard(input);
+    return (await this.loadCatalog()).page(input);
   }
   private async setup(service: string): Promise<RuntimeProviderSetup> {
     const setup = await this.input.connector.call({
@@ -82,7 +94,17 @@ export class ProviderService {
               (metadata.status === 'active' || metadata.status === 'reauth_required')
                 ? metadata.status
                 : 'unknown';
-            return { id, account, status };
+            const authType = metadata?.authType;
+            const supportedAuth: RuntimeProviderSetup['auth'][number]['type'] | undefined =
+              authType === 'oauth2' || authType === 'api_key' || authType === 'custom_credential'
+                ? authType
+                : undefined;
+            return {
+              id,
+              account,
+              status,
+              authType: supportedAuth,
+            };
           }),
       ),
     };
@@ -107,11 +129,30 @@ export class ProviderService {
     });
     return { configured: true };
   }
-  async start(input: ProviderScope & { service: string; authorizationOptionIds?: string[] }) {
+  private async reconnectTarget(input: ProviderScope & { service: string; connectionId?: string }) {
+    if (!input.connectionId) {
+      return undefined;
+    }
+    const target = await this.input.repository.connection({ ...input, id: input.connectionId });
+    if (!target || target.service !== input.service) {
+      fail('not_found');
+    }
+    return target;
+  }
+  async start(
+    input: ProviderScope & {
+      service: string;
+      authorizationOptionIds?: string[];
+      connectionId?: string;
+    },
+  ) {
     this.guard(input);
-    const id = `connection_${crypto.randomUUID()}`;
+    const target = await this.reconnectTarget(input);
+    const id = target?.id ?? `connection_${crypto.randomUUID()}`;
     const result = await this.input.connector.call({
-      path: `/v1/connections/${encodeURIComponent(input.service)}/connect`,
+      path: target
+        ? `/v1/connections/by-id/${encodeURIComponent(target.connectorId)}/connect`
+        : `/v1/connections/${encodeURIComponent(input.service)}/connect`,
       method: 'POST',
       body: {
         returnUri: this.input.returnUrl({ service: input.service, id }),
@@ -135,33 +176,51 @@ export class ProviderService {
     input: ProviderScope & {
       service: string;
       authType: 'api_key' | 'custom_credential';
+      connectionId?: string;
       values: Record<string, string>;
     },
   ) {
     this.guard(input);
+    const target = await this.reconnectTarget(input);
+    const path = target
+      ? `/v1/connections/by-id/${encodeURIComponent(target.connectorId)}/connect`
+      : `/v1/connections/${encodeURIComponent(input.service)}/connect`;
     const { apiKey, ...extra } = input.values;
     const result = await this.input.connector.call({
-      path: `/v1/connections/${encodeURIComponent(input.service)}/connect/${input.authType === 'api_key' ? 'api-key' : 'custom-credential'}`,
+      path: `${path}/${input.authType === 'api_key' ? 'api-key' : 'custom-credential'}`,
       method: 'POST',
       body: input.authType === 'api_key' ? { apiKey, extra } : { values: input.values },
     });
     const connection = publicConnection({ metadata: result, service: input.service });
     this.input.signal.throwIfAborted();
-    await this.input.repository.add({
+    const id = target?.id ?? `connection_${crypto.randomUUID()}`;
+    const owned = { actorId: input.actorId, ownerId: input.ownerId, id, ...connection };
+    if (target) {
+      if (connection.connectorId !== target.connectorId) {
+        fail('connection_unavailable');
+      }
+      await this.input.repository.updateAccount(owned);
+    } else {
+      await this.input.repository.add(owned);
+    }
+    await this.input.onConnected?.({
       actorId: input.actorId,
       ownerId: input.ownerId,
-      id: `connection_${crypto.randomUUID()}`,
-      ...connection,
+      connection: { id, service: input.service },
     });
     return { connected: true };
   }
   async complete(input: ProviderScope & { service: string; id: string }): Promise<void> {
     this.guard(input);
     const existing = await this.input.repository.connection(input);
-    if (existing?.service === input.service) {
+    const pending = await this.input.repository.pending(input);
+    if (!pending && existing?.service === input.service) {
+      await this.input.onConnected?.({
+        ...input,
+        connection: { id: input.id, service: input.service },
+      });
       return;
     }
-    const pending = await this.input.repository.pending(input);
     if (!pending || pending.service !== input.service) {
       fail('not_found');
     }
@@ -178,13 +237,20 @@ export class ProviderService {
       path: `/v1/connections/by-id/${encodeURIComponent(result.appId)}`,
     });
     const connection = publicConnection({ metadata, service: input.service });
-    if (connection.connectorId !== result.appId) {
+    if (
+      connection.connectorId !== result.appId ||
+      (existing && existing.connectorId !== connection.connectorId)
+    ) {
       throw new SyncError({ code: 'provider_request_failed', message: 'Invalid connection.' });
     }
     this.input.signal.throwIfAborted();
-    await this.input.repository.complete({ ...input, ...connection });
+    await this.input.repository.complete({ ...input, ...connection, requestId: pending.requestId });
     if (!(await this.input.repository.connection(input))) {
       fail('not_found');
     }
+    await this.input.onConnected?.({
+      ...input,
+      connection: { id: input.id, service: input.service },
+    });
   }
 }