@oomol-lab/open-connector 1.7.0 → 1.9.0

package/src/triggers/store.d.ts

@@ -0,0 +1,29 @@
+import type { JsonValue } from "./common/types.ts";
+export interface TriggerSubscription {
+    mode: "webhook" | "resource" | "resource-set";
+    id: string;
+    tokenId: string;
+    service: string;
+    connectionId: string;
+    triggerId: string;
+    connectionRevision: string;
+    providerAccountId: string;
+    requestKey: string;
+    config: Readonly<Record<string, JsonValue>>;
+    endpointUrl: string;
+    callbackNonce: string;
+    callbackSecret: string;
+    checkpoint: JsonValue;
+    subscription: Readonly<Record<string, JsonValue>>;
+    reconcileAt: number;
+    status: "active" | "deleting" | "deleted" | "abandoned";
+}
+export interface TriggerStore {
+    list(): Promise<TriggerSubscription[]>;
+    listFlowTriggersForMaintenance(now: number, limit: number): Promise<TriggerSubscription[]>;
+    insertFlowTrigger(record: TriggerSubscription): Promise<TriggerSubscription>;
+    getFlowTrigger(id: string): Promise<TriggerSubscription | null>;
+    claimFlowTrigger(id: string, owner: string, now: number, until: number): Promise<boolean>;
+    saveFlowTrigger(record: TriggerSubscription, owner: string, now: number): Promise<boolean>;
+    releaseFlowTrigger(id: string, owner: string): Promise<void>;
+}

package/src/triggers/store.js

@@ -0,0 +1 @@
+export {};

package/src/triggers/subscriptions.d.ts

@@ -0,0 +1,22 @@
+import type { IntegrationDefinition, IntegrationReceiveResult, IntegrationReconcileResult } from "./common/integration.ts";
+import type { ConnectorProxy } from "./common/proxy.ts";
+import type { TriggerRequest } from "./request.ts";
+import type { TriggerSubscription, TriggerStore } from "./store.ts";
+export type SubscriptionOwner = Pick<TriggerSubscription, "tokenId" | "service" | "connectionId" | "triggerId" | "connectionRevision" | "providerAccountId">;
+export declare function subscriptionId(owner: SubscriptionOwner, requestKey: string): Promise<string>;
+export declare function withSubscription<T>(store: TriggerStore, id: string, signal: AbortSignal, run: (record: TriggerSubscription, save: () => Promise<void>) => Promise<T>): Promise<T>;
+interface SubscriptionState {
+    checkpoint: import("./common/types.ts").JsonValue;
+    subscription: Readonly<Record<string, import("./common/types.ts").JsonValue>>;
+    reconcileAt: number;
+}
+export type SubscriptionResult = SubscriptionState & (IntegrationReconcileResult | {
+    result: IntegrationReceiveResult;
+});
+export declare function executeSubscription(store: TriggerStore, definition: IntegrationDefinition, owner: SubscriptionOwner, input: Extract<TriggerRequest, {
+    operation: "receive" | "reconcile";
+}>, proxy: ConnectorProxy, signal: AbortSignal): Promise<SubscriptionResult>;
+export declare function executeSubscriptionOperation(record: TriggerSubscription, save: () => Promise<void>, definition: IntegrationDefinition, input: Extract<TriggerRequest, {
+    operation: "receive" | "reconcile";
+}>, proxy: ConnectorProxy, signal: AbortSignal): Promise<SubscriptionResult>;
+export {};

package/src/triggers/subscriptions.js

@@ -0,0 +1,220 @@
+import { randomBytes } from "node:crypto";
+import { isDeepStrictEqual } from "node:util";
+import { randomUUIDv7 } from "../core/uuid-v7.js";
+import { HttpRequestError } from "../server/api/http-utils.js";
+import { canonicalJsonBytes, digestBytes } from "./common/encoding.js";
+export async function subscriptionId(owner, requestKey) {
+    return digestBytes(canonicalJsonBytes([
+        owner.tokenId,
+        owner.service,
+        owner.connectionId,
+        owner.providerAccountId,
+        owner.triggerId,
+        requestKey,
+    ]));
+}
+export async function withSubscription(store, id, signal, run) {
+    const lease = randomUUIDv7();
+    if (!(await store.claimFlowTrigger(id, lease, Date.now(), Date.now() + 60_000)))
+        throw new HttpRequestError("invalid_input", "Trigger subscription is busy", 409);
+    try {
+        const record = await store.getFlowTrigger(id);
+        if (!record)
+            throw new HttpRequestError("trigger_not_found", "Subscription not found", 404);
+        const save = async () => {
+            signal.throwIfAborted();
+            if (!(await store.saveFlowTrigger(record, lease, Date.now())))
+                throw new HttpRequestError("invalid_input", "Trigger subscription lease expired", 409);
+        };
+        return await run(record, save);
+    }
+    finally {
+        await store.releaseFlowTrigger(id, lease);
+    }
+}
+export async function executeSubscription(store, definition, owner, input, proxy, signal) {
+    const id = input.operation === "receive"
+        ? input.subscriptionId
+        : (input.subscriptionId ?? (await subscriptionId(owner, input.requestKey)));
+    if (input.operation === "reconcile" && input.active && !input.subscriptionId) {
+        const url = new URL(input.endpointUrl);
+        if (url.protocol !== "https:" || url.username || url.password || url.hash)
+            throw new HttpRequestError("invalid_input", "A HTTPS callback without credentials or fragment is required", 400);
+        await store.insertFlowTrigger({
+            mode: "webhook",
+            id,
+            ...owner,
+            requestKey: input.requestKey,
+            config: input.config,
+            endpointUrl: url.href,
+            callbackNonce: randomUUIDv7(),
+            callbackSecret: randomBytes(32).toString("base64url"),
+            checkpoint: definition.initialState?.checkpoint ?? null,
+            subscription: definition.initialState?.subscription ?? {},
+            reconcileAt: Date.now(),
+            status: "active",
+        });
+    }
+    const existing = await store.getFlowTrigger(id);
+    if (!existing) {
+        if (input.operation === "reconcile" && !input.active)
+            return {
+                outcome: "ready",
+                checkpoint: null,
+                subscription: {},
+                reconcileAt: Date.now() + 60_000,
+            };
+        throw new HttpRequestError("trigger_not_found", "Subscription not found", 404);
+    }
+    if (existing.tokenId !== owner.tokenId ||
+        existing.service !== owner.service ||
+        existing.connectionId !== owner.connectionId ||
+        existing.triggerId !== owner.triggerId ||
+        existing.providerAccountId !== owner.providerAccountId)
+        throw new HttpRequestError("trigger_not_found", "Subscription not found", 404);
+    return withSubscription(store, id, signal, async (record, save) => {
+        return executeSubscriptionOperation(record, save, definition, input, proxy, signal);
+    });
+}
+// The caller must hold the subscription lease until this operation completes.
+export async function executeSubscriptionOperation(record, save, definition, input, proxy, signal) {
+    const id = record.id;
+    const state = {
+        get checkpoint() {
+            return record.checkpoint;
+        },
+        get subscription() {
+            return record.subscription;
+        },
+        saveCheckpoint: async (checkpoint) => {
+            record.checkpoint = checkpoint;
+            await save();
+        },
+        saveSubscription: async (subscription, reconcileAt) => {
+            record.subscription = subscription;
+            record.reconcileAt = reconcileAt.getTime();
+            await save();
+        },
+    };
+    if (input.operation === "reconcile") {
+        if (record.status === "deleted" && !input.active)
+            return {
+                outcome: "ready",
+                checkpoint: record.checkpoint,
+                subscription: { id },
+                reconcileAt: record.reconcileAt,
+            };
+        const url = new URL(input.endpointUrl);
+        if (url.protocol !== "https:" || url.username || url.password || url.hash)
+            throw new HttpRequestError("invalid_input", "A HTTPS callback without credentials or fragment is required", 400);
+        if (record.status === "abandoned")
+            throw new HttpRequestError("trigger_abandoned", "Create a new request key after abandoning cleanup.", 409);
+        if (record.status === "deleted") {
+            record.config = input.config;
+            record.endpointUrl = url.href;
+            record.callbackNonce = randomUUIDv7();
+            record.callbackSecret = randomBytes(32).toString("base64url");
+            record.checkpoint = definition.initialState?.checkpoint ?? null;
+            record.subscription = definition.initialState?.subscription ?? {};
+            record.reconcileAt = Date.now();
+            record.status = "active";
+            await save();
+        }
+        if (!isDeepStrictEqual(record.config, input.config) || record.endpointUrl !== url.href)
+            throw new HttpRequestError("invalid_input", "Subscription request key is already bound to another configuration", 409);
+        if (!input.active) {
+            record.status = "deleting";
+            await save();
+        }
+        if (input.active && record.status !== "active")
+            throw new HttpRequestError("invalid_input", "Subscription is being deleted", 409);
+        const callback = new URL(record.endpointUrl);
+        callback.searchParams.set("connector_subscription", record.callbackNonce);
+        const result = await definition.reconcile({
+            active: input.active,
+            config: record.config,
+            endpointUrl: callback.href,
+            callbackSecret: record.callbackSecret,
+            idempotencyKey: record.callbackNonce,
+            connector: proxy,
+            now: new Date(),
+            state,
+            signal,
+        });
+        if (!input.active && result.outcome === "ready") {
+            record.status = "deleted";
+            await save();
+        }
+        return {
+            ...result,
+            checkpoint: record.checkpoint,
+            subscription: { id },
+            reconcileAt: record.reconcileAt,
+        };
+    }
+    if (record.status !== "active")
+        return {
+            result: { outcome: "ignored", reason: "Subscription is not active" },
+            checkpoint: record.checkpoint,
+            subscription: { id },
+            reconcileAt: record.reconcileAt,
+        };
+    if (input.query.connector_subscription !== record.callbackNonce)
+        return {
+            result: { outcome: "respond", status: 404, body: "", contentType: "text/plain" },
+            checkpoint: record.checkpoint,
+            subscription: { id },
+            reconcileAt: record.reconcileAt,
+        };
+    if (!definition.snapshot.endpoint.methods.includes(input.method))
+        throw new HttpRequestError("invalid_input", "Unsupported webhook method.");
+    const rawBody = Uint8Array.from(Buffer.from(input.rawBody, "base64"));
+    if (rawBody.byteLength > 64 * 1024)
+        throw new HttpRequestError("invalid_input", "Webhook body is too large", 400);
+    if (!rawBody.byteLength && !definition.snapshot.endpoint.body.allowEmpty)
+        throw new HttpRequestError("invalid_input", "Webhook body is required.");
+    const type = input.headers["content-type"]?.split(";")[0]?.trim().toLowerCase();
+    if (rawBody.byteLength && type !== "application/json")
+        throw new HttpRequestError("invalid_input", "Webhook content type must be application/json.");
+    let payload = null;
+    if (rawBody.byteLength > 0) {
+        const text = new TextDecoder().decode(rawBody);
+        try {
+            payload =
+                type === "application/x-www-form-urlencoded" ? Object.fromEntries(new URLSearchParams(text)) : JSON.parse(text);
+        }
+        catch (error) {
+            if (error instanceof SyntaxError)
+                throw new HttpRequestError("invalid_input", "Invalid webhook JSON payload", 400);
+            throw error;
+        }
+    }
+    if (Array.isArray(payload) && !definition.snapshot.endpoint.body.allowArray)
+        throw new HttpRequestError("invalid_input", "Webhook body must not be an array.");
+    const result = await definition.receive({
+        bindingId: id,
+        callbackSecret: record.callbackSecret,
+        config: record.config,
+        connector: proxy,
+        current: input.current,
+        admit: input.admit,
+        method: input.method,
+        now: new Date(),
+        payload: payload,
+        rawBody,
+        header: (name) => input.headers[name.toLowerCase()],
+        query: (name) => input.query[name],
+        state,
+        signal,
+    });
+    if ("checkpoint" in result && result.checkpoint !== undefined) {
+        record.checkpoint = result.checkpoint;
+        await save();
+    }
+    return {
+        result,
+        checkpoint: record.checkpoint,
+        subscription: { id },
+        reconcileAt: record.reconcileAt,
+    };
+}

package/src/triggers/trigger-runner.d.ts

@@ -0,0 +1,38 @@
+import type { CatalogStore } from "../catalog-store.ts";
+import type { ConnectionService } from "../connection-service.ts";
+import type { ActionPolicySnapshot } from "../core/action-policy.ts";
+import type { ProviderHttpDispatchOptions } from "../core/provider-http-dispatch.ts";
+import type { IProviderLoader } from "../providers/provider-loader.ts";
+import type { RuntimeGrant } from "../server/storage/runtime-token-service.ts";
+import type { TriggerRequest } from "./request.ts";
+import type { TriggerSubscription, TriggerStore } from "./store.ts";
+interface TriggerRunnerOptions {
+    providerHttpDispatch?: ProviderHttpDispatchOptions;
+    catalog: CatalogStore;
+    providerLoader: IProviderLoader;
+    connections: ConnectionService;
+    store: TriggerStore;
+}
+export interface RunTriggerInput {
+    service: string;
+    triggerId: string;
+    connectionName?: string;
+    connectionId?: string;
+    policy: ActionPolicySnapshot;
+    grant?: RuntimeGrant;
+    request: TriggerRequest;
+    signal: AbortSignal;
+}
+export declare class TriggerRunner {
+    private readonly options;
+    constructor(options: TriggerRunnerOptions);
+    run(input: RunTriggerInput): Promise<unknown>;
+    private runTrigger;
+    cleanup(record: TriggerSubscription, save: () => Promise<void>, signal: AbortSignal): Promise<void>;
+    abandonResource(record: TriggerSubscription, signal: AbortSignal): Promise<void>;
+    private definition;
+    private target;
+    private providerOperation;
+    private runProviderOperation;
+}
+export {};

package/src/triggers/trigger-runner.js

@@ -0,0 +1,183 @@
+import { ConnectionError } from "../connection-service.js";
+import { optionalInteger, optionalRecord } from "../core/cast.js";
+import { withProviderHttpDispatch } from "../core/provider-http-dispatch.js";
+import { withProviderHttpDispatchResult } from "../providers/provider-runtime.js";
+import { HttpRequestError } from "../server/api/http-utils.js";
+import { mapConnectionErrorStatus } from "../server/api/runtime-api.js";
+import { resolveTriggerConfig } from "./common/config.js";
+import { validateListenerPage } from "./common/integration.js";
+import { executeResourceSubscription, executeResourceSubscriptionOperation, abandonResourceSubscription, } from "./resources.js";
+import { executeSubscription, executeSubscriptionOperation } from "./subscriptions.js";
+export class TriggerRunner {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+    async run(input) {
+        const provider = this.options.catalog.providers.find((candidate) => candidate.service === input.service);
+        return withProviderHttpDispatch({ operation: "trigger", service: provider?.service }, () => this.runTrigger(input), this.options.providerHttpDispatch);
+    }
+    async runTrigger(input) {
+        const decision = input.policy.evaluateTrigger(input.triggerId);
+        if (!decision.allowed)
+            throw new HttpRequestError(decision.code, decision.message, 403);
+        const stateful = input.request.operation !== "read" && input.request.operation !== "options";
+        if (stateful && !input.grant)
+            throw new HttpRequestError("trigger_owner_required", "Stateful Trigger operations require a persistent runtime token.", 403);
+        const definition = await this.definition(input.service, input.triggerId);
+        const target = await this.target(input.service, input.connectionName, input.connectionId, input.signal, input.policy);
+        const request = input.request;
+        if (request.operation === "receive") {
+            if (definition.snapshot.type !== "integration" || ("eventSource" in definition && definition.eventSource))
+                throw new HttpRequestError("invalid_input", "This Trigger has no remote webhook.");
+            return this.providerOperation(input.service, () => executeSubscription(this.options.store, definition, { ...target.owner, tokenId: input.grant.tokenId, triggerId: input.triggerId }, request, target.proxy, input.signal));
+        }
+        const config = resolveTriggerConfig(definition.snapshot.configInputs, request.config, request.operation === "options");
+        const context = { config, connector: target.proxy, now: new Date(), signal: input.signal };
+        return this.providerOperation(input.service, async () => {
+            switch (request.operation) {
+                case "options": {
+                    if (!definition.configOptions ||
+                        !definition.snapshot.configInputs.some((field) => "handle" in field && field.handle === request.field))
+                        throw new HttpRequestError("invalid_input", "This Trigger has no options for the selected field.");
+                    return definition.configOptions({ ...context, field: request.field });
+                }
+                case "read": {
+                    if (definition.snapshot.type === "poll")
+                        return definition.poll({ ...context, checkpoint: request.checkpoint });
+                    const listener = definition.listener;
+                    if (!listener)
+                        throw new HttpRequestError("invalid_input", "This Trigger cannot be read.");
+                    const page = await listener.read({ ...context, checkpoint: request.checkpoint });
+                    validateListenerPage(page);
+                    return page;
+                }
+                case "reconcile":
+                    if (definition.snapshot.type !== "integration" || definition.eventSource)
+                        throw new HttpRequestError("invalid_input", "This Trigger has no remote webhook.");
+                    return executeSubscription(this.options.store, definition, { ...target.owner, tokenId: input.grant.tokenId, triggerId: input.triggerId }, { ...request, config }, target.proxy, input.signal);
+                case "resource":
+                    if (!definition.resources)
+                        throw new HttpRequestError("invalid_input", "This Trigger has no shared resource.");
+                    return executeResourceSubscription(this.options.store, definition.resources, { ...target.owner, tokenId: input.grant.tokenId, triggerId: input.triggerId }, { ...request, config }, target.proxy, input.signal);
+            }
+        });
+    }
+    async cleanup(record, save, signal) {
+        const target = await this.target(record.service, undefined, record.connectionId, signal);
+        if (target.owner.providerAccountId !== record.providerAccountId)
+            throw new HttpRequestError("trigger_connection_error", "The subscription's original provider account is unavailable.", 409);
+        record.connectionRevision = target.owner.connectionRevision;
+        const definition = await this.definition(record.service, record.triggerId);
+        if (record.mode === "resource") {
+            if (!definition.resources)
+                throw new HttpRequestError("trigger_not_supported", "Resource subscriptions are unavailable.", 501);
+            await this.providerOperation(record.service, () => executeResourceSubscriptionOperation(this.options.store, definition.resources, record, save, { operation: "resource", config: record.config, active: false, requestKey: record.requestKey }, target.proxy, signal));
+        }
+        else {
+            if (definition.snapshot.type !== "integration")
+                throw new HttpRequestError("trigger_not_found", "The remote Trigger definition is unavailable.", 404);
+            await this.providerOperation(record.service, () => executeSubscriptionOperation(record, save, definition, {
+                operation: "reconcile",
+                config: record.config,
+                endpointUrl: record.endpointUrl,
+                requestKey: record.requestKey,
+                subscriptionId: record.id,
+                active: false,
+            }, target.proxy, signal));
+        }
+    }
+    async abandonResource(record, signal) {
+        const definition = await this.definition(record.service, record.triggerId);
+        const resources = definition.resources;
+        if (!resources)
+            throw new HttpRequestError("trigger_not_supported", "Resource subscriptions are unavailable.", 501);
+        await abandonResourceSubscription(this.options.store, resources, record, signal);
+    }
+    async definition(service, triggerId) {
+        if (!this.options.catalog.providers.some((provider) => provider.service === service))
+            throw new HttpRequestError("provider_not_found", "Provider not found.", 404);
+        const definitions = await this.options.providerLoader.loadTriggerDefinitions?.(service);
+        const definition = definitions?.find((candidate) => candidate.snapshot.key === triggerId && candidate.snapshot.provider === service);
+        if (!definition)
+            throw new HttpRequestError("trigger_not_found", "Trigger not found.", 404);
+        return definition;
+    }
+    async target(service, connectionName, connectionId, signal, policy) {
+        try {
+            const summary = await this.options.connections.getConnectionSummary(service, connectionName, connectionId);
+            const permission = policy?.evaluateConnection(summary?.id);
+            if (permission && !permission.allowed)
+                throw new HttpRequestError(permission.code, permission.message, 403);
+            const target = await this.options.connections.resolveForExecution(service, connectionName, connectionId);
+            if (target.kind !== "local")
+                throw new HttpRequestError("trigger_source_not_supported", "Triggers require a local connection; this connection source is not supported.", 501);
+            const selected = policy?.evaluateConnection(target.summary?.id);
+            if (selected && !selected.allowed)
+                throw new HttpRequestError(selected.code, selected.message, 403);
+            const credential = await target.getCredential(service);
+            if (!credential || credential.authType === "no_auth" || !target.summary)
+                throw new HttpRequestError("connection_not_found", "Trigger connection not found.", 404);
+            const stored = await this.options.connections.getStoredConnection(target.summary.id);
+            if (!stored ||
+                stored.source === "saas" ||
+                stored.credential.authType === "no_auth" ||
+                stored.credential.profile.accountId !== credential.profile.accountId)
+                throw new HttpRequestError("trigger_connection_error", "The selected connection changed during Trigger execution.", 409);
+            const executor = await this.options.providerLoader.loadProxyExecutor(service);
+            if (!executor)
+                throw new HttpRequestError("trigger_not_supported", "The provider has no Trigger transport.", 501);
+            const proxy = {
+                execute: async (request, requestSignal) => {
+                    signal.throwIfAborted();
+                    const result = await withProviderHttpDispatch({
+                        operation: "trigger",
+                        service,
+                        connectionId: stored.id,
+                        connectionName: stored.connectionName,
+                    }, () => executor(request, {
+                        getCredential: target.getCredential,
+                        signal: requestSignal ?? signal,
+                    }), this.options.providerHttpDispatch);
+                    if (result.ok)
+                        return result.response;
+                    const status = optionalInteger(optionalRecord(result.error.details)?.status) ?? 502;
+                    return { status, data: { error: result.error.message } };
+                },
+            };
+            return {
+                proxy,
+                owner: {
+                    service,
+                    connectionId: stored.id,
+                    connectionRevision: stored.revision,
+                    providerAccountId: credential.profile.accountId,
+                },
+            };
+        }
+        catch (error) {
+            if (error instanceof ConnectionError)
+                throw new HttpRequestError(error.code, error.message, mapConnectionErrorStatus(error));
+            throw error;
+        }
+    }
+    async providerOperation(service, run) {
+        return withProviderHttpDispatchResult({ operation: "trigger", service }, () => this.runProviderOperation(run), this.options.providerHttpDispatch);
+    }
+    async runProviderOperation(run) {
+        try {
+            return await run();
+        }
+        catch (error) {
+            if (error instanceof Error) {
+                if (error.name === "IntegrationConnectionError" || error.name === "PollConnectionError")
+                    throw new HttpRequestError("trigger_connection_error", "The Trigger connection could not be used.", 409);
+                if (error.name === "TransientIntegrationError" || error.name === "TransientPollError")
+                    throw new HttpRequestError("proxy_upstream_error", "The Trigger provider is temporarily unavailable.", 503);
+                if (error.name === "PermanentIntegrationError" || error.name === "PermanentPollError")
+                    throw new HttpRequestError("invalid_input", error.message);
+            }
+            throw error;
+        }
+    }
+}