@shipfox/api-integration-core 2.0.0

package/src/db/webhook-deliveries.ts

@@ -0,0 +1,207 @@
+import {
+  INTEGRATION_EVENT_RECEIVED,
+  INTEGRATION_SOURCE_COMMIT_PUSHED,
+  type IntegrationEventReceivedEvent,
+  type IntegrationsEventMap,
+  type SourcePushPayload,
+} from '@shipfox/api-integration-core-dto';
+import {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';
+import {lt} from 'drizzle-orm';
+import {db} from './db.js';
+import {integrationsOutbox} from './schema/outbox.js';
+import {integrationsWebhookDeliveries} from './schema/webhook-deliveries.js';
+
+type IntegrationDb = ReturnType<typeof db>;
+type IntegrationTx = Parameters<Parameters<IntegrationDb['transaction']>[0]>[0];
+type Executor = IntegrationDb | IntegrationTx;
+
+function connectionDedupScope(connectionId: string): string {
+  return `connection:${connectionId}`;
+}
+
+function providerDedupScope(provider: string): string {
+  return `provider:${provider}`;
+}
+
+function receivedEventDedupScope(event: IntegrationEventReceivedEvent): string {
+  if (event.provider === 'webhook') return connectionDedupScope(event.connectionId);
+  return providerDedupScope(event.provider);
+}
+
+export interface PublishIntegrationEventReceivedParams {
+  tx: Executor;
+  event: IntegrationEventReceivedEvent;
+}
+
+export interface PublishIntegrationEventReceivedResult {
+  published: boolean;
+}
+
+export async function publishIntegrationEventReceived(
+  params: PublishIntegrationEventReceivedParams,
+): Promise<PublishIntegrationEventReceivedResult> {
+  const inserted = await params.tx
+    .insert(integrationsWebhookDeliveries)
+    .values({
+      provider: params.event.provider,
+      dedupScope: receivedEventDedupScope(params.event),
+      deliveryId: params.event.deliveryId,
+    })
+    .onConflictDoNothing({
+      target: [
+        integrationsWebhookDeliveries.provider,
+        integrationsWebhookDeliveries.dedupScope,
+        integrationsWebhookDeliveries.deliveryId,
+      ],
+    })
+    .returning({deliveryId: integrationsWebhookDeliveries.deliveryId});
+
+  if (inserted.length === 0) return {published: false};
+
+  await writeOutboxEvent(params.tx, integrationsOutbox, {
+    type: INTEGRATION_EVENT_RECEIVED,
+    payload: params.event,
+  });
+
+  return {published: true};
+}
+
+export interface PublishSourcePushParams {
+  tx: IntegrationTx;
+  provider: string;
+  source: string;
+  workspaceId: string;
+  connectionId: string;
+  connectionName: string;
+  deliveryId: string;
+  receivedAt: string;
+  rawPayload: unknown;
+  push: SourcePushPayload;
+}
+
+// Emits a single source-control push as two outbox rows: the generic
+// `INTEGRATION_EVENT_RECEIVED` envelope with the raw provider payload for triggers, and the
+// typed `INTEGRATION_SOURCE_COMMIT_PUSHED` event for domain modules. One delivery-dedup gates
+// both, so a redelivered webhook writes nothing. Requires a transaction: the dedup insert and
+// both outbox rows must commit or roll back together.
+export async function publishSourcePush(
+  params: PublishSourcePushParams,
+): Promise<{published: boolean}> {
+  const inserted = await params.tx
+    .insert(integrationsWebhookDeliveries)
+    .values({
+      provider: params.provider,
+      dedupScope: providerDedupScope(params.provider),
+      deliveryId: params.deliveryId,
+    })
+    .onConflictDoNothing({
+      target: [
+        integrationsWebhookDeliveries.provider,
+        integrationsWebhookDeliveries.dedupScope,
+        integrationsWebhookDeliveries.deliveryId,
+      ],
+    })
+    .returning({deliveryId: integrationsWebhookDeliveries.deliveryId});
+
+  if (inserted.length === 0) return {published: false};
+
+  await writeOutboxEvents<IntegrationsEventMap>(params.tx, integrationsOutbox, [
+    {
+      type: INTEGRATION_EVENT_RECEIVED,
+      payload: {
+        provider: params.provider,
+        source: params.source,
+        event: 'push',
+        workspaceId: params.workspaceId,
+        connectionId: params.connectionId,
+        connectionName: params.connectionName,
+        deliveryId: params.deliveryId,
+        receivedAt: params.receivedAt,
+        payload: params.rawPayload,
+      },
+    },
+    {
+      type: INTEGRATION_SOURCE_COMMIT_PUSHED,
+      payload: {
+        provider: params.provider,
+        workspaceId: params.workspaceId,
+        connectionId: params.connectionId,
+        deliveryId: params.deliveryId,
+        receivedAt: params.receivedAt,
+        push: params.push,
+      },
+    },
+  ]);
+
+  return {published: true};
+}
+
+export interface PublishSourceCommitPushedParams {
+  provider: string;
+  workspaceId: string;
+  connectionId: string;
+  deliveryId: string;
+  receivedAt: string;
+  push: SourcePushPayload;
+}
+
+// Emits ONLY the typed `INTEGRATION_SOURCE_COMMIT_PUSHED` event, intentionally skipping the
+// `INTEGRATION_EVENT_RECEIVED` envelope so triggers do not run workflows, and skipping the
+// delivery-dedup row so it is never suppressed. Used to force a definitions re-sync without
+// simulating an inbound webhook.
+export async function publishSourceCommitPushed(
+  params: PublishSourceCommitPushedParams,
+): Promise<void> {
+  await writeOutboxEvent<IntegrationsEventMap>(db(), integrationsOutbox, {
+    type: INTEGRATION_SOURCE_COMMIT_PUSHED,
+    payload: {
+      provider: params.provider,
+      workspaceId: params.workspaceId,
+      connectionId: params.connectionId,
+      deliveryId: params.deliveryId,
+      receivedAt: params.receivedAt,
+      push: params.push,
+    },
+  });
+}
+
+export interface RecordDeliveryOnlyParams {
+  tx: Executor;
+  provider: string;
+  deliveryId: string;
+}
+
+export async function recordDeliveryOnly(params: RecordDeliveryOnlyParams): Promise<void> {
+  await params.tx
+    .insert(integrationsWebhookDeliveries)
+    .values({
+      provider: params.provider,
+      dedupScope: providerDedupScope(params.provider),
+      deliveryId: params.deliveryId,
+    })
+    .onConflictDoNothing({
+      target: [
+        integrationsWebhookDeliveries.provider,
+        integrationsWebhookDeliveries.dedupScope,
+        integrationsWebhookDeliveries.deliveryId,
+      ],
+    });
+}
+
+export interface PruneWebhookDeliveriesParams {
+  olderThan: Date;
+}
+
+export async function pruneWebhookDeliveries(
+  params: PruneWebhookDeliveriesParams,
+): Promise<{deleted: number}> {
+  const result = await db()
+    .delete(integrationsWebhookDeliveries)
+    .where(lt(integrationsWebhookDeliveries.receivedAt, params.olderThan));
+  return {deleted: result.rowCount ?? 0};
+}
+
+export type PublishIntegrationEventReceivedFn = typeof publishIntegrationEventReceived;
+export type PublishSourcePushFn = typeof publishSourcePush;
+export type PublishSourceCommitPushedFn = typeof publishSourceCommitPushed;
+export type RecordDeliveryOnlyFn = typeof recordDeliveryOnly;

package/src/index.ts

@@ -0,0 +1,212 @@
+import {dirname, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {integrationsEventSchemas} from '@shipfox/api-integration-core-dto';
+import type {ShipfoxModule} from '@shipfox/node-module';
+import {logger} from '@shipfox/node-opentelemetry';
+import type {IntegrationProvider} from '#core/entities/provider.js';
+import {
+  createIntegrationProviderRegistry,
+  type IntegrationProviderRegistry,
+} from '#core/providers/registry.js';
+import {
+  createSourceControlIntegrationService,
+  type IntegrationSourceControlService,
+} from '#core/source-control-service.js';
+import {getIntegrationConnectionById} from '#db/connections.js';
+import {db} from '#db/db.js';
+import {migrationsPath} from '#db/migrations.js';
+import {integrationsOutbox} from '#db/schema/outbox.js';
+import {createIntegrationRoutes, type LeasedAgentStepLoader} from '#presentation/routes/index.js';
+import {loadEnabledProviderModules} from '#providers/modules.js';
+import type {IntegrationModuleParts, IntegrationProviderSecrets} from '#providers/types.js';
+import {createIntegrationsMaintenanceActivities} from '#temporal/activities/index.js';
+import {INTEGRATIONS_MAINTENANCE_TASK_QUEUE} from '#temporal/constants.js';
+
+const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const maintenanceWorkflowsPath = resolve(packageRoot, 'dist/temporal/workflows/index.js');
+
+export {
+  buildProviderRepositoryId,
+  MAX_REPOSITORY_FILE_BYTES,
+  parseProviderRepositoryId,
+} from '@shipfox/api-integration-core-dto';
+export type {
+  AgentToolCatalogs,
+  AgentToolSelectionCatalogs,
+  LoadWorkspaceConnectionSnapshot,
+  WorkspaceConnectionSnapshot,
+  WorkspaceConnectionSnapshotEntry,
+} from '#core/agent-tool-selection.js';
+export {
+  buildAgentToolCatalogs,
+  buildAgentToolSelectionCatalogs,
+  createWorkspaceConnectionSnapshotLoader,
+} from '#core/agent-tool-selection.js';
+export type {
+  IntegrationConnection,
+  IntegrationConnectionLifecycleStatus,
+} from '#core/entities/connection.js';
+export type {
+  IntegrationCapability,
+  IntegrationProvider,
+  IntegrationProviderAdapters,
+  IntegrationProviderKind,
+  RegisteredIntegrationProvider,
+} from '#core/entities/provider.js';
+export type {IntegrationProviderErrorReason} from '#core/errors.js';
+export {
+  ConnectionSlugConflictError,
+  IntegrationCapabilityUnavailableError,
+  IntegrationCheckoutUnsupportedError,
+  IntegrationConnectionInactiveError,
+  IntegrationConnectionNotFoundError,
+  IntegrationConnectionWorkspaceMismatchError,
+  IntegrationProviderError,
+  IntegrationProviderUnavailableError,
+} from '#core/errors.js';
+export type {
+  AgentToolCallInput,
+  AgentToolCatalogEntry,
+  AgentToolCatalogMethod,
+  AgentToolJsonSchema,
+  AgentToolSensitivity,
+  AgentToolSession,
+  AgentToolsProvider,
+  OpenAgentToolsSessionInput,
+} from '#core/providers/agent-tools.js';
+export {redactCheckoutSpec} from '#core/providers/redact-checkout-spec.js';
+export type {IntegrationProviderRegistry} from '#core/providers/registry.js';
+export type {
+  CheckoutCredentials,
+  CheckoutPermissions,
+  CheckoutSpec,
+  CreateCheckoutSpecInput,
+  FetchFileInput,
+  FileEntry,
+  FilePage,
+  FileSnapshot,
+  ListFilesInput,
+  ListRepositoriesInput,
+  RepositoryPage,
+  RepositorySnapshot,
+  RepositoryVisibility,
+  ResolveRepositoryInput,
+  SourceControlProvider,
+} from '#core/providers/source-control.js';
+export type {IntegrationSourceControlService} from '#core/source-control-service.js';
+export {createSourceControlIntegrationService} from '#core/source-control-service.js';
+export type {GetIntegrationConnectionByIdFn} from '#db/connections.js';
+export {getIntegrationConnectionById} from '#db/connections.js';
+export type {
+  PublishIntegrationEventReceivedFn,
+  PublishIntegrationEventReceivedParams,
+  PublishIntegrationEventReceivedResult,
+  PublishSourcePushFn,
+  PublishSourcePushParams,
+  RecordDeliveryOnlyFn,
+  RecordDeliveryOnlyParams,
+} from '#db/webhook-deliveries.js';
+export {pruneWebhookDeliveries} from '#db/webhook-deliveries.js';
+export {integrationRouteErrorHandler} from '#presentation/routes/errors.js';
+
+export interface CreateIntegrationsModuleOptions {
+  providers?: IntegrationProvider[] | undefined;
+  /**
+   * Pre-built module parts, bypassing config-gated provider loading. Test-only seam
+   * for exercising a provider's database, workers, or startup tasks directly. Takes
+   * precedence over `providers`.
+   */
+  parts?: IntegrationModuleParts[] | undefined;
+  secrets?: IntegrationProviderSecrets | undefined;
+  agentTools?:
+    | {
+        loadLeasedAgentStep: LeasedAgentStepLoader;
+      }
+    | undefined;
+}
+
+export interface IntegrationsContext {
+  module: ShipfoxModule;
+  registry: IntegrationProviderRegistry;
+  capabilities: {
+    sourceControl: IntegrationSourceControlService;
+  };
+  sourceControl: IntegrationSourceControlService;
+  /**
+   * Runs every enabled provider's one-shot boot-time tasks, after modules are initialized
+   * (migrations done). Failures are isolated and logged, never rethrown, so a provider task
+   * can never gate API boot. No-op when no enabled provider contributes a task.
+   */
+  runStartupTasks: () => Promise<void>;
+}
+
+export async function createIntegrationsModule(
+  options: CreateIntegrationsModuleOptions = {},
+): Promise<ShipfoxModule> {
+  return (await createIntegrationsContext(options)).module;
+}
+
+export async function createIntegrationsContext(
+  options: CreateIntegrationsModuleOptions = {},
+): Promise<IntegrationsContext> {
+  const parts: IntegrationModuleParts[] =
+    options.parts ??
+    (options.providers
+      ? options.providers.map((provider) => ({provider}))
+      : await loadEnabledProviderModules({secrets: options.secrets}));
+
+  const registry = createIntegrationProviderRegistry(parts.map((part) => part.provider));
+  const sourceControl = createSourceControlIntegrationService({
+    registry,
+    getIntegrationConnectionById,
+  });
+
+  async function runStartupTasks(): Promise<void> {
+    for (const task of parts.flatMap((part) => part.startupTasks ?? [])) {
+      // A provider convenience must never gate API boot.
+      try {
+        await task();
+      } catch (error) {
+        logger().error({err: error}, 'Integration startup task failed, continuing boot');
+      }
+    }
+  }
+
+  const module: ShipfoxModule = {
+    name: 'integrations',
+    startupTasks: runStartupTasks,
+    database: [
+      {db, migrationsPath},
+      ...parts.flatMap((part) => (part.database ? [part.database] : [])),
+    ],
+    routes: createIntegrationRoutes(registry, sourceControl, {
+      agentTools: options.agentTools
+        ? {
+            loadLeasedAgentStep: options.agentTools.loadLeasedAgentStep,
+            getIntegrationConnectionById,
+          }
+        : undefined,
+    }),
+    e2eRoutes: parts.flatMap((part) => part.e2eRoutes ?? []),
+    publishers: [
+      {name: 'integrations', table: integrationsOutbox, db, eventSchemas: integrationsEventSchemas},
+    ],
+    workers: [
+      {
+        taskQueue: INTEGRATIONS_MAINTENANCE_TASK_QUEUE,
+        workflowsPath: maintenanceWorkflowsPath,
+        activities: createIntegrationsMaintenanceActivities,
+        workflows: [
+          {
+            name: 'pruneWebhookDeliveriesCron',
+            id: 'integrations-prune-webhook-deliveries',
+            cronSchedule: '0 3 * * *',
+          },
+        ],
+      },
+      ...parts.flatMap((part) => part.workers ?? []),
+    ],
+  };
+
+  return {module, registry, capabilities: {sourceControl}, sourceControl, runStartupTasks};
+}

package/src/metrics/index.ts

@@ -0,0 +1 @@
+export * from './instance.js';

package/src/metrics/instance.ts

@@ -0,0 +1,35 @@
+import {instanceMetrics} from '@shipfox/node-opentelemetry';
+
+const meter = instanceMetrics.getMeter('integrations');
+
+export type IntegrationAgentToolCallOutcome =
+  | 'success'
+  | 'tool-error'
+  | 'invalid-request'
+  | 'exception';
+
+const agentToolCallCount = meter.createCounter<{
+  provider: string;
+  tool: string;
+  method: string;
+  outcome: IntegrationAgentToolCallOutcome;
+}>('integrations_agent_tool_call', {
+  description: 'Integration agent tool calls by provider, tool, method, and outcome',
+});
+
+function recordMetric(record: () => void): void {
+  try {
+    record();
+  } catch {
+    // Metrics must not affect integration tool call outcomes.
+  }
+}
+
+export function recordIntegrationAgentToolCall(params: {
+  provider: string;
+  tool: string;
+  method: string;
+  outcome: IntegrationAgentToolCallOutcome;
+}): void {
+  recordMetric(() => agentToolCallCount.add(1, params));
+}

package/src/presentation/dto/integrations.ts

@@ -0,0 +1,39 @@
+import {toIntegrationConnectionDto as toCoreIntegrationConnectionDto} from '@shipfox/api-integration-core-dto';
+import type {IntegrationConnection} from '#core/entities/connection.js';
+import type {
+  IntegrationCapability,
+  RegisteredIntegrationProvider,
+} from '#core/entities/provider.js';
+import type {RepositorySnapshot} from '#core/providers/source-control.js';
+
+export function toIntegrationProviderDto(provider: RegisteredIntegrationProvider) {
+  return {
+    provider: provider.provider,
+    display_name: provider.displayName,
+    capabilities: provider.capabilities,
+  };
+}
+
+export function toIntegrationConnectionDto(
+  connection: IntegrationConnection,
+  options: {
+    capabilities: IntegrationCapability[];
+    externalUrl?: string | undefined;
+  },
+) {
+  return toCoreIntegrationConnectionDto(connection, options);
+}
+
+export function toRepositoryDto(connectionId: string, repository: RepositorySnapshot) {
+  return {
+    connection_id: connectionId,
+    external_repository_id: repository.externalRepositoryId,
+    owner: repository.owner,
+    name: repository.name,
+    full_name: repository.fullName,
+    default_branch: repository.defaultBranch,
+    visibility: repository.visibility,
+    clone_url: repository.cloneUrl,
+    html_url: repository.htmlUrl,
+  };
+}