@shipfox/api-definitions 2.0.0

package/src/temporal/activities/sync-activities.ts

@@ -0,0 +1,206 @@
+import type {
+  AgentToolSelectionCatalogs,
+  IntegrationSourceControlService,
+  LoadWorkspaceConnectionSnapshot,
+} from '@shipfox/api-integration-core';
+import {Context} from '@temporalio/activity';
+import {ApplicationFailure} from '@temporalio/common';
+import type {DefinitionSyncErrorCode} from '#core/entities/sync-state.js';
+import {
+  classifySyncFailure,
+  discoverWorkflowFiles,
+  fetchAndParseWorkflows,
+  resolveSyncSource,
+  UNRESOLVED_SYNC_REF,
+} from '#core/index.js';
+import {applyVcsDefinitionsBatch, markDefinitionSyncState} from '#db/index.js';
+
+export interface SyncWorkflowInput {
+  projectId: string;
+  workspaceId: string;
+  sourceConnectionId: string;
+  sourceExternalRepositoryId: string;
+  sourceRef?: string | undefined;
+  sourceCommitSha?: string | undefined;
+}
+
+export interface SyncRefScopedInput extends SyncWorkflowInput {
+  sourceRef: string;
+}
+
+export interface FetchAndApplyActivityInput extends SyncRefScopedInput {
+  paths: string[];
+}
+
+export interface MarkSyncFailedActivityInput extends Omit<SyncWorkflowInput, 'sourceRef'> {
+  sourceRef: string | null;
+  code: DefinitionSyncErrorCode;
+  message: string;
+}
+
+export interface PrepareSyncResult {
+  sourceRef: string;
+  sourceCommitSha?: string | undefined;
+}
+
+export interface DiscoverWorkflowsActivityResult {
+  paths: string[];
+}
+
+export interface FetchAndApplyActivityResult {
+  appliedCount: number;
+  deletedCount: number;
+}
+
+export interface DefinitionSyncIntegrationValidationOptions {
+  agentToolSelectionCatalogs: AgentToolSelectionCatalogs;
+  loadWorkspaceConnectionSnapshot: LoadWorkspaceConnectionSnapshot;
+  getIntegrationConnectionById: (id: string) => Promise<{slug: string} | undefined>;
+}
+
+export function createDefinitionSyncActivities(
+  sourceControl: IntegrationSourceControlService,
+  integrationValidation?: DefinitionSyncIntegrationValidationOptions | undefined,
+) {
+  return {
+    prepareDefinitionSync: createPrepareDefinitionSyncActivity(sourceControl),
+    discoverDefinitionWorkflows: createDiscoverDefinitionWorkflowsActivity(sourceControl),
+    fetchAndApplyDefinitionWorkflows: createFetchAndApplyActivity(
+      sourceControl,
+      integrationValidation,
+    ),
+    markDefinitionSyncSucceeded: createMarkSyncSucceededActivity(),
+    markDefinitionSyncFailed: createMarkSyncFailedActivity(),
+  };
+}
+
+function createPrepareDefinitionSyncActivity(sourceControl: IntegrationSourceControlService) {
+  return async function prepareDefinitionSync(
+    input: SyncWorkflowInput,
+  ): Promise<PrepareSyncResult> {
+    return await runWithPermanentTranslation(async () => {
+      const sourceRef = input.sourceRef ?? (await resolveSyncSource({...input, sourceControl})).ref;
+
+      await markDefinitionSyncState({
+        projectId: input.projectId,
+        sourceConnectionId: input.sourceConnectionId,
+        sourceExternalRepositoryId: input.sourceExternalRepositoryId,
+        ref: sourceRef,
+        status: 'syncing',
+        lastErrorCode: null,
+        lastErrorMessage: null,
+        startedAt: new Date(),
+        finishedAt: null,
+      });
+
+      return {sourceRef, sourceCommitSha: input.sourceCommitSha};
+    });
+  };
+}
+
+function createDiscoverDefinitionWorkflowsActivity(sourceControl: IntegrationSourceControlService) {
+  return async function discoverDefinitionWorkflows(
+    input: SyncRefScopedInput,
+  ): Promise<DiscoverWorkflowsActivityResult> {
+    return await runWithPermanentTranslation(async () => {
+      return await discoverWorkflowFiles({
+        ...input,
+        ref: input.sourceCommitSha ?? input.sourceRef,
+        sourceControl,
+      });
+    });
+  };
+}
+
+function createFetchAndApplyActivity(
+  sourceControl: IntegrationSourceControlService,
+  integrationValidation?: DefinitionSyncIntegrationValidationOptions | undefined,
+) {
+  return async function fetchAndApplyDefinitionWorkflows(
+    input: FetchAndApplyActivityInput,
+  ): Promise<FetchAndApplyActivityResult> {
+    return await runWithPermanentTranslation(async () => {
+      const definitions = await fetchAndParseWorkflows({
+        ...input,
+        ref: input.sourceCommitSha ?? input.sourceRef,
+        sourceControl,
+        onProgress: (path) => Context.current().heartbeat({path}),
+        loadIntegrationValidationContext:
+          integrationValidation === undefined
+            ? undefined
+            : async () => {
+                const sourceConnection = await integrationValidation.getIntegrationConnectionById(
+                  input.sourceConnectionId,
+                );
+                return {
+                  agentToolSelectionCatalogs: integrationValidation.agentToolSelectionCatalogs,
+                  workspaceConnectionSnapshot:
+                    await integrationValidation.loadWorkspaceConnectionSnapshot(input.workspaceId),
+                  defaultConnectionSlug: sourceConnection?.slug,
+                };
+              },
+      });
+
+      return await applyVcsDefinitionsBatch({
+        projectId: input.projectId,
+        workspaceId: input.workspaceId,
+        ref: input.sourceRef,
+        upserts: definitions.map((entry) => ({
+          configPath: entry.path,
+          name: entry.name,
+          document: entry.definition.document,
+          model: entry.definition.model,
+          sourceSnapshot: entry.definition.sourceSnapshot,
+          contentHash: entry.contentHash,
+        })),
+      });
+    });
+  };
+}
+
+function createMarkSyncSucceededActivity() {
+  return async function markDefinitionSyncSucceeded(input: SyncRefScopedInput): Promise<void> {
+    await markDefinitionSyncState({
+      projectId: input.projectId,
+      sourceConnectionId: input.sourceConnectionId,
+      sourceExternalRepositoryId: input.sourceExternalRepositoryId,
+      ref: input.sourceRef,
+      status: 'succeeded',
+      lastErrorCode: null,
+      lastErrorMessage: null,
+      finishedAt: new Date(),
+    });
+  };
+}
+
+function createMarkSyncFailedActivity() {
+  return async function markDefinitionSyncFailed(
+    input: MarkSyncFailedActivityInput,
+  ): Promise<void> {
+    await markDefinitionSyncState({
+      projectId: input.projectId,
+      sourceConnectionId: input.sourceConnectionId,
+      sourceExternalRepositoryId: input.sourceExternalRepositoryId,
+      ref: input.sourceRef ?? UNRESOLVED_SYNC_REF,
+      status: 'failed',
+      lastErrorCode: input.code,
+      lastErrorMessage: input.message,
+      finishedAt: new Date(),
+    });
+  };
+}
+
+async function runWithPermanentTranslation<T>(operation: () => Promise<T>): Promise<T> {
+  try {
+    return await operation();
+  } catch (error) {
+    if (error instanceof ApplicationFailure) {
+      throw error;
+    }
+    const failure = classifySyncFailure(error);
+    if (!failure.retryable) {
+      throw ApplicationFailure.nonRetryable(failure.message, failure.code);
+    }
+    throw ApplicationFailure.retryable(failure.message, failure.code);
+  }
+}

package/src/temporal/constants.ts

@@ -0,0 +1,2 @@
+export const DEFINITIONS_TASK_QUEUE = 'definitions-sync';
+export const DEFINITION_SYNC_WORKFLOW = 'definitionSyncWorkflow';

package/src/temporal/index.ts

@@ -0,0 +1,2 @@
+export {createDefinitionSyncActivities} from './activities/index.js';
+export {DEFINITION_SYNC_WORKFLOW, DEFINITIONS_TASK_QUEUE} from './constants.js';

package/src/temporal/workflows/definition-sync.test.ts

@@ -0,0 +1,59 @@
+import {ActivityFailure, ApplicationFailure, RetryState} from '@temporalio/common';
+import {classifyWorkflowError} from './definition-sync.js';
+
+describe('definitionSyncWorkflow error classification', () => {
+  it('unwraps activity application failures before persisting sync failure metadata', () => {
+    const cause = ApplicationFailure.nonRetryable(
+      'Invalid workflow definition at .shipfox/workflows/bad.yml',
+      'invalid-definition',
+    );
+    const error = new ActivityFailure(
+      'Activity failed',
+      'fetchAndApplyDefinitionWorkflows',
+      'activity-id',
+      RetryState.NON_RETRYABLE_FAILURE,
+      'worker',
+      cause,
+    );
+
+    const result = classifyWorkflowError(error);
+
+    expect(result).toEqual({
+      code: 'invalid-definition',
+      message: 'Invalid workflow definition at .shipfox/workflows/bad.yml',
+    });
+  });
+
+  it('classifies a bare ApplicationFailure using its type', () => {
+    const error = ApplicationFailure.retryable('GitHub request timed out', 'provider-timeout');
+
+    const result = classifyWorkflowError(error);
+
+    expect(result).toEqual({
+      code: 'provider-timeout',
+      message: 'GitHub request timed out',
+    });
+  });
+
+  it('falls back to unknown when ApplicationFailure.type is not a known sync error code', () => {
+    const error = ApplicationFailure.nonRetryable('boom', 'something-unexpected');
+
+    const result = classifyWorkflowError(error);
+
+    expect(result).toEqual({
+      code: 'unknown',
+      message: 'boom',
+    });
+  });
+
+  it('classifies non-temporal errors as unknown and preserves the message', () => {
+    const error = new Error('connection reset');
+
+    const result = classifyWorkflowError(error);
+
+    expect(result).toEqual({
+      code: 'unknown',
+      message: 'connection reset',
+    });
+  });
+});

package/src/temporal/workflows/definition-sync.ts

@@ -0,0 +1,116 @@
+import {ActivityFailure, ApplicationFailure, proxyActivities} from '@temporalio/workflow';
+import {
+  type DefinitionSyncErrorCode,
+  isDefinitionSyncErrorCode,
+} from '#core/entities/sync-state.js';
+import type {createDefinitionSyncActivities} from '../activities/index.js';
+
+export interface DefinitionSyncWorkflowInput {
+  projectId: string;
+  workspaceId: string;
+  sourceConnectionId: string;
+  sourceExternalRepositoryId: string;
+  sourceRef?: string | undefined;
+  sourceCommitSha?: string | undefined;
+}
+
+export interface DefinitionSyncWorkflowResult {
+  sourceRef: string;
+  appliedCount: number;
+  deletedCount: number;
+}
+
+const PROVIDER_RETRY = {
+  initialInterval: '5 seconds',
+  backoffCoefficient: 2,
+  maximumInterval: '1 minute',
+  maximumAttempts: 5,
+} as const;
+
+const DB_RETRY = {
+  initialInterval: '1 second',
+  backoffCoefficient: 2,
+  maximumInterval: '15 seconds',
+  maximumAttempts: 5,
+} as const;
+
+const {prepareDefinitionSync, discoverDefinitionWorkflows, fetchAndApplyDefinitionWorkflows} =
+  proxyActivities<ReturnType<typeof createDefinitionSyncActivities>>({
+    startToCloseTimeout: '5 minutes',
+    heartbeatTimeout: '30 seconds',
+    retry: PROVIDER_RETRY,
+  });
+
+const {markDefinitionSyncSucceeded, markDefinitionSyncFailed} = proxyActivities<
+  ReturnType<typeof createDefinitionSyncActivities>
+>({
+  startToCloseTimeout: '30 seconds',
+  retry: DB_RETRY,
+});
+
+export async function definitionSyncWorkflow(
+  input: DefinitionSyncWorkflowInput,
+): Promise<DefinitionSyncWorkflowResult> {
+  let sourceRef: string | null = null;
+
+  try {
+    const prepared = await prepareDefinitionSync(input);
+    sourceRef = prepared.sourceRef;
+
+    const source = {
+      ...input,
+      sourceRef: prepared.sourceRef,
+      sourceCommitSha: prepared.sourceCommitSha,
+    };
+    const {paths} = await discoverDefinitionWorkflows(source);
+    const applied = await fetchAndApplyDefinitionWorkflows({...source, paths});
+
+    await markDefinitionSyncSucceeded(source);
+
+    return {
+      sourceRef,
+      appliedCount: applied.appliedCount,
+      deletedCount: applied.deletedCount,
+    };
+  } catch (error) {
+    const {code, message} = classifyWorkflowError(error);
+    try {
+      await markDefinitionSyncFailed({...input, sourceRef, code, message});
+    } catch (markFailedError) {
+      const failureOptions = {
+        message: `Definition sync failed with ${code}: ${message}; additionally failed to persist failure state: ${formatWorkflowError(markFailedError)}`,
+        type: 'definition-sync-failure-persistence-failed',
+        nonRetryable: true,
+        details: [
+          {
+            syncFailureCode: code,
+            syncFailureMessage: message,
+            failurePersistenceMessage: formatWorkflowError(markFailedError),
+          },
+        ],
+      };
+      throw ApplicationFailure.create(
+        error instanceof Error ? {...failureOptions, cause: error} : failureOptions,
+      );
+    }
+    throw error;
+  }
+}
+
+export function classifyWorkflowError(error: unknown): {
+  code: DefinitionSyncErrorCode;
+  message: string;
+} {
+  if (error instanceof ActivityFailure && error.cause instanceof ApplicationFailure) {
+    return classifyWorkflowError(error.cause);
+  }
+  if (error instanceof ApplicationFailure) {
+    const code = isDefinitionSyncErrorCode(error.type) ? error.type : 'unknown';
+    return {code, message: error.message ?? code};
+  }
+  return {code: 'unknown', message: error instanceof Error ? error.message : String(error)};
+}
+
+function formatWorkflowError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/temporal/workflows/index.ts

@@ -0,0 +1 @@
+export {definitionSyncWorkflow} from './definition-sync.js';

package/test/env.ts

@@ -0,0 +1,7 @@
+process.env.POSTGRES_HOST ??= 'localhost';
+process.env.POSTGRES_PORT ??= '5432';
+process.env.POSTGRES_USERNAME ??= 'shipfox';
+process.env.POSTGRES_PASSWORD ??= 'password';
+process.env.POSTGRES_DATABASE = 'api_test';
+process.env.POSTGRES_MAX_CONNECTIONS ??= '5';
+process.env.TZ = 'UTC';

package/test/factories/definition.ts

@@ -0,0 +1,66 @@
+import {Factory} from 'fishery';
+import type {
+  WorkflowDefinition,
+  WorkflowDefinitionPayload,
+} from '#core/entities/workflow-definition.js';
+import {normalizeWorkflowDocument} from '#core/workflow-model/index.js';
+import {upsertDefinition} from '#db/definitions.js';
+
+function defaultDefinition(): WorkflowDefinitionPayload {
+  const document = {
+    name: 'Test Workflow',
+    runner: 'ubuntu-latest',
+    jobs: {
+      build: {
+        steps: [{run: 'echo hello'}],
+      },
+    },
+  };
+  return {document, model: normalizeWorkflowDocument(document)};
+}
+
+interface DefinitionTransients {
+  workspaceId: string;
+}
+
+export const definitionFactory = Factory.define<WorkflowDefinition, DefinitionTransients>(
+  ({sequence, onCreate, transientParams}) => {
+    const projectId = crypto.randomUUID();
+
+    onCreate((definition) => {
+      return upsertDefinition({
+        projectId: definition.projectId,
+        workspaceId: transientParams.workspaceId ?? crypto.randomUUID(),
+        configPath: definition.configPath,
+        source: definition.source,
+        name: definition.name,
+        document: definition.document,
+        model: definition.model,
+        sourceSnapshot: definition.sourceSnapshot,
+        contentHash: definition.contentHash ?? undefined,
+        sha: definition.sha ?? undefined,
+        ref: definition.ref ?? undefined,
+      });
+    });
+
+    const definition = defaultDefinition();
+
+    return {
+      id: crypto.randomUUID(),
+      projectId,
+      configPath: `.shipfox/workflows/test-${sequence}.yml`,
+      source: 'manual',
+      sha: null,
+      ref: null,
+      name: `Test Workflow ${sequence}`,
+      definition: definition.document,
+      ...definition,
+      sourceSnapshot: definition.sourceSnapshot ?? null,
+      contentHash: null,
+      fetchedAt: new Date(),
+      createdAt: new Date(),
+      updatedAt: new Date(),
+      deletedAt: null,
+    };
+  },
+);

package/test/fixtures/invalid-cycle.yml

@@ -0,0 +1,13 @@
+name: Cyclic workflow
+runner: ubuntu-latest
+
+jobs:
+  a:
+    needs: b
+    steps:
+      - run: echo a
+
+  b:
+    needs: a
+    steps:
+      - run: echo b

package/test/fixtures/invalid-missing-name.yml

@@ -0,0 +1,4 @@
+jobs:
+  build:
+    steps:
+      - run: echo hello

package/test/fixtures/invalid-yaml-syntax.yml

@@ -0,0 +1,6 @@
+name: Bad YAML
+jobs:
+  build:
+    steps:
+      - run: echo hello
+      invalid indentation here

package/test/fixtures/valid-dag.yml

@@ -0,0 +1,25 @@
+name: Multi-job pipeline
+runner: ubuntu-latest
+
+jobs:
+  build:
+    steps:
+      - run: npm install
+      - run: npm run build
+
+  test-unit:
+    needs: build
+    steps:
+      - run: npm run test:unit
+
+  test-integration:
+    needs: build
+    steps:
+      - run: npm run test:integration
+
+  deploy:
+    needs:
+      - test-unit
+      - test-integration
+    steps:
+      - run: ./deploy.sh staging

package/test/fixtures/valid-listening-job.yml

@@ -0,0 +1,21 @@
+name: Listening workflow
+runner: ubuntu-latest
+jobs:
+  review:
+    name: Review batch ${{ execution.index }}
+    listening:
+      on:
+        - source: github
+          event: pull_request_review
+      until:
+        - source: github
+          event: pull_request
+      timeout: 30d
+      max_executions: 3
+      batch:
+        debounce: 5s
+        max_size: 10
+        max_wait: 1h
+      on_resolve: cancel
+    steps:
+      - prompt: Review ${{ execution.events[0].data.body }}

package/test/fixtures/valid-simple.yml

@@ -0,0 +1,18 @@
+name: Simple build
+runner: ubuntu-latest
+
+triggers:
+  on_demand:
+    source: manual
+    event: fire
+  on_push:
+    source: github_acme
+    event: push
+    filter: event.ref == "refs/heads/main"
+
+jobs:
+  build:
+    steps:
+      - run: npm install
+      - run: npm run build
+        name: build_output

package/test/fixtures/workflow-yaml/invalid/invalid-yaml-syntax.yaml

@@ -0,0 +1,6 @@
+name: invalid syntax
+jobs:
+  build:
+    steps:
+      - run npm test
+broken

package/test/fixtures/workflow-yaml/invalid/missing-step-run.yaml

@@ -0,0 +1,5 @@
+name: missing step run
+jobs:
+  build:
+    steps:
+      - name: test

package/test/fixtures/workflow-yaml/valid/simple-build.document.json

@@ -0,0 +1,26 @@
+{
+  "name": "simple build",
+  "triggers": {
+    "main_push": {
+      "source": "github_acme",
+      "event": "push",
+      "filter": "event.ref == \"refs/heads/main\""
+    }
+  },
+  "jobs": {
+    "build": {
+      "steps": [
+        {
+          "run": "npm install"
+        },
+        {
+          "name": "build",
+          "run": "npm run build",
+          "gate": {
+            "success": "step.exit_code == 0"
+          }
+        }
+      ]
+    }
+  }
+}

package/test/fixtures/workflow-yaml/valid/simple-build.yaml

@@ -0,0 +1,14 @@
+name: simple build
+triggers:
+  main_push:
+    source: github_acme
+    event: push
+    filter: event.ref == "refs/heads/main"
+jobs:
+  build:
+    steps:
+      - run: npm install
+      - name: build
+        run: npm run build
+        gate:
+          success: step.exit_code == 0

package/test/globalSetup.ts

@@ -0,0 +1,25 @@
+import './env.js';
+import {runMigrations} from '@shipfox/node-drizzle';
+import {closePostgresClient, createPostgresClient} from '@shipfox/node-postgres';
+import {sql} from 'drizzle-orm';
+import {closeDb, db, migrationsPath} from '#db/index.js';
+
+export async function setup() {
+  createPostgresClient();
+
+  await db().execute(sql`DROP TABLE IF EXISTS definitions_workflow_definitions CASCADE`);
+  await db().execute(sql`DROP TABLE IF EXISTS definitions_sync_states CASCADE`);
+  await db().execute(sql`DROP TABLE IF EXISTS definitions_outbox CASCADE`);
+  await db().execute(sql`DROP TYPE IF EXISTS definitions_sync_status CASCADE`);
+  await db().execute(sql`DROP TYPE IF EXISTS definitions_sync_error_code CASCADE`);
+  await db().execute(sql`DROP TYPE IF EXISTS definitions_source CASCADE`);
+  await db().execute(sql`DROP TABLE IF EXISTS __drizzle_migrations_definitions CASCADE`);
+  await db().execute(sql`DROP TABLE IF EXISTS drizzle.__drizzle_migrations_definitions CASCADE`);
+  await runMigrations(db(), migrationsPath, '__drizzle_migrations_definitions');
+  await db().execute(sql`TRUNCATE TABLE definitions_workflow_definitions CASCADE`);
+  await db().execute(sql`TRUNCATE TABLE definitions_sync_states CASCADE`);
+  await db().execute(sql`TRUNCATE TABLE definitions_outbox CASCADE`);
+
+  closeDb();
+  await closePostgresClient();
+}

package/test/index.ts

@@ -0,0 +1 @@
+export {definitionFactory} from './factories/definition.js';

package/test/setup.ts

@@ -0,0 +1,17 @@
+import './env.js';
+import {closePostgresClient, createPostgresClient} from '@shipfox/node-postgres';
+import {afterAll, afterEach, beforeAll, vi} from '@shipfox/vitest/vi';
+import {closeDb} from '#db/index.js';
+
+beforeAll(() => {
+  createPostgresClient();
+});
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+afterAll(async () => {
+  closeDb();
+  await closePostgresClient();
+});

package/tsconfig.build.json

@@ -0,0 +1,9 @@
+{
+  "extends": "@shipfox/ts-config/node",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": ["src"],
+  "exclude": ["**/*.test.tsx", "**/*.test.ts"]
+}