@shipfox/api-definitions 2.0.0

package/src/core/parse-definition.ts

@@ -0,0 +1,28 @@
+import type {HarnessToolDeploymentConfig} from '@shipfox/api-agent-dto';
+import type {IntegrationValidationContext} from './entities/integration-context.js';
+import type {WorkflowDefinitionPayload} from './entities/workflow-definition.js';
+import {DefinitionParseError} from './errors.js';
+import {validateDefinition} from './validate-definition.js';
+
+export function parseDefinition(
+  yamlString: string,
+  options?: {
+    defaultRunnerLabels?: readonly string[];
+    harnessToolDeploymentConfig?: HarnessToolDeploymentConfig;
+    integrationValidationContext?: IntegrationValidationContext;
+  },
+): WorkflowDefinitionPayload {
+  const result = validateDefinition(yamlString, options);
+
+  if (!result.valid) {
+    throw new DefinitionParseError(
+      result.errors[0]?.message ?? 'Invalid definition',
+      result.errors,
+    );
+  }
+
+  return {
+    ...result.definition,
+    sourceSnapshot: {content: yamlString, format: 'yaml'},
+  };
+}

package/src/core/sync-definitions.test.ts

@@ -0,0 +1,390 @@
+import {
+  IntegrationConnectionInactiveError,
+  IntegrationConnectionNotFoundError,
+  IntegrationConnectionWorkspaceMismatchError,
+  IntegrationProviderError,
+  type IntegrationSourceControlService,
+} from '@shipfox/api-integration-core';
+import {LOWERCASE_SHA256_HEX_RE} from '@shipfox/regex';
+import type {IntegrationValidationContext} from './entities/integration-context.js';
+import {DefinitionSyncPermanentError} from './errors.js';
+import {
+  classifySyncFailure,
+  discoverWorkflowFiles,
+  fetchAndParseWorkflows,
+  resolveSyncSource,
+} from './sync-definitions.js';
+
+const validYaml = `
+name: CI
+runner: ubuntu-latest
+jobs:
+  build:
+    steps:
+      - run: pnpm test
+`;
+
+const validIntegrationYaml = `
+name: Agent CI
+runner: ubuntu-latest
+jobs:
+  build:
+    steps:
+      - prompt: Fix the issue
+        integrations:
+          - connection: github-main
+            include: [issue_read]
+`;
+
+const invalidIntegrationYaml = `
+name: Agent CI
+runner: ubuntu-latest
+jobs:
+  build:
+    steps:
+      - prompt: Fix the issue
+        integrations:
+          - connection: github-main
+            include: [issue_read.missing]
+`;
+
+const integrationValidationContext = {
+  agentToolSelectionCatalogs: new Map([
+    [
+      'github',
+      {
+        selectors: [
+          {token: 'issue_read', kind: 'family', sensitivity: 'read', sensitive: false},
+          {token: 'issue_read.get', kind: 'method', sensitivity: 'read', sensitive: false},
+        ],
+      },
+    ],
+  ]),
+  workspaceConnectionSnapshot: new Map([
+    ['github-main', {id: 'connection-1', provider: 'github', capabilities: ['agent_tools']}],
+  ]),
+} satisfies IntegrationValidationContext;
+
+function sourceControl(
+  overrides: Partial<IntegrationSourceControlService> = {},
+): IntegrationSourceControlService {
+  return {
+    getConnection: vi.fn(),
+    listRepositories: vi.fn(),
+    resolveRepository: vi.fn(() =>
+      Promise.resolve({
+        connection: {
+          id: 'connection-1',
+          workspaceId: 'workspace-1',
+          provider: 'gitea',
+          externalAccountId: 'gitea-owner',
+          slug: 'gitea_owner',
+          displayName: 'Gitea',
+          lifecycleStatus: 'active' as const,
+          createdAt: new Date(),
+          updatedAt: new Date(),
+        },
+        repository: {
+          externalRepositoryId: 'gitea:gitea-owner/platform',
+          owner: 'gitea-owner',
+          name: 'platform',
+          fullName: 'gitea-owner/platform',
+          defaultBranch: 'main',
+          visibility: 'private' as const,
+          cloneUrl: 'https://gitea.local/gitea-owner/platform.git',
+          htmlUrl: 'https://gitea.local/gitea-owner/platform',
+        },
+      }),
+    ),
+    listFiles: vi.fn(() =>
+      Promise.resolve({
+        files: [{path: '.shipfox/workflows/ci.yml', type: 'file' as const, size: validYaml.length}],
+        nextCursor: null,
+      }),
+    ),
+    fetchFile: vi.fn(() =>
+      Promise.resolve({path: '.shipfox/workflows/ci.yml', ref: 'main', content: validYaml}),
+    ),
+    createCheckoutSpec: vi.fn(),
+    ...overrides,
+  };
+}
+
+const baseContext = {
+  workspaceId: 'workspace-1',
+  sourceConnectionId: 'connection-1',
+  sourceExternalRepositoryId: 'gitea:gitea-owner/platform',
+};
+
+describe('resolveSyncSource', () => {
+  it('returns the repository default branch as ref', async () => {
+    const result = await resolveSyncSource({...baseContext, sourceControl: sourceControl()});
+
+    expect(result).toEqual({ref: 'main'});
+  });
+});
+
+describe('discoverWorkflowFiles', () => {
+  it('returns yaml/yml workflow paths', async () => {
+    const result = await discoverWorkflowFiles({
+      ...baseContext,
+      ref: 'main',
+      sourceControl: sourceControl({
+        listFiles: vi.fn(() =>
+          Promise.resolve({
+            files: [
+              {path: '.shipfox/workflows/ci.yml', type: 'file' as const, size: 64},
+              {path: '.shipfox/workflows/deploy.yaml', type: 'file' as const, size: 64},
+              {path: '.shipfox/workflows/README.md', type: 'file' as const, size: 64},
+            ],
+            nextCursor: null,
+          }),
+        ),
+      }),
+    });
+
+    expect(result.paths).toEqual(['.shipfox/workflows/ci.yml', '.shipfox/workflows/deploy.yaml']);
+  });
+
+  it('throws no-workflow-files when nothing matches the yaml extensions', async () => {
+    const result = discoverWorkflowFiles({
+      ...baseContext,
+      ref: 'main',
+      sourceControl: sourceControl({
+        listFiles: vi.fn(() =>
+          Promise.resolve({
+            files: [{path: '.shipfox/workflows/README.md', type: 'file' as const, size: 1}],
+            nextCursor: null,
+          }),
+        ),
+      }),
+    });
+
+    await expect(result).rejects.toMatchObject({code: 'no-workflow-files'});
+  });
+
+  it('throws too-many-files when the listing reports more pages', async () => {
+    const result = discoverWorkflowFiles({
+      ...baseContext,
+      ref: 'main',
+      sourceControl: sourceControl({
+        listFiles: vi.fn(() =>
+          Promise.resolve({
+            files: [{path: '.shipfox/workflows/ci.yml', type: 'file' as const, size: 1}],
+            nextCursor: '1',
+          }),
+        ),
+      }),
+    });
+
+    await expect(result).rejects.toMatchObject({code: 'too-many-files'});
+  });
+});
+
+describe('fetchAndParseWorkflows', () => {
+  it('fetches and parses each provided path', async () => {
+    const result = await fetchAndParseWorkflows({
+      ...baseContext,
+      ref: 'main',
+      paths: ['.shipfox/workflows/ci.yml'],
+      sourceControl: sourceControl(),
+    });
+
+    expect(result).toHaveLength(1);
+    expect(result[0]?.name).toBe('CI');
+    expect(result[0]?.path).toBe('.shipfox/workflows/ci.yml');
+    expect(result[0]?.contentHash).toMatch(LOWERCASE_SHA256_HEX_RE);
+  });
+
+  it('produces stable content hashes for identical content', async () => {
+    const sourceControlA = sourceControl();
+    const sourceControlB = sourceControl();
+
+    const a = await fetchAndParseWorkflows({
+      ...baseContext,
+      ref: 'main',
+      paths: ['.shipfox/workflows/ci.yml'],
+      sourceControl: sourceControlA,
+    });
+    const b = await fetchAndParseWorkflows({
+      ...baseContext,
+      ref: 'main',
+      paths: ['.shipfox/workflows/ci.yml'],
+      sourceControl: sourceControlB,
+    });
+
+    expect(a[0]?.contentHash).toBe(b[0]?.contentHash);
+  });
+
+  it('rejects oversized contents as content-too-large', async () => {
+    const huge = 'x'.repeat(1_000_001);
+    const result = fetchAndParseWorkflows({
+      ...baseContext,
+      ref: 'main',
+      paths: ['.shipfox/workflows/big.yml'],
+      sourceControl: sourceControl({
+        fetchFile: vi.fn(() =>
+          Promise.resolve({path: '.shipfox/workflows/big.yml', ref: 'main', content: huge}),
+        ),
+      }),
+    });
+
+    await expect(result).rejects.toBeInstanceOf(DefinitionSyncPermanentError);
+    await expect(result).rejects.toMatchObject({code: 'content-too-large'});
+  });
+
+  it('rejects invalid YAML as invalid-definition', async () => {
+    const result = fetchAndParseWorkflows({
+      ...baseContext,
+      ref: 'main',
+      paths: ['.shipfox/workflows/bad.yml'],
+      sourceControl: sourceControl({
+        fetchFile: vi.fn(() =>
+          Promise.resolve({
+            path: '.shipfox/workflows/bad.yml',
+            ref: 'main',
+            content: 'name: Bad\n  broken:\nindent',
+          }),
+        ),
+      }),
+    });
+
+    await expect(result).rejects.toMatchObject({code: 'invalid-definition'});
+  });
+
+  it('emits onProgress for every path', async () => {
+    const onProgress = vi.fn();
+    await fetchAndParseWorkflows({
+      ...baseContext,
+      ref: 'main',
+      paths: ['.shipfox/workflows/ci.yml'],
+      sourceControl: sourceControl(),
+      onProgress,
+    });
+
+    expect(onProgress).toHaveBeenCalledWith('.shipfox/workflows/ci.yml');
+  });
+
+  it('does not load integration validation context when no workflow uses integrations', async () => {
+    const loadIntegrationValidationContext = vi.fn(() =>
+      Promise.resolve(integrationValidationContext),
+    );
+
+    const result = await fetchAndParseWorkflows({
+      ...baseContext,
+      ref: 'main',
+      paths: ['.shipfox/workflows/ci.yml'],
+      sourceControl: sourceControl(),
+      loadIntegrationValidationContext,
+    });
+
+    expect(loadIntegrationValidationContext).not.toHaveBeenCalled();
+    expect(result[0]).not.toHaveProperty('rawContent');
+  });
+
+  it('loads integration validation context once and reparses integration workflows', async () => {
+    const loadIntegrationValidationContext = vi.fn(() =>
+      Promise.resolve(integrationValidationContext),
+    );
+
+    const result = await fetchAndParseWorkflows({
+      ...baseContext,
+      ref: 'main',
+      paths: ['.shipfox/workflows/ci.yml', '.shipfox/workflows/agent.yml'],
+      sourceControl: sourceControl({
+        fetchFile: vi.fn(({path}) =>
+          Promise.resolve({
+            path,
+            ref: 'main',
+            content: path.endsWith('/agent.yml') ? validIntegrationYaml : validYaml,
+          }),
+        ),
+      }),
+      loadIntegrationValidationContext,
+    });
+
+    expect(loadIntegrationValidationContext).toHaveBeenCalledTimes(1);
+    expect(result).toHaveLength(2);
+    expect(result[0]).not.toHaveProperty('rawContent');
+    expect(result[1]?.definition.model.jobs[0]?.steps[0]).toMatchObject({
+      kind: 'agent',
+      integrations: [{connection: 'github-main', include: ['issue_read']}],
+    });
+  });
+
+  it('rejects integration catalog issues after loading validation context', async () => {
+    const loadIntegrationValidationContext = vi.fn(() =>
+      Promise.resolve(integrationValidationContext),
+    );
+
+    const result = fetchAndParseWorkflows({
+      ...baseContext,
+      ref: 'main',
+      paths: ['.shipfox/workflows/agent.yml'],
+      sourceControl: sourceControl({
+        fetchFile: vi.fn(() =>
+          Promise.resolve({
+            path: '.shipfox/workflows/agent.yml',
+            ref: 'main',
+            content: invalidIntegrationYaml,
+          }),
+        ),
+      }),
+      loadIntegrationValidationContext,
+    });
+
+    await expect(result).rejects.toMatchObject({code: 'invalid-definition'});
+    expect(loadIntegrationValidationContext).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('classifySyncFailure', () => {
+  it.each([
+    ['rate-limited', true, 'provider-rate-limited'],
+    ['timeout', true, 'provider-timeout'],
+    ['provider-unavailable', true, 'provider-unavailable'],
+    ['access-denied', false, 'provider-access-denied'],
+    ['repository-not-found', false, 'provider-repository-not-found'],
+    ['file-not-found', false, 'provider-file-not-found'],
+    ['malformed-provider-response', false, 'provider-malformed-response'],
+    ['content-too-large', false, 'content-too-large'],
+    ['too-many-files', false, 'too-many-files'],
+  ])('maps IntegrationProviderError(%s) to retryable=%s code=%s', (reason, retryable, code) => {
+    const result = classifySyncFailure(
+      // biome-ignore lint/suspicious/noExplicitAny: enumerated reason
+      new IntegrationProviderError(reason as any, 'boom'),
+    );
+
+    expect(result).toEqual({code, message: 'boom', retryable});
+  });
+
+  it('classifies DefinitionSyncPermanentError as non-retryable', () => {
+    const result = classifySyncFailure(
+      new DefinitionSyncPermanentError('invalid-definition', 'bad yaml'),
+    );
+
+    expect(result).toEqual({code: 'invalid-definition', message: 'bad yaml', retryable: false});
+  });
+
+  it.each([
+    () => new IntegrationConnectionNotFoundError('connection-1'),
+    () => new IntegrationConnectionInactiveError('connection-1'),
+    () => new IntegrationConnectionWorkspaceMismatchError('connection-1'),
+  ])('classifies connection lifecycle errors as non-retryable connection-unavailable', (build) => {
+    const error = build();
+
+    const result = classifySyncFailure(error);
+
+    expect(result).toEqual({
+      code: 'connection-unavailable',
+      message: error.message,
+      retryable: false,
+    });
+  });
+
+  it('falls back to unknown + retryable for plain errors', () => {
+    const result = classifySyncFailure(new Error('boom'));
+
+    expect(result).toEqual({code: 'unknown', message: 'boom', retryable: true});
+  });
+});

package/src/core/sync-definitions.ts

@@ -0,0 +1,216 @@
+import {createHash} from 'node:crypto';
+import {
+  IntegrationConnectionInactiveError,
+  IntegrationConnectionNotFoundError,
+  IntegrationConnectionWorkspaceMismatchError,
+  IntegrationProviderError,
+  type IntegrationSourceControlService,
+  MAX_REPOSITORY_FILE_BYTES,
+} from '@shipfox/api-integration-core';
+import {boundedMap} from '@shipfox/node-module';
+import type {IntegrationValidationContext} from './entities/integration-context.js';
+import type {DefinitionSyncErrorCode} from './entities/sync-state.js';
+import type {WorkflowDefinitionPayload} from './entities/workflow-definition.js';
+import {DefinitionParseError, DefinitionSyncPermanentError} from './errors.js';
+import {hasAgentStepIntegrations} from './has-agent-step-integrations.js';
+import {parseDefinition} from './parse-definition.js';
+
+export const WORKFLOW_PREFIX = '.shipfox/workflows/';
+export const MAX_WORKFLOW_FILES = 100;
+export const FILE_FETCH_CONCURRENCY = 4;
+export const UNRESOLVED_SYNC_REF = '__unresolved__';
+
+export interface SyncSourceContext {
+  workspaceId: string;
+  sourceConnectionId: string;
+  sourceExternalRepositoryId: string;
+  sourceControl: IntegrationSourceControlService;
+}
+
+export interface ResolvedSyncSource {
+  ref: string;
+}
+
+export async function resolveSyncSource(params: SyncSourceContext): Promise<ResolvedSyncSource> {
+  const source = await params.sourceControl.resolveRepository({
+    workspaceId: params.workspaceId,
+    connectionId: params.sourceConnectionId,
+    externalRepositoryId: params.sourceExternalRepositoryId,
+  });
+  return {ref: source.repository.defaultBranch};
+}
+
+export interface DiscoverWorkflowFilesParams extends SyncSourceContext {
+  ref: string;
+}
+
+export async function discoverWorkflowFiles(
+  params: DiscoverWorkflowFilesParams,
+): Promise<{paths: string[]}> {
+  const page = await params.sourceControl.listFiles({
+    workspaceId: params.workspaceId,
+    connectionId: params.sourceConnectionId,
+    externalRepositoryId: params.sourceExternalRepositoryId,
+    ref: params.ref,
+    prefix: WORKFLOW_PREFIX,
+    limit: MAX_WORKFLOW_FILES,
+  });
+  if (page.nextCursor) {
+    throw new DefinitionSyncPermanentError(
+      'too-many-files',
+      `More than ${MAX_WORKFLOW_FILES} workflow files were found`,
+    );
+  }
+
+  const paths = page.files
+    .filter((file) => file.path.endsWith('.yml') || file.path.endsWith('.yaml'))
+    .map((file) => file.path);
+  if (paths.length === 0) {
+    throw new DefinitionSyncPermanentError(
+      'no-workflow-files',
+      `No workflow files were found under ${WORKFLOW_PREFIX}`,
+    );
+  }
+
+  return {paths};
+}
+
+export interface ParsedWorkflow {
+  path: string;
+  name: string;
+  definition: WorkflowDefinitionPayload;
+  contentHash: string;
+}
+
+export interface FetchAndParseWorkflowsParams extends SyncSourceContext {
+  ref: string;
+  paths: string[];
+  onProgress?: ((path: string) => void) | undefined;
+  loadIntegrationValidationContext?: (() => Promise<IntegrationValidationContext>) | undefined;
+}
+
+export async function fetchAndParseWorkflows(
+  params: FetchAndParseWorkflowsParams,
+): Promise<ParsedWorkflow[]> {
+  const parsed = await boundedMap(
+    params.paths,
+    FILE_FETCH_CONCURRENCY,
+    async (path) => {
+      params.onProgress?.(path);
+
+      const snapshot = await params.sourceControl.fetchFile({
+        workspaceId: params.workspaceId,
+        connectionId: params.sourceConnectionId,
+        externalRepositoryId: params.sourceExternalRepositoryId,
+        ref: params.ref,
+        path,
+      });
+
+      if (Buffer.byteLength(snapshot.content, 'utf8') > MAX_REPOSITORY_FILE_BYTES) {
+        throw new DefinitionSyncPermanentError(
+          'content-too-large',
+          `Workflow file is larger than ${MAX_REPOSITORY_FILE_BYTES} bytes: ${snapshot.path}`,
+        );
+      }
+
+      return {
+        ...parseWorkflowSnapshot({path: snapshot.path, content: snapshot.content}),
+        rawContent: snapshot.content,
+      };
+    },
+    {stopOnError: true},
+  );
+
+  if (
+    !params.loadIntegrationValidationContext ||
+    !parsed.some((entry) => hasAgentStepIntegrations(entry.definition.document))
+  ) {
+    return parsed.map(({rawContent: _rawContent, ...entry}) => entry);
+  }
+
+  const integrationValidationContext = await params.loadIntegrationValidationContext();
+  return parsed.map((entry) =>
+    parseWorkflowSnapshot({
+      path: entry.path,
+      content: entry.rawContent,
+      integrationValidationContext,
+    }),
+  );
+}
+
+function parseWorkflowSnapshot(params: {
+  path: string;
+  content: string;
+  integrationValidationContext?: IntegrationValidationContext | undefined;
+}): ParsedWorkflow {
+  try {
+    const definition =
+      params.integrationValidationContext === undefined
+        ? parseDefinition(params.content)
+        : parseDefinition(params.content, {
+            integrationValidationContext: params.integrationValidationContext,
+          });
+    const contentHash = sha256Hex(params.content);
+    return {path: params.path, name: definition.document.name, definition, contentHash};
+  } catch (error) {
+    if (error instanceof DefinitionParseError) {
+      throw new DefinitionSyncPermanentError(
+        'invalid-definition',
+        `Invalid workflow definition at ${params.path}: ${error.message}`,
+      );
+    }
+    throw error;
+  }
+}
+
+export interface SyncFailureClassification {
+  code: DefinitionSyncErrorCode;
+  message: string;
+  retryable: boolean;
+}
+
+export function classifySyncFailure(error: unknown): SyncFailureClassification {
+  if (error instanceof DefinitionSyncPermanentError) {
+    return {code: error.code, message: error.message, retryable: false};
+  }
+  if (
+    error instanceof IntegrationConnectionNotFoundError ||
+    error instanceof IntegrationConnectionInactiveError ||
+    error instanceof IntegrationConnectionWorkspaceMismatchError
+  ) {
+    return {code: 'connection-unavailable', message: error.message, retryable: false};
+  }
+  if (error instanceof IntegrationProviderError) {
+    return {
+      code: providerErrorCode(error.reason),
+      message: error.message,
+      retryable: isProviderReasonRetryable(error.reason),
+    };
+  }
+  return {
+    code: 'unknown',
+    message: error instanceof Error ? error.message : String(error),
+    retryable: true,
+  };
+}
+
+function isProviderReasonRetryable(reason: string): boolean {
+  return reason === 'rate-limited' || reason === 'timeout' || reason === 'provider-unavailable';
+}
+
+function providerErrorCode(reason: string): DefinitionSyncErrorCode {
+  if (reason === 'repository-not-found') return 'provider-repository-not-found';
+  if (reason === 'file-not-found') return 'provider-file-not-found';
+  if (reason === 'access-denied') return 'provider-access-denied';
+  if (reason === 'rate-limited') return 'provider-rate-limited';
+  if (reason === 'timeout') return 'provider-timeout';
+  if (reason === 'provider-unavailable') return 'provider-unavailable';
+  if (reason === 'malformed-provider-response') return 'provider-malformed-response';
+  if (reason === 'content-too-large') return 'content-too-large';
+  if (reason === 'too-many-files') return 'too-many-files';
+  return 'unknown';
+}
+
+function sha256Hex(content: string): string {
+  return createHash('sha256').update(content, 'utf8').digest('hex');
+}