@shipfox/api-definitions 12.3.0 → 14.0.0

package/src/core/resolve-definition-at-ref.ts

@@ -0,0 +1,551 @@
+import type {AgentInterModuleClient} from '@shipfox/api-agent-dto/inter-module';
+import type {
+  TriggerDto,
+  WorkflowModelSnapshot,
+  WorkflowSourceSnapshot,
+} from '@shipfox/api-definitions-dto';
+import {createWorkflowModelSnapshot} from '@shipfox/api-definitions-dto';
+import {
+  type IntegrationsModuleClient,
+  integrationsInterModuleContract,
+} from '@shipfox/api-integration-core-dto/inter-module';
+import type {ProjectsModuleClient} from '@shipfox/api-projects-dto/inter-module';
+import {isInterModuleKnownError} from '@shipfox/inter-module';
+import {boundedMap} from '@shipfox/node-module';
+import {definitionTriggersFor} from '#db/definition-triggers.js';
+import {findOrCreateWorkflowLineage} from '#db/definitions.js';
+import {recordDefinitionRefResolution} from '#metrics/index.js';
+import type {ValidationDiagnostic} from './entities/validation-diagnostic.js';
+import {
+  DefinitionAtRefError,
+  type DefinitionAtRefErrorCode,
+  DefinitionParseError,
+} from './errors.js';
+import {hasAgentStepIntegrations} from './has-agent-step-integrations.js';
+import {loadIntegrationValidationContext} from './integrations.js';
+import type {ParsedDefinition} from './parse-definition.js';
+import {parseDefinitionWithDiagnostics} from './parse-definition.js';
+import {
+  DEFAULT_WORKFLOW_PATH,
+  FILE_FETCH_CONCURRENCY,
+  MAX_WORKFLOW_FILE_BYTES,
+  MAX_WORKFLOW_FILES,
+} from './sync-definitions.js';
+import type {ValidationError} from './validate-definition.js';
+
+export interface ResolveDefinitionAtRefParams {
+  projectId: string;
+  ref: string;
+  configPath: string;
+  expectedCommit?: string | undefined;
+  projects: ProjectsModuleClient;
+  agent: AgentInterModuleClient;
+  integrations: IntegrationsModuleClient;
+  signal?: AbortSignal;
+}
+
+export interface ValidationWarning {
+  code: string;
+  message: string;
+  path?: string | undefined;
+}
+
+export interface ResolvedDefinitionAtRef {
+  workflow: {id: string; configPath: string};
+  commit: string;
+  model: WorkflowModelSnapshot;
+  sourceSnapshot: WorkflowSourceSnapshot;
+  triggers: Record<string, TriggerDto>;
+  warnings: ValidationWarning[];
+}
+
+export interface ListDefinitionsAtRefParams {
+  projectId: string;
+  ref: string;
+  projects: ProjectsModuleClient;
+  agent: AgentInterModuleClient;
+  integrations: IntegrationsModuleClient;
+  signal?: AbortSignal;
+}
+
+export interface DefinitionAtRefFile {
+  configPath: string;
+  name: string | null;
+  valid: boolean;
+  errors: ValidationError[];
+  warnings: ValidationWarning[];
+  triggers: Record<string, TriggerDto>;
+}
+
+export interface DefinitionsAtRefListing {
+  commit: string;
+  files: DefinitionAtRefFile[];
+}
+
+interface ResolvedProjectSource {
+  workspaceId: string;
+  connectionId: string;
+  externalRepositoryId: string;
+}
+
+/**
+ * Resolves a workflow definition file at a git ref without persisting it.
+ * The ref is pinned to a commit, the file is fetched at that commit, and the
+ * result is validated with the sync pipeline. Only the workflow lineage row is
+ * created so the dev run can be numbered; no definition row and no outbox
+ * event are written.
+ */
+export async function resolveDefinitionAtRef(
+  params: ResolveDefinitionAtRefParams,
+): Promise<ResolvedDefinitionAtRef> {
+  try {
+    return await resolveDefinitionAtRefUnsafe(params);
+  } catch (error) {
+    if (error instanceof DefinitionAtRefError) {
+      recordDefinitionRefResolution(error.code);
+    }
+    throw error;
+  }
+}
+
+async function resolveDefinitionAtRefUnsafe(
+  params: ResolveDefinitionAtRefParams,
+): Promise<ResolvedDefinitionAtRef> {
+  throwIfAborted(params.signal);
+  const source = await requireProjectSource(params.projects, params.projectId, params.signal);
+  const commit = await resolveRefToCommit({
+    integrations: params.integrations,
+    source,
+    ref: params.ref,
+    signal: params.signal,
+  });
+  throwIfAborted(params.signal);
+  if (params.expectedCommit !== undefined && commit !== params.expectedCommit) {
+    throw new DefinitionAtRefError(
+      'ref-moved',
+      `Git ref ${params.ref} no longer resolves to the expected commit`,
+      {ref: params.ref, expectedCommit: params.expectedCommit},
+    );
+  }
+
+  const snapshot = await fetchFileAtCommit({
+    integrations: params.integrations,
+    source,
+    commit,
+    ref: params.ref,
+    configPath: params.configPath,
+    signal: params.signal,
+  });
+  assertFileSize(snapshot.content, snapshot.path);
+
+  const parsed = await parseDefinitionAtRef({
+    content: snapshot.content,
+    agent: params.agent,
+    integrations: params.integrations,
+    source,
+    signal: params.signal,
+  });
+  throwIfAborted(params.signal);
+  const workflowId = await findOrCreateWorkflowLineage({
+    projectId: params.projectId,
+    configPath: params.configPath,
+  });
+  recordDefinitionRefResolution('resolved');
+
+  return {
+    workflow: {id: workflowId, configPath: params.configPath},
+    commit,
+    model: createWorkflowModelSnapshot(parsed.model),
+    sourceSnapshot: {content: snapshot.content, format: 'yaml'},
+    triggers: definitionTriggersFor(parsed.model),
+    warnings: warningsFor(parsed.diagnostics),
+  };
+}
+
+/**
+ * Lists the workflow files at a git ref with their validation state. Applies
+ * the sync limits (100 files, 1 MB per file). One invalid file does not fail
+ * the listing; it is reported as invalid.
+ */
+export async function listDefinitionsAtRef(
+  params: ListDefinitionsAtRefParams,
+): Promise<DefinitionsAtRefListing> {
+  try {
+    return await listDefinitionsAtRefUnsafe(params);
+  } catch (error) {
+    if (error instanceof DefinitionAtRefError) {
+      recordDefinitionRefResolution(error.code);
+    }
+    throw error;
+  }
+}
+
+type ListingEntry =
+  | {path: string; content: string; definition: ParsedDefinition}
+  | {path: string; errors: ValidationError[]};
+
+async function listDefinitionsAtRefUnsafe(
+  params: ListDefinitionsAtRefParams,
+): Promise<DefinitionsAtRefListing> {
+  throwIfAborted(params.signal);
+  const source = await requireProjectSource(params.projects, params.projectId, params.signal);
+  const commit = await resolveRefToCommit({
+    integrations: params.integrations,
+    source,
+    ref: params.ref,
+    signal: params.signal,
+  });
+  throwIfAborted(params.signal);
+  const paths = await listWorkflowFilesAtCommit({
+    integrations: params.integrations,
+    source,
+    commit,
+    signal: params.signal,
+  });
+
+  const fetched = await boundedMap(
+    paths,
+    FILE_FETCH_CONCURRENCY,
+    (path) =>
+      fetchListingFile({
+        integrations: params.integrations,
+        source,
+        commit,
+        ref: params.ref,
+        path,
+        signal: params.signal,
+      }),
+    {stopOnError: true, signal: params.signal},
+  );
+  throwIfAborted(params.signal);
+  const agentValidationCatalog = await callWithSignal(
+    params.agent.getValidationCatalog,
+    {},
+    params.signal,
+  );
+  throwIfAborted(params.signal);
+  let entries = fetched.map((entry) => {
+    throwIfAborted(params.signal);
+    return parseListingEntry(entry, {agentValidationCatalog});
+  });
+
+  const needsIntegrationContext = entries.some(
+    (entry) => 'definition' in entry && hasAgentStepIntegrations(entry.definition.document),
+  );
+  if (needsIntegrationContext) {
+    const integrationValidationContext = await loadAtRefIntegrationValidationContext({
+      integrations: params.integrations,
+      source,
+      signal: params.signal,
+    });
+    entries = entries.map((entry) =>
+      'definition' in entry && hasAgentStepIntegrations(entry.definition.document)
+        ? parseListingEntry(
+            {path: entry.path, content: entry.content},
+            {agentValidationCatalog, integrationValidationContext},
+          )
+        : entry,
+    );
+  }
+
+  throwIfAborted(params.signal);
+  recordDefinitionRefResolution('resolved');
+  return {commit, files: entries.map((entry) => listingFileFor(entry))};
+}
+
+async function requireProjectSource(
+  projects: ProjectsModuleClient,
+  projectId: string,
+  signal: AbortSignal | undefined,
+): Promise<ResolvedProjectSource> {
+  const {project} = await callWithSignal(projects.getProjectById, {projectId}, signal);
+  if (project === null) {
+    throw new DefinitionAtRefError('project-not-found', `Project not found: ${projectId}`, {
+      projectId,
+    });
+  }
+  return {
+    workspaceId: project.workspaceId,
+    connectionId: project.sourceConnectionId,
+    externalRepositoryId: project.sourceExternalRepositoryId,
+  };
+}
+
+async function resolveRefToCommit(params: {
+  integrations: IntegrationsModuleClient;
+  source: ResolvedProjectSource;
+  ref: string;
+  signal: AbortSignal | undefined;
+}): Promise<string> {
+  try {
+    const resolved = await callWithSignal(
+      params.integrations.resolveSourceRef,
+      {
+        workspaceId: params.source.workspaceId,
+        connectionId: params.source.connectionId,
+        externalRepositoryId: params.source.externalRepositoryId,
+        ref: params.ref,
+      },
+      params.signal,
+    );
+    return resolved.commit;
+  } catch (error) {
+    if (isInterModuleKnownError(integrationsInterModuleContract.methods.resolveSourceRef, error)) {
+      if (error.code === 'ref-not-found') {
+        throw new DefinitionAtRefError('ref-not-found', `Git ref not found: ${params.ref}`, {
+          ref: params.ref,
+        });
+      }
+      if (error.code === 'ref-invalid') {
+        throw new DefinitionAtRefError(
+          'ref-invalid',
+          `Git ref is not a resolvable branch or tag name: ${params.ref}`,
+          {ref: params.ref},
+        );
+      }
+      throw sourceUnavailable(error, 'The source repository is unavailable');
+    }
+    throw error;
+  }
+}
+
+async function listWorkflowFilesAtCommit(params: {
+  integrations: IntegrationsModuleClient;
+  source: ResolvedProjectSource;
+  commit: string;
+  signal: AbortSignal | undefined;
+}): Promise<string[]> {
+  let page: Awaited<ReturnType<IntegrationsModuleClient['listSourceFiles']>>;
+  try {
+    page = await callWithSignal(
+      params.integrations.listSourceFiles,
+      {
+        workspaceId: params.source.workspaceId,
+        connectionId: params.source.connectionId,
+        externalRepositoryId: params.source.externalRepositoryId,
+        ref: params.commit,
+        prefix: DEFAULT_WORKFLOW_PATH,
+        limit: MAX_WORKFLOW_FILES,
+      },
+      params.signal,
+    );
+  } catch (error) {
+    throwIfAborted(params.signal);
+    throw sourceUnavailable(error, 'The workflow files at the ref could not be listed');
+  }
+  if (page.nextCursor) {
+    throw new DefinitionAtRefError(
+      'too-many-files',
+      `More than ${MAX_WORKFLOW_FILES} workflow files were found`,
+    );
+  }
+  return page.files
+    .filter((file) => file.path.endsWith('.yml') || file.path.endsWith('.yaml'))
+    .map((file) => file.path);
+}
+
+async function fetchFileAtCommit(params: {
+  integrations: IntegrationsModuleClient;
+  source: ResolvedProjectSource;
+  commit: string;
+  ref: string;
+  configPath: string;
+  signal: AbortSignal | undefined;
+}): Promise<{path: string; content: string}> {
+  try {
+    return await callWithSignal(
+      params.integrations.fetchSourceFile,
+      {
+        workspaceId: params.source.workspaceId,
+        connectionId: params.source.connectionId,
+        externalRepositoryId: params.source.externalRepositoryId,
+        ref: params.commit,
+        path: params.configPath,
+      },
+      params.signal,
+    );
+  } catch (error) {
+    throwIfAborted(params.signal);
+    if (isInterModuleKnownError(integrationsInterModuleContract.methods.fetchSourceFile, error)) {
+      if (error.code === 'provider-failure' && error.details.reason === 'file-not-found') {
+        throw new DefinitionAtRefError(
+          'file-not-found',
+          `Workflow file not found at ${params.ref}: ${params.configPath}`,
+          {ref: params.ref, configPath: params.configPath},
+        );
+      }
+      throw sourceUnavailable(error, 'The workflow file at the ref could not be fetched');
+    }
+    throw error;
+  }
+}
+
+function assertFileSize(content: string, path: string): void {
+  if (Buffer.byteLength(content, 'utf8') > MAX_WORKFLOW_FILE_BYTES) {
+    throw new DefinitionAtRefError(
+      'content-too-large',
+      `Workflow file is larger than ${MAX_WORKFLOW_FILE_BYTES} bytes: ${path}`,
+      {configPath: path},
+    );
+  }
+}
+
+async function parseDefinitionAtRef(params: {
+  content: string;
+  agent: AgentInterModuleClient;
+  integrations: IntegrationsModuleClient;
+  source: ResolvedProjectSource;
+  signal: AbortSignal | undefined;
+}): Promise<ParsedDefinition> {
+  const agentValidationCatalog = await callWithSignal(
+    params.agent.getValidationCatalog,
+    {},
+    params.signal,
+  );
+  throwIfAborted(params.signal);
+  const firstPass = parseWorkflowDefinition(params.content, {agentValidationCatalog});
+  if (!hasAgentStepIntegrations(firstPass.document)) return firstPass;
+
+  const integrationValidationContext = await loadAtRefIntegrationValidationContext({
+    integrations: params.integrations,
+    source: params.source,
+    signal: params.signal,
+  });
+  return parseWorkflowDefinition(params.content, {
+    agentValidationCatalog,
+    integrationValidationContext,
+  });
+}
+
+function parseWorkflowDefinition(
+  content: string,
+  options: Parameters<typeof parseDefinitionWithDiagnostics>[1],
+): ParsedDefinition {
+  try {
+    return parseDefinitionWithDiagnostics(content, options);
+  } catch (error) {
+    if (error instanceof DefinitionParseError) {
+      throw new DefinitionAtRefError(
+        'invalid-definition',
+        `Invalid workflow definition: ${error.message}`,
+        {errors: (error.details ?? []) as ValidationError[]},
+      );
+    }
+    throw error;
+  }
+}
+
+async function fetchListingFile(params: {
+  integrations: IntegrationsModuleClient;
+  source: ResolvedProjectSource;
+  commit: string;
+  ref: string;
+  path: string;
+  signal: AbortSignal | undefined;
+}): Promise<{path: string; content: string} | {path: string; errors: ValidationError[]}> {
+  try {
+    const snapshot = await fetchFileAtCommit({
+      integrations: params.integrations,
+      source: params.source,
+      commit: params.commit,
+      ref: params.ref,
+      configPath: params.path,
+      signal: params.signal,
+    });
+    assertFileSize(snapshot.content, snapshot.path);
+    return {path: snapshot.path, content: snapshot.content};
+  } catch (error) {
+    if (error instanceof DefinitionAtRefError && isPerFileListingError(error.code)) {
+      return {path: params.path, errors: [{message: error.message}]};
+    }
+    throw error;
+  }
+}
+
+function parseListingEntry(
+  entry: {path: string; content: string} | {path: string; errors: ValidationError[]},
+  options: Parameters<typeof parseDefinitionWithDiagnostics>[1],
+): ListingEntry {
+  if ('errors' in entry) return entry;
+  try {
+    const definition = parseDefinitionWithDiagnostics(entry.content, options);
+    return {path: entry.path, content: entry.content, definition};
+  } catch (error) {
+    if (error instanceof DefinitionParseError) {
+      return {path: entry.path, errors: (error.details ?? []) as ValidationError[]};
+    }
+    throw error;
+  }
+}
+
+function listingFileFor(entry: ListingEntry): DefinitionAtRefFile {
+  if ('definition' in entry) {
+    return {
+      configPath: entry.path,
+      name: entry.definition.document.name,
+      valid: true,
+      errors: [],
+      warnings: warningsFor(entry.definition.diagnostics),
+      triggers: definitionTriggersFor(entry.definition.model),
+    };
+  }
+  return {
+    configPath: entry.path,
+    name: null,
+    valid: false,
+    errors: entry.errors,
+    warnings: [],
+    triggers: {},
+  };
+}
+
+function warningsFor(diagnostics: readonly ValidationDiagnostic[]): ValidationWarning[] {
+  return diagnostics
+    .filter((diagnostic) => diagnostic.severity === 'warning')
+    .map((diagnostic) => ({
+      code: diagnostic.code,
+      message: diagnostic.message,
+      ...(diagnostic.path === undefined ? {} : {path: diagnostic.path}),
+    }));
+}
+
+async function loadAtRefIntegrationValidationContext(params: {
+  integrations: IntegrationsModuleClient;
+  source: ResolvedProjectSource;
+  signal: AbortSignal | undefined;
+}) {
+  try {
+    return await loadIntegrationValidationContext(
+      params.integrations,
+      params.source.workspaceId,
+      params.source.connectionId,
+      params.signal,
+    );
+  } catch (error) {
+    throwIfAborted(params.signal);
+    throw sourceUnavailable(error, 'Integration validation context is unavailable');
+  }
+}
+
+function isPerFileListingError(code: DefinitionAtRefErrorCode): boolean {
+  return code === 'file-not-found' || code === 'content-too-large' || code === 'invalid-definition';
+}
+
+function callWithSignal<Input, Output>(
+  method: (input: Input, options?: {signal?: AbortSignal}) => Promise<Output>,
+  input: Input,
+  signal: AbortSignal | undefined,
+): Promise<Output> {
+  return signal === undefined ? method(input) : method(input, {signal});
+}
+
+function throwIfAborted(signal: AbortSignal | undefined): void {
+  if (signal?.aborted) throw signal.reason ?? new Error('Operation aborted');
+}
+
+function sourceUnavailable(error: unknown, message: string): DefinitionAtRefError {
+  return new DefinitionAtRefError(
+    'source-unavailable',
+    `${message}: ${error instanceof Error ? error.message : String(error)}`,
+  );
+}

package/src/core/sync-definitions.test.ts

@@ -79,6 +79,11 @@ const integrationValidationContext = {
   workspaceConnectionSnapshot: new Map([
     ['github-main', {id: 'connection-1', provider: 'github', capabilities: ['agent_tools']}],
   ]),
+  eventCatalogs: new Map([
+    ['github', new Set(['push', 'pull_request.opened'])],
+    ['webhook', new Set(['received'])],
+  ]),
+  fixedEventProviders: new Set(['webhook']),
 } satisfies IntegrationValidationContext;
 
 function sourceControl(
@@ -243,7 +248,7 @@ describe('fetchAndParseWorkflows', () => {
     expect(result[0]?.name).toBe('CI');
     expect(result[0]?.path).toBe('.shipfox/workflows/ci.yml');
     expect(result[0]?.contentHash).toMatch(LOWERCASE_SHA256_HEX_RE);
-    expect(result[0]?.warnings).toEqual([]);
+    expect(result[0]?.diagnostics).toEqual([]);
   });
 
   it('keeps warning-only definitions available to the sync path', async () => {
@@ -262,15 +267,69 @@ describe('fetchAndParseWorkflows', () => {
       }),
     });
 
-    expect(result[0]?.warnings).toMatchObject([
+    expect(result[0]?.diagnostics).toMatchObject([
       {
         code: 're-evaluating-command',
         path: 'jobs.build.steps.0.run',
+        severity: 'warning',
       },
     ]);
     expect(result[0]?.definition.model.jobs[0]?.steps).toHaveLength(1);
   });
 
+  it('keeps definitions with inert trigger-scoped errors available to the sync path', async () => {
+    const brokenTriggerPath = '.shipfox/workflows/broken-trigger.yml';
+    const validTriggerPath = '.shipfox/workflows/valid-trigger.yml';
+    const brokenTriggerYaml = `
+name: Broken trigger
+runner: ubuntu-latest
+triggers:
+  nightly:
+    source: cron
+    event: tick
+    config:
+      schedule: "not a cron"
+  on_demand:
+    source: manual
+    event: fire
+jobs:
+  build:
+    steps:
+      - run: pnpm test
+`;
+
+    const fetchFile = vi.fn(({path}: {path: string}) =>
+      Promise.resolve({
+        path,
+        ref: 'main',
+        content: path === brokenTriggerPath ? brokenTriggerYaml : validYaml,
+      }),
+    );
+    const result = await fetchAndParseWorkflows({
+      ...baseContext,
+      ref: 'main',
+      paths: [brokenTriggerPath, validTriggerPath],
+      sourceControl: sourceControl({fetchFile}),
+    });
+
+    expect(result).toHaveLength(2);
+    expect(result[0]?.diagnostics).toEqual([
+      {
+        code: 'invalid-cron-schedule',
+        message: 'Cron trigger schedule must be a valid 5-field cron expression.',
+        path: 'triggers.nightly.config.schedule',
+        severity: 'error',
+      },
+    ]);
+    // The broken cron trigger is inert; the manual trigger stays active.
+    expect(result[0]?.definition.model.triggers.map((trigger) => trigger.key)).toEqual([
+      'on_demand',
+    ]);
+    expect(result[1]?.path).toBe(validTriggerPath);
+    expect(result[1]?.diagnostics).toEqual([]);
+    expect(result[1]?.definition.model.jobs[0]?.steps).toHaveLength(1);
+  });
+
   it('produces stable content hashes for identical content', async () => {
     const sourceControlA = sourceControl();
     const sourceControlB = sourceControl();

package/src/core/sync-definitions.ts

@@ -5,12 +5,12 @@ import {isInterModuleKnownError} from '@shipfox/inter-module';
 import {boundedMap} from '@shipfox/node-module';
 import type {IntegrationValidationContext} from './entities/integration-context.js';
 import type {DefinitionSyncErrorCode} from './entities/sync-state.js';
-import type {ValidationWarning} from './entities/validation-warning.js';
+import type {ValidationDiagnostic} from './entities/validation-diagnostic.js';
 import type {WorkflowDefinitionPayload} from './entities/workflow-definition.js';
 import {DefinitionParseError, DefinitionSyncPermanentError} from './errors.js';
 import {hasAgentStepIntegrations} from './has-agent-step-integrations.js';
 import type {DefinitionsSourceControl} from './integrations.js';
-import {parseDefinitionWithWarnings, stripDefinitionWarnings} from './parse-definition.js';
+import {parseDefinitionWithDiagnostics, stripDefinitionDiagnostics} from './parse-definition.js';
 
 export const DEFAULT_WORKFLOW_PATH = '.shipfox/workflows/';
 export const MAX_WORKFLOW_FILES = 100;
@@ -80,7 +80,7 @@ export interface ParsedWorkflow {
   name: string;
   definition: WorkflowDefinitionPayload;
   contentHash: string;
-  warnings: ValidationWarning[];
+  diagnostics: ValidationDiagnostic[];
 }
 
 export interface FetchAndParseWorkflowsParams extends SyncSourceContext {
@@ -154,10 +154,10 @@ function parseWorkflowSnapshot(params: {
   try {
     const definition =
       params.integrationValidationContext === undefined
-        ? parseDefinitionWithWarnings(params.content, {
+        ? parseDefinitionWithDiagnostics(params.content, {
             agentValidationCatalog: params.agentValidationCatalog,
           })
-        : parseDefinitionWithWarnings(params.content, {
+        : parseDefinitionWithDiagnostics(params.content, {
             agentValidationCatalog: params.agentValidationCatalog,
             integrationValidationContext: params.integrationValidationContext,
           });
@@ -165,9 +165,9 @@ function parseWorkflowSnapshot(params: {
     return {
       path: params.path,
       name: definition.document.name,
-      definition: stripDefinitionWarnings(definition),
+      definition: stripDefinitionDiagnostics(definition),
       contentHash,
-      warnings: definition.warnings,
+      diagnostics: definition.diagnostics,
     };
   } catch (error) {
     if (error instanceof DefinitionParseError) {