@shipfox/api-definitions 2.0.0

package/src/db/definitions.ts

@@ -0,0 +1,357 @@
+import {
+  DEFINITION_DELETED,
+  DEFINITION_RESOLVED,
+  type DefinitionsEventMap,
+} from '@shipfox/api-definitions-dto';
+import {writeOutboxEvent} from '@shipfox/node-outbox';
+import type {WorkflowDocument} from '@shipfox/workflow-document';
+import {and, asc, eq, gt, isNull, notInArray, or, type SQL, sql} from 'drizzle-orm';
+import type {
+  WorkflowDefinition,
+  WorkflowDefinitionPayload,
+  WorkflowSourceSnapshot,
+} from '#core/entities/workflow-definition.js';
+import type {WorkflowModel} from '#core/entities/workflow-model.js';
+import {db} from './db.js';
+import {definitionTriggersFor} from './definition-triggers.js';
+import {toDefinition, workflowDefinitions} from './schema/definitions.js';
+import {definitionsOutbox} from './schema/outbox.js';
+
+type Tx = Parameters<Parameters<ReturnType<typeof db>['transaction']>[0]>[0];
+
+export interface UpsertDefinitionParams {
+  projectId: string;
+  workspaceId: string;
+  configPath?: string | null | undefined;
+  source?: 'manual' | 'vcs' | undefined;
+  name: string;
+  document: WorkflowDocument;
+  model: WorkflowModel;
+  sourceSnapshot?: WorkflowSourceSnapshot | null | undefined;
+  contentHash?: string | null | undefined;
+  sha?: string | undefined;
+  ref?: string | undefined;
+}
+
+function buildUpsertQuery(tx: Tx, params: UpsertDefinitionParams) {
+  const source = params.source ?? 'manual';
+  if (source === 'vcs' && !params.configPath) {
+    throw new Error('configPath is required for VCS definitions');
+  }
+  // Keep source aligned with the conflict arbiter (chosen by ref/sha presence):
+  // vcs rows are versioned by ref/sha, manual rows have neither.
+  const hasRefOrSha = params.ref != null || params.sha != null;
+  if ((source === 'vcs') !== hasRefOrSha) {
+    throw new Error(
+      'Definition source/ref/sha mismatch: vcs requires a ref or sha, manual requires neither',
+    );
+  }
+
+  const definition: WorkflowDefinitionPayload = {
+    document: params.document,
+    model: params.model,
+    sourceSnapshot: params.sourceSnapshot ?? null,
+  };
+
+  const set = {
+    name: params.name,
+    source,
+    definition,
+    contentHash: params.contentHash ?? null,
+    fetchedAt: sql`now()`,
+    updatedAt: sql`now()`,
+    deletedAt: null,
+  };
+
+  const values = {
+    projectId: params.projectId,
+    configPath: params.configPath ?? null,
+    source,
+    sha: params.sha ?? null,
+    ref: params.ref ?? null,
+    name: params.name,
+    definition,
+    contentHash: params.contentHash ?? null,
+  };
+
+  if (params.sha) {
+    return tx
+      .insert(workflowDefinitions)
+      .values(values)
+      .onConflictDoUpdate({
+        target: [
+          workflowDefinitions.projectId,
+          workflowDefinitions.sha,
+          workflowDefinitions.configPath,
+        ],
+        targetWhere: sql`"sha" IS NOT NULL`,
+        set,
+      })
+      .returning();
+  }
+
+  if (params.ref) {
+    return tx
+      .insert(workflowDefinitions)
+      .values(values)
+      .onConflictDoUpdate({
+        target: [
+          workflowDefinitions.projectId,
+          workflowDefinitions.ref,
+          workflowDefinitions.configPath,
+        ],
+        targetWhere: sql`"ref" IS NOT NULL`,
+        set,
+      })
+      .returning();
+  }
+
+  return tx
+    .insert(workflowDefinitions)
+    .values(values)
+    .onConflictDoUpdate({
+      target: [workflowDefinitions.projectId, workflowDefinitions.configPath],
+      targetWhere: sql`"config_path" IS NOT NULL AND "ref" IS NULL AND "sha" IS NULL`,
+      set,
+    })
+    .returning();
+}
+
+export async function upsertDefinition(
+  params: UpsertDefinitionParams,
+): Promise<WorkflowDefinition> {
+  return await db().transaction(async (tx) => {
+    const rows = await buildUpsertQuery(tx, params);
+    const row = rows[0];
+    if (!row) throw new Error('Upsert returned no rows');
+
+    await writeOutboxEvent<DefinitionsEventMap>(tx, definitionsOutbox, {
+      type: DEFINITION_RESOLVED,
+      payload: {
+        definitionId: row.id,
+        projectId: row.projectId,
+        workspaceId: params.workspaceId,
+        configPath: row.configPath,
+        triggers: definitionTriggersFor(row.definition.model),
+      },
+    });
+
+    return toDefinition(row);
+  });
+}
+
+export async function getDefinitionById(id: string): Promise<WorkflowDefinition | undefined> {
+  const rows = await db()
+    .select()
+    .from(workflowDefinitions)
+    .where(and(eq(workflowDefinitions.id, id), isNull(workflowDefinitions.deletedAt)))
+    .limit(1);
+  const row = rows[0];
+
+  if (!row) return undefined;
+  return toDefinition(row);
+}
+
+export interface DefinitionCursor {
+  value: string;
+  id: string;
+}
+
+export interface ListDefinitionsParams {
+  projectId: string;
+  limit: number;
+  cursor?: DefinitionCursor | undefined;
+}
+
+export interface ListDefinitionsResult {
+  definitions: WorkflowDefinition[];
+  nextCursor: DefinitionCursor | null;
+}
+
+function cursorWhere(cursor: DefinitionCursor | undefined): SQL | undefined {
+  if (!cursor) return undefined;
+  return or(
+    gt(workflowDefinitions.name, cursor.value),
+    and(eq(workflowDefinitions.name, cursor.value), gt(workflowDefinitions.id, cursor.id)),
+  );
+}
+
+export async function listDefinitions(
+  params: ListDefinitionsParams,
+): Promise<ListDefinitionsResult> {
+  const conditions = [
+    eq(workflowDefinitions.projectId, params.projectId),
+    isNull(workflowDefinitions.deletedAt),
+  ];
+  const cursorCondition = cursorWhere(params.cursor);
+  if (cursorCondition) conditions.push(cursorCondition);
+
+  const rows = await db()
+    .select()
+    .from(workflowDefinitions)
+    .where(and(...conditions))
+    .orderBy(asc(workflowDefinitions.name), asc(workflowDefinitions.id))
+    .limit(params.limit + 1);
+
+  const hasMore = rows.length > params.limit;
+  const pageRows = hasMore ? rows.slice(0, params.limit) : rows;
+  const last = pageRows.at(-1);
+
+  return {
+    definitions: pageRows.map(toDefinition),
+    nextCursor: hasMore && last ? {value: last.name, id: last.id} : null,
+  };
+}
+
+export async function listDefinitionsByProject(projectId: string): Promise<WorkflowDefinition[]> {
+  const result = await listDefinitions({projectId, limit: 100});
+  return result.definitions;
+}
+
+export interface SoftDeleteVcsDefinitionsParams {
+  projectId: string;
+  workspaceId: string;
+  ref: string;
+  keepConfigPaths: string[];
+}
+
+async function softDeleteVcsDefinitionsNotInTx(
+  tx: Tx,
+  params: SoftDeleteVcsDefinitionsParams,
+): Promise<number> {
+  const now = sql`now()`;
+  const baseWhere = and(
+    eq(workflowDefinitions.projectId, params.projectId),
+    eq(workflowDefinitions.source, 'vcs'),
+    eq(workflowDefinitions.ref, params.ref),
+    isNull(workflowDefinitions.deletedAt),
+  );
+  const where =
+    params.keepConfigPaths.length > 0
+      ? and(baseWhere, notInArray(workflowDefinitions.configPath, params.keepConfigPaths))
+      : baseWhere;
+
+  const rows = await tx
+    .update(workflowDefinitions)
+    .set({deletedAt: now, updatedAt: now})
+    .where(where)
+    .returning({id: workflowDefinitions.id});
+
+  for (const row of rows) {
+    await writeOutboxEvent<DefinitionsEventMap>(tx, definitionsOutbox, {
+      type: DEFINITION_DELETED,
+      payload: {
+        definitionId: row.id,
+        projectId: params.projectId,
+        workspaceId: params.workspaceId,
+      },
+    });
+  }
+
+  return rows.length;
+}
+
+export async function softDeleteVcsDefinitionsNotIn(
+  params: SoftDeleteVcsDefinitionsParams,
+): Promise<number> {
+  return await db().transaction((tx) => softDeleteVcsDefinitionsNotInTx(tx, params));
+}
+
+export interface ApplyVcsDefinitionsBatchParams {
+  projectId: string;
+  workspaceId: string;
+  ref: string;
+  upserts: Array<{
+    configPath: string;
+    name: string;
+    document: WorkflowDocument;
+    model: WorkflowModel;
+    sourceSnapshot?: WorkflowSourceSnapshot | null | undefined;
+    contentHash: string;
+  }>;
+}
+
+export interface ApplyVcsDefinitionsBatchResult {
+  appliedCount: number;
+  deletedCount: number;
+}
+
+export async function applyVcsDefinitionsBatch(
+  params: ApplyVcsDefinitionsBatchParams,
+): Promise<ApplyVcsDefinitionsBatchResult> {
+  return await db().transaction(async (tx) => {
+    let appliedCount = 0;
+    for (const item of params.upserts) {
+      const existing = await tx
+        .select({
+          contentHash: workflowDefinitions.contentHash,
+          deletedAt: workflowDefinitions.deletedAt,
+        })
+        .from(workflowDefinitions)
+        .where(
+          and(
+            eq(workflowDefinitions.projectId, params.projectId),
+            eq(workflowDefinitions.ref, params.ref),
+            eq(workflowDefinitions.configPath, item.configPath),
+          ),
+        )
+        .limit(1);
+
+      const previous = existing[0];
+      const unchanged =
+        previous !== undefined &&
+        previous.deletedAt === null &&
+        previous.contentHash === item.contentHash;
+
+      const rows = await buildUpsertQuery(tx, {
+        projectId: params.projectId,
+        workspaceId: params.workspaceId,
+        configPath: item.configPath,
+        source: 'vcs',
+        ref: params.ref,
+        name: item.name,
+        document: item.document,
+        model: item.model,
+        sourceSnapshot: item.sourceSnapshot ?? null,
+        contentHash: item.contentHash,
+      });
+      const row = rows[0];
+      if (!row) throw new Error('Upsert returned no rows');
+
+      if (!unchanged) {
+        await writeOutboxEvent<DefinitionsEventMap>(tx, definitionsOutbox, {
+          type: DEFINITION_RESOLVED,
+          payload: {
+            definitionId: row.id,
+            projectId: row.projectId,
+            workspaceId: params.workspaceId,
+            configPath: row.configPath,
+            triggers: definitionTriggersFor(row.definition.model),
+          },
+        });
+        appliedCount += 1;
+      }
+    }
+
+    const keepConfigPaths = params.upserts.map((upsert) => upsert.configPath);
+    const deletedCount = await softDeleteVcsDefinitionsNotInTx(tx, {
+      projectId: params.projectId,
+      workspaceId: params.workspaceId,
+      ref: params.ref,
+      keepConfigPaths,
+    });
+
+    return {appliedCount, deletedCount};
+  });
+}
+
+export async function invalidateCache(params: {projectId: string; ref: string}): Promise<void> {
+  await db()
+    .delete(workflowDefinitions)
+    .where(
+      and(
+        eq(workflowDefinitions.projectId, params.projectId),
+        eq(workflowDefinitions.ref, params.ref),
+      ),
+    );
+}

package/src/db/index.ts

@@ -0,0 +1,31 @@
+import {dirname, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+export {closeDb, db, schema} from './db.js';
+export type {
+  ApplyVcsDefinitionsBatchParams,
+  ApplyVcsDefinitionsBatchResult,
+  ListDefinitionsParams,
+  ListDefinitionsResult,
+  SoftDeleteVcsDefinitionsParams,
+  UpsertDefinitionParams,
+} from './definitions.js';
+export {
+  applyVcsDefinitionsBatch,
+  getDefinitionById,
+  invalidateCache,
+  listDefinitions,
+  listDefinitionsByProject,
+  softDeleteVcsDefinitionsNotIn,
+  upsertDefinition,
+} from './definitions.js';
+export {definitionsOutbox} from './schema/outbox.js';
+export {definitionSyncStates} from './schema/sync-states.js';
+export {
+  type DefinitionSyncStateKey,
+  getLatestDefinitionSyncState,
+  type MarkDefinitionSyncParams,
+  markDefinitionSyncState,
+} from './sync-states.js';
+
+export const migrationsPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../drizzle');

package/src/db/schema/common.ts

@@ -0,0 +1,3 @@
+import {pgTableCreator} from 'drizzle-orm/pg-core';
+
+export const pgTable = pgTableCreator((name) => `definitions_${name}`);

package/src/db/schema/definitions.test.ts

@@ -0,0 +1,36 @@
+import {normalizeWorkflowDocument} from '#core/workflow-model/index.js';
+import type {DefinitionDb} from './definitions.js';
+import {toDefinition} from './definitions.js';
+
+describe('toDefinition', () => {
+  it('maps legacy definition payloads without source snapshots to null', () => {
+    const document = {
+      name: 'Manual workflow',
+      runner: 'ubuntu-latest',
+      jobs: {
+        build: {
+          steps: [{run: 'pnpm build'}],
+        },
+      },
+    };
+    const row: DefinitionDb = {
+      id: '019e98ab-6656-7ca1-b9ad-1ca4442c479d',
+      projectId: '019e98ab-b90f-7265-b13c-8b441c991381',
+      configPath: '.shipfox/workflows/manual.yml',
+      source: 'manual',
+      sha: null,
+      ref: null,
+      name: 'Manual workflow',
+      definition: {document, model: normalizeWorkflowDocument(document)},
+      contentHash: null,
+      fetchedAt: new Date('2026-06-09T10:00:00.000Z'),
+      createdAt: new Date('2026-06-09T10:00:01.000Z'),
+      updatedAt: new Date('2026-06-09T10:00:02.000Z'),
+      deletedAt: null,
+    };
+
+    const result = toDefinition(row);
+
+    expect(result.sourceSnapshot).toBeNull();
+  });
+});

package/src/db/schema/definitions.ts

@@ -0,0 +1,76 @@
+import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';
+import {sql} from 'drizzle-orm';
+import {check, index, jsonb, pgEnum, text, timestamp, uniqueIndex, uuid} from 'drizzle-orm/pg-core';
+import type {
+  WorkflowDefinition,
+  WorkflowDefinitionPayload,
+} from '#core/entities/workflow-definition.js';
+import {pgTable} from './common.js';
+
+export const definitionSourceEnum = pgEnum('definitions_source', ['manual', 'vcs']);
+
+export const workflowDefinitions = pgTable(
+  'workflow_definitions',
+  {
+    id: uuidv7PrimaryKey(),
+    projectId: uuid('project_id').notNull(),
+    configPath: text('config_path'),
+    source: definitionSourceEnum('source').notNull().default('manual'),
+    sha: text('sha'),
+    ref: text('ref'),
+    name: text('name').notNull(),
+    definition: jsonb('definition').notNull().$type<WorkflowDefinitionPayload>(),
+    contentHash: text('content_hash'),
+    fetchedAt: timestamp('fetched_at', {withTimezone: true}).notNull().defaultNow(),
+    createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
+    updatedAt: timestamp('updated_at', {withTimezone: true}).notNull().defaultNow(),
+    deletedAt: timestamp('deleted_at', {withTimezone: true}),
+  },
+  (table) => [
+    // Manual definitions only (no ref, no sha); VCS uniqueness is sha_lookup/ref_lookup.
+    // A source-agnostic predicate here collides VCS upserts on a non-arbiter index. ENG-659.
+    uniqueIndex('definitions_wd_manual_unique')
+      .on(table.projectId, table.configPath)
+      .where(sql`"config_path" IS NOT NULL AND "ref" IS NULL AND "sha" IS NULL`),
+    uniqueIndex('definitions_wd_sha_lookup')
+      .on(table.projectId, table.sha, table.configPath)
+      .where(sql`"sha" IS NOT NULL`),
+    uniqueIndex('definitions_wd_ref_lookup')
+      .on(table.projectId, table.ref, table.configPath)
+      .where(sql`"ref" IS NOT NULL`),
+    index('definitions_wd_project_name_id_idx')
+      .on(table.projectId, table.name, table.id)
+      .where(sql`"deleted_at" IS NULL`),
+    // Binds source to its git coordinates so the partial indexes above stay
+    // unambiguous: vcs rows carry a ref or sha; manual rows carry neither.
+    check(
+      'definitions_wd_source_ref_sha_consistent',
+      sql`("source" = 'vcs') = ("ref" IS NOT NULL OR "sha" IS NOT NULL)`,
+    ),
+  ],
+);
+
+export type DefinitionDb = typeof workflowDefinitions.$inferSelect;
+export type DefinitionCreateDb = typeof workflowDefinitions.$inferInsert;
+export type DefinitionUpdateDb = Partial<DefinitionCreateDb>;
+
+export function toDefinition(row: DefinitionDb): WorkflowDefinition {
+  return {
+    id: row.id,
+    projectId: row.projectId,
+    configPath: row.configPath,
+    source: row.source,
+    sha: row.sha,
+    ref: row.ref,
+    name: row.name,
+    definition: row.definition.document,
+    document: row.definition.document,
+    model: row.definition.model,
+    sourceSnapshot: row.definition.sourceSnapshot ?? null,
+    contentHash: row.contentHash,
+    fetchedAt: row.fetchedAt,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+    deletedAt: row.deletedAt,
+  };
+}

package/src/db/schema/outbox.ts

@@ -0,0 +1,4 @@
+import {createOutboxTable} from '@shipfox/node-outbox';
+import {pgTable} from './common.js';
+
+export const definitionsOutbox = createOutboxTable(pgTable);

package/src/db/schema/sync-states.ts

@@ -0,0 +1,79 @@
+import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';
+import {sql} from 'drizzle-orm';
+import {index, pgEnum, text, timestamp, uniqueIndex, uuid} from 'drizzle-orm/pg-core';
+import type {
+  DefinitionSyncErrorCode,
+  DefinitionSyncState,
+  DefinitionSyncStatus,
+} from '#core/entities/sync-state.js';
+import {pgTable} from './common.js';
+
+export const definitionSyncStatusEnum = pgEnum('definitions_sync_status', [
+  'pending',
+  'syncing',
+  'succeeded',
+  'failed',
+]);
+
+export const definitionSyncErrorCodeEnum = pgEnum('definitions_sync_error_code', [
+  'no-workflow-files',
+  'invalid-definition',
+  'provider-repository-not-found',
+  'provider-file-not-found',
+  'provider-access-denied',
+  'provider-rate-limited',
+  'provider-timeout',
+  'provider-unavailable',
+  'provider-malformed-response',
+  'content-too-large',
+  'too-many-files',
+  'connection-unavailable',
+  'unknown',
+]);
+
+export const definitionSyncStates = pgTable(
+  'sync_states',
+  {
+    id: uuidv7PrimaryKey(),
+    projectId: uuid('project_id').notNull(),
+    sourceConnectionId: uuid('source_connection_id').notNull(),
+    sourceExternalRepositoryId: text('source_external_repository_id').notNull(),
+    ref: text('ref').notNull(),
+    status: definitionSyncStatusEnum('status').notNull().default('pending'),
+    lastErrorCode: definitionSyncErrorCodeEnum('last_error_code'),
+    lastErrorMessage: text('last_error_message'),
+    startedAt: timestamp('started_at', {withTimezone: true}),
+    finishedAt: timestamp('finished_at', {withTimezone: true}),
+    createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
+    updatedAt: timestamp('updated_at', {withTimezone: true}).notNull().defaultNow(),
+  },
+  (table) => [
+    uniqueIndex('definitions_sync_states_source_unique').on(
+      table.projectId,
+      table.sourceConnectionId,
+      table.sourceExternalRepositoryId,
+      table.ref,
+    ),
+    index('definitions_sync_states_failed_idx').on(table.updatedAt).where(sql`"status" = 'failed'`),
+  ],
+);
+
+export type DefinitionSyncStateDb = typeof definitionSyncStates.$inferSelect;
+export type DefinitionSyncStateCreateDb = typeof definitionSyncStates.$inferInsert;
+
+export function toDefinitionSyncState(row: DefinitionSyncStateDb): DefinitionSyncState {
+  return {
+    id: row.id,
+    projectId: row.projectId,
+    sourceConnectionId: row.sourceConnectionId,
+    sourceExternalRepositoryId: row.sourceExternalRepositoryId,
+    ref: row.ref,
+    status: row.status as DefinitionSyncStatus,
+    lastErrorCode: row.lastErrorCode as DefinitionSyncErrorCode | null,
+    lastErrorMessage: row.lastErrorMessage,
+    startedAt: row.startedAt,
+    finishedAt: row.finishedAt,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+  };
+}

package/src/db/sync-states.test.ts

@@ -0,0 +1,90 @@
+import {eq} from 'drizzle-orm';
+import {db} from './db.js';
+import {definitionSyncStates} from './schema/sync-states.js';
+import {markDefinitionSyncState} from './sync-states.js';
+
+describe('definition sync state queries', () => {
+  let projectId: string;
+  let sourceConnectionId: string;
+
+  beforeEach(() => {
+    projectId = crypto.randomUUID();
+    sourceConnectionId = crypto.randomUUID();
+  });
+
+  it('creates a sync-state row', async () => {
+    const state = await markDefinitionSyncState({
+      projectId,
+      sourceConnectionId,
+      sourceExternalRepositoryId: 'gitea-owner/platform',
+      ref: 'main',
+      status: 'syncing',
+      startedAt: new Date(),
+    });
+
+    expect(state.projectId).toBe(projectId);
+    expect(state.status).toBe('syncing');
+    expect(state.lastErrorCode).toBeNull();
+  });
+
+  it('updates the same logical sync-state row', async () => {
+    const first = await markDefinitionSyncState({
+      projectId,
+      sourceConnectionId,
+      sourceExternalRepositoryId: 'gitea-owner/platform',
+      ref: 'main',
+      status: 'syncing',
+      startedAt: new Date(),
+    });
+
+    const second = await markDefinitionSyncState({
+      projectId,
+      sourceConnectionId,
+      sourceExternalRepositoryId: 'gitea-owner/platform',
+      ref: 'main',
+      status: 'failed',
+      lastErrorCode: 'invalid-definition',
+      lastErrorMessage: 'Invalid YAML',
+      finishedAt: new Date(),
+    });
+
+    const rows = await db()
+      .select()
+      .from(definitionSyncStates)
+      .where(eq(definitionSyncStates.projectId, projectId));
+    expect(rows).toHaveLength(1);
+    expect(second.id).toBe(first.id);
+    expect(second.status).toBe('failed');
+    expect(second.lastErrorCode).toBe('invalid-definition');
+    expect(second.startedAt?.getTime()).toBe(first.startedAt?.getTime());
+  });
+
+  it('clears stale finish data when a sync starts again', async () => {
+    await markDefinitionSyncState({
+      projectId,
+      sourceConnectionId,
+      sourceExternalRepositoryId: 'gitea-owner/platform',
+      ref: 'main',
+      status: 'failed',
+      lastErrorCode: 'invalid-definition',
+      lastErrorMessage: 'Invalid YAML',
+      startedAt: new Date(),
+      finishedAt: new Date(),
+    });
+
+    const state = await markDefinitionSyncState({
+      projectId,
+      sourceConnectionId,
+      sourceExternalRepositoryId: 'gitea-owner/platform',
+      ref: 'main',
+      status: 'syncing',
+      startedAt: new Date(),
+      finishedAt: null,
+    });
+
+    expect(state.status).toBe('syncing');
+    expect(state.lastErrorCode).toBeNull();
+    expect(state.lastErrorMessage).toBeNull();
+    expect(state.finishedAt).toBeNull();
+  });
+});