@moretea-labs/forge 1.5.0-rc.1

package/src/runtime/workflow/findings/store.ts

@@ -0,0 +1,165 @@
+import { createHash, randomUUID } from 'crypto';
+import { existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { repositoryControllerRoot } from '../../../cli/repositories/controller-home';
+import { withControllerLock } from '../../../cli/repositories/locks';
+import { appendRuntimeEvent } from '../../evidence/event-ledger';
+import { readJsonFile, sanitizeFileComponent, writeJsonAtomic } from '../../shared/json-files';
+import type { CandidateFinding, RecordCandidateFindingInput } from './types';
+
+interface FindingIndex {
+  schemaVersion: 1;
+  updatedAt: string;
+  candidates: Array<{ findingId: string; semanticKey: string; severity: CandidateFinding['severity']; lastSeenAt: string }>;
+  recent: Array<{ findingId: string; semanticKey: string; status: CandidateFinding['status']; lastSeenAt: string }>;
+}
+
+function root(controllerHome: string, repoId: string): string {
+  return join(repositoryControllerRoot(controllerHome, repoId), 'candidate-findings');
+}
+function recordPath(controllerHome: string, repoId: string, findingId: string): string {
+  return join(root(controllerHome, repoId), 'records', `${sanitizeFileComponent(findingId)}.json`);
+}
+function semanticPath(controllerHome: string, repoId: string, semanticKey: string): string {
+  return join(root(controllerHome, repoId), 'indexes', 'semantic', `${createHash('sha256').update(semanticKey).digest('hex')}.json`);
+}
+function indexPath(controllerHome: string, repoId: string): string {
+  return join(root(controllerHome, repoId), 'indexes', 'findings.json');
+}
+function emptyIndex(): FindingIndex {
+  return { schemaVersion: 1, updatedAt: new Date().toISOString(), candidates: [], recent: [] };
+}
+function readIndex(controllerHome: string, repoId: string): FindingIndex {
+  return readJsonFile<FindingIndex>(indexPath(controllerHome, repoId), emptyIndex());
+}
+function upsertIndexUnlocked(controllerHome: string, finding: CandidateFinding): void {
+  const index = readIndex(controllerHome, finding.repoId);
+  index.candidates = index.candidates.filter((entry) => entry.findingId !== finding.findingId);
+  index.recent = index.recent.filter((entry) => entry.findingId !== finding.findingId);
+  if (finding.status === 'candidate') {
+    index.candidates.push({ findingId: finding.findingId, semanticKey: finding.semanticKey, severity: finding.severity, lastSeenAt: finding.lastSeenAt });
+  }
+  index.recent.push({ findingId: finding.findingId, semanticKey: finding.semanticKey, status: finding.status, lastSeenAt: finding.lastSeenAt });
+  index.candidates.sort((a, b) => b.lastSeenAt.localeCompare(a.lastSeenAt));
+  index.recent.sort((a, b) => b.lastSeenAt.localeCompare(a.lastSeenAt));
+  index.candidates = index.candidates.slice(0, 5000);
+  index.recent = index.recent.slice(0, 5000);
+  index.updatedAt = new Date().toISOString();
+  writeJsonAtomic(indexPath(controllerHome, finding.repoId), index);
+}
+
+export function getCandidateFinding(controllerHome: string, repoId: string, findingId: string): CandidateFinding {
+  return readJsonFile<CandidateFinding>(recordPath(controllerHome, repoId, findingId));
+}
+
+export function recordCandidateFinding(controllerHome: string, input: RecordCandidateFindingInput): CandidateFinding {
+  const semanticKey = input.semanticKey.trim();
+  const title = input.title.trim();
+  const requestId = input.requestId.trim();
+  if (!semanticKey) throw new Error('CANDIDATE_SEMANTIC_KEY_REQUIRED');
+  if (!title) throw new Error('CANDIDATE_TITLE_REQUIRED');
+  if (!requestId) throw new Error('CANDIDATE_REQUEST_ID_REQUIRED');
+  const lockId = createHash('sha256').update(semanticKey).digest('hex').slice(0, 20);
+  return withControllerLock(controllerHome, { scope: 'task', repoId: input.repoId, taskId: `candidate-${lockId}` }, `record-candidate:${lockId}`, () => {
+    const semanticIndex = semanticPath(controllerHome, input.repoId, semanticKey);
+    const timestamp = input.evidence?.observedAt ?? new Date().toISOString();
+    let finding: CandidateFinding;
+    if (existsSync(semanticIndex)) {
+      const reference = readJsonFile<{ findingId: string }>(semanticIndex);
+      const current = getCandidateFinding(controllerHome, input.repoId, reference.findingId);
+      const evidence = input.evidence
+        ? [...current.evidence, { ...input.evidence, observedAt: timestamp }].slice(-100)
+        : current.evidence;
+      finding = {
+        ...current,
+        revision: current.revision + 1,
+        title,
+        summary: input.summary ?? current.summary,
+        severity: input.severity ?? current.severity,
+        status: current.status === 'dismissed' ? 'candidate' : current.status,
+        observationCount: current.observationCount + 1,
+        evidence,
+        lastSeenAt: timestamp,
+        dismissedReason: current.status === 'dismissed' ? undefined : current.dismissedReason,
+      };
+    } else {
+      finding = {
+        schemaVersion: 1,
+        revision: 1,
+        findingId: `FIND-${Date.now()}-${randomUUID().slice(0, 8)}`,
+        repoId: input.repoId,
+        semanticKey,
+        title,
+        summary: input.summary,
+        severity: input.severity ?? 'medium',
+        status: 'candidate',
+        observationCount: 1,
+        evidence: input.evidence ? [{ ...input.evidence, observedAt: timestamp }] : [],
+        firstSeenAt: timestamp,
+        lastSeenAt: timestamp,
+      };
+      writeJsonAtomic(semanticIndex, { semanticKey, findingId: finding.findingId, createdAt: timestamp });
+    }
+    writeJsonAtomic(recordPath(controllerHome, finding.repoId, finding.findingId), finding);
+    upsertIndexUnlocked(controllerHome, finding);
+    appendRuntimeEvent(controllerHome, {
+      repoId: finding.repoId,
+      entityType: 'candidate-finding',
+      entityId: finding.findingId,
+      eventType: finding.observationCount === 1 ? 'candidate_finding_created' : 'candidate_finding_observed',
+      requestId,
+      revision: finding.revision,
+      data: { semanticKey, observationCount: finding.observationCount, severity: finding.severity },
+    });
+    return finding;
+  }, 10_000);
+}
+
+export function updateCandidateFinding(
+  controllerHome: string,
+  repoId: string,
+  findingId: string,
+  updater: (current: CandidateFinding) => CandidateFinding,
+  requestId: string,
+  eventType: string,
+): CandidateFinding {
+  return withControllerLock(controllerHome, { scope: 'task', repoId, taskId: `candidate-${findingId}` }, `update-candidate:${findingId}`, () => {
+    const current = getCandidateFinding(controllerHome, repoId, findingId);
+    const next = updater(structuredClone(current));
+    if (next.findingId !== current.findingId || next.repoId !== current.repoId || next.semanticKey !== current.semanticKey) {
+      throw new Error('CANDIDATE_IDENTITY_IMMUTABLE');
+    }
+    next.revision = current.revision + 1;
+    next.lastSeenAt = new Date().toISOString();
+    writeJsonAtomic(recordPath(controllerHome, repoId, findingId), next);
+    upsertIndexUnlocked(controllerHome, next);
+    appendRuntimeEvent(controllerHome, {
+      repoId,
+      entityType: 'candidate-finding',
+      entityId: findingId,
+      eventType,
+      requestId,
+      revision: next.revision,
+      data: { status: next.status, promotedJobId: next.promotedJobId },
+    });
+    return next;
+  }, 10_000);
+}
+
+export function listCandidateFindings(controllerHome: string, repoId: string, options: { includeTerminal?: boolean; limit?: number } = {}): CandidateFinding[] {
+  const bounded = Math.max(1, Math.min(options.limit ?? 100, 1000));
+  const index = readIndex(controllerHome, repoId);
+  const entries = options.includeTerminal ? index.recent : index.candidates;
+  const indexed = entries.slice(0, bounded).flatMap((entry) => {
+    try { return [getCandidateFinding(controllerHome, repoId, entry.findingId)]; } catch { return []; }
+  });
+  if (indexed.length > 0) return indexed;
+  try {
+    return readdirSync(join(root(controllerHome, repoId), 'records'))
+      .filter((name) => name.endsWith('.json'))
+      .map((name) => readJsonFile<CandidateFinding>(join(root(controllerHome, repoId), 'records', name)))
+      .filter((finding) => options.includeTerminal || finding.status === 'candidate')
+      .sort((a, b) => b.lastSeenAt.localeCompare(a.lastSeenAt))
+      .slice(0, bounded);
+  } catch { return []; }
+}

package/src/runtime/workflow/findings/types.ts

@@ -0,0 +1,36 @@
+export type CandidateFindingStatus = 'candidate' | 'promoted' | 'dismissed';
+
+export interface CandidateFindingEvidence {
+  source: string;
+  reference?: string;
+  observedAt: string;
+  details?: Record<string, unknown>;
+}
+
+export interface CandidateFinding {
+  schemaVersion: 1;
+  revision: number;
+  findingId: string;
+  repoId: string;
+  semanticKey: string;
+  title: string;
+  summary?: string;
+  severity: 'low' | 'medium' | 'high' | 'critical';
+  status: CandidateFindingStatus;
+  observationCount: number;
+  evidence: CandidateFindingEvidence[];
+  firstSeenAt: string;
+  lastSeenAt: string;
+  promotedJobId?: string;
+  dismissedReason?: string;
+}
+
+export interface RecordCandidateFindingInput {
+  repoId: string;
+  semanticKey: string;
+  title: string;
+  summary?: string;
+  severity?: CandidateFinding['severity'];
+  evidence?: Omit<CandidateFindingEvidence, 'observedAt'> & { observedAt?: string };
+  requestId: string;
+}

package/src/runtime/workflow/portfolio/engine.ts

@@ -0,0 +1,20 @@
+import type { PortfolioWorkflow } from './types';
+import { listActivePortfolioWorkflows, savePortfolioWorkflow } from './store';
+
+const EXTERNAL_CONTROLLER_REQUIRED = 'Portfolio execution requires an explicitly claimed external Controller.';
+
+/**
+ * Portfolio definitions remain durable planning records. Kernel-owned DAG execution
+ * is retired, so ticking only records the explicit handoff state and creates no Job.
+ */
+export function tickPortfolioWorkflow(controllerHome: string, source: PortfolioWorkflow): PortfolioWorkflow {
+  const steps = source.steps.map((step) => step.status === 'pending' || step.status === 'queued'
+    ? { ...step, status: 'blocked' as const, error: EXTERNAL_CONTROLLER_REQUIRED }
+    : step);
+  return savePortfolioWorkflow(controllerHome, { ...source, status: 'paused', steps });
+}
+
+export function tickPortfolioWorkflows(controllerHome: string): PortfolioWorkflow[] {
+  return listActivePortfolioWorkflows(controllerHome)
+    .map((workflow) => tickPortfolioWorkflow(controllerHome, workflow));
+}

package/src/runtime/workflow/portfolio/store.ts

@@ -0,0 +1,185 @@
+import { createHash, randomUUID } from 'crypto';
+import { existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { ensureControllerHome } from '../../../cli/repositories/controller-home';
+import { withControllerLock } from '../../../cli/repositories/locks';
+import { readJsonFile, writeJsonAtomic } from '../../shared/json-files';
+import { appendRuntimeEvent } from '../../evidence/event-ledger';
+import type { PortfolioWorkflow } from './types';
+
+interface PortfolioIndexEntry {
+  workflowId: string;
+  status: PortfolioWorkflow['status'];
+  createdAt: string;
+  updatedAt: string;
+}
+
+
+interface PortfolioRequestRecord {
+  schemaVersion: 1;
+  requestId: string;
+  semanticKey: string;
+  workflowId: string;
+  createdAt: string;
+}
+
+interface PortfolioIndex {
+  schemaVersion: 1;
+  updatedAt: string;
+  active: PortfolioIndexEntry[];
+  recent: PortfolioIndexEntry[];
+}
+
+function portfolioRoot(controllerHome: string): string {
+  return join(ensureControllerHome(controllerHome), 'portfolio');
+}
+function root(controllerHome: string): string { return join(portfolioRoot(controllerHome), 'workflows'); }
+function path(controllerHome: string, workflowId: string): string { return join(root(controllerHome), `${workflowId}.json`); }
+function indexPath(controllerHome: string): string { return join(portfolioRoot(controllerHome), 'indexes', 'workflows.json'); }
+function requestPath(controllerHome: string, requestId: string): string { return join(portfolioRoot(controllerHome), 'indexes', 'requests', `${createHash('sha256').update(requestId).digest('hex')}.json`); }
+function canonical(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(canonical);
+  if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value as Record<string, unknown>).sort(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => [key, canonical(entry)]));
+  return value;
+}
+function semanticKey(input: Omit<PortfolioWorkflow, 'schemaVersion' | 'revision' | 'workflowId' | 'createdAt' | 'updatedAt' | 'status'>): string {
+  return createHash('sha256').update(JSON.stringify(canonical({ name: input.name, failurePolicy: input.failurePolicy, steps: input.steps }))).digest('hex');
+}
+function emptyIndex(): PortfolioIndex {
+  return { schemaVersion: 1, updatedAt: new Date().toISOString(), active: [], recent: [] };
+}
+function readIndex(controllerHome: string): PortfolioIndex {
+  return readJsonFile<PortfolioIndex>(indexPath(controllerHome), emptyIndex());
+}
+function activeStatus(status: PortfolioWorkflow['status']): boolean {
+  return status === 'active' || status === 'compensating';
+}
+function upsertIndexUnlocked(controllerHome: string, workflow: PortfolioWorkflow): void {
+  const index = readIndex(controllerHome);
+  const entry: PortfolioIndexEntry = {
+    workflowId: workflow.workflowId,
+    status: workflow.status,
+    createdAt: workflow.createdAt,
+    updatedAt: workflow.updatedAt,
+  };
+  index.active = index.active.filter((candidate) => candidate.workflowId !== workflow.workflowId);
+  index.recent = index.recent.filter((candidate) => candidate.workflowId !== workflow.workflowId);
+  if (activeStatus(workflow.status)) index.active.push(entry);
+  index.recent.push(entry);
+  index.active.sort((left, right) => left.updatedAt.localeCompare(right.updatedAt));
+  index.recent.sort((left, right) => right.createdAt.localeCompare(left.createdAt));
+  index.active = index.active.slice(-5000);
+  index.recent = index.recent.slice(0, 5000);
+  index.updatedAt = new Date().toISOString();
+  writeJsonAtomic(indexPath(controllerHome), index);
+}
+function readIndexedWorkflows(controllerHome: string, entries: PortfolioIndexEntry[], limit: number): PortfolioWorkflow[] {
+  return entries.slice(0, limit).flatMap((entry) => {
+    const candidate = path(controllerHome, entry.workflowId);
+    if (!existsSync(candidate)) return [];
+    try { return [readJsonFile<PortfolioWorkflow>(candidate)]; } catch { return []; }
+  });
+}
+function backfillIndex(controllerHome: string, workflows: PortfolioWorkflow[]): void {
+  if (workflows.length === 0) return;
+  withControllerLock(controllerHome, { scope: 'global' }, 'portfolio-index-backfill', () => {
+    for (const workflow of workflows) upsertIndexUnlocked(controllerHome, workflow);
+  }, 10_000);
+}
+
+export function createPortfolioWorkflow(
+  controllerHome: string,
+  input: Omit<PortfolioWorkflow, 'schemaVersion' | 'revision' | 'workflowId' | 'createdAt' | 'updatedAt' | 'status'>,
+): PortfolioWorkflow {
+  const requestId = input.requestId.trim();
+  if (!requestId) throw new Error('PORTFOLIO_REQUEST_ID_REQUIRED');
+  const inputSemanticKey = semanticKey(input);
+  const ids = new Set(input.steps.map((step) => step.stepId));
+  if (ids.size !== input.steps.length) throw new Error('PORTFOLIO_STEP_DUPLICATE: stepId values must be unique');
+  for (const step of input.steps) {
+    for (const dependency of step.dependsOn) if (!ids.has(dependency)) throw new Error(`PORTFOLIO_DEPENDENCY_UNKNOWN: ${dependency}`);
+  }
+  const dependencies = new Map(input.steps.map((step) => [step.stepId, step.dependsOn]));
+  const visiting = new Set<string>();
+  const visited = new Set<string>();
+  const visit = (stepId: string): void => {
+    if (visiting.has(stepId)) throw new Error(`PORTFOLIO_DEPENDENCY_CYCLE: ${stepId}`);
+    if (visited.has(stepId)) return;
+    visiting.add(stepId);
+    for (const dependency of dependencies.get(stepId) ?? []) visit(dependency);
+    visiting.delete(stepId);
+    visited.add(stepId);
+  };
+  for (const stepId of ids) visit(stepId);
+  return withControllerLock(controllerHome, { scope: 'global' }, `create-portfolio:${requestId}`, () => {
+    const recordPath = requestPath(controllerHome, requestId);
+    if (existsSync(recordPath)) {
+      const record = readJsonFile<PortfolioRequestRecord>(recordPath);
+      if (record.semanticKey !== inputSemanticKey) throw new Error(`PORTFOLIO_REQUEST_ID_CONFLICT: ${requestId}`);
+      return getPortfolioWorkflow(controllerHome, record.workflowId);
+    }
+    const timestamp = new Date().toISOString();
+    const workflow: PortfolioWorkflow = {
+      ...input,
+      requestId,
+      schemaVersion: 1,
+      revision: 1,
+      workflowId: `PFW-${Date.now()}-${randomUUID().slice(0, 8)}`,
+      status: 'active',
+      createdAt: timestamp,
+      updatedAt: timestamp,
+    };
+    writeJsonAtomic(path(controllerHome, workflow.workflowId), workflow);
+    writeJsonAtomic(recordPath, { schemaVersion: 1, requestId, semanticKey: inputSemanticKey, workflowId: workflow.workflowId, createdAt: timestamp } satisfies PortfolioRequestRecord);
+    upsertIndexUnlocked(controllerHome, workflow);
+    for (const repoId of new Set(workflow.steps.map((step) => step.repoId))) {
+      appendRuntimeEvent(controllerHome, { repoId, entityType: 'portfolio', entityId: workflow.workflowId, eventType: 'portfolio_created', requestId, revision: workflow.revision, data: { name: workflow.name, failurePolicy: workflow.failurePolicy } });
+    }
+    return workflow;
+  }, 10_000);
+}
+
+export function savePortfolioWorkflow(controllerHome: string, workflow: PortfolioWorkflow): PortfolioWorkflow {
+  return withControllerLock(controllerHome, { scope: 'global' }, `save-portfolio:${workflow.workflowId}`, () => {
+    const current = getPortfolioWorkflow(controllerHome, workflow.workflowId);
+    const next = { ...workflow, revision: current.revision + 1, updatedAt: new Date().toISOString() };
+    writeJsonAtomic(path(controllerHome, workflow.workflowId), next);
+    upsertIndexUnlocked(controllerHome, next);
+    for (const repoId of new Set(next.steps.map((step) => step.repoId))) {
+      appendRuntimeEvent(controllerHome, { repoId, entityType: 'portfolio', entityId: next.workflowId, eventType: `portfolio_${next.status}`, requestId: next.requestId, revision: next.revision, data: { status: next.status } });
+    }
+    return next;
+  }, 10_000);
+}
+
+export function getPortfolioWorkflow(controllerHome: string, workflowId: string): PortfolioWorkflow {
+  const workflow = readJsonFile<PortfolioWorkflow>(path(controllerHome, workflowId));
+  return { ...workflow, revision: Number.isFinite(workflow.revision) ? workflow.revision : 1 };
+}
+
+export function listActivePortfolioWorkflows(controllerHome: string): PortfolioWorkflow[] {
+  const indexed = readIndexedWorkflows(controllerHome, readIndex(controllerHome).active, 5000)
+    .filter((workflow) => activeStatus(workflow.status));
+  if (indexed.length > 0) return indexed;
+  const legacy = listLegacyPortfolioWorkflows(controllerHome, 5000).filter((workflow) => activeStatus(workflow.status));
+  backfillIndex(controllerHome, legacy);
+  return legacy;
+}
+
+function listLegacyPortfolioWorkflows(controllerHome: string, limit: number): PortfolioWorkflow[] {
+  try {
+    return readdirSync(root(controllerHome)).filter((name) => name.endsWith('.json'))
+      .map((name) => readJsonFile<PortfolioWorkflow>(join(root(controllerHome), name)))
+      .sort((a, b) => b.createdAt.localeCompare(a.createdAt))
+      .slice(0, limit);
+  } catch { return []; }
+}
+
+export function listPortfolioWorkflows(controllerHome: string, limit = 100): PortfolioWorkflow[] {
+  const bounded = Math.max(1, Math.min(limit, 1000));
+  const indexed = readIndexedWorkflows(controllerHome, readIndex(controllerHome).recent, bounded);
+  if (indexed.length > 0) return indexed;
+  const legacy = listLegacyPortfolioWorkflows(controllerHome, bounded);
+  backfillIndex(controllerHome, legacy);
+  return legacy;
+}

package/src/runtime/workflow/portfolio/types.ts

@@ -0,0 +1,29 @@
+import type { ExecutionJobPriority, ResourceClaimSpec } from '../../execution/jobs/types';
+
+export interface PortfolioStep {
+  stepId: string;
+  repoId: string;
+  operation: string;
+  arguments?: Record<string, unknown>;
+  dependsOn: string[];
+  priority: ExecutionJobPriority;
+  resourceClaims: ResourceClaimSpec[];
+  compensation?: { operation: string; arguments?: Record<string, unknown> };
+  status: 'pending' | 'queued' | 'running' | 'succeeded' | 'failed' | 'blocked' | 'compensating' | 'compensated';
+  jobId?: string;
+  compensationJobId?: string;
+  error?: string;
+}
+
+export interface PortfolioWorkflow {
+  schemaVersion: 1;
+  revision: number;
+  workflowId: string;
+  name: string;
+  status: 'active' | 'succeeded' | 'failed' | 'compensating' | 'compensated' | 'paused';
+  failurePolicy: 'stop' | 'compensate';
+  steps: PortfolioStep[];
+  createdAt: string;
+  updatedAt: string;
+  requestId: string;
+}