forgeos 0.1.0-alpha.0

package/src/forge/runtime/telemetry/spans.ts

@@ -0,0 +1,58 @@
+import type { DbAdapter, DbTransaction } from "../db/adapter.ts";
+import type { TelemetryRuntimeKind } from "./types.ts";
+
+type QueryTarget = DbTransaction | DbAdapter;
+
+export async function startTraceSpan(
+  target: QueryTarget,
+  input: {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    kind: TelemetryRuntimeKind;
+    attributes?: Record<string, unknown>;
+    startedAt?: string;
+  },
+): Promise<void> {
+  await target.query(
+    `INSERT INTO _forge_trace_spans (trace_id, parent_span_id, span_id, name, kind, attributes, status, started_at)
+     VALUES ($1, $2, $3, $4, $5, $6::jsonb, 'ok', $7)`,
+    [
+      input.traceId,
+      input.parentSpanId ?? null,
+      input.spanId,
+      input.name,
+      input.kind,
+      JSON.stringify(input.attributes ?? {}),
+      input.startedAt ?? new Date().toISOString(),
+    ],
+  );
+}
+
+export async function endTraceSpan(
+  target: QueryTarget,
+  input: {
+    traceId: string;
+    spanId: string;
+    attributes?: Record<string, unknown>;
+    error?: string;
+    endedAt?: string;
+  },
+): Promise<void> {
+  const status = input.error ? "error" : "ok";
+  await target.query(
+    `UPDATE _forge_trace_spans
+     SET ended_at = $1, status = $2, error = $3,
+         attributes = COALESCE(attributes, '{}'::jsonb) || $4::jsonb
+     WHERE trace_id = $5 AND span_id = $6`,
+    [
+      input.endedAt ?? new Date().toISOString(),
+      status,
+      input.error ?? null,
+      JSON.stringify(input.attributes ?? {}),
+      input.traceId,
+      input.spanId,
+    ],
+  );
+}

package/src/forge/runtime/telemetry/types.ts

@@ -0,0 +1,64 @@
+export type TelemetryEnvelopeType =
+  | "event"
+  | "exception"
+  | "log"
+  | "span.start"
+  | "span.end"
+  | "workflow_step";
+
+export type TelemetryEnvironment = "dev" | "test" | "preview" | "prod";
+
+export type TelemetryRuntimeKind =
+  | "command"
+  | "action"
+  | "workflow"
+  | "endpoint"
+  | "dev"
+  | "query";
+
+export interface ForgeTelemetryEnvelope {
+  schemaVersion: "0.1";
+  type: TelemetryEnvelopeType;
+  traceId: string;
+  requestId?: string;
+  environment: TelemetryEnvironment;
+  runtime: { kind: TelemetryRuntimeKind; name?: string };
+  workflow?: { runId?: string; stepName?: string };
+  outbox?: { eventId?: string; deliveryId?: string };
+  release?: { releaseId?: string; deployId?: string; environment?: string };
+  event?: { name: string; properties: Record<string, unknown> };
+  exception?: { name?: string; message: string; stack?: string };
+  log?: { level: string; message: string; fields?: Record<string, unknown> };
+  span?: {
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    attributes?: Record<string, unknown>;
+  };
+  createdAt: string;
+}
+
+export interface TelemetrySpanHandle {
+  spanId: string;
+  end: (attributes?: Record<string, unknown>) => Promise<void>;
+}
+
+export interface TelemetryContext {
+  traceId: string;
+  requestId?: string;
+  capture: (name: string, properties?: Record<string, unknown>) => Promise<void>;
+  captureException: (
+    error: unknown,
+    context?: Record<string, unknown>,
+  ) => Promise<void>;
+  log: (
+    level: string,
+    message: string,
+    fields?: Record<string, unknown>,
+  ) => Promise<void>;
+  span: (
+    name: string,
+    attributes?: Record<string, unknown>,
+  ) => Promise<TelemetrySpanHandle>;
+  flush: (sink?: string) => Promise<void>;
+}

package/src/forge/runtime/workflows/cancel.ts

@@ -0,0 +1,26 @@
+import type { DbAdapter } from "../db/adapter.ts";
+
+export async function cancelWorkflowRun(
+  adapter: DbAdapter,
+  runId: number,
+): Promise<boolean> {
+  const result = await adapter.query(
+    `UPDATE _forge_workflow_runs
+     SET status = 'canceled', canceled_at = now(), updated_at = now()
+     WHERE id = $1 AND status NOT IN ('completed', 'canceled', 'dead')`,
+    [runId],
+  );
+
+  if (result.rowCount === 0) {
+    return false;
+  }
+
+  await adapter.query(
+    `UPDATE _forge_workflow_steps
+     SET status = 'skipped', locked_at = NULL, locked_by = NULL
+     WHERE run_id = $1 AND status IN ('pending', 'running')`,
+    [runId],
+  );
+
+  return true;
+}

package/src/forge/runtime/workflows/create-run.ts

@@ -0,0 +1,98 @@
+import type { WorkflowDefinition } from "../../compiler/types/workflow-registry.ts";
+import type { DbAdapter } from "../db/adapter.ts";
+import type { CreateWorkflowRunInput, WorkflowRunRow } from "./types.ts";
+import { findWorkflowDefinition } from "./registry.ts";
+
+function parseJsonValue(value: unknown): unknown {
+  if (typeof value === "string") {
+    try {
+      return JSON.parse(value);
+    } catch {
+      return value;
+    }
+  }
+  return value;
+}
+
+function rowToRun(row: Record<string, unknown>): WorkflowRunRow {
+  return {
+    id: Number(row.id),
+    workflow_name: String(row.workflow_name),
+    trigger_type: String(row.trigger_type),
+    trigger_outbox_id:
+      row.trigger_outbox_id != null ? Number(row.trigger_outbox_id) : null,
+    idempotency_key: String(row.idempotency_key),
+    input: parseJsonValue(row.input),
+    auth_context: parseJsonValue(row.auth_context),
+    status: String(row.status) as WorkflowRunRow["status"],
+    current_step: row.current_step != null ? String(row.current_step) : null,
+    last_error: row.last_error != null ? String(row.last_error) : null,
+    created_at: String(row.created_at),
+    updated_at: String(row.updated_at),
+    started_at: row.started_at != null ? String(row.started_at) : null,
+    completed_at: row.completed_at != null ? String(row.completed_at) : null,
+    canceled_at: row.canceled_at != null ? String(row.canceled_at) : null,
+  };
+}
+
+export async function createWorkflowRun(
+  adapter: DbAdapter,
+  registry: WorkflowDefinition[],
+  input: CreateWorkflowRunInput,
+): Promise<{ created: boolean; run: WorkflowRunRow }> {
+  const definition = findWorkflowDefinition(registry, input.workflowName);
+  if (!definition) {
+    throw new Error(`workflow '${input.workflowName}' not found in registry`);
+  }
+
+  const insertResult = await adapter.query(
+    `INSERT INTO _forge_workflow_runs (workflow_name, trigger_type, trigger_outbox_id, idempotency_key, input, auth_context, status)
+     VALUES ($1, $2, $3, $4, $5, $6::jsonb, 'pending')
+     ON CONFLICT (idempotency_key) DO NOTHING
+     RETURNING id`,
+    [
+      input.workflowName,
+      input.triggerType,
+      input.triggerOutboxId ?? null,
+      input.idempotencyKey,
+      JSON.stringify(input.input),
+      JSON.stringify(input.authContext ?? null),
+    ],
+  );
+
+  let runId: number;
+
+  if (insertResult.rows.length > 0) {
+    runId = Number(insertResult.rows[0]!.id);
+
+    for (const step of definition.steps) {
+      await adapter.query(
+        `INSERT INTO _forge_workflow_steps (run_id, step_name, step_index, status)
+         VALUES ($1, $2, $3, 'pending')`,
+        [runId, step.name, step.index],
+      );
+    }
+
+    const runResult = await adapter.query(
+      `SELECT id, workflow_name, trigger_type, trigger_outbox_id, idempotency_key, input, auth_context, status, current_step, last_error, created_at, updated_at, started_at, completed_at, canceled_at
+       FROM _forge_workflow_runs WHERE id = $1`,
+      [runId],
+    );
+
+    return {
+      created: true,
+      run: rowToRun(runResult.rows[0]!),
+    };
+  }
+
+  const existing = await adapter.query(
+    `SELECT id, workflow_name, trigger_type, trigger_outbox_id, idempotency_key, input, status, current_step, last_error, created_at, updated_at, started_at, completed_at, canceled_at
+     FROM _forge_workflow_runs WHERE idempotency_key = $1`,
+    [input.idempotencyKey],
+  );
+
+  return {
+    created: false,
+    run: rowToRun(existing.rows[0]!),
+  };
+}

package/src/forge/runtime/workflows/process-run.ts

@@ -0,0 +1,182 @@
+import type { TableMapEntry } from "../../compiler/data-graph/sql/serialize.ts";
+import type { WorkflowDefinition } from "../../compiler/types/workflow-registry.ts";
+import type { DbAdapter } from "../db/adapter.ts";
+import { processWorkflowStep } from "./process-step.ts";
+import { sanitizeWorkflowError } from "./sanitize.ts";
+import type { WorkflowRunRow, WorkflowStepRow } from "./types.ts";
+
+function rowToStep(row: Record<string, unknown>): WorkflowStepRow {
+  return {
+    id: Number(row.id),
+    run_id: Number(row.run_id),
+    step_name: String(row.step_name),
+    step_index: Number(row.step_index),
+    status: String(row.status) as WorkflowStepRow["status"],
+    input: row.input,
+    output: row.output,
+    attempts: Number(row.attempts),
+    max_attempts: Number(row.max_attempts),
+    next_attempt_at: String(row.next_attempt_at),
+    locked_at: row.locked_at != null ? String(row.locked_at) : null,
+    locked_by: row.locked_by != null ? String(row.locked_by) : null,
+    last_error: row.last_error != null ? String(row.last_error) : null,
+    started_at: row.started_at != null ? String(row.started_at) : null,
+    completed_at: row.completed_at != null ? String(row.completed_at) : null,
+    created_at: String(row.created_at),
+  };
+}
+
+async function findNextPendingStep(
+  adapter: DbAdapter,
+  runId: number,
+): Promise<WorkflowStepRow | null> {
+  const result = await adapter.query(
+    `SELECT s.id, s.run_id, s.step_name, s.step_index, s.status, s.input, s.output, s.attempts, s.max_attempts, s.next_attempt_at, s.locked_at, s.locked_by, s.last_error, s.started_at, s.completed_at, s.created_at
+     FROM _forge_workflow_steps s
+     WHERE s.run_id = $1 AND s.status = 'pending' AND s.next_attempt_at <= now()
+     ORDER BY s.step_index
+     LIMIT 1`,
+    [runId],
+  );
+
+  if (result.rows.length === 0) {
+    return null;
+  }
+
+  return rowToStep(result.rows[0]!);
+}
+
+async function allStepsCompleted(adapter: DbAdapter, runId: number): Promise<boolean> {
+  const result = await adapter.query(
+    `SELECT COUNT(*)::int AS pending FROM _forge_workflow_steps
+     WHERE run_id = $1 AND status NOT IN ('completed', 'skipped')`,
+    [runId],
+  );
+  return Number(result.rows[0]?.pending ?? 0) === 0;
+}
+
+async function hasDeadStep(adapter: DbAdapter, runId: number): Promise<boolean> {
+  const result = await adapter.query(
+    `SELECT COUNT(*)::int AS dead FROM _forge_workflow_steps
+     WHERE run_id = $1 AND status = 'dead'`,
+    [runId],
+  );
+  return Number(result.rows[0]?.dead ?? 0) > 0;
+}
+
+export async function processWorkflowRun(
+  adapter: DbAdapter,
+  workspaceRoot: string,
+  tableMap: Record<string, TableMapEntry>,
+  workflow: WorkflowDefinition,
+  run: WorkflowRunRow,
+  workerId: string,
+  mock: boolean,
+): Promise<"completed" | "dead" | "retry" | "noop" | "canceled"> {
+  if (run.status === "canceled" || run.status === "completed" || run.status === "dead") {
+    return "noop";
+  }
+
+  const freshRun = await adapter.query(
+    `SELECT status, canceled_at FROM _forge_workflow_runs WHERE id = $1`,
+    [run.id],
+  );
+  const status = String(freshRun.rows[0]?.status ?? run.status);
+  if (status === "canceled") {
+    return "canceled";
+  }
+
+  const nextStep = await findNextPendingStep(adapter, run.id);
+  if (!nextStep) {
+    if (await hasDeadStep(adapter, run.id)) {
+      await adapter.query(
+        `UPDATE _forge_workflow_runs SET status = 'dead', updated_at = now() WHERE id = $1`,
+        [run.id],
+      );
+      return "dead";
+    }
+
+    if (await allStepsCompleted(adapter, run.id)) {
+      await adapter.query(
+        `UPDATE _forge_workflow_runs
+         SET status = 'completed', completed_at = now(), updated_at = now(), current_step = NULL
+         WHERE id = $1`,
+        [run.id],
+      );
+      return "completed";
+    }
+
+    return "noop";
+  }
+
+  const priorCompleted = await adapter.query(
+    `SELECT COUNT(*)::int AS incomplete FROM _forge_workflow_steps
+     WHERE run_id = $1 AND step_index < $2 AND status != 'completed'`,
+    [run.id, nextStep.step_index],
+  );
+  if (Number(priorCompleted.rows[0]?.incomplete ?? 0) > 0) {
+    return "noop";
+  }
+
+  const claimResult = await adapter.query(
+    `UPDATE _forge_workflow_steps
+     SET status = 'running', locked_at = now(), locked_by = $1, started_at = COALESCE(started_at, now())
+     WHERE id = $2 AND status = 'pending'`,
+    [workerId, nextStep.id],
+  );
+
+  if (claimResult.rowCount === 0) {
+    return "noop";
+  }
+
+  await adapter.query(
+    `UPDATE _forge_workflow_runs
+     SET status = 'running', started_at = COALESCE(started_at, now()), current_step = $1, updated_at = now()
+     WHERE id = $2`,
+    [nextStep.step_name, run.id],
+  );
+
+  const stepResult = await processWorkflowStep(
+    adapter,
+    workspaceRoot,
+    tableMap,
+    workflow,
+    run,
+    nextStep,
+    mock,
+  );
+
+  if (stepResult.status === "dead") {
+    await adapter.query(
+      `UPDATE _forge_workflow_runs
+       SET status = 'dead', last_error = $1, updated_at = now()
+       WHERE id = $2`,
+      [sanitizeWorkflowError(stepResult.error ?? "step dead"), run.id],
+    );
+    return "dead";
+  }
+
+  if (stepResult.status === "retry") {
+    await adapter.query(
+      `UPDATE _forge_workflow_runs SET status = 'failed', last_error = $1, updated_at = now() WHERE id = $2`,
+      [sanitizeWorkflowError(stepResult.error ?? "step failed"), run.id],
+    );
+    return "retry";
+  }
+
+  if (await allStepsCompleted(adapter, run.id)) {
+    await adapter.query(
+      `UPDATE _forge_workflow_runs
+       SET status = 'completed', completed_at = now(), updated_at = now(), current_step = NULL, last_error = NULL
+       WHERE id = $1`,
+      [run.id],
+    );
+    return "completed";
+  }
+
+  await adapter.query(
+    `UPDATE _forge_workflow_runs SET status = 'running', updated_at = now() WHERE id = $1`,
+    [run.id],
+  );
+  return "noop";
+}

package/src/forge/runtime/workflows/process-step.ts

@@ -0,0 +1,190 @@
+import type { TableMapEntry } from "../../compiler/data-graph/sql/serialize.ts";
+import type { WorkflowDefinition } from "../../compiler/types/workflow-registry.ts";
+import { createDiagnostic } from "../../compiler/diagnostics/create.ts";
+import {
+  FORGE_WORKFLOW_STEP_DEAD,
+  FORGE_WORKFLOW_STEP_FAILED,
+} from "../../compiler/diagnostics/codes.ts";
+import { adapterAsTransaction } from "../db/adapter.ts";
+import type { DbAdapter } from "../db/adapter.ts";
+import { createGeneratedDbClient } from "../db/generated-client.ts";
+import { createActionContext } from "../context/create-context.ts";
+import { systemAuthFromSnapshot } from "../auth/resolve.ts";
+import type { AuthContext } from "../auth/types.ts";
+import { resolveWorkflowStepHandler } from "./resolve-step.ts";
+import { computeNextAttemptAt, formatTimestamp } from "./retry.ts";
+import { sanitizeWorkflowError } from "./sanitize.ts";
+import { createTelemetryContext } from "../telemetry/context.ts";
+import { generateTraceId } from "../telemetry/correlation.ts";
+import type { WorkflowRunRow, WorkflowStepRow } from "./types.ts";
+
+export interface ProcessStepResult {
+  ok: boolean;
+  status: "completed" | "retry" | "dead";
+  output?: unknown;
+  error?: string;
+}
+
+async function loadCompletedStepOutputs(
+  adapter: DbAdapter,
+  runId: number,
+): Promise<Record<string, { output: unknown }>> {
+  const result = await adapter.query(
+    `SELECT step_name, output FROM _forge_workflow_steps
+     WHERE run_id = $1 AND status = 'completed'
+     ORDER BY step_index`,
+    [runId],
+  );
+
+  const steps: Record<string, { output: unknown }> = {};
+  for (const row of result.rows) {
+    const output = row.output;
+    steps[String(row.step_name)] = {
+      output:
+        typeof output === "string"
+          ? (() => {
+              try {
+                return JSON.parse(output);
+              } catch {
+                return output;
+              }
+            })()
+          : output,
+    };
+  }
+  return steps;
+}
+
+export async function processWorkflowStep(
+  adapter: DbAdapter,
+  workspaceRoot: string,
+  tableMap: Record<string, TableMapEntry>,
+  workflow: WorkflowDefinition,
+  run: WorkflowRunRow,
+  step: WorkflowStepRow,
+  mock: boolean,
+): Promise<ProcessStepResult> {
+  const resolved = await resolveWorkflowStepHandler(
+    workspaceRoot,
+    workflow,
+    step.step_name,
+    mock,
+  );
+
+  if (!resolved.ok) {
+    return await handleStepFailure(adapter, step, resolved.error);
+  }
+
+  try {
+    const tx = adapterAsTransaction(adapter);
+    const snapshot = (run.auth_context as AuthContext | undefined) ?? { kind: "anonymous" };
+    const tenantId =
+      snapshot.kind === "user"
+        ? snapshot.tenantId
+        : snapshot.kind === "system"
+          ? snapshot.tenantId
+          : undefined;
+    const auth = systemAuthFromSnapshot(snapshot, tenantId);
+    const db = createGeneratedDbClient(tx, tableMap, { auth });
+    const completedSteps = await loadCompletedStepOutputs(adapter, run.id);
+
+    const inputObj =
+      run.input && typeof run.input === "object"
+        ? (run.input as Record<string, unknown>)
+        : {};
+    const traceId =
+      typeof inputObj.traceId === "string" ? inputObj.traceId : generateTraceId();
+
+    const telemetry = createTelemetryContext({
+      adapter,
+      traceId,
+      runtime: { kind: "workflow", name: run.workflow_name },
+      workflow: { runId: String(run.id), stepName: step.step_name },
+      bufferInTransaction: false,
+      workspaceRoot,
+      sinks: ["local"],
+    });
+
+    const baseCtx = createActionContext(db, telemetry, auth, {
+      workspaceRoot,
+      runtimeKind: "workflow",
+      mockAi: mock,
+    });
+
+    const ctx = {
+      input: run.input,
+      steps: completedSteps,
+      db: baseCtx.db,
+      env: baseCtx.env,
+      telemetry,
+      auth,
+      ai: baseCtx.ai,
+    };
+
+    const runRecord = {
+      id: run.id,
+      workflowName: run.workflow_name,
+      status: run.status,
+      input: run.input,
+      currentStep: step.step_name,
+    };
+
+    const output = await resolved.handler(ctx, runRecord);
+
+    await telemetry.flush("local");
+
+    await adapter.query(
+      `UPDATE _forge_workflow_steps
+       SET status = 'completed', output = $1, completed_at = now(), locked_at = NULL, locked_by = NULL, last_error = NULL
+       WHERE id = $2`,
+      [JSON.stringify(output ?? null), step.id],
+    );
+
+    return { ok: true, status: "completed", output };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "workflow step failed";
+    return await handleStepFailure(adapter, step, message);
+  }
+}
+
+async function handleStepFailure(
+  adapter: DbAdapter,
+  step: WorkflowStepRow,
+  rawError: string,
+): Promise<ProcessStepResult> {
+  const errorMessage = sanitizeWorkflowError(rawError);
+  const nextAttempts = step.attempts + 1;
+
+  if (nextAttempts >= step.max_attempts) {
+    await adapter.query(
+      `UPDATE _forge_workflow_steps
+       SET status = 'dead', attempts = $1, last_error = $2, locked_at = NULL, locked_by = NULL
+       WHERE id = $3`,
+      [nextAttempts, errorMessage, step.id],
+    );
+
+    createDiagnostic({
+      severity: "error",
+      code: FORGE_WORKFLOW_STEP_DEAD,
+      message: `step ${step.step_name} dead after ${nextAttempts} attempts: ${errorMessage}`,
+    });
+
+    return { ok: false, status: "dead", error: errorMessage };
+  }
+
+  const nextAttemptAt = formatTimestamp(computeNextAttemptAt(nextAttempts));
+  await adapter.query(
+    `UPDATE _forge_workflow_steps
+     SET status = 'pending', attempts = $1, last_error = $2, next_attempt_at = $3, locked_at = NULL, locked_by = NULL
+     WHERE id = $4`,
+    [nextAttempts, errorMessage, nextAttemptAt, step.id],
+  );
+
+  createDiagnostic({
+    severity: "warning",
+    code: FORGE_WORKFLOW_STEP_FAILED,
+    message: `step ${step.step_name} scheduled retry ${nextAttempts}: ${errorMessage}`,
+  });
+
+  return { ok: false, status: "retry", error: errorMessage };
+}