forgeos 0.1.0-alpha.0

package/src/forge/runtime/workflows/process.ts

@@ -0,0 +1,260 @@
+import type { TableMapEntry } from "../../compiler/data-graph/sql/serialize.ts";
+import type { DbAdapter } from "../db/adapter.ts";
+import { processWorkflowRun } from "./process-run.ts";
+import { findWorkflowDefinition, loadWorkflowRegistry } from "./registry.ts";
+import { startWorkflowRunsForPendingOutbox } from "./start-from-outbox.ts";
+import type {
+  ProcessWorkflowBatchOptions,
+  ProcessWorkflowBatchResult,
+  WorkflowRunRow,
+} from "./types.ts";
+
+function defaultWorkerId(): string {
+  return `forge-worker-${process.pid}`;
+}
+
+function parseJsonValue(value: unknown): unknown {
+  if (typeof value === "string") {
+    try {
+      return JSON.parse(value);
+    } catch {
+      return value;
+    }
+  }
+  return value;
+}
+
+function rowToRun(row: Record<string, unknown>): WorkflowRunRow {
+  let input: unknown = row.input;
+  if (typeof input === "string") {
+    try {
+      input = JSON.parse(input);
+    } catch {
+      /* keep raw */
+    }
+  }
+
+  return {
+    id: Number(row.id),
+    workflow_name: String(row.workflow_name),
+    trigger_type: String(row.trigger_type),
+    trigger_outbox_id:
+      row.trigger_outbox_id != null ? Number(row.trigger_outbox_id) : null,
+    idempotency_key: String(row.idempotency_key),
+    input,
+    auth_context: parseJsonValue(row.auth_context),
+    status: String(row.status) as WorkflowRunRow["status"],
+    current_step: row.current_step != null ? String(row.current_step) : null,
+    last_error: row.last_error != null ? String(row.last_error) : null,
+    created_at: String(row.created_at),
+    updated_at: String(row.updated_at),
+    started_at: row.started_at != null ? String(row.started_at) : null,
+    completed_at: row.completed_at != null ? String(row.completed_at) : null,
+    canceled_at: row.canceled_at != null ? String(row.canceled_at) : null,
+  };
+}
+
+export async function processWorkflowBatch(
+  adapter: DbAdapter,
+  workspaceRoot: string,
+  tableMap: Record<string, TableMapEntry>,
+  options: ProcessWorkflowBatchOptions = {},
+): Promise<ProcessWorkflowBatchResult> {
+  const limit = options.limit ?? 10;
+  const workerId = options.workerId ?? defaultWorkerId();
+  const mock = options.mock ?? false;
+  const { workflows } = loadWorkflowRegistry(workspaceRoot);
+
+  const pendingRuns = await adapter.query(
+    `SELECT id, workflow_name, trigger_type, trigger_outbox_id, idempotency_key, input, auth_context, status, current_step, last_error, created_at, updated_at, started_at, completed_at, canceled_at
+     FROM _forge_workflow_runs
+     WHERE status IN ('pending', 'running', 'failed')
+     ORDER BY id
+     LIMIT $1`,
+    [limit],
+  );
+
+  const result: ProcessWorkflowBatchResult = {
+    claimed: pendingRuns.rows.length,
+    completed: 0,
+    failed: 0,
+    dead: 0,
+    runsStarted: 0,
+    errors: [],
+  };
+
+  for (const row of pendingRuns.rows) {
+    const run = rowToRun(row);
+    const workflow = findWorkflowDefinition(workflows, run.workflow_name);
+    if (!workflow) {
+      result.errors.push(`workflow '${run.workflow_name}' not found`);
+      continue;
+    }
+
+    const outcome = await processWorkflowRun(
+      adapter,
+      workspaceRoot,
+      tableMap,
+      workflow,
+      run,
+      workerId,
+      mock,
+    );
+
+    if (outcome === "completed") {
+      result.completed += 1;
+    } else if (outcome === "dead") {
+      result.dead += 1;
+    } else if (outcome === "retry") {
+      result.failed += 1;
+    }
+  }
+
+  return result;
+}
+
+export async function runWorkerTick(
+  adapter: DbAdapter,
+  workspaceRoot: string,
+  tableMap: Record<string, TableMapEntry>,
+  runtimeGraphEntries: Parameters<typeof import("../outbox/process.ts").processOutboxBatch>[3],
+  options: ProcessWorkflowBatchOptions = {},
+): Promise<{
+  workflows: { started: number; skipped: number };
+  outbox: Awaited<ReturnType<typeof import("../outbox/process.ts").processOutboxBatch>>;
+  workflowBatch: ProcessWorkflowBatchResult;
+}> {
+  const { processOutboxBatch } = await import("../outbox/process.ts");
+
+  const workflowStart = await startWorkflowRunsForPendingOutbox(adapter, workspaceRoot);
+  const outbox = await processOutboxBatch(
+    adapter,
+    workspaceRoot,
+    tableMap,
+    runtimeGraphEntries,
+    options,
+  );
+  const workflowBatch = await processWorkflowBatch(
+    adapter,
+    workspaceRoot,
+    tableMap,
+    options,
+  );
+
+  if (options.telemetrySinks?.includes("local")) {
+    const { processTelemetryBatch } = await import("../telemetry/process.ts");
+    await processTelemetryBatch(adapter, workspaceRoot, ["local"]);
+  }
+
+  return {
+    workflows: workflowStart,
+    outbox,
+    workflowBatch: {
+      ...workflowBatch,
+      runsStarted: workflowStart.started,
+    },
+  };
+}
+
+export async function getWorkflowSummary(adapter: DbAdapter): Promise<{
+  pending: number;
+  running: number;
+  completed: number;
+  failed: number;
+  dead: number;
+  canceled: number;
+}> {
+  const result = await adapter.query(
+    `SELECT status, COUNT(*)::int AS count FROM _forge_workflow_runs GROUP BY status`,
+  );
+
+  const counts: Record<string, number> = {};
+  for (const row of result.rows) {
+    counts[String(row.status)] = Number(row.count);
+  }
+
+  return {
+    pending: counts.pending ?? 0,
+    running: counts.running ?? 0,
+    completed: counts.completed ?? 0,
+    failed: counts.failed ?? 0,
+    dead: counts.dead ?? 0,
+    canceled: counts.canceled ?? 0,
+  };
+}
+
+export async function listWorkflowRuns(
+  adapter: DbAdapter,
+): Promise<Record<string, unknown>[]> {
+  const result = await adapter.query(
+    `SELECT id, workflow_name, trigger_type, trigger_outbox_id, idempotency_key, input, auth_context, status, current_step, last_error, created_at, updated_at, started_at, completed_at, canceled_at
+     FROM _forge_workflow_runs
+     ORDER BY id`,
+  );
+  return result.rows;
+}
+
+export async function inspectWorkflowRun(
+  adapter: DbAdapter,
+  runId: number,
+): Promise<{ run: Record<string, unknown> | null; steps: Record<string, unknown>[] }> {
+  const runResult = await adapter.query(
+    `SELECT id, workflow_name, trigger_type, trigger_outbox_id, idempotency_key, input, auth_context, status, current_step, last_error, created_at, updated_at, started_at, completed_at, canceled_at
+     FROM _forge_workflow_runs WHERE id = $1`,
+    [runId],
+  );
+
+  if (runResult.rows.length === 0) {
+    return { run: null, steps: [] };
+  }
+
+  const stepsResult = await adapter.query(
+    `SELECT id, run_id, step_name, step_index, status, input, output, attempts, max_attempts, next_attempt_at, last_error, started_at, completed_at, created_at
+     FROM _forge_workflow_steps WHERE run_id = $1 ORDER BY step_index`,
+    [runId],
+  );
+
+  return {
+    run: runResult.rows[0] ?? null,
+    steps: stepsResult.rows,
+  };
+}
+
+export function startWorkflowWorker(
+  adapter: DbAdapter,
+  workspaceRoot: string,
+  tableMap: Record<string, TableMapEntry>,
+  runtimeGraphEntries: Parameters<typeof import("../outbox/process.ts").processOutboxBatch>[3],
+  options: ProcessWorkflowBatchOptions & { intervalMs?: number } = {},
+): { stop: () => void; isRunning: () => boolean } {
+  const intervalMs = options.intervalMs ?? 2_000;
+  let running = true;
+  let timer: ReturnType<typeof setInterval> | null = null;
+
+  const tick = async () => {
+    if (!running) {
+      return;
+    }
+    try {
+      await runWorkerTick(adapter, workspaceRoot, tableMap, runtimeGraphEntries, options);
+    } catch {
+      /* worker loop continues on batch errors */
+    }
+  };
+
+  void tick();
+  timer = setInterval(() => {
+    void tick();
+  }, intervalMs);
+
+  return {
+    stop: () => {
+      running = false;
+      if (timer) {
+        clearInterval(timer);
+        timer = null;
+      }
+    },
+    isRunning: () => running,
+  };
+}

package/src/forge/runtime/workflows/registry.ts

@@ -0,0 +1,51 @@
+import { nodeFileSystem } from "../../compiler/fs/index.ts";
+import { join } from "node:path";
+import type { WorkflowDefinition } from "../../compiler/types/workflow-registry.ts";
+import type { WorkflowSubscription } from "../../compiler/types/workflow-registry.ts";
+import { GENERATED_DIR } from "../../compiler/emitter/constants.ts";
+import { stripDeterministicHeader } from "../../compiler/primitives/header.ts";
+
+interface WorkflowRegistryJson {
+  workflows: WorkflowDefinition[];
+}
+
+interface WorkflowSubscriptionsJson {
+  subscriptions: WorkflowSubscription[];
+  byEvent: Record<string, WorkflowSubscription[]>;
+}
+
+export function loadWorkflowRegistry(workspaceRoot: string): WorkflowRegistryJson {
+  const absolute = join(workspaceRoot, GENERATED_DIR, "workflowRegistry.json");
+  if (!nodeFileSystem.exists(absolute)) {
+    return { workflows: [] };
+  }
+
+  const raw = stripDeterministicHeader((nodeFileSystem.readText(absolute) ?? ""));
+  const parsed = JSON.parse(raw) as WorkflowRegistryJson;
+  return {
+    workflows: parsed.workflows ?? [],
+  };
+}
+
+export function loadWorkflowSubscriptions(
+  workspaceRoot: string,
+): WorkflowSubscriptionsJson {
+  const absolute = join(workspaceRoot, GENERATED_DIR, "workflowSubscriptions.json");
+  if (!nodeFileSystem.exists(absolute)) {
+    return { subscriptions: [], byEvent: {} };
+  }
+
+  const raw = stripDeterministicHeader((nodeFileSystem.readText(absolute) ?? ""));
+  const parsed = JSON.parse(raw) as WorkflowSubscriptionsJson;
+  return {
+    subscriptions: parsed.subscriptions ?? [],
+    byEvent: parsed.byEvent ?? {},
+  };
+}
+
+export function findWorkflowDefinition(
+  registry: WorkflowDefinition[],
+  workflowName: string,
+): WorkflowDefinition | undefined {
+  return registry.find((workflow) => workflow.name === workflowName);
+}

package/src/forge/runtime/workflows/resolve-step.ts

@@ -0,0 +1,46 @@
+import { join } from "node:path";
+import type { WorkflowDefinition } from "../../compiler/types/workflow-registry.ts";
+import { prepareRuntimeEnvironment } from "../executor.ts";
+
+interface WorkflowStepHandler {
+  name: string;
+  handler: (ctx: unknown, run: unknown) => unknown | Promise<unknown>;
+}
+
+interface ExportedWorkflow {
+  __forge?: { kind: string };
+  steps?: WorkflowStepHandler[];
+}
+
+export async function resolveWorkflowStepHandler(
+  workspaceRoot: string,
+  workflow: WorkflowDefinition,
+  stepName: string,
+  mock: boolean,
+): Promise<
+  | { ok: true; handler: (ctx: unknown, run: unknown) => unknown | Promise<unknown> }
+  | { ok: false; error: string }
+> {
+  await prepareRuntimeEnvironment(workspaceRoot, { mock });
+
+  const absolutePath = join(workspaceRoot, workflow.file);
+  const mod = (await import(absolutePath)) as Record<string, unknown>;
+  const exported = mod[workflow.exportName] as ExportedWorkflow | undefined;
+
+  if (!exported || exported.__forge?.kind !== "workflow") {
+    return {
+      ok: false,
+      error: `export '${workflow.exportName}' is not a workflow`,
+    };
+  }
+
+  const step = exported.steps?.find((candidate) => candidate.name === stepName);
+  if (!step || typeof step.handler !== "function") {
+    return {
+      ok: false,
+      error: `step '${stepName}' not found in workflow '${workflow.name}'`,
+    };
+  }
+
+  return { ok: true, handler: step.handler };
+}

package/src/forge/runtime/workflows/retry-run.ts

@@ -0,0 +1,44 @@
+import type { DbAdapter } from "../db/adapter.ts";
+
+export async function retryWorkflowRun(
+  adapter: DbAdapter,
+  runId: number,
+  stepName?: string,
+): Promise<boolean> {
+  const runResult = await adapter.query(
+    `SELECT id, status FROM _forge_workflow_runs WHERE id = $1`,
+    [runId],
+  );
+
+  if (runResult.rows.length === 0) {
+    return false;
+  }
+
+  if (stepName) {
+    const stepResult = await adapter.query(
+      `UPDATE _forge_workflow_steps
+       SET status = 'pending', attempts = 0, next_attempt_at = now(), last_error = NULL, locked_at = NULL, locked_by = NULL, output = NULL, completed_at = NULL
+       WHERE run_id = $1 AND step_name = $2`,
+      [runId, stepName],
+    );
+    if (stepResult.rowCount === 0) {
+      return false;
+    }
+  } else {
+    await adapter.query(
+      `UPDATE _forge_workflow_steps
+       SET status = 'pending', attempts = 0, next_attempt_at = now(), last_error = NULL, locked_at = NULL, locked_by = NULL
+       WHERE run_id = $1 AND status IN ('failed', 'dead')`,
+      [runId],
+    );
+  }
+
+  await adapter.query(
+    `UPDATE _forge_workflow_runs
+     SET status = 'pending', last_error = NULL, completed_at = NULL, canceled_at = NULL, updated_at = now()
+     WHERE id = $1`,
+    [runId],
+  );
+
+  return true;
+}

package/src/forge/runtime/workflows/retry.ts

@@ -0,0 +1,8 @@
+export function computeNextAttemptAt(attempts: number, now = new Date()): Date {
+  const delayMs = attempts * 5_000;
+  return new Date(now.getTime() + delayMs);
+}
+
+export function formatTimestamp(date: Date): string {
+  return date.toISOString();
+}

package/src/forge/runtime/workflows/sanitize.ts

@@ -0,0 +1,19 @@
+const SECRET_PATTERNS = [
+  /sk_[a-zA-Z0-9]+/g,
+  /Bearer\s+[a-zA-Z0-9._-]+/gi,
+  /password\s*[:=]\s*\S+/gi,
+  /api[_-]?key\s*[:=]\s*\S+/gi,
+];
+
+const MAX_ERROR_LENGTH = 500;
+
+export function sanitizeWorkflowError(message: string): string {
+  let sanitized = message;
+  for (const pattern of SECRET_PATTERNS) {
+    sanitized = sanitized.replace(pattern, "[redacted]");
+  }
+  if (sanitized.length > MAX_ERROR_LENGTH) {
+    return `${sanitized.slice(0, MAX_ERROR_LENGTH)}…`;
+  }
+  return sanitized;
+}

package/src/forge/runtime/workflows/start-from-outbox.ts

@@ -0,0 +1,71 @@
+import type { TableMapEntry } from "../../compiler/data-graph/sql/serialize.ts";
+import type { DbAdapter } from "../db/adapter.ts";
+import { createWorkflowRun } from "./create-run.ts";
+import { loadWorkflowRegistry, loadWorkflowSubscriptions } from "./registry.ts";
+
+function parsePayload(value: unknown): unknown {
+  if (typeof value === "string") {
+    try {
+      return JSON.parse(value);
+    } catch {
+      return value;
+    }
+  }
+  return value;
+}
+
+export async function startWorkflowRunsForPendingOutbox(
+  adapter: DbAdapter,
+  workspaceRoot: string,
+): Promise<{ started: number; skipped: number }> {
+  const { workflows } = loadWorkflowRegistry(workspaceRoot);
+  const { byEvent } = loadWorkflowSubscriptions(workspaceRoot);
+
+  if (workflows.length === 0) {
+    return { started: 0, skipped: 0 };
+  }
+
+  const outboxEvents = await adapter.query(
+    `SELECT o.id, o.event_type, o.payload, o.auth_context
+     FROM _forge_outbox o
+     ORDER BY o.id`,
+  );
+
+  let started = 0;
+  let skipped = 0;
+
+  for (const row of outboxEvents.rows) {
+    const outboxId = Number(row.id);
+    const eventType = String(row.event_type);
+    const subscriptions = byEvent[eventType] ?? [];
+
+    if (subscriptions.length === 0) {
+      continue;
+    }
+
+    const payload = parsePayload(row.payload);
+    const authContext = parsePayload(row.auth_context);
+
+    for (const subscription of subscriptions) {
+      const idempotencyKey = `${subscription.workflowName}:outbox:${outboxId}`;
+      const result = await createWorkflowRun(adapter, workflows, {
+        workflowName: subscription.workflowName,
+        input: payload,
+        triggerType: "event",
+        triggerOutboxId: outboxId,
+        idempotencyKey,
+        authContext,
+      });
+
+      if (result.created) {
+        started += 1;
+      } else {
+        skipped += 1;
+      }
+    }
+  }
+
+  return { started, skipped };
+}
+
+export type { TableMapEntry };

package/src/forge/runtime/workflows/types.ts

@@ -0,0 +1,77 @@
+export type WorkflowRunStatus =
+  | "pending"
+  | "running"
+  | "completed"
+  | "failed"
+  | "canceled"
+  | "dead";
+
+export type WorkflowStepStatus =
+  | "pending"
+  | "running"
+  | "completed"
+  | "failed"
+  | "dead"
+  | "skipped";
+
+export interface WorkflowRunRow {
+  id: number;
+  workflow_name: string;
+  trigger_type: string;
+  trigger_outbox_id: number | null;
+  idempotency_key: string;
+  input: unknown;
+  auth_context?: unknown;
+  status: WorkflowRunStatus;
+  current_step: string | null;
+  last_error: string | null;
+  created_at: string;
+  updated_at: string;
+  started_at: string | null;
+  completed_at: string | null;
+  canceled_at: string | null;
+}
+
+export interface WorkflowStepRow {
+  id: number;
+  run_id: number;
+  step_name: string;
+  step_index: number;
+  status: WorkflowStepStatus;
+  input: unknown;
+  output: unknown;
+  attempts: number;
+  max_attempts: number;
+  next_attempt_at: string;
+  locked_at: string | null;
+  locked_by: string | null;
+  last_error: string | null;
+  started_at: string | null;
+  completed_at: string | null;
+  created_at: string;
+}
+
+export interface CreateWorkflowRunInput {
+  workflowName: string;
+  input: unknown;
+  triggerType: string;
+  triggerOutboxId?: number;
+  idempotencyKey: string;
+  authContext?: unknown;
+}
+
+export interface ProcessWorkflowBatchOptions {
+  limit?: number;
+  workerId?: string;
+  mock?: boolean;
+  telemetrySinks?: string[];
+}
+
+export interface ProcessWorkflowBatchResult {
+  claimed: number;
+  completed: number;
+  failed: number;
+  dead: number;
+  runsStarted: number;
+  errors: string[];
+}

package/src/forge/server.ts

@@ -0,0 +1,96 @@
+export type ForgeAuthRule =
+  | { kind: "policy"; policy: string }
+  | { kind: "roles"; roles: string[] }
+  | { kind: "public" };
+
+export type ForgeDefinition<T extends Record<string, unknown>> = T;
+
+export type ForgeRecord = Record<string, any>;
+
+export interface ForgeTelemetry {
+  capture(name: string, payload?: Record<string, unknown>): Promise<void> | void;
+}
+
+export interface ForgeContext {
+  db: ForgeRecord;
+  emit(event: string, payload?: Record<string, unknown>): Promise<void> | void;
+  telemetry: ForgeTelemetry;
+  secrets: ForgeRecord;
+  auth?: {
+    userId?: string;
+    tenantId?: string;
+    roles?: string[];
+    claims?: Record<string, unknown>;
+  };
+}
+
+export type ForgeCommandDefinition<TArgs = unknown, TResult = unknown> = Record<string, unknown> & {
+  auth?: ForgeAuthRule;
+  handler: (ctx: ForgeContext, args: TArgs) => TResult | Promise<TResult>;
+};
+
+export type ForgeQueryDefinition<TArgs = unknown, TResult = unknown> = Record<string, unknown> & {
+  auth?: ForgeAuthRule;
+  handler: (ctx: ForgeContext, args?: TArgs) => TResult | Promise<TResult>;
+};
+
+export type ForgeLiveQueryDefinition<TArgs = unknown, TResult = unknown> = Record<string, unknown> & {
+  auth?: ForgeAuthRule;
+  handler: (ctx: ForgeContext, args?: TArgs) => TResult | Promise<TResult>;
+};
+
+export type ForgeActionDefinition<TEvent = unknown, TResult = unknown> = Record<string, unknown> & {
+  event?: string;
+  handler: (ctx: ForgeContext, event: TEvent) => TResult | Promise<TResult>;
+};
+
+export function defineTable<T extends Record<string, unknown>>(definition: T): T {
+  return definition;
+}
+
+export function definePolicies<T extends Record<string, unknown>>(definition: T): T {
+  return definition;
+}
+
+export function can(policy: string): ForgeAuthRule {
+  return { kind: "policy", policy };
+}
+
+export function canRole(...roles: string[]): ForgeAuthRule {
+  return { kind: "roles", roles };
+}
+
+export function public_(): ForgeAuthRule {
+  return { kind: "public" };
+}
+
+export function command<T extends ForgeCommandDefinition>(definition: T): ForgeDefinition<T> {
+  return definition;
+}
+
+export function query<T extends ForgeQueryDefinition>(definition: T): ForgeDefinition<T> {
+  return definition;
+}
+
+export function liveQuery<T extends ForgeLiveQueryDefinition>(definition: T): ForgeDefinition<T> {
+  return definition;
+}
+
+export function action<T extends ForgeActionDefinition>(definition: T): ForgeDefinition<T> {
+  return definition;
+}
+
+export function event(name: string): { kind: "event"; name: string } {
+  return { kind: "event", name };
+}
+
+export function step<T extends (...args: any[]) => unknown>(
+  name: string,
+  handler: T,
+): { kind: "step"; name: string; handler: T } {
+  return { kind: "step", name, handler };
+}
+
+export function workflow<T extends Record<string, unknown>>(definition: T): ForgeDefinition<T> {
+  return definition;
+}