forgeos 0.1.0-alpha.0

package/src/forge/runtime/outbox/claim.ts

@@ -0,0 +1,153 @@
+import type { DbAdapter } from "../db/adapter.ts";
+import type { ClaimedDelivery, OutboxDeliveryRow } from "./types.ts";
+
+function rowToDelivery(row: Record<string, unknown>): OutboxDeliveryRow {
+  return {
+    id: Number(row.id),
+    outbox_id: Number(row.outbox_id),
+    action_name: String(row.action_name),
+    status: String(row.status) as OutboxDeliveryRow["status"],
+    attempts: Number(row.attempts),
+    max_attempts: Number(row.max_attempts),
+    next_attempt_at: String(row.next_attempt_at),
+    locked_at: row.locked_at != null ? String(row.locked_at) : null,
+    locked_by: row.locked_by != null ? String(row.locked_by) : null,
+    last_error: row.last_error != null ? String(row.last_error) : null,
+    processed_at: row.processed_at != null ? String(row.processed_at) : null,
+    created_at: String(row.created_at),
+  };
+}
+
+export async function claimPendingDeliveries(
+  adapter: DbAdapter,
+  limit: number,
+  workerId: string,
+): Promise<ClaimedDelivery[]> {
+  const pending = await adapter.query(
+    `SELECT id, outbox_id, action_name, status, attempts, max_attempts, next_attempt_at, locked_at, locked_by, last_error, processed_at, created_at
+     FROM _forge_outbox_deliveries
+     WHERE status = 'pending' AND next_attempt_at <= now()
+     ORDER BY id
+     LIMIT $1`,
+    [limit],
+  );
+
+  const claimed: ClaimedDelivery[] = [];
+
+  for (const row of pending.rows) {
+    const deliveryId = Number(row.id);
+    const claimResult = await adapter.query(
+      `UPDATE _forge_outbox_deliveries
+       SET status = 'processing', locked_at = now(), locked_by = $1
+       WHERE id = $2 AND status = 'pending'`,
+      [workerId, deliveryId],
+    );
+
+    if (claimResult.rowCount === 0) {
+      continue;
+    }
+
+    const eventResult = await adapter.query(
+      `SELECT o.id, o.event_type, o.payload, o.auth_context, o.created_at
+       FROM _forge_outbox o
+       WHERE o.id = $1`,
+      [Number(row.outbox_id)],
+    );
+
+    const eventRow = eventResult.rows[0];
+    if (!eventRow) {
+      continue;
+    }
+
+    let payload: unknown = eventRow.payload;
+    if (typeof payload === "string") {
+      try {
+        payload = JSON.parse(payload);
+      } catch {
+        payload = eventRow.payload;
+      }
+    }
+
+    let authContext: unknown = eventRow.auth_context;
+    if (typeof authContext === "string") {
+      try {
+        authContext = JSON.parse(authContext);
+      } catch {
+        authContext = eventRow.auth_context;
+      }
+    }
+
+    claimed.push({
+      ...rowToDelivery(row),
+      event_type: String(eventRow.event_type),
+      payload,
+      auth_context: authContext,
+    });
+  }
+
+  return claimed;
+}
+
+export async function markDeliveryProcessed(
+  adapter: DbAdapter,
+  deliveryId: number,
+): Promise<void> {
+  await adapter.query(
+    `UPDATE _forge_outbox_deliveries
+     SET status = 'processed', processed_at = now(), locked_at = NULL, locked_by = NULL
+     WHERE id = $1`,
+    [deliveryId],
+  );
+}
+
+export async function markDeliveryRetry(
+  adapter: DbAdapter,
+  deliveryId: number,
+  attempts: number,
+  lastError: string,
+  nextAttemptAt: string,
+): Promise<void> {
+  await adapter.query(
+    `UPDATE _forge_outbox_deliveries
+     SET status = 'pending', attempts = $1, last_error = $2, next_attempt_at = $3, locked_at = NULL, locked_by = NULL
+     WHERE id = $4`,
+    [attempts, lastError, nextAttemptAt, deliveryId],
+  );
+}
+
+export async function markDeliveryDead(
+  adapter: DbAdapter,
+  deliveryId: number,
+  lastError: string,
+  attempts?: number,
+): Promise<void> {
+  if (attempts !== undefined) {
+    await adapter.query(
+      `UPDATE _forge_outbox_deliveries
+       SET status = 'dead', attempts = $1, last_error = $2, locked_at = NULL, locked_by = NULL
+       WHERE id = $3`,
+      [attempts, lastError, deliveryId],
+    );
+    return;
+  }
+
+  await adapter.query(
+    `UPDATE _forge_outbox_deliveries
+     SET status = 'dead', last_error = $1, locked_at = NULL, locked_by = NULL
+     WHERE id = $2`,
+    [lastError, deliveryId],
+  );
+}
+
+export async function resetDeliveryForRetry(
+  adapter: DbAdapter,
+  deliveryId: number,
+): Promise<boolean> {
+  const result = await adapter.query(
+    `UPDATE _forge_outbox_deliveries
+     SET status = 'pending', attempts = 0, next_attempt_at = now(), last_error = NULL, locked_at = NULL, locked_by = NULL, processed_at = NULL
+     WHERE id = $1`,
+    [deliveryId],
+  );
+  return result.rowCount > 0;
+}

package/src/forge/runtime/outbox/process.ts

@@ -0,0 +1,298 @@
+import { join } from "node:path";
+import { createDiagnostic } from "../../compiler/diagnostics/create.ts";
+import {
+  FORGE_OUTBOX_PROCESS_FAILED,
+  FORGE_RUNTIME_NOT_FOUND,
+} from "../../compiler/diagnostics/codes.ts";
+import type { TableMapEntry } from "../../compiler/data-graph/sql/serialize.ts";
+import type { RuntimeEntry } from "../../compiler/types/runtime-graph.ts";
+import { adapterAsTransaction } from "../db/adapter.ts";
+import type { DbAdapter } from "../db/adapter.ts";
+import { createGeneratedDbClient } from "../db/generated-client.ts";
+import { createActionContext } from "../context/create-context.ts";
+import { systemAuthFromSnapshot } from "../auth/resolve.ts";
+import type { AuthContext } from "../auth/types.ts";
+import { prepareRuntimeEnvironment } from "../executor.ts";
+import { loadActionSubscriptions } from "./subscriptions.ts";
+import { createTelemetryContext } from "../telemetry/context.ts";
+import { generateTraceId } from "../telemetry/correlation.ts";
+import {
+  claimPendingDeliveries,
+  markDeliveryDead,
+  markDeliveryProcessed,
+  markDeliveryRetry,
+} from "./claim.ts";
+import { computeNextAttemptAt, formatTimestamp } from "./retry.ts";
+import type {
+  ClaimedDelivery,
+  ProcessOutboxBatchOptions,
+  ProcessOutboxBatchResult,
+} from "./types.ts";
+
+function defaultWorkerId(): string {
+  return `forge-worker-${process.pid}`;
+}
+
+function resolveSystemAuthFromDelivery(delivery: ClaimedDelivery): AuthContext {
+  const snapshot = (delivery.auth_context as AuthContext | undefined) ?? { kind: "anonymous" };
+  const tenantId =
+    snapshot.kind === "user"
+      ? snapshot.tenantId
+      : snapshot.kind === "system"
+        ? snapshot.tenantId
+        : undefined;
+  return systemAuthFromSnapshot(snapshot, tenantId);
+}
+
+async function runDeliveryAction(
+  workspaceRoot: string,
+  delivery: ClaimedDelivery,
+  adapter: DbAdapter,
+  tableMap: Record<string, TableMapEntry>,
+  runtimeGraphEntries: RuntimeEntry[],
+  mock: boolean,
+): Promise<{ ok: true; result: unknown } | { ok: false; error: string }> {
+  const entry = runtimeGraphEntries.find(
+    (candidate) => candidate.name === delivery.action_name,
+  );
+
+  if (!entry || entry.kind !== "action") {
+    return {
+      ok: false,
+      error: `action '${delivery.action_name}' not found in runtime graph`,
+    };
+  }
+
+  await prepareRuntimeEnvironment(workspaceRoot, { mock, db: adapter });
+
+  const absolutePath = join(workspaceRoot, entry.file);
+  const mod = (await import(absolutePath)) as Record<string, unknown>;
+  const exported = mod[entry.name];
+
+  if (
+    !exported ||
+    typeof exported !== "object" ||
+    !("handler" in exported) ||
+    typeof (exported as { handler: unknown }).handler !== "function"
+  ) {
+    return {
+      ok: false,
+      error: `export '${delivery.action_name}' is not an action handler`,
+    };
+  }
+
+  const handler = (exported as {
+    handler: (ctx: unknown, args: unknown) => unknown | Promise<unknown>;
+  }).handler;
+
+  try {
+    const tx = adapterAsTransaction(adapter);
+    const auth = resolveSystemAuthFromDelivery(delivery);
+    const db = createGeneratedDbClient(tx, tableMap, { auth });
+
+    const payloadObj =
+      delivery.payload && typeof delivery.payload === "object"
+        ? (delivery.payload as Record<string, unknown>)
+        : {};
+    const traceId =
+      typeof payloadObj.traceId === "string" ? payloadObj.traceId : generateTraceId();
+
+    const telemetry = createTelemetryContext({
+      adapter,
+      traceId,
+      runtime: { kind: "action", name: delivery.action_name },
+      outbox: {
+        eventId: String(delivery.outbox_id),
+        deliveryId: String(delivery.id),
+      },
+      bufferInTransaction: false,
+      workspaceRoot,
+      sinks: ["local"],
+    });
+
+    const ctx = createActionContext(db, telemetry, auth);
+    const result = await handler(ctx, delivery.payload);
+    await telemetry.flush("local");
+    return { ok: true, result };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "action handler failed";
+    return { ok: false, error: message };
+  }
+}
+
+export async function processOutboxBatch(
+  adapter: DbAdapter,
+  workspaceRoot: string,
+  tableMap: Record<string, TableMapEntry>,
+  runtimeGraphEntries: RuntimeEntry[],
+  options: ProcessOutboxBatchOptions = {},
+): Promise<ProcessOutboxBatchResult> {
+  const limit = options.limit ?? 10;
+  const workerId = options.workerId ?? defaultWorkerId();
+  const mock = options.mock ?? false;
+
+  loadActionSubscriptions(workspaceRoot);
+
+  const claimed = await claimPendingDeliveries(adapter, limit, workerId);
+  const result: ProcessOutboxBatchResult = {
+    processed: 0,
+    failed: 0,
+    dead: 0,
+    claimed: claimed.length,
+    errors: [],
+  };
+
+  for (const delivery of claimed) {
+    const executed = await runDeliveryAction(
+      workspaceRoot,
+      delivery,
+      adapter,
+      tableMap,
+      runtimeGraphEntries,
+      mock,
+    );
+
+    if (executed.ok) {
+      await markDeliveryProcessed(adapter, delivery.id);
+      result.processed += 1;
+      continue;
+    }
+
+    const nextAttempts = delivery.attempts + 1;
+    const errorMessage = executed.error;
+
+    if (nextAttempts >= delivery.max_attempts) {
+      await markDeliveryDead(adapter, delivery.id, errorMessage, nextAttempts);
+      result.dead += 1;
+      result.errors.push(
+        createDiagnostic({
+          severity: "error",
+          code: FORGE_OUTBOX_PROCESS_FAILED,
+          message: `delivery ${delivery.id} dead after ${nextAttempts} attempts: ${errorMessage}`,
+        }).message,
+      );
+      continue;
+    }
+
+    const nextAttemptAt = formatTimestamp(computeNextAttemptAt(nextAttempts));
+    await markDeliveryRetry(
+      adapter,
+      delivery.id,
+      nextAttempts,
+      errorMessage,
+      nextAttemptAt,
+    );
+    result.failed += 1;
+    result.errors.push(
+      createDiagnostic({
+        severity: "warning",
+        code: FORGE_OUTBOX_PROCESS_FAILED,
+        message: `delivery ${delivery.id} scheduled retry ${nextAttempts}: ${errorMessage}`,
+      }).message,
+    );
+  }
+
+  return result;
+}
+
+export function startOutboxWorker(
+  adapter: DbAdapter,
+  workspaceRoot: string,
+  tableMap: Record<string, TableMapEntry>,
+  runtimeGraphEntries: RuntimeEntry[],
+  options: ProcessOutboxBatchOptions & { intervalMs?: number } = {},
+): { stop: () => void; isRunning: () => boolean } {
+  const intervalMs = options.intervalMs ?? 2_000;
+  let running = true;
+  let timer: ReturnType<typeof setInterval> | null = null;
+
+  const tick = async () => {
+    if (!running) {
+      return;
+    }
+    try {
+      const { runWorkerTick } = await import("../workflows/process.ts");
+      await runWorkerTick(adapter, workspaceRoot, tableMap, runtimeGraphEntries, options);
+    } catch {
+      /* worker loop continues on batch errors */
+    }
+  };
+
+  void tick();
+  timer = setInterval(() => {
+    void tick();
+  }, intervalMs);
+
+  return {
+    stop: () => {
+      running = false;
+      if (timer) {
+        clearInterval(timer);
+        timer = null;
+      }
+    },
+    isRunning: () => running,
+  };
+}
+
+export async function getOutboxSummary(adapter: DbAdapter): Promise<{
+  pending: number;
+  dead: number;
+  processing: number;
+  processed: number;
+  failed: number;
+  events: number;
+}> {
+  const deliveries = await adapter.query(
+    `SELECT status, COUNT(*)::int AS count FROM _forge_outbox_deliveries GROUP BY status`,
+  );
+  const events = await adapter.query(`SELECT COUNT(*)::int AS count FROM _forge_outbox`);
+
+  const counts: Record<string, number> = {};
+  for (const row of deliveries.rows) {
+    counts[String(row.status)] = Number(row.count);
+  }
+
+  return {
+    pending: counts.pending ?? 0,
+    processing: counts.processing ?? 0,
+    processed: counts.processed ?? 0,
+    failed: counts.failed ?? 0,
+    dead: counts.dead ?? 0,
+    events: Number(events.rows[0]?.count ?? 0),
+  };
+}
+
+export async function listOutboxDeliveries(
+  adapter: DbAdapter,
+): Promise<Record<string, unknown>[]> {
+  const result = await adapter.query(
+    `SELECT d.id, d.outbox_id, d.action_name, d.status, d.attempts, d.max_attempts, d.next_attempt_at, d.last_error, d.processed_at, o.event_type, o.created_at AS event_created_at
+     FROM _forge_outbox_deliveries d
+     JOIN _forge_outbox o ON o.id = d.outbox_id
+     ORDER BY d.id`,
+  );
+  return result.rows;
+}
+
+export async function listDeadDeliveries(
+  adapter: DbAdapter,
+): Promise<Record<string, unknown>[]> {
+  const result = await adapter.query(
+    `SELECT d.id, d.outbox_id, d.action_name, d.attempts, d.last_error, d.created_at, o.event_type
+     FROM _forge_outbox_deliveries d
+     JOIN _forge_outbox o ON o.id = d.outbox_id
+     WHERE d.status = 'dead'
+     ORDER BY d.id`,
+  );
+  return result.rows;
+}
+
+export async function clearDeadDeliveries(adapter: DbAdapter): Promise<number> {
+  const result = await adapter.query(
+    `DELETE FROM _forge_outbox_deliveries WHERE status = 'dead'`,
+  );
+  return result.rowCount;
+}
+
+export { FORGE_RUNTIME_NOT_FOUND };

package/src/forge/runtime/outbox/retry.ts

@@ -0,0 +1,8 @@
+export function computeNextAttemptAt(attempts: number, now = new Date()): Date {
+  const delayMs = attempts * 5_000;
+  return new Date(now.getTime() + delayMs);
+}
+
+export function formatTimestamp(date: Date): string {
+  return date.toISOString();
+}

package/src/forge/runtime/outbox/subscriptions.ts

@@ -0,0 +1,33 @@
+import { nodeFileSystem } from "../../compiler/fs/index.ts";
+import { join } from "node:path";
+import type { ActionSubscription } from "../../compiler/types/action-subscriptions.ts";
+import { GENERATED_DIR } from "../../compiler/emitter/constants.ts";
+import { stripDeterministicHeader } from "../../compiler/primitives/header.ts";
+
+interface ActionSubscriptionsJson {
+  subscriptions: ActionSubscription[];
+  byEvent: Record<string, ActionSubscription[]>;
+}
+
+export function loadActionSubscriptions(
+  workspaceRoot: string,
+): ActionSubscriptionsJson {
+  const absolute = join(workspaceRoot, GENERATED_DIR, "actionSubscriptions.json");
+  if (!nodeFileSystem.exists(absolute)) {
+    return { subscriptions: [], byEvent: {} };
+  }
+
+  const raw = stripDeterministicHeader((nodeFileSystem.readText(absolute) ?? ""));
+  const parsed = JSON.parse(raw) as ActionSubscriptionsJson;
+  return {
+    subscriptions: parsed.subscriptions ?? [],
+    byEvent: parsed.byEvent ?? {},
+  };
+}
+
+export function subscriptionsForEvent(
+  subscriptions: ActionSubscription[],
+  eventType: string,
+): ActionSubscription[] {
+  return subscriptions.filter((subscription) => subscription.eventType === eventType);
+}

package/src/forge/runtime/outbox/types.ts

@@ -0,0 +1,69 @@
+export type OutboxDeliveryStatus =
+  | "pending"
+  | "processing"
+  | "processed"
+  | "failed"
+  | "dead";
+
+export interface OutboxEventRow {
+  id: number;
+  event_type: string;
+  payload: unknown;
+  auth_context?: unknown;
+  created_at: string;
+}
+
+export interface OutboxDeliveryRow {
+  id: number;
+  outbox_id: number;
+  action_name: string;
+  status: OutboxDeliveryStatus;
+  attempts: number;
+  max_attempts: number;
+  next_attempt_at: string;
+  locked_at: string | null;
+  locked_by: string | null;
+  last_error: string | null;
+  processed_at: string | null;
+  created_at: string;
+}
+
+export interface ClaimedDelivery extends OutboxDeliveryRow {
+  payload: unknown;
+  event_type: string;
+  auth_context?: unknown;
+}
+
+export interface ProcessOutboxBatchOptions {
+  limit?: number;
+  workerId?: string;
+  mock?: boolean;
+  telemetrySinks?: string[];
+  workspaceRoot?: string;
+}
+
+export interface ProcessOutboxBatchResult {
+  processed: number;
+  failed: number;
+  dead: number;
+  claimed: number;
+  errors: string[];
+}
+
+export interface OutboxWorkerOptions extends ProcessOutboxBatchOptions {
+  intervalMs?: number;
+}
+
+export interface OutboxWorkerHandle {
+  stop: () => void;
+  isRunning: () => boolean;
+}
+
+export interface OutboxSummary {
+  pending: number;
+  processing: number;
+  processed: number;
+  failed: number;
+  dead: number;
+  events: number;
+}