forgeos 0.1.0-alpha.0

package/src/forge/runtime/executor.ts

@@ -0,0 +1,446 @@
+import { nodeFileSystem } from "../compiler/fs/index.ts";
+import { join } from "node:path";
+import { buildAppGraph } from "../compiler/app-graph/build.ts";
+import { buildRuntimeMatrix } from "../compiler/classifier/runtime-matrix.ts";
+import { classify } from "../compiler/classifier/classify.ts";
+import { createDiagnostic } from "../compiler/diagnostics/create.ts";
+import {
+  FORGE_RUNTIME_NOT_FOUND,
+} from "../compiler/diagnostics/codes.ts";
+import type { TableMapEntry } from "../compiler/data-graph/sql/serialize.ts";
+import { GENERATED_DIR, FORGE_LOCK_PATH } from "../compiler/emitter/constants.ts";
+import { checkImportGuards } from "../compiler/guards/check-import-guards.ts";
+import { discover } from "../compiler/orchestrator/discover.ts";
+import { loadManifest } from "../compiler/orchestrator/manifest.ts";
+import { PackageGraphCompiler } from "../compiler/package-graph/compiler.ts";
+import { stripDeterministicHeader } from "../compiler/primitives/header.ts";
+import { resolveByPackageName } from "../compiler/recipes/registry.ts";
+import type { ModuleGraph } from "../compiler/types/app-graph.ts";
+import type { Diagnostic } from "../compiler/types/diagnostic.ts";
+import type { ForgeLock } from "../compiler/types/lock.ts";
+import type { RuntimeEntry, RuntimeGraph } from "../compiler/types/runtime-graph.ts";
+import type { RuntimeMatrix } from "../compiler/types/runtime-matrix.ts";
+import type { DbAdapter } from "./db/adapter.ts";
+import type { AuthContext } from "./auth/types.ts";
+import type { LiveSubscriptionManager } from "./live/types.ts";
+import { resolveAuthFromCli } from "./auth/resolve.ts";
+import {
+  executeResolvedEntry,
+  guardBlockedDiagnostics,
+  resolveHandlerFromModule,
+  type RunEntryRuntime,
+} from "./runner/run-entry.ts";
+
+export interface RunEntryOptions {
+  json: boolean;
+  mock: boolean;
+  args?: unknown;
+  db?: DbAdapter | null;
+  auth?: AuthContext;
+  userId?: string;
+  tenantId?: string;
+  role?: string;
+  liveManager?: LiveSubscriptionManager;
+}
+
+export interface RunEntryResult {
+  ok: boolean;
+  result?: unknown;
+  entry?: RuntimeEntry;
+  diagnostics: Diagnostic[];
+  exitCode: 0 | 1;
+  traceId?: string;
+}
+
+export interface ListEntriesResult {
+  entries: RuntimeEntry[];
+  diagnostics: Diagnostic[];
+  exitCode: 0 | 1;
+}
+
+export interface PrepareRuntimeEnvironmentOptions {
+  mock: boolean;
+  mockAi?: boolean;
+  db?: DbAdapter | null;
+}
+
+function readGeneratedJson<T>(workspaceRoot: string, relative: string): T | null {
+  const absolute = join(workspaceRoot, relative);
+  if (!nodeFileSystem.exists(absolute)) {
+    return null;
+  }
+  const raw = stripDeterministicHeader((nodeFileSystem.readText(absolute) ?? ""));
+  return JSON.parse(raw) as T;
+}
+
+function loadRuntimeGraph(workspaceRoot: string): {
+  graph: RuntimeGraph | null;
+  diagnostics: Diagnostic[];
+} {
+  const graph = readGeneratedJson<RuntimeGraph>(
+    workspaceRoot,
+    `${GENERATED_DIR}/runtimeGraph.json`,
+  );
+
+  if (!graph) {
+    return {
+      graph: null,
+      diagnostics: [
+        createDiagnostic({
+          severity: "error",
+          code: FORGE_RUNTIME_NOT_FOUND,
+          message: `missing ${GENERATED_DIR}/runtimeGraph.json; run forge generate first`,
+          file: `${GENERATED_DIR}/runtimeGraph.json`,
+        }),
+      ],
+    };
+  }
+
+  return { graph, diagnostics: [...(graph.diagnostics ?? [])] };
+}
+
+function loadTableMap(workspaceRoot: string): Record<string, TableMapEntry> | null {
+  const dbJson = readGeneratedJson<{ tableMap: Record<string, TableMapEntry> }>(
+    workspaceRoot,
+    `${GENERATED_DIR}/db.json`,
+  );
+  return dbJson?.tableMap ?? null;
+}
+
+function moduleClosure(moduleGraph: ModuleGraph, rootModuleId: string): Set<string> {
+  const closure = new Set<string>();
+  const nodesById = new Map(moduleGraph.nodes.map((node) => [node.id, node]));
+  const queue = [rootModuleId];
+
+  while (queue.length > 0) {
+    const moduleId = queue.pop()!;
+    if (closure.has(moduleId)) {
+      continue;
+    }
+    closure.add(moduleId);
+
+    const node = nodesById.get(moduleId);
+    if (!node) {
+      continue;
+    }
+
+    for (const imp of node.localImports) {
+      queue.push(imp.toModuleId);
+    }
+  }
+
+  return closure;
+}
+
+function filesForModuleClosure(
+  moduleGraph: ModuleGraph,
+  closure: Set<string>,
+): Set<string> {
+  const files = new Set<string>();
+  for (const node of moduleGraph.nodes) {
+    if (closure.has(node.id)) {
+      files.add(node.file);
+    }
+  }
+  return files;
+}
+
+async function loadRuntimeMatrix(workspaceRoot: string): Promise<RuntimeMatrix> {
+  const fromDisk = readGeneratedJson<RuntimeMatrix>(
+    workspaceRoot,
+    `${GENERATED_DIR}/runtimeMatrix.json`,
+  );
+  if (fromDisk) {
+    return fromDisk;
+  }
+
+  const ctx = discover({ workspaceRoot });
+  const compiler = new PackageGraphCompiler();
+  const classified = await Promise.all(
+    ctx.dependencies.map(async (dep) => {
+      const recipe = resolveByPackageName(dep.name) ?? undefined;
+      const api = await compiler.analyze(dep, {
+        runtimeInspect: false,
+        resolutionMode: "nodenext",
+        cacheDir: ctx.cacheDir,
+        recipeVersion: recipe?.recipeVersion,
+      });
+      return {
+        api,
+        classification: classify(api, recipe),
+        recipe,
+      };
+    }),
+  );
+
+  return buildRuntimeMatrix(classified);
+}
+
+function loadForgeLock(workspaceRoot: string): ForgeLock | null {
+  const absolute = join(workspaceRoot, FORGE_LOCK_PATH);
+  if (!nodeFileSystem.exists(absolute)) {
+    return null;
+  }
+  const raw = stripDeterministicHeader((nodeFileSystem.readText(absolute) ?? ""));
+  return JSON.parse(raw) as ForgeLock;
+}
+
+function loadMockMap(workspaceRoot: string): Record<string, string> {
+  const fromJson = readGeneratedJson<{ entries: { packageName: string; mockFile: string }[] }>(
+    workspaceRoot,
+    `${GENERATED_DIR}/mockMap.json`,
+  );
+  if (fromJson?.entries) {
+    const map: Record<string, string> = {};
+    for (const entry of fromJson.entries) {
+      map[entry.packageName] = entry.mockFile;
+    }
+    return map;
+  }
+
+  return {};
+}
+
+function capitalizeSegment(value: string): string {
+  return value
+    .split(/[^a-zA-Z0-9]+/)
+    .filter(Boolean)
+    .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+    .join("");
+}
+
+async function applyMocks(workspaceRoot: string, lock: ForgeLock | null): Promise<void> {
+  const mockMap = loadMockMap(workspaceRoot);
+
+  for (const [packageName, relativePath] of Object.entries(mockMap).sort()) {
+    const absolute = join(workspaceRoot, relativePath);
+    if (!nodeFileSystem.exists(absolute)) {
+      continue;
+    }
+
+    const mod = (await import(absolute)) as Record<string, unknown>;
+    const factoryName = `create${capitalizeSegment(packageName)}Mock`;
+    const factory = mod[factoryName];
+
+    const bunMock = Bun as typeof Bun & {
+      mock: { module: (specifier: string, factory: () => unknown) => void };
+    };
+
+    bunMock.mock.module(packageName, () => {
+      if (typeof factory === "function") {
+        const mockValue = (factory as () => unknown)();
+        if (typeof mockValue === "function") {
+          return { default: mockValue };
+        }
+        const MockConstructor = function MockConstructor() {
+          return mockValue;
+        };
+        return { default: MockConstructor, ...(mockValue as object) };
+      }
+      return mod;
+    });
+  }
+
+  if (lock) {
+    for (const pkg of lock.packages) {
+      for (const secret of pkg.secrets) {
+        if (!process.env[secret.envVar]) {
+          process.env[secret.envVar] = `sk_forge_mock_${secret.envVar.toLowerCase()}`;
+        }
+      }
+    }
+  }
+}
+
+let activeDbAdapter: DbAdapter | null = null;
+
+function snapshotEnv(): Record<string, string | undefined> {
+  return { ...process.env };
+}
+
+function restoreEnv(snapshot: Record<string, string | undefined>): void {
+  for (const key of Object.keys(process.env)) {
+    if (!(key in snapshot)) {
+      delete process.env[key];
+    }
+  }
+  for (const [key, value] of Object.entries(snapshot)) {
+    if (value === undefined) {
+      delete process.env[key];
+    } else {
+      process.env[key] = value;
+    }
+  }
+}
+
+export async function prepareRuntimeEnvironment(
+  workspaceRoot: string,
+  options: PrepareRuntimeEnvironmentOptions,
+): Promise<() => void> {
+  const envSnapshot = snapshotEnv();
+  const previousActiveDbAdapter = activeDbAdapter;
+
+  const useMock = options.mock || process.env.FORGE_MOCK === "1";
+  if (useMock) {
+    await applyMocks(workspaceRoot, loadForgeLock(workspaceRoot));
+  }
+
+  if (options.mockAi || process.env.FORGE_MOCK_AI === "1") {
+    process.env.FORGE_MOCK_AI = "1";
+  }
+
+  activeDbAdapter = options.db ?? null;
+
+  return () => {
+    activeDbAdapter = previousActiveDbAdapter;
+    restoreEnv(envSnapshot);
+  };
+}
+
+export function getActiveDbAdapter(): DbAdapter | null {
+  return activeDbAdapter;
+}
+
+async function guardPreflight(
+  workspaceRoot: string,
+  entry: RuntimeEntry,
+): Promise<Diagnostic[]> {
+  const ctx = discover({ workspaceRoot });
+  const manifest = loadManifest(ctx.cacheDir);
+  const appGraph = await buildAppGraph({
+    workspaceRoot: ctx.workspaceRoot,
+    sources: ctx.sources,
+    prior: manifest.priorAppGraph,
+    tsconfigPath: ctx.tsconfigPath ?? undefined,
+  });
+
+  const matrix = await loadRuntimeMatrix(workspaceRoot);
+  const violations = checkImportGuards(appGraph.moduleGraph, matrix);
+  const closure = moduleClosure(appGraph.moduleGraph, entry.moduleId);
+  const allowedFiles = filesForModuleClosure(appGraph.moduleGraph, closure);
+
+  return violations.filter(
+    (diagnostic) => diagnostic.file !== undefined && allowedFiles.has(diagnostic.file),
+  );
+}
+
+export function listEntries(workspaceRoot: string): ListEntriesResult {
+  const { graph, diagnostics } = loadRuntimeGraph(workspaceRoot);
+  if (!graph) {
+    return { entries: [], diagnostics, exitCode: 1 };
+  }
+
+  return {
+    entries: graph.entries,
+    diagnostics,
+    exitCode: 0,
+  };
+}
+
+export async function runEntry(
+  workspaceRoot: string,
+  name: string,
+  options: RunEntryOptions,
+): Promise<RunEntryResult> {
+  const { graph, diagnostics } = loadRuntimeGraph(workspaceRoot);
+  if (!graph) {
+    return { ok: false, diagnostics, exitCode: 1 };
+  }
+
+  const entry = graph.entries.find((candidate) => candidate.name === name);
+  if (!entry) {
+    return {
+      ok: false,
+      diagnostics: [
+        ...diagnostics,
+        createDiagnostic({
+          severity: "error",
+          code: FORGE_RUNTIME_NOT_FOUND,
+          message: `runtime entry '${name}' not found`,
+        }),
+      ],
+      exitCode: 1,
+    };
+  }
+
+  const guardViolations = await guardPreflight(workspaceRoot, entry);
+  if (guardViolations.length > 0) {
+    return {
+      ok: false,
+      entry,
+      diagnostics: guardBlockedDiagnostics(entry, guardViolations, diagnostics),
+      exitCode: 1,
+    };
+  }
+
+  const db = Object.prototype.hasOwnProperty.call(options, "db")
+    ? (options.db ?? null)
+    : activeDbAdapter;
+
+  const envSnapshot = snapshotEnv();
+  const previousActiveDbAdapter = activeDbAdapter;
+
+  try {
+    await prepareRuntimeEnvironment(workspaceRoot, {
+      mock: options.mock,
+      db,
+    });
+
+    const absolutePath = join(workspaceRoot, entry.file);
+    const mod = (await import(absolutePath)) as Record<string, unknown>;
+    const resolved = resolveHandlerFromModule(mod, entry.name);
+
+    if (!resolved) {
+      return {
+        ok: false,
+        entry,
+        diagnostics: [
+          ...diagnostics,
+          createDiagnostic({
+            severity: "error",
+            code: FORGE_RUNTIME_NOT_FOUND,
+            message: `export '${entry.name}' is not a callable handler`,
+            file: entry.file,
+          }),
+        ],
+        exitCode: 1,
+      };
+    }
+
+    const auth =
+      options.auth ??
+      resolveAuthFromCli({
+        userId: options.userId,
+        tenantId: options.tenantId,
+        role: options.role,
+      });
+
+    const runtime: RunEntryRuntime = {
+      adapter: db,
+      tableMap: loadTableMap(workspaceRoot) ?? undefined,
+      workspaceRoot,
+      auth,
+      liveManager: options.liveManager,
+    };
+
+    const executed = await executeResolvedEntry(
+      workspaceRoot,
+      entry,
+      resolved,
+      { ...options, auth },
+      runtime,
+    );
+
+    return {
+      ok: executed.ok,
+      result: executed.result,
+      entry,
+      diagnostics: [...diagnostics, ...executed.diagnostics],
+      exitCode: executed.ok ? 0 : 1,
+      traceId: "traceId" in executed ? (executed as { traceId?: string }).traceId : undefined,
+    };
+  } finally {
+    activeDbAdapter = previousActiveDbAdapter;
+    restoreEnv(envSnapshot);
+  }
+}

package/src/forge/runtime/live/dependency-tracker.ts

@@ -0,0 +1,57 @@
+import type { AuthContext } from "../auth/types.ts";
+import type { DataChange, DataDependency, WriteTracker } from "./types.ts";
+
+export class DependencyTracker {
+  private readonly dependenciesByKey = new Map<string, DataDependency>();
+
+  record(table: string, tenantId: string | null): void {
+    if (!tenantId) {
+      return;
+    }
+    const key = `${table}:${tenantId}`;
+    this.dependenciesByKey.set(key, { table, tenantId });
+  }
+
+  snapshot(): DataDependency[] {
+    return [...this.dependenciesByKey.values()].sort((a, b) => {
+      if (a.table !== b.table) {
+        return a.table < b.table ? -1 : 1;
+      }
+      return a.tenantId < b.tenantId ? -1 : a.tenantId > b.tenantId ? 1 : 0;
+    });
+  }
+}
+
+export function tenantIdFromAuth(auth: AuthContext | undefined): string | null {
+  if (auth?.kind === "user") {
+    return auth.tenantId ?? null;
+  }
+  if (auth?.kind === "system" && auth.tenantId) {
+    return auth.tenantId;
+  }
+  return null;
+}
+
+export function createWriteTracker(): WriteTracker {
+  const byKey = new Map<string, DataChange>();
+
+  return {
+    get changes() {
+      return [...byKey.values()].sort((a, b) => {
+        const tableA = a.tables.join(",");
+        const tableB = b.tables.join(",");
+        if (tableA !== tableB) {
+          return tableA < tableB ? -1 : 1;
+        }
+        return a.tenantId < b.tenantId ? -1 : a.tenantId > b.tenantId ? 1 : 0;
+      });
+    },
+    record(table, tenantId, operation = "write") {
+      if (!tenantId) {
+        return;
+      }
+      const key = `${table}:${tenantId}:${operation}`;
+      byKey.set(key, { tables: [table], tenantId, operation });
+    },
+  };
+}

package/src/forge/runtime/live/invalidation-log.ts

@@ -0,0 +1,189 @@
+import type { DbAdapter, DbTransaction } from "../db/adapter.ts";
+import { currentReleaseInfo } from "../release/runtime.ts";
+import type { DataChange, LiveInvalidation } from "./types.ts";
+
+export const LIVE_INVALIDATION_SEQUENCE_SQL =
+  "CREATE SEQUENCE IF NOT EXISTS _forge_live_revision_seq START WITH 2";
+
+export const LIVE_INVALIDATION_TABLE_SQL =
+  "CREATE TABLE IF NOT EXISTS _forge_live_invalidations (id bigserial PRIMARY KEY, revision bigint NOT NULL, table_name text NOT NULL, tenant_id text, operation text NOT NULL, source_kind text NOT NULL, source_name text, trace_id text, release_id text, deploy_id text, payload jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT now())";
+
+export const LIVE_SUBSCRIPTION_DEBUG_TABLE_SQL =
+  "CREATE TABLE IF NOT EXISTS _forge_live_subscription_debug (id text PRIMARY KEY, name text NOT NULL, tenant_id text, dependencies jsonb NOT NULL, last_revision bigint, runtime_id text, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now())";
+
+const LIVE_INVALIDATION_INDEX_SQL = [
+  "CREATE INDEX IF NOT EXISTS forge_live_invalidations_revision_idx ON _forge_live_invalidations (revision)",
+  "CREATE INDEX IF NOT EXISTS forge_live_invalidations_table_tenant_revision_idx ON _forge_live_invalidations (table_name, tenant_id, revision)",
+];
+
+export interface WriteLiveInvalidationsInput {
+  changes: DataChange[];
+  sourceKind: string;
+  sourceName?: string;
+  traceId?: string;
+  payload?: Record<string, unknown>;
+}
+
+export async function ensureLiveInvalidationSchema(
+  db: DbAdapter | DbTransaction,
+): Promise<void> {
+  await db.query(LIVE_INVALIDATION_SEQUENCE_SQL);
+  await db.query(LIVE_INVALIDATION_TABLE_SQL);
+  await db.query(LIVE_SUBSCRIPTION_DEBUG_TABLE_SQL);
+  for (const sql of LIVE_INVALIDATION_INDEX_SQL) {
+    await db.query(sql);
+  }
+}
+
+async function nextRevision(tx: DbTransaction): Promise<number> {
+  const result = await tx.query(
+    "SELECT nextval('_forge_live_revision_seq') AS revision",
+  );
+  const revision = Number(result.rows[0]?.revision ?? result.rows[0]?.nextval);
+  if (!Number.isFinite(revision)) {
+    throw new Error("live invalidation revision allocation failed");
+  }
+  return revision;
+}
+
+export async function writeLiveInvalidations(
+  tx: DbTransaction,
+  input: WriteLiveInvalidationsInput,
+): Promise<LiveInvalidation[]> {
+  if (input.changes.length === 0) {
+    return [];
+  }
+
+  await ensureLiveInvalidationSchema(tx);
+  const release = currentReleaseInfo();
+  const written: LiveInvalidation[] = [];
+
+  for (const change of input.changes) {
+    for (const table of change.tables) {
+      const revision = await nextRevision(tx);
+      const payload = input.payload ?? {};
+      const result = await tx.query(
+        `INSERT INTO _forge_live_invalidations
+          (revision, table_name, tenant_id, operation, source_kind, source_name, trace_id, release_id, deploy_id, payload)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb)
+         RETURNING id, created_at`,
+        [
+          revision,
+          table,
+          change.tenantId,
+          change.operation ?? "write",
+          input.sourceKind,
+          input.sourceName ?? null,
+          input.traceId ?? change.traceId ?? null,
+          release.releaseId ?? null,
+          release.deployId ?? null,
+          JSON.stringify(payload),
+        ],
+      );
+
+      written.push({
+        id: Number(result.rows[0]?.id ?? written.length + 1),
+        revision,
+        tableName: table,
+        tenantId: change.tenantId,
+        operation: change.operation ?? "write",
+        sourceKind: input.sourceKind,
+        ...(input.sourceName ? { sourceName: input.sourceName } : {}),
+        ...(input.traceId ?? change.traceId ? { traceId: input.traceId ?? change.traceId } : {}),
+        ...(release.releaseId ? { releaseId: release.releaseId } : {}),
+        ...(release.deployId ? { deployId: release.deployId } : {}),
+        payload,
+        createdAt: String(result.rows[0]?.created_at ?? new Date(0).toISOString()),
+      });
+    }
+  }
+
+  return written;
+}
+
+export async function notifyLiveWakeup(
+  db: DbAdapter | DbTransaction,
+  fromRevision: number,
+): Promise<void> {
+  try {
+    await db.query(`SELECT pg_notify('forge_live', $1)`, [
+      JSON.stringify({ fromRevision }),
+    ]);
+  } catch {
+    // Wakeups are best-effort. The durable invalidation log is the source of truth.
+  }
+}
+
+function parsePayload(value: unknown): Record<string, unknown> {
+  if (typeof value === "string") {
+    try {
+      const parsed = JSON.parse(value);
+      return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+        ? (parsed as Record<string, unknown>)
+        : {};
+    } catch {
+      return {};
+    }
+  }
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : {};
+}
+
+function rowToInvalidation(row: Record<string, unknown>): LiveInvalidation {
+  return {
+    id: Number(row.id),
+    revision: Number(row.revision),
+    tableName: String(row.table_name),
+    tenantId: row.tenant_id === null || row.tenant_id === undefined ? null : String(row.tenant_id),
+    operation: String(row.operation),
+    sourceKind: String(row.source_kind),
+    ...(row.source_name ? { sourceName: String(row.source_name) } : {}),
+    ...(row.trace_id ? { traceId: String(row.trace_id) } : {}),
+    ...(row.release_id ? { releaseId: String(row.release_id) } : {}),
+    ...(row.deploy_id ? { deployId: String(row.deploy_id) } : {}),
+    payload: parsePayload(row.payload),
+    createdAt: String(row.created_at ?? ""),
+  };
+}
+
+export async function getLatestLiveRevision(db: DbAdapter): Promise<number> {
+  await ensureLiveInvalidationSchema(db);
+  const result = await db.query(
+    "SELECT COALESCE(MAX(revision), 0) AS revision FROM _forge_live_invalidations",
+  );
+  const revision = Number(result.rows[0]?.revision ?? 0);
+  return Number.isFinite(revision) ? revision : 0;
+}
+
+export async function readLiveInvalidations(
+  db: DbAdapter,
+  afterRevision = 0,
+  limit = 100,
+): Promise<LiveInvalidation[]> {
+  await ensureLiveInvalidationSchema(db);
+  const result = await db.query(
+    `SELECT id, revision, table_name, tenant_id, operation, source_kind, source_name, trace_id, release_id, deploy_id, payload, created_at
+     FROM _forge_live_invalidations
+     WHERE revision > $1
+     ORDER BY revision ASC
+     LIMIT $2`,
+    [afterRevision, limit],
+  );
+  return result.rows.map(rowToInvalidation);
+}
+
+export async function listLiveInvalidations(
+  db: DbAdapter,
+  limit = 50,
+): Promise<LiveInvalidation[]> {
+  await ensureLiveInvalidationSchema(db);
+  const result = await db.query(
+    `SELECT id, revision, table_name, tenant_id, operation, source_kind, source_name, trace_id, release_id, deploy_id, payload, created_at
+     FROM _forge_live_invalidations
+     ORDER BY revision ASC
+     LIMIT $1`,
+    [limit],
+  );
+  return result.rows.map(rowToInvalidation);
+}