everything-dev 0.3.2 → 1.3.2

package/src/dev-logs.ts

@@ -0,0 +1,85 @@
+import { existsSync } from "node:fs";
+import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+export interface LogEntry {
+  timestamp: number;
+  source: string;
+  line: string;
+  isError?: boolean;
+}
+
+export interface DevLogger {
+  logFile: string;
+  latestFile: string;
+  write: (entry: LogEntry) => Promise<void>;
+  readLatest: (opts?: { tail?: number }) => Promise<string>;
+}
+
+export function getBosDir(configDir: string): string {
+  return join(configDir, ".bos");
+}
+
+export function getLogsDir(configDir: string): string {
+  return join(getBosDir(configDir), "logs");
+}
+
+function formatLogLine(entry: LogEntry): string {
+  const ts = new Date(entry.timestamp).toISOString();
+  const prefix = entry.isError ? "ERR" : "OUT";
+  return `[${ts}] [${entry.source}] [${prefix}] ${entry.line}`;
+}
+
+export async function createDevLogger(configDir: string, description: string): Promise<DevLogger> {
+  const dir = getLogsDir(configDir);
+  if (!existsSync(dir)) {
+    await mkdir(dir, { recursive: true });
+  }
+
+  const now = new Date();
+  const ts = now.toISOString().replace(/[:.]/g, "-").slice(0, 19);
+  const logFile = join(dir, `dev-${ts}.log`);
+  const latestFile = join(dir, "dev-latest.log");
+
+  const header =
+    `# everything-dev dev session: ${description}\n` + `# Started: ${now.toISOString()}\n\n`;
+  // Overwrite each run so dev-latest.log is always actionable.
+  await writeFile(logFile, header, "utf8");
+  await writeFile(latestFile, header, "utf8");
+
+  let chain = Promise.resolve();
+  const enqueue = (fn: () => Promise<void>) => {
+    chain = chain.then(fn, fn);
+    return chain;
+  };
+
+  return {
+    logFile,
+    latestFile,
+    write: (entry) =>
+      enqueue(async () => {
+        const line = `${formatLogLine(entry)}\n`;
+        await appendFile(logFile, line);
+        await appendFile(latestFile, line);
+      }),
+    readLatest: async (opts) => {
+      const text = await readFile(latestFile, "utf8").catch(() => "");
+      const tail = opts?.tail;
+      if (!tail || tail <= 0) return text;
+      const lines = text.split("\n");
+      return lines.slice(Math.max(0, lines.length - tail)).join("\n");
+    },
+  };
+}
+
+export async function readDevLatestLog(
+  configDir: string,
+  opts?: { tail?: number },
+): Promise<string> {
+  const latestFile = join(getLogsDir(configDir), "dev-latest.log");
+  const text = await readFile(latestFile, "utf8").catch(() => "");
+  const tail = opts?.tail;
+  if (!tail || tail <= 0) return text;
+  const lines = text.split("\n");
+  return lines.slice(Math.max(0, lines.length - tail)).join("\n");
+}

package/src/dev-session.ts

@@ -0,0 +1,318 @@
+import { Effect } from "effect";
+import {
+  type DevViewHandle,
+  type LogEntry,
+  type ProcessState,
+  renderDevView,
+} from "./components/dev-view";
+import { renderStreamingView } from "./components/streaming-view";
+import { getProjectRoot } from "./config";
+import { createDevLogger } from "./dev-logs";
+import {
+  getProcessConfig,
+  makeDevProcess,
+  type ProcessCallbacks,
+  type ProcessHandle,
+} from "./orchestrator";
+import { makeProcessRegistry } from "./process-registry";
+import type { BosConfig, RuntimeConfig, SourceMode } from "./types";
+
+export interface AppConfig {
+  host: SourceMode;
+  ui: SourceMode;
+  api: SourceMode;
+  proxy?: boolean;
+  ssr?: boolean;
+}
+
+export interface AppOrchestrator {
+  packages: string[];
+  env: Record<string, string>;
+  description: string;
+  appConfig: AppConfig;
+  bosConfig: BosConfig;
+  runtimeConfig: RuntimeConfig;
+  port?: number;
+  interactive?: boolean;
+  noLogs?: boolean;
+}
+
+const LOG_NOISE_PATTERNS = [
+  /\[ Federation Runtime \] Version .* from (host|ui) of shared singleton module/,
+  /Executing an Effect versioned \d+\.\d+\.\d+ with a Runtime of version/,
+  /you may want to dedupe the effect dependencies/,
+];
+
+const SSR_LOG_ALLOWLIST = [
+  /\bready\s+built in\b/i,
+  /\bcompiled\b.*successfully/i,
+  /\berror\b/i,
+  /\bfailed\b/i,
+  /\bexception\b/i,
+];
+
+const shouldDisplayLog = (source: string, line: string, isError?: boolean): boolean => {
+  if (process.env.DEBUG === "true" || process.env.DEBUG === "1") return true;
+  if (source === "ui-ssr") {
+    if (isError) return true;
+    return SSR_LOG_ALLOWLIST.some((pattern) => pattern.test(line));
+  }
+  return !LOG_NOISE_PATTERNS.some((pattern) => pattern.test(line));
+};
+
+const isInteractiveSupported = (): boolean => {
+  return process.stdin.isTTY === true && process.stdout.isTTY === true;
+};
+
+const STARTUP_ORDER = ["ui-ssr", "ui", "api", "plugin", "host-build", "host"];
+
+const sortByOrder = (packages: string[]): string[] => {
+  return [...packages].sort((a, b) => {
+    const aIdx = a.startsWith("plugin:")
+      ? STARTUP_ORDER.indexOf("plugin")
+      : STARTUP_ORDER.indexOf(a);
+    const bIdx = b.startsWith("plugin:")
+      ? STARTUP_ORDER.indexOf("plugin")
+      : STARTUP_ORDER.indexOf(b);
+    if (aIdx === -1 && bIdx === -1) return 0;
+    if (aIdx === -1) return 1;
+    if (bIdx === -1) return -1;
+    return aIdx - bIdx;
+  });
+};
+
+function formatLogLine(entry: LogEntry): string {
+  const ts = new Date(entry.timestamp).toISOString();
+  const prefix = entry.isError ? "ERR" : "OUT";
+  return `[${ts}] [${entry.source}] [${prefix}] ${entry.line}`;
+}
+
+export const runDevSession = (
+  orchestrator: AppOrchestrator,
+  onCleanupReady?: (cleanup: () => Promise<void>) => void,
+) =>
+  Effect.gen(function* () {
+    const configDir = getProjectRoot();
+    const orderedPackages = sortByOrder(orchestrator.packages);
+    const initialProcesses: ProcessState[] = orderedPackages.map((pkg) => {
+      const portOverride = pkg === "host" ? orchestrator.port : undefined;
+      const config = getProcessConfig(
+        pkg,
+        undefined,
+        portOverride,
+        orchestrator.bosConfig,
+        orchestrator.runtimeConfig,
+      );
+      const source =
+        pkg === "host"
+          ? orchestrator.appConfig.host
+          : pkg === "ui"
+            ? orchestrator.appConfig.ui
+            : pkg === "api"
+              ? orchestrator.appConfig.api
+              : undefined;
+      return {
+        name: pkg,
+        status: "pending" as const,
+        port: config?.port ?? 0,
+        source,
+      };
+    });
+
+    const registry = yield* makeProcessRegistry(configDir);
+    yield* registry.killAll().pipe(Effect.ignore);
+
+    const logger = yield* Effect.promise(() =>
+      createDevLogger(configDir, orchestrator.description),
+    );
+    const handles: ProcessHandle[] = [];
+    const allLogs: LogEntry[] = [];
+    let view: DevViewHandle | null = null;
+    let shuttingDown = false;
+
+    const killAll = async () => {
+      const reversed = [...handles].reverse();
+      for (const handle of reversed) {
+        try {
+          await handle.kill();
+        } catch {}
+      }
+      await Effect.runPromise(registry.killAll(true)).catch(() => {});
+    };
+
+    const exportLogs = async () => {
+      console.log("\n");
+      console.log("═".repeat(70));
+      console.log(`  SESSION LOGS: ${orchestrator.description}`);
+      console.log(`  Started: ${new Date(allLogs[0]?.timestamp || Date.now()).toISOString()}`);
+      console.log(`  Total entries: ${allLogs.length}`);
+      console.log("═".repeat(70));
+      console.log("");
+      for (const entry of allLogs) {
+        console.log(formatLogLine(entry));
+      }
+      console.log("");
+      console.log("═".repeat(70));
+      console.log(`  Full logs saved to: ${logger.logFile}`);
+      console.log("═".repeat(70));
+      console.log("");
+    };
+
+    const cleanup = async (showLogs = false) => {
+      if (shuttingDown) return;
+      shuttingDown = true;
+      view?.unmount();
+      await killAll();
+      if (showLogs) {
+        await exportLogs();
+      }
+    };
+
+    onCleanupReady?.(cleanup);
+
+    const useInteractive = orchestrator.interactive ?? isInteractiveSupported();
+    view = useInteractive
+      ? renderDevView(
+          initialProcesses,
+          orchestrator.description,
+          orchestrator.env,
+          () => cleanup(false),
+          () => cleanup(true),
+        )
+      : renderStreamingView(initialProcesses, orchestrator.description, orchestrator.env, () =>
+          cleanup(false),
+        );
+
+    const callbacks: ProcessCallbacks = {
+      onStatus: (name, status, message) => {
+        view?.updateProcess(name, status, message);
+      },
+      onLog: (name, line, isError) => {
+        const entry: LogEntry = {
+          id: `${Date.now()}-${allLogs.length + 1}`,
+          source: name,
+          line,
+          timestamp: Date.now(),
+          isError,
+        };
+        allLogs.push(entry);
+        if (shouldDisplayLog(name, line, isError)) {
+          view?.addLog(name, line, isError);
+        }
+        if (!orchestrator.noLogs) {
+          void logger.write(entry);
+        }
+      },
+    };
+
+    const startProcess = (pkg: string) => {
+      const portOverride = pkg === "host" ? orchestrator.port : undefined;
+      return makeDevProcess(
+        pkg,
+        orchestrator.env,
+        callbacks,
+        portOverride,
+        orchestrator.bosConfig,
+        orchestrator.runtimeConfig,
+        registry,
+      );
+    };
+
+    const startGroup = (packages: string[]) =>
+      Effect.forEach(packages, startProcess, { concurrency: "unbounded" });
+
+    const awaitReady = (pkg: string, handle: ProcessHandle) =>
+      Effect.race(
+        handle.waitForReady,
+        Effect.sleep("30 seconds").pipe(
+          Effect.andThen(
+            Effect.sync(() => {
+              callbacks.onLog(pkg, "Timeout waiting for ready, continuing...", true);
+            }),
+          ),
+        ),
+      );
+
+    const nonHostPackages = orderedPackages.filter((pkg) => pkg !== "host");
+    const hostPackages = orderedPackages.filter((pkg) => pkg === "host");
+
+    const nonHostHandles = yield* startGroup(nonHostPackages);
+    handles.push(...nonHostHandles);
+
+    yield* Effect.forEach(
+      nonHostHandles.map((handle, index) => ({
+        handle,
+        pkg: nonHostPackages[index] ?? handle.name,
+      })),
+      ({ handle, pkg }) => awaitReady(pkg, handle),
+      { concurrency: "unbounded" },
+    );
+
+    const hostHandles = yield* startGroup(hostPackages);
+    handles.push(...hostHandles);
+
+    yield* Effect.forEach(
+      hostHandles.map((handle, index) => ({ handle, pkg: hostPackages[index] ?? handle.name })),
+      ({ handle, pkg }) => awaitReady(pkg, handle),
+      { concurrency: "unbounded" },
+    );
+
+    yield* Effect.addFinalizer(() => Effect.promise(() => cleanup(false)));
+    yield* Effect.never;
+  });
+
+export const startApp = (orchestrator: AppOrchestrator) => {
+  let activeCleanup: (() => Promise<void>) | null = null;
+
+  const program = Effect.scoped(
+    runDevSession(orchestrator, (cleanup) => {
+      activeCleanup = cleanup;
+    }),
+  ).pipe(
+    Effect.catchAll((e) =>
+      Effect.sync(() => {
+        if (e instanceof Error) {
+          console.error("App server error:", e.message);
+          if (e.stack) console.error(e.stack);
+        } else {
+          console.error("App server error:", e);
+        }
+      }),
+    ),
+  );
+
+  const handleSignal = async () => {
+    if (activeCleanup) await activeCleanup();
+  };
+
+  const forceExit = () => {
+    console.log("\n[Dev] Force exit");
+    process.exit(0);
+  };
+
+  let signalCount = 0;
+  process.on("SIGINT", () => {
+    signalCount++;
+    if (signalCount > 1) {
+      forceExit();
+      return;
+    }
+    const timeout = setTimeout(forceExit, 5000);
+    void handleSignal().finally(() => {
+      clearTimeout(timeout);
+    });
+  });
+  process.on("SIGTERM", () => {
+    signalCount++;
+    if (signalCount > 1) {
+      forceExit();
+      return;
+    }
+    const timeout = setTimeout(forceExit, 5000);
+    void handleSignal().finally(() => {
+      clearTimeout(timeout);
+    });
+  });
+
+  void Effect.runPromise(program);
+};

package/src/fastkv.ts

@@ -0,0 +1,153 @@
+export type NetworkId = "mainnet" | "testnet";
+
+interface FastKvEntry {
+  value: unknown;
+}
+
+interface FastKvListResponse {
+  entries?: Array<FastKvEntry | null>;
+}
+
+const FASTKV_TIMEOUT_MS = 10_000;
+
+function getNetworkIdForAccount(accountId: string): NetworkId {
+  return accountId.endsWith(".testnet") ? "testnet" : "mainnet";
+}
+
+export function getFastKvBaseUrlForNetwork(network: NetworkId): string {
+  return network === "testnet"
+    ? process.env.REGISTRY_FASTKV_TESTNET_URL || "https://kv.test.fastnear.com"
+    : process.env.REGISTRY_FASTKV_MAINNET_URL || "https://kv.main.fastnear.com";
+}
+
+function getFastKvBaseUrlForAccount(accountId: string): string {
+  return getNetworkIdForAccount(accountId) === "testnet"
+    ? getFastKvBaseUrlForNetwork("testnet")
+    : getFastKvBaseUrlForNetwork("mainnet");
+}
+
+export function buildRegistryConfigUrl(accountId: string, gatewayId: string): string {
+  const baseUrl = getFastKvBaseUrlForAccount(accountId);
+  const namespace = getRegistryNamespaceForAccount(accountId);
+  const key = encodeURIComponent(getRegistryConfigKey(accountId, gatewayId));
+  return `${baseUrl}/v0/latest/${encodeURIComponent(namespace)}/${encodeURIComponent(accountId)}/${key}`;
+}
+
+export function buildRegistryConfigUrlForNetwork(
+  network: NetworkId,
+  accountId: string,
+  gatewayId: string,
+): string {
+  const baseUrl = getFastKvBaseUrlForNetwork(network);
+  const namespace = getRegistryNamespaceForNetwork(network);
+  const key = encodeURIComponent(getRegistryConfigKey(accountId, gatewayId));
+  return `${baseUrl}/v0/latest/${encodeURIComponent(namespace)}/${encodeURIComponent(accountId)}/${key}`;
+}
+
+export function getRegistryNamespaceForAccount(accountId: string): string {
+  return accountId.endsWith(".testnet")
+    ? process.env.REGISTRY_FASTKV_TESTNET_NAMESPACE || "dev.everything.near"
+    : process.env.REGISTRY_FASTKV_MAINNET_NAMESPACE || accountId;
+}
+
+export function getRegistryNamespaceForNetwork(network: NetworkId): string {
+  return network === "testnet"
+    ? process.env.REGISTRY_FASTKV_TESTNET_NAMESPACE || "dev.everything.near"
+    : process.env.REGISTRY_FASTKV_MAINNET_NAMESPACE || "dev.everything.near";
+}
+
+function getRegistryConfigKey(
+  accountId: string,
+  gatewayId: string,
+  pathSegments: string[] = [],
+): string {
+  const suffix =
+    pathSegments.length > 0
+      ? `/${pathSegments.map((segment) => encodeURIComponent(segment)).join("/")}`
+      : "";
+  return `apps/${accountId}/${gatewayId}${suffix}/bos.config.json`;
+}
+
+function parseBosUrl(bosUrl: string): {
+  accountId: string;
+  gatewayId: string;
+  pathSegments: string[];
+} {
+  const match = bosUrl.match(/^bos:\/\/([^/]+)\/(.+)$/);
+  if (!match?.[1] || !match[2]) {
+    throw new Error(`Invalid BOS URL: ${bosUrl}`);
+  }
+
+  const pathSegments = match[2]
+    .split("/")
+    .filter(Boolean)
+    .map((segment) => decodeURIComponent(segment));
+  if (pathSegments.length === 0) {
+    throw new Error(`Invalid BOS URL: ${bosUrl}`);
+  }
+
+  const [gatewayId, ...pathSegmentsTail] = pathSegments;
+  if (!gatewayId) {
+    throw new Error(`Invalid BOS URL: ${bosUrl}`);
+  }
+
+  return {
+    accountId: match[1],
+    gatewayId,
+    pathSegments: pathSegmentsTail,
+  };
+}
+
+export async function fetchBosConfigFromFastKv<T>(bosUrl: string): Promise<T> {
+  const { accountId, gatewayId, pathSegments } = parseBosUrl(bosUrl);
+  const payload = await fetchJson<FastKvListResponse>(
+    `${getFastKvBaseUrlForAccount(accountId)}/v0/latest/${encodeURIComponent(getRegistryNamespaceForAccount(accountId))}/${encodeURIComponent(accountId)}`,
+    {
+      method: "POST",
+      body: JSON.stringify({
+        key: getRegistryConfigKey(accountId, gatewayId, pathSegments),
+        limit: 1,
+      }),
+    },
+  );
+  const value = payload?.entries?.find(Boolean)?.value;
+
+  if (!value) {
+    throw new Error(`No config found for ${bosUrl}`);
+  }
+
+  if (typeof value === "string") {
+    return JSON.parse(value) as T;
+  }
+
+  if (typeof value !== "object") {
+    throw new Error(`Invalid config value for ${bosUrl}`);
+  }
+
+  return value as T;
+}
+
+async function fetchJson<T>(url: string, init?: RequestInit): Promise<T | null> {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), FASTKV_TIMEOUT_MS);
+
+  try {
+    const response = await fetch(url, {
+      ...init,
+      headers: {
+        accept: "application/json",
+        "content-type": "application/json",
+        ...(init?.headers ?? {}),
+      },
+      signal: controller.signal,
+    });
+
+    if (!response.ok) {
+      return null;
+    }
+
+    return (await response.json()) as T;
+  } finally {
+    clearTimeout(timeout);
+  }
+}

package/src/federation.server.ts

@@ -0,0 +1,43 @@
+import { verifySriForUrl } from "./integrity";
+import { ensureNodeRuntimePlugin, loadRemoteModule, registerRemote } from "./mf";
+import type { RuntimeConfig } from "./types";
+import type { RouterModule } from "./ui/types";
+
+export type { RouterModule };
+
+export async function loadRouterModule(config: RuntimeConfig): Promise<RouterModule> {
+  const isLocalDev = config.ui.source === "local";
+  const ssrUrl = config.ui.ssrUrl ?? (isLocalDev ? config.ui.url : undefined);
+
+  if (!ssrUrl) {
+    if (!isLocalDev) {
+      throw new Error(
+        "SSR URL not configured in production. Set app.ui.ssr in bos.config.json to enable SSR.",
+      );
+    }
+
+    throw new Error(
+      "SSR URL not configured. In local dev, set app.ui.ssr or use a UI package with SSR support.",
+    );
+  }
+
+  const ssrEntryUrl = `${ssrUrl.replace(/\/$/, "")}/remoteEntry.server.js`;
+
+  if (config.ui.ssrIntegrity) {
+    await verifySriForUrl(ssrEntryUrl, config.ui.ssrIntegrity);
+  }
+
+  await ensureNodeRuntimePlugin();
+  await registerRemote({
+    name: config.ui.name,
+    entry: ssrEntryUrl,
+    type: "script",
+  });
+
+  const loadedModule = await loadRemoteModule<any>(`${config.ui.name}/Router`, { from: "build" });
+  if (!loadedModule) {
+    throw new Error(`Module not found: ${config.ui.name}/Router`);
+  }
+
+  return loadedModule.default as RouterModule;
+}