@aion0/forge 0.5.49 → 0.5.50

package/lib/crafts/registry.ts

@@ -0,0 +1,272 @@
+// Craft marketplace — slim version of skills.ts.
+//
+// Mirrors the Forge skills marketplace pattern: a GitHub-hosted repo with
+// `registry.json` at the root + per-craft folders containing the manifest
+// and ui.tsx/server.ts. Default repo: aiwatching/forge-crafts. Override via
+// settings.craftsRepoUrl.
+//
+// Install state for crafts is implicit (file-system): if
+// <project>/.forge/crafts/<name>/ exists, it's installed. No DB rows needed.
+
+import { existsSync, mkdirSync, writeFileSync, readFileSync, rmSync, readdirSync, statSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import * as YAML from 'yaml';
+import { loadSettings } from '../settings';
+import type { CraftManifest, CraftRequirements } from './types';
+import { matchesRequirements, listProjectCrafts } from './loader';
+
+export interface RegistryEntry {
+  name: string;
+  displayName: string;
+  description?: string;
+  version: string;
+  author?: string;
+  tags?: string[];
+  requires?: CraftRequirements;
+  files: string[];                   // relative paths inside the craft dir on the registry
+  sourceUrl?: string;                // GitHub web URL for browsing
+}
+
+export interface RegistryItem extends RegistryEntry {
+  // Per-project install info (populated by listMarketplace for a given project)
+  installed: boolean;
+  installedVersion?: string;
+  hasUpdate: boolean;
+  compatible: boolean;               // requires gate against the project
+}
+
+function getBaseUrl(): string {
+  const s = loadSettings();
+  return (s as any).craftsRepoUrl || 'https://raw.githubusercontent.com/aiwatching/forge-crafts/main';
+}
+
+function compareVersions(a: string, b: string): number {
+  const pa = (a || '0.0.0').split('.').map(Number);
+  const pb = (b || '0.0.0').split('.').map(Number);
+  for (let i = 0; i < 3; i++) {
+    const d = (pa[i] || 0) - (pb[i] || 0);
+    if (d !== 0) return d;
+  }
+  return 0;
+}
+
+// ── Registry fetch ─────────────────────────────────────
+
+let cachedRegistry: { fetchedAt: number; entries: RegistryEntry[] } | null = null;
+// Short TTL — registry.json is tiny + GitHub Pages caches independently anyway.
+// 5-minute cache used to leave users staring at stale data after their own
+// publish landed; 30s is the right "barely noticeable but still saves repeat
+// hits during a single UI session" balance.
+const REGISTRY_TTL_MS = 30_000;
+
+export function invalidateRegistry() {
+  cachedRegistry = null;
+}
+
+export async function fetchRegistry(force = false): Promise<RegistryEntry[]> {
+  if (!force && cachedRegistry && Date.now() - cachedRegistry.fetchedAt < REGISTRY_TTL_MS) {
+    return cachedRegistry.entries;
+  }
+  const baseUrl = getBaseUrl();
+  try {
+    const ctrl = new AbortController();
+    const t = setTimeout(() => ctrl.abort(), 10000);
+    const res = await fetch(`${baseUrl}/registry.json?_t=${Date.now()}`, {
+      signal: ctrl.signal,
+      headers: { 'Accept': 'application/json', 'Cache-Control': 'no-cache' },
+    });
+    clearTimeout(t);
+    if (!res.ok) {
+      console.warn(`[craft-registry] fetch failed: ${res.status}`);
+      return cachedRegistry?.entries || [];
+    }
+    const data = await res.json() as { crafts?: RegistryEntry[] };
+    const entries = (data.crafts || []).filter(e => e.name && e.version && Array.isArray(e.files));
+    cachedRegistry = { fetchedAt: Date.now(), entries };
+    return entries;
+  } catch (e) {
+    console.warn('[craft-registry] fetch error', e);
+    return cachedRegistry?.entries || [];
+  }
+}
+
+// ── Per-project marketplace listing ────────────────────
+
+export async function listMarketplace(projectPath: string): Promise<RegistryItem[]> {
+  const [entries, projectCrafts] = await Promise.all([
+    fetchRegistry(),
+    Promise.resolve(listProjectCrafts(projectPath)),
+  ]);
+  const installedByName = new Map(projectCrafts.filter(c => c.__scope === 'project').map(c => [c.name, c]));
+
+  return entries.map(e => {
+    const inst = installedByName.get(e.name);
+    const installedVersion = inst?.version;
+    const hasUpdate = !!installedVersion && compareVersions(e.version, installedVersion) > 0;
+    const compatible = e.requires ? matchesRequirements(e.requires, projectPath) : true;
+    return {
+      ...e,
+      installed: !!inst,
+      installedVersion,
+      hasUpdate,
+      compatible,
+    };
+  });
+}
+
+// ── Install / uninstall ────────────────────────────────
+
+export async function installCraft(name: string, projectPath: string): Promise<{ ok: boolean; error?: string }> {
+  // Always pull fresh on install — user just clicked install, can't be stale.
+  const entries = await fetchRegistry(true);
+  const entry = entries.find(e => e.name === name);
+  if (!entry) return { ok: false, error: `craft "${name}" not in registry` };
+
+  const baseUrl = getBaseUrl();
+  const targetDir = join(projectPath, '.forge', 'crafts', name);
+  if (existsSync(targetDir)) {
+    return { ok: false, error: `craft "${name}" already installed at ${targetDir} (uninstall first to upgrade)` };
+  }
+  mkdirSync(targetDir, { recursive: true });
+  mkdirSync(join(targetDir, 'data'), { recursive: true });
+
+  for (const rel of entry.files) {
+    try {
+      const url = `${baseUrl}/${name}/${rel}?_t=${Date.now()}`;
+      const res = await fetch(url, { headers: { 'Cache-Control': 'no-cache' } });
+      if (!res.ok) throw new Error(`${url}: ${res.status}`);
+      const text = await res.text();
+      const dest = join(targetDir, rel);
+      mkdirSync(join(dest, '..'), { recursive: true });
+      writeFileSync(dest, text, 'utf8');
+    } catch (e: any) {
+      // Roll back partial install
+      try { rmSync(targetDir, { recursive: true, force: true }); } catch {}
+      return { ok: false, error: `download failed: ${e?.message || e}` };
+    }
+  }
+  return { ok: true };
+}
+
+export function uninstallCraft(name: string, projectPath: string): { ok: boolean; error?: string } {
+  const targetDir = join(projectPath, '.forge', 'crafts', name);
+  if (!existsSync(targetDir)) return { ok: false, error: 'not installed' };
+  rmSync(targetDir, { recursive: true, force: true });
+  return { ok: true };
+}
+
+// In-place update — preserves <project>/.forge/crafts/<name>/data/ (which holds
+// useStore JSON) and overwrites everything else. Atomic: downloads first, then
+// writes; if any download fails the existing install is untouched.
+export async function updateCraft(name: string, projectPath: string): Promise<{ ok: boolean; error?: string; from?: string; to?: string }> {
+  const targetDir = join(projectPath, '.forge', 'crafts', name);
+  if (!existsSync(targetDir)) return { ok: false, error: 'not installed' };
+
+  const entries = await fetchRegistry(true);
+  const entry = entries.find(e => e.name === name);
+  if (!entry) return { ok: false, error: `craft "${name}" not in registry` };
+
+  // Read current version from local craft.yaml for the response
+  let fromVersion: string | undefined;
+  try {
+    const m = YAML.parse(readFileSync(join(targetDir, 'craft.yaml'), 'utf8')) as any;
+    fromVersion = m?.version;
+  } catch {}
+
+  // Download everything into memory first
+  const baseUrl = getBaseUrl();
+  const downloaded: { rel: string; content: string }[] = [];
+  for (const rel of entry.files) {
+    try {
+      const res = await fetch(`${baseUrl}/${name}/${rel}?_t=${Date.now()}`, { headers: { 'Cache-Control': 'no-cache' } });
+      if (!res.ok) throw new Error(`${rel}: HTTP ${res.status}`);
+      downloaded.push({ rel, content: await res.text() });
+    } catch (e: any) {
+      return { ok: false, error: `download failed for ${rel}: ${e?.message || e}` };
+    }
+  }
+
+  // All downloads succeeded — write them. data/ stays intact.
+  const fs = require('node:fs');
+  for (const d of downloaded) {
+    const dest = join(targetDir, d.rel);
+    mkdirSync(dest.split('/').slice(0, -1).join('/'), { recursive: true });
+    fs.writeFileSync(dest, d.content, 'utf8');
+  }
+  return { ok: true, from: fromVersion, to: entry.version };
+}
+
+// Returns just the names of installed crafts that have updates available.
+// Used by the dropdown badge — cheap, no per-craft enrichment beyond version
+// compare.
+export async function listAvailableUpdates(projectPath: string): Promise<{ name: string; from?: string; to: string }[]> {
+  const items = await listMarketplace(projectPath);
+  return items
+    .filter(it => it.installed && it.hasUpdate)
+    .map(it => ({ name: it.name, from: it.installedVersion, to: it.version }));
+}
+
+// ── File scanner ───────────────────────────────────────
+
+const EXCLUDE_DIRS = new Set(['data', '.git', 'node_modules', '.next', 'dist', 'build', '.cache']);
+const EXCLUDE_FILES = new Set(['prompt.md', '.DS_Store']);
+const INCLUDE_EXT = new Set(['.yaml', '.yml', '.tsx', '.ts', '.jsx', '.js', '.md', '.json', '.css']);
+const MAX_FILE_BYTES = 1024 * 1024;   // 1 MB per file
+
+function collectCraftFiles(dir: string): { path: string; content: string }[] {
+  const out: { path: string; content: string }[] = [];
+  const walk = (d: string) => {
+    for (const name of readdirSync(d)) {
+      if (name.startsWith('.') && name !== '.gitignore') continue;
+      if (EXCLUDE_FILES.has(name)) continue;
+      const full = join(d, name);
+      const st = statSync(full);
+      if (st.isDirectory()) {
+        if (EXCLUDE_DIRS.has(name)) continue;
+        walk(full);
+        continue;
+      }
+      if (!st.isFile()) continue;
+      if (st.size > MAX_FILE_BYTES) continue;
+      const ext = name.lastIndexOf('.') >= 0 ? name.slice(name.lastIndexOf('.')) : '';
+      if (ext && !INCLUDE_EXT.has(ext)) continue;
+      out.push({ path: relative(dir, full), content: readFileSync(full, 'utf8') });
+    }
+  };
+  walk(dir);
+  return out;
+}
+
+// ── Publish helper ─────────────────────────────────────
+// MVP: returns a registry-entry JSON snippet + a tar of the craft dir for
+// the user to attach to a PR on the registry repo. Real "press a button to
+// publish" requires GitHub auth + write — out of scope for v1.
+
+export function bundleCraftForPublish(projectPath: string, craftName: string): { entry: RegistryEntry; files: { path: string; content: string }[]; error?: string } {
+  const dir = join(projectPath, '.forge', 'crafts', craftName);
+  if (!existsSync(dir)) return { entry: null as any, files: [], error: 'craft not found' };
+
+  let manifest: CraftManifest;
+  try {
+    manifest = YAML.parse(readFileSync(join(dir, 'craft.yaml'), 'utf8'));
+  } catch {
+    return { entry: null as any, files: [], error: 'craft.yaml unreadable' };
+  }
+
+  // Recursive scan — pick up arbitrary multi-file crafts (e.g. server.ts split
+  // into _types.ts / _runner.ts / etc.) while excluding runtime data, history,
+  // and anything obviously not source.
+  const files = collectCraftFiles(dir);
+
+  const entry: RegistryEntry = {
+    name: manifest.name,
+    displayName: manifest.displayName || manifest.name,
+    description: manifest.description,
+    version: manifest.version || '0.1.0',
+    author: manifest.author,
+    tags: manifest.tags,
+    requires: manifest.requires,
+    files: files.map(f => f.path),
+  };
+  return { entry, files };
+}

package/lib/crafts/runtime.ts

@@ -0,0 +1,208 @@
+// Runtime: load + cache craft server modules; build the ForgeServerApi per request.
+
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { execSync } from 'node:child_process';
+import { tmpdir } from 'node:os';
+import * as esbuild from 'esbuild';
+import { pathToFileURL } from 'node:url';
+import { createTask } from '@/lib/task-manager';
+import type { CraftDescriptor, CraftServerDef, CraftRouteHandler, ForgeServerApi } from './types';
+
+// Module cache: dir → { mtimeMs, mod }
+interface CachedMod { mtimeMs: number; def: CraftServerDef; }
+const cache = new Map<string, CachedMod>();
+
+// Function-wrapped dynamic import so Turbopack doesn't try to statically resolve the URL.
+const dynamicImport = new Function('u', 'return import(u)') as (u: string) => Promise<any>;
+
+async function transpileToFile(src: string, resolveDir: string): Promise<string> {
+  // Compile TS → JS, bundling the @forge/craft/server SDK inline (node_modules stays external).
+  // Cache key includes a salt so older transpile outputs are skipped after format changes.
+  const hash = require('node:crypto').createHash('md5').update('v2:' + src).digest('hex').slice(0, 16);
+  const out = join(tmpdir(), `forge-craft-${hash}.mjs`);
+  if (existsSync(out)) return out;
+  const sdkServerEntry = require('node:path').resolve(process.cwd(), 'lib/craft-sdk/server.ts');
+  const result = await esbuild.build({
+    stdin: { contents: src, loader: 'ts', resolveDir },
+    bundle: true,
+    format: 'esm',
+    platform: 'node',
+    target: 'node20',
+    packages: 'external',
+    alias: {
+      '@forge/craft/server': sdkServerEntry,
+    },
+    write: false,
+  });
+  writeFileSync(out, result.outputFiles[0].text, 'utf8');
+  return out;
+}
+
+export async function loadServer(craft: CraftDescriptor): Promise<CraftServerDef | null> {
+  if (!craft.hasServer) return null;
+  const file = join(craft.__dir, craft.server?.entry || 'server.ts');
+  const stat = statSync(file);
+  const cached = cache.get(craft.__dir);
+  if (cached && cached.mtimeMs === stat.mtimeMs) return cached.def;
+
+  const src = readFileSync(file, 'utf8');
+  const compiled = await transpileToFile(src, craft.__dir);
+  // Bust ESM cache by adding a query string (Node ESM caches by URL).
+  // Use Function() so Turbopack doesn't try to statically analyze the import.
+  const url = pathToFileURL(compiled).href + `?t=${stat.mtimeMs}`;
+  const mod = await dynamicImport(url);
+  const def: CraftServerDef = mod.default ?? mod.craft ?? mod;
+  if (!def || typeof def !== 'object' || !def.routes) {
+    throw new Error(`Craft "${craft.name}" server.ts must export default defineCraftServer({...})`);
+  }
+  cache.set(craft.__dir, { mtimeMs: stat.mtimeMs, def });
+  return def;
+}
+
+// Match a route key like "GET /items/:id" against a request method + path.
+function matchRoute(routeKey: string, method: string, path: string): Record<string, string> | null {
+  const [m, pat] = routeKey.split(/\s+/, 2);
+  if (m.toUpperCase() !== method.toUpperCase()) return null;
+  // Normalize: ensure leading slash on both
+  const cleanPath = path.startsWith('/') ? path : '/' + path;
+  const cleanPat = pat.startsWith('/') ? pat : '/' + pat;
+  const pSegs = cleanPath.split('/').filter(Boolean);
+  const tSegs = cleanPat.split('/').filter(Boolean);
+  if (pSegs.length !== tSegs.length) return null;
+  const params: Record<string, string> = {};
+  for (let i = 0; i < tSegs.length; i++) {
+    const t = tSegs[i];
+    const p = pSegs[i];
+    if (t.startsWith(':')) params[t.slice(1)] = decodeURIComponent(p);
+    else if (t !== p) return null;
+  }
+  return params;
+}
+
+export function findHandler(def: CraftServerDef, method: string, path: string): { handler: CraftRouteHandler; params: Record<string, string> } | null {
+  for (const [key, handler] of Object.entries(def.routes)) {
+    const params = matchRoute(key, method, path);
+    if (params) return { handler, params };
+  }
+  return null;
+}
+
+// ── Forge server-side API ────────────────────────────────
+
+export function buildForgeApi(craft: CraftDescriptor, projectPath: string, projectName?: string): ForgeServerApi {
+  const dataDir = join(craft.__dir, 'data');
+  // For builtin crafts, redirect storage to the project so writes don't go into the install
+  const writableDataDir = craft.__scope === 'builtin'
+    ? join(projectPath, '.forge', 'crafts', craft.name, 'data')
+    : dataDir;
+
+  return {
+    project: { path: projectPath, name: projectName },
+
+    storage: {
+      read<T>(file: string): T | null {
+        const f = join(writableDataDir, file);
+        if (!existsSync(f)) return null;
+        try { return JSON.parse(readFileSync(f, 'utf8')); } catch { return null; }
+      },
+      write(file: string, data: any): void {
+        if (!existsSync(writableDataDir)) mkdirSync(writableDataDir, { recursive: true });
+        writeFileSync(join(writableDataDir, file), JSON.stringify(data, null, 2), 'utf8');
+      },
+      listFiles(): string[] {
+        if (!existsSync(writableDataDir)) return [];
+        return readdirSync(writableDataDir);
+      },
+    },
+
+    exec(cmd, opts = {}) {
+      try {
+        const stdout = execSync(cmd, {
+          cwd: projectPath,
+          timeout: opts.timeout ?? 30000,
+          input: opts.input,
+          encoding: 'utf8',
+          maxBuffer: 10 * 1024 * 1024,
+        }).toString();
+        return { stdout, stderr: '', code: 0 };
+      } catch (e: any) {
+        return { stdout: (e?.stdout || '').toString(), stderr: (e?.stderr || e?.message || '').toString(), code: e?.status ?? 1 };
+      }
+    },
+
+    task(opts) {
+      const task = createTask({
+        projectName: projectName || projectPath.split('/').filter(Boolean).pop() || 'project',
+        projectPath,
+        prompt: opts.prompt,
+        agent: opts.agent,
+      });
+      return { id: task.id };
+    },
+
+    inject(text, opts = {}) {
+      // Reuse migration session-resolver logic inline
+      try {
+        const sessionName = opts.sessionName || resolveBoundSession(projectPath);
+        if (!sessionName) return { ok: false };
+        const buf = join(tmpdir(), `forge-craft-inject-${Date.now()}.txt`);
+        writeFileSync(buf, text);
+        execSync(`tmux load-buffer -t "${sessionName}" "${buf}" && tmux paste-buffer -t "${sessionName}" && sleep 0.2 && tmux send-keys -t "${sessionName}" Enter`, { timeout: 5000 });
+        try { require('node:fs').unlinkSync(buf); } catch {}
+        return { ok: true, sessionName };
+      } catch (e) {
+        return { ok: false };
+      }
+    },
+
+    openapi(specPath) {
+      try {
+        const file = join(projectPath, specPath);
+        if (!existsSync(file)) return null;
+        return JSON.parse(readFileSync(file, 'utf8'));
+      } catch { return null; }
+    },
+
+    log: (...args) => console.log(`[craft:${craft.name}]`, ...args),
+  };
+}
+
+function resolveBoundSession(projectPath: string): string | null {
+  try {
+    const sessions = execSync(`tmux list-sessions -F '#{session_name}'`, { encoding: 'utf8', timeout: 2000 })
+      .trim().split('\n').filter(Boolean).filter(n => /^mw[a-z0-9]*-/.test(n));
+    for (const s of sessions) {
+      try {
+        const cwd = execSync(`tmux display-message -p -t '${s}' '#{pane_current_path}'`, { encoding: 'utf8', timeout: 2000 }).trim();
+        if (cwd === projectPath || cwd.startsWith(projectPath + '/')) return s;
+      } catch {}
+    }
+  } catch {}
+  return null;
+}
+
+// ── UI transpile (TSX → JS) for browser dynamic import ──
+
+export async function transpileUi(craft: CraftDescriptor): Promise<string> {
+  const file = join(craft.__dir, craft.ui?.tab || 'ui.tsx');
+  if (!existsSync(file)) throw new Error('No UI file');
+  const src = readFileSync(file, 'utf8');
+  const result = await esbuild.build({
+    stdin: { contents: src, loader: 'tsx', resolveDir: craft.__dir },
+    bundle: true,
+    format: 'esm',
+    platform: 'browser',
+    target: 'es2022',
+    jsx: 'automatic',
+    write: false,
+    // Mark React + SDK as externals so they share the host page's instances
+    external: ['react', 'react/jsx-runtime', 'react-dom', '@forge/craft'],
+  });
+  // Rewrite SDK + react bare imports → absolute URLs that Forge serves.
+  let code = result.outputFiles[0].text;
+  code = code.replace(/from\s*["']react["']/g, 'from "/api/craft-system/runtime/react"');
+  code = code.replace(/from\s*["']react\/jsx-runtime["']/g, 'from "/api/craft-system/runtime/react-jsx"');
+  code = code.replace(/from\s*["']@forge\/craft["']/g, 'from "/api/craft-system/runtime/sdk"');
+  return code;
+}

package/lib/crafts/types.ts

@@ -0,0 +1,92 @@
+// Craft = a project-scoped mini-app: UI tab + optional API routes.
+// Lives at <project>/.forge/crafts/<name>/ or in lib/builtin-crafts/<name>/.
+
+export interface CraftRequirements {
+  hasFile?: string[];                 // any of these paths must exist (relative to project)
+  hasGlob?: string[];                 // any glob must match (e.g. "**/*.java")
+  // future: language, framework detection
+}
+
+export interface CraftManifest {
+  name: string;                       // unique slug, dir name
+  displayName?: string;               // tab label (default = name)
+  icon?: string;                      // emoji shown in tab
+  description?: string;
+  version?: string;
+  author?: string;                    // marketplace metadata
+  tags?: string[];                    // for marketplace browsing/filtering
+  requires?: CraftRequirements;       // project-type compatibility gate
+
+  ui?: {
+    tab?: string;                     // path to ui.tsx (default 'ui.tsx')
+    showWhen?: string;                // expression: hasFile("X") | always (v2)
+  };
+
+  server?: {
+    entry?: string;                   // path to server.ts (default 'server.ts')
+  };
+
+  // Source of truth — set by loader, not authored.
+  __dir?: string;                     // absolute dir
+  __scope?: 'builtin' | 'project';
+}
+
+export interface CraftDescriptor extends CraftManifest {
+  __dir: string;
+  __scope: 'builtin' | 'project';
+  hasUi: boolean;
+  hasServer: boolean;
+}
+
+export interface CraftRouteHandlerCtx {
+  projectPath: string;
+  projectName?: string;
+  query: Record<string, string>;
+  params: Record<string, string>;
+  body?: any;
+  headers: Record<string, string>;
+  forge: ForgeServerApi;
+}
+
+export type CraftRouteHandler = (ctx: CraftRouteHandlerCtx) => Promise<any> | any;
+
+export interface CraftServerDef {
+  routes: Record<string, CraftRouteHandler>;  // key = "METHOD path", e.g. "GET /items"
+  onLoad?: (ctx: { projectPath: string; forge: ForgeServerApi }) => Promise<void> | void;
+  schedule?: string;                          // optional cron — v2
+}
+
+// Server-side helper bag passed into every handler. Keep small + stable.
+export interface ForgeServerApi {
+  // Project context
+  project: {
+    path: string;
+    name?: string;
+  };
+
+  // Storage scoped to <project>/.forge/crafts/<name>/data/
+  storage: {
+    read<T = any>(file: string): T | null;
+    write(file: string, data: any): void;
+    listFiles(): string[];
+  };
+
+  // Run a shell command in the project cwd
+  exec(cmd: string, opts?: { timeout?: number; input?: string }): {
+    stdout: string;
+    stderr: string;
+    code: number;
+  };
+
+  // Spawn a Forge background task in this project
+  task(opts: { prompt: string; agent?: string }): { id: string };
+
+  // Inject text into the project's bound tmux session (auto-resolves)
+  inject(text: string, opts?: { sessionName?: string }): { ok: boolean; sessionName?: string };
+
+  // Lazy access to OpenAPI loader (when project has one configured)
+  openapi(specPath: string): any | null;
+
+  // Structured logging — visible in Forge logs
+  log: (...args: any[]) => void;
+}