@aion0/forge 0.8.0 → 0.8.2

package/app/api/connectors/install-local/route.ts

@@ -0,0 +1,211 @@
+/**
+ * POST /api/connectors/install-local
+ *
+ * Install a connector from a user-supplied manifest. Two flavours:
+ *
+ *   multipart/form-data with field `file`:
+ *     - .yaml / .yml  → single-file manifest
+ *     - .zip          → multi-file bundle (must contain manifest.yaml
+ *                       at the root; may add README.md, icon.svg,
+ *                       tools/*.js, etc.)
+ *
+ *   application/json body `{ yaml: "...", id?: "..." }`:
+ *     - convenience path for callers that already have the YAML in
+ *       memory (Forge Help AI, internal scripts). `id` is optional
+ *       — if missing, parsed from the manifest.
+ *
+ * Behaviour:
+ *   1. Validate the manifest has `id`, `name`, and at least `tools` or
+ *      `connectors[]`.
+ *   2. Write the manifest (and any zip siblings) to
+ *      <dataDir>/connectors/<id>/.
+ *   3. Register/upsert the row in connector-configs.json with the
+ *      manifest's version as installed_version, preserving any prior
+ *      user-supplied settings (so re-installing a tweaked version
+ *      doesn't blow away a PAT).
+ *
+ * Marketplace UI distinguishes "local" connectors from "registry"
+ * connectors by membership in the registry cache — no extra source
+ * field on disk.
+ */
+
+import { NextResponse } from 'next/server';
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { join, normalize, sep } from 'node:path';
+import AdmZip from 'adm-zip';
+import YAML from 'yaml';
+import { getDataDir } from '@/lib/dirs';
+import { installConnector } from '@/lib/connectors/registry';
+
+const MAX_ZIP_BYTES = 5 * 1024 * 1024;          // 5 MB
+const MAX_FILE_BYTES = 1 * 1024 * 1024;         // 1 MB per file inside the zip
+const MAX_FILES_IN_ZIP = 50;
+
+interface ParsedManifest {
+  id: string;
+  name: string;
+  version?: string;
+}
+
+function parseManifest(yaml: string): ParsedManifest {
+  let def: any;
+  try {
+    def = YAML.parse(yaml);
+  } catch (e) {
+    throw new Error(`invalid YAML: ${(e as Error).message}`);
+  }
+  if (!def?.id || typeof def.id !== 'string') {
+    throw new Error('manifest.id is required');
+  }
+  if (!/^[a-z0-9][a-z0-9_-]*$/.test(def.id)) {
+    throw new Error(`manifest.id "${def.id}" must be lowercase alphanumerics + hyphens/underscores`);
+  }
+  if (!def.name || typeof def.name !== 'string') {
+    throw new Error('manifest.name is required');
+  }
+  if (!def.tools && !(Array.isArray(def.connectors) && def.connectors.length)) {
+    throw new Error('manifest must declare `tools` or `connectors[]`');
+  }
+  return { id: def.id, name: def.name, version: def.version };
+}
+
+function safeEntryPath(name: string): string | null {
+  // Reject absolute paths, parent-dir escapes, leading slashes.
+  if (!name || name.startsWith('/') || name.includes('..')) return null;
+  const normalized = normalize(name).replace(/^[/\\]+/, '');
+  if (normalized.split(sep).some((seg) => seg === '..' || seg === '')) return null;
+  return normalized;
+}
+
+function ensureDir(p: string): void {
+  if (!existsSync(p)) mkdirSync(p, { recursive: true, mode: 0o700 });
+}
+
+interface InstallResult {
+  ok: boolean;
+  id?: string;
+  version?: string;
+  files_written?: number;
+  error?: string;
+}
+
+function installFromFiles(
+  manifestYaml: string,
+  extraFiles: Array<{ path: string; data: Buffer }>,
+): InstallResult {
+  let parsed: ParsedManifest;
+  try { parsed = parseManifest(manifestYaml); }
+  catch (e) { return { ok: false, error: (e as Error).message }; }
+
+  const dir = join(getDataDir(), 'connectors', parsed.id);
+  ensureDir(dir);
+
+  // Wipe stale siblings from a prior install of the same id (but
+  // preserve nothing — local install replaces the directory).
+  // For safety, we only write inside `dir`; safeEntryPath gated it.
+
+  writeFileSync(join(dir, 'manifest.yaml'), manifestYaml, { mode: 0o600 });
+  let written = 1;
+  for (const f of extraFiles) {
+    const target = join(dir, f.path);
+    ensureDir(join(target, '..'));
+    writeFileSync(target, f.data, { mode: 0o600 });
+    written += 1;
+  }
+
+  const ok = installConnector(parsed.id, manifestYaml);
+  if (!ok) {
+    return { ok: false, error: 'installConnector rejected the manifest (re-parse failed)' };
+  }
+  return {
+    ok: true,
+    id: parsed.id,
+    version: parsed.version || '0.0.0',
+    files_written: written,
+  };
+}
+
+export async function POST(req: Request) {
+  const ct = req.headers.get('content-type') || '';
+
+  // ── JSON path: { yaml: "...", id?: "..." } ─────────────────
+  if (ct.includes('application/json')) {
+    let body: any = {};
+    try { body = await req.json(); } catch {}
+    const yaml = typeof body?.yaml === 'string' ? body.yaml : '';
+    if (!yaml) {
+      return NextResponse.json({ ok: false, error: 'yaml field is required' }, { status: 400 });
+    }
+    const r = installFromFiles(yaml, []);
+    return NextResponse.json(r, { status: r.ok ? 200 : 400 });
+  }
+
+  // ── multipart: file = .yaml | .yml | .zip ──────────────────
+  if (ct.includes('multipart/form-data')) {
+    let form: FormData;
+    try { form = await req.formData(); }
+    catch (e) {
+      return NextResponse.json({ ok: false, error: 'invalid multipart body' }, { status: 400 });
+    }
+    const file = form.get('file');
+    if (!(file instanceof File)) {
+      return NextResponse.json({ ok: false, error: 'file field missing' }, { status: 400 });
+    }
+    if (file.size > MAX_ZIP_BYTES) {
+      return NextResponse.json({ ok: false, error: `file too large (max ${MAX_ZIP_BYTES} bytes)` }, { status: 413 });
+    }
+
+    const name = file.name.toLowerCase();
+    const buf = Buffer.from(await file.arrayBuffer());
+
+    if (name.endsWith('.yaml') || name.endsWith('.yml')) {
+      const r = installFromFiles(buf.toString('utf-8'), []);
+      return NextResponse.json(r, { status: r.ok ? 200 : 400 });
+    }
+
+    if (name.endsWith('.zip')) {
+      let zip: AdmZip;
+      try { zip = new AdmZip(buf); }
+      catch (e) {
+        return NextResponse.json({ ok: false, error: `invalid zip: ${(e as Error).message}` }, { status: 400 });
+      }
+      const entries = zip.getEntries().filter((e) => !e.isDirectory);
+      if (entries.length === 0) {
+        return NextResponse.json({ ok: false, error: 'zip is empty' }, { status: 400 });
+      }
+      if (entries.length > MAX_FILES_IN_ZIP) {
+        return NextResponse.json({ ok: false, error: `zip has too many files (max ${MAX_FILES_IN_ZIP})` }, { status: 400 });
+      }
+
+      // Locate manifest.yaml at the root.
+      const manifestEntry = entries.find((e) => {
+        const safe = safeEntryPath(e.entryName);
+        return safe === 'manifest.yaml' || safe === 'manifest.yml';
+      });
+      if (!manifestEntry) {
+        return NextResponse.json({ ok: false, error: 'zip must contain manifest.yaml at the root' }, { status: 400 });
+      }
+      const manifestYaml = manifestEntry.getData().toString('utf-8');
+
+      const extras: Array<{ path: string; data: Buffer }> = [];
+      for (const e of entries) {
+        if (e === manifestEntry) continue;
+        const safe = safeEntryPath(e.entryName);
+        if (!safe) {
+          return NextResponse.json({ ok: false, error: `unsafe path in zip: ${e.entryName}` }, { status: 400 });
+        }
+        const data = e.getData();
+        if (data.length > MAX_FILE_BYTES) {
+          return NextResponse.json({ ok: false, error: `entry too large: ${safe}` }, { status: 413 });
+        }
+        extras.push({ path: safe, data });
+      }
+      const r = installFromFiles(manifestYaml, extras);
+      return NextResponse.json(r, { status: r.ok ? 200 : 400 });
+    }
+
+    return NextResponse.json({ ok: false, error: 'unsupported file extension (expected .yaml/.yml/.zip)' }, { status: 400 });
+  }
+
+  return NextResponse.json({ ok: false, error: 'expected JSON or multipart/form-data' }, { status: 400 });
+}

package/app/api/connectors/marketplace/route.ts

@@ -0,0 +1,79 @@
+/**
+ * Connector Marketplace API
+ *
+ *   GET  /api/connectors/marketplace
+ *     → { fetched_at, base_url, entries: ConnectorMarketEntry[] }
+ *     Pure read — last cached registry merged with installed state.
+ *     Returns an empty list when sync has never succeeded.
+ *
+ *   POST /api/connectors/marketplace
+ *     body: { action: 'sync' | 'install' | 'uninstall' | 'update', id?: string }
+ *
+ *     - sync       refresh registry.json (and installed manifests)
+ *     - install    pull manifest + write to <dataDir>/connectors/<id>/
+ *     - uninstall  drop manifest (config retained by default)
+ *     - update     alias for install when a newer version exists
+ *
+ * Auth handled by middleware (cookie or X-Forge-Token). All operations
+ * are local — secrets never leave the host.
+ */
+
+import { NextResponse } from 'next/server';
+import {
+  listMarketplace,
+  syncRegistry,
+  installFromRegistry,
+} from '@/lib/connectors/sync';
+import {
+  uninstallConnector,
+  getInstalledConnector,
+} from '@/lib/connectors/registry';
+
+export async function GET() {
+  return NextResponse.json(listMarketplace());
+}
+
+export async function POST(req: Request) {
+  let body: any = {};
+  try { body = await req.json(); } catch {}
+  const action = String(body?.action || '').trim();
+  const id = body?.id ? String(body.id).trim() : '';
+
+  switch (action) {
+    case 'sync': {
+      const r = await syncRegistry({ refreshInstalled: !!body?.refreshInstalled });
+      return NextResponse.json(r);
+    }
+
+    case 'install':
+    case 'update': {
+      if (!id) return NextResponse.json({ error: 'id is required' }, { status: 400 });
+      const r = await installFromRegistry(id);
+      if (!r.ok) return NextResponse.json(r, { status: 502 });
+      return NextResponse.json(r);
+    }
+
+    case 'uninstall': {
+      if (!id) return NextResponse.json({ error: 'id is required' }, { status: 400 });
+      const keepConfig = body?.keepConfig !== false; // default true
+      const ok = uninstallConnector(id, keepConfig);
+      if (!ok) return NextResponse.json({ ok: false, error: 'uninstall failed' }, { status: 500 });
+      return NextResponse.json({ ok: true });
+    }
+
+    case 'status': {
+      // Quick "is this id installed and at what version?" probe — used
+      // by the UI to refresh a single row without reloading the list.
+      if (!id) return NextResponse.json({ error: 'id is required' }, { status: 400 });
+      const inst = getInstalledConnector(id);
+      return NextResponse.json({
+        installed: !!inst,
+        installed_version: inst?.installed_version,
+        enabled: inst?.enabled ?? false,
+      });
+    }
+
+    default:
+      return NextResponse.json({ error: `unknown action: ${action}` }, { status: 400 });
+  }
+}

package/app/api/connectors/route.ts

@@ -1,26 +1,30 @@
 /**
  * Connectors API — discovery surface for the Forge browser extension.
  *
- * The extension hits this endpoint to learn what connector plugins are
- * installed and what tools they advertise. The new declarative-script
- * design (see docs/Connector-DeclarativeExtract-Spec.md) means each tool
- * carries its own `page` (where to navigate) and `script` (function body
- * to run in the user's tab). Forge expands {base_url} / {settings.*}
- * tokens here using the user's saved settings; {args.*} is left literal
- * for the extension's runner to expand at execution time.
+ * Reads from the standalone connector registry (lib/connectors/), no
+ * longer from the plugin registry. The extension hits this endpoint to
+ * learn what connectors are installed and what tools each exposes;
+ * `page` / `host_match` / `login_redirect` are expanded server-side
+ * with the user's saved settings, {args.*} stays literal for the
+ * extension runner.
+ *
+ * Response shape is preserved 1:1 from the old plugin-backed
+ * implementation so the extension and Forge chat keep working without
+ * any client change.
  */
 
 import { NextResponse } from 'next/server';
 import {
-  listPlugins,
-  getPlugin,
-  getInstalledPlugin,
-  getConnectorsForPlugin,
-} from '@/lib/plugins/registry';
+  listInstalledConnectors,
+  getInstalledConnector,
+  getConnectorEntries,
+} from '@/lib/connectors/registry';
 import { expandSettingsTokens } from '@/lib/plugins/templates';
-import type { Connector, ConnectorTool, PluginDefinition } from '@/lib/plugins/types';
-
-// Auth is handled by middleware (session cookie OR X-Forge-Token).
+import type {
+  ConnectorDefinition,
+  ConnectorEntry,
+  ConnectorTool,
+} from '@/lib/connectors/types';
 
 function expandTool(tool: ConnectorTool, settings: Record<string, any> | undefined): ConnectorTool {
   if (!tool.page && !tool.script) return tool;
@@ -35,8 +39,8 @@ function expandTool(tool: ConnectorTool, settings: Record<string, any> | undefin
   return { ...tool, ...(page ? { page } : {}) };
 }
 
-function expandConnector(entry: Connector, settings: Record<string, any> | undefined): Connector {
-  const out: Connector = {
+function expandEntry(entry: ConnectorEntry, settings: Record<string, any> | undefined): ConnectorEntry {
+  const out: ConnectorEntry = {
     ...entry,
     tools: Object.fromEntries(
       Object.entries(entry.tools).map(([name, t]) => [name, expandTool(t, settings)]),
@@ -47,36 +51,33 @@ function expandConnector(entry: Connector, settings: Record<string, any> | undef
   return out;
 }
 
-function toConnectorPayload(def: PluginDefinition) {
-  const inst = getInstalledPlugin(def.id);
-  const settings = inst?.config;
-  const entries = getConnectorsForPlugin(def).map(e => expandConnector(e, settings));
+function toConnectorPayload(def: ConnectorDefinition, config: Record<string, any> | undefined, installed: boolean) {
+  const entries = getConnectorEntries(def).map(e => expandEntry(e, config));
 
-  // Hoist host_match / login_redirect to plugin level for 1:1 cases — the
-  // common case (and what the spec example shows). Pull from the first entry
-  // when those came from the top-level plugin fields.
   const hostMatch = def.host_match
-    ? expandSettingsTokens(def.host_match, settings)
+    ? expandSettingsTokens(def.host_match, config)
     : entries[0]?.host_match;
   const loginRedirect = def.login_redirect
-    ? expandSettingsTokens(def.login_redirect, settings)
+    ? expandSettingsTokens(def.login_redirect, config)
     : entries[0]?.login_redirect;
-  // Default to 'main' so existing connectors (Mantis, GitLab) keep their
-  // current execution path. Plugins on strict-CSP sites opt into 'isolated'.
   const runner = def.runner || entries[0]?.runner || 'main';
 
   return {
-    plugin_id: def.id,
+    plugin_id: def.id,                        // legacy field name preserved
     name: def.name,
     icon: def.icon,
     version: def.version,
     author: def.author || 'forge',
     description: def.description || '',
-    mode: def.mode || 'browser-side',
-    installed: !!inst,
+    mode: 'browser-side',                     // every connector is browser-facing
+    installed,
     ...(hostMatch ? { host_match: hostMatch } : {}),
     ...(loginRedirect ? { login_redirect: loginRedirect } : {}),
     runner,
+    /** Hint for the marketplace UI: a Test button is wired for this connector. */
+    has_test: !!def.test,
+    /** Optional human description of what the test does (no request shape exposed). */
+    test_description: def.test?.description,
     entries,
   };
 }
@@ -85,24 +86,18 @@ export async function GET(req: Request) {
   const url = new URL(req.url);
   const id = url.searchParams.get('id');
 
-  // Single-connector detail
   if (id) {
-    const def = getPlugin(id);
-    if (!def || def.category !== 'connector') {
-      return NextResponse.json({ error: 'connector not found' }, { status: 404 });
-    }
-    return NextResponse.json({ connector: toConnectorPayload(def) });
+    const inst = getInstalledConnector(id);
+    if (!inst) return NextResponse.json({ error: 'connector not found' }, { status: 404 });
+    return NextResponse.json({ connector: toConnectorPayload(inst.definition, inst.config, true) });
   }
 
-  // List all connectors (installed or not — extension surfaces both for marketplace)
-  const onlyInstalled = url.searchParams.get('installed') === 'true';
-  const sources = listPlugins().filter(s => s.category === 'connector');
-
-  const connectors = sources
-    .map(s => getPlugin(s.id))
-    .filter((d): d is PluginDefinition => !!d)
-    .filter(d => !onlyInstalled || !!getInstalledPlugin(d.id))
-    .map(toConnectorPayload);
+  // The extension always wants the installed list — there is no "not
+  // installed but visible" path on this endpoint anymore (that lives
+  // in /api/connectors/marketplace, coming in P1.6).
+  const connectors = listInstalledConnectors()
+    .filter(c => c.enabled)
+    .map(c => toConnectorPayload(c.definition, c.config, true));
 
   return NextResponse.json({ connectors });
 }

package/app/api/jobs/route.ts

@@ -30,6 +30,7 @@ export async function POST(req: Request) {
     dedup_field: String(body.dedup_field),
     dispatch_type: body.dispatch_type,
     dispatch_params: body.dispatch_params,
+    skills: Array.isArray(body.skills) ? body.skills.map((s: unknown) => String(s)) : [],
     mark_existing_as_seen: body.mark_existing_as_seen !== false,
   });
   return NextResponse.json({ job });

package/app/api/monitor/route.ts

@@ -22,8 +22,25 @@ export async function GET() {
   const terminal = countProcess('terminal-standalone');
   const telegram = countProcess('telegram-standalone');
   const workspace = countProcess('workspace-standalone');
+  const chat = countProcess('chat-standalone');
+  const browserBridge = countProcess('browser-bridge-standalone');
   const tunnel = countProcess('cloudflared tunnel');
 
+  // Chat backend health (port 8408 — process can be alive but crashed)
+  let chatStatus: { running: boolean; sessions: number; port: number } = {
+    running: false,
+    sessions: 0,
+    port: Number(process.env.CHAT_PORT) || 8408,
+  };
+  try {
+    const chatRes = run(`curl -s http://127.0.0.1:${chatStatus.port}/api/status 2>/dev/null`);
+    if (chatRes) {
+      const data = JSON.parse(chatRes);
+      chatStatus.running = true;
+      chatStatus.sessions = data.active_sse_sessions ?? 0;
+    }
+  } catch {}
+
   // MCP Server (runs inside workspace process, check /health endpoint)
   let mcpStatus = { running: false, sessions: 0 };
   try {
@@ -65,6 +82,15 @@ export async function GET() {
       terminal: { running: terminal.count > 0, pid: terminal.pid, startedAt: terminal.startedAt },
       telegram: { running: telegram.count > 0, pid: telegram.pid, startedAt: telegram.startedAt },
       workspace: { running: workspace.count > 0, pid: workspace.pid, startedAt: workspace.startedAt },
+      browserBridge: { running: browserBridge.count > 0, pid: browserBridge.pid, startedAt: browserBridge.startedAt },
+      chat: {
+        running: chatStatus.running,
+        pid: chat.pid,
+        startedAt: chat.startedAt,
+        port: chatStatus.port,
+        sessions: chatStatus.sessions,
+        processAlive: chat.count > 0,
+      },
       tunnel: { running: tunnel.count > 0, pid: tunnel.pid, url: tunnelUrl, startedAt: tunnel.startedAt },
       mcp: { running: mcpStatus.running, port: 8406, sessions: mcpStatus.sessions },
     },