@aion0/forge 0.8.1 → 0.8.2

package/app/api/connectors/marketplace/route.ts

@@ -0,0 +1,79 @@
+/**
+ * Connector Marketplace API
+ *
+ *   GET  /api/connectors/marketplace
+ *     → { fetched_at, base_url, entries: ConnectorMarketEntry[] }
+ *     Pure read — last cached registry merged with installed state.
+ *     Returns an empty list when sync has never succeeded.
+ *
+ *   POST /api/connectors/marketplace
+ *     body: { action: 'sync' | 'install' | 'uninstall' | 'update', id?: string }
+ *
+ *     - sync       refresh registry.json (and installed manifests)
+ *     - install    pull manifest + write to <dataDir>/connectors/<id>/
+ *     - uninstall  drop manifest (config retained by default)
+ *     - update     alias for install when a newer version exists
+ *
+ * Auth handled by middleware (cookie or X-Forge-Token). All operations
+ * are local — secrets never leave the host.
+ */
+
+import { NextResponse } from 'next/server';
+import {
+  listMarketplace,
+  syncRegistry,
+  installFromRegistry,
+} from '@/lib/connectors/sync';
+import {
+  uninstallConnector,
+  getInstalledConnector,
+} from '@/lib/connectors/registry';
+
+export async function GET() {
+  return NextResponse.json(listMarketplace());
+}
+
+export async function POST(req: Request) {
+  let body: any = {};
+  try { body = await req.json(); } catch {}
+  const action = String(body?.action || '').trim();
+  const id = body?.id ? String(body.id).trim() : '';
+
+  switch (action) {
+    case 'sync': {
+      const r = await syncRegistry({ refreshInstalled: !!body?.refreshInstalled });
+      return NextResponse.json(r);
+    }
+
+    case 'install':
+    case 'update': {
+      if (!id) return NextResponse.json({ error: 'id is required' }, { status: 400 });
+      const r = await installFromRegistry(id);
+      if (!r.ok) return NextResponse.json(r, { status: 502 });
+      return NextResponse.json(r);
+    }
+
+    case 'uninstall': {
+      if (!id) return NextResponse.json({ error: 'id is required' }, { status: 400 });
+      const keepConfig = body?.keepConfig !== false; // default true
+      const ok = uninstallConnector(id, keepConfig);
+      if (!ok) return NextResponse.json({ ok: false, error: 'uninstall failed' }, { status: 500 });
+      return NextResponse.json({ ok: true });
+    }
+
+    case 'status': {
+      // Quick "is this id installed and at what version?" probe — used
+      // by the UI to refresh a single row without reloading the list.
+      if (!id) return NextResponse.json({ error: 'id is required' }, { status: 400 });
+      const inst = getInstalledConnector(id);
+      return NextResponse.json({
+        installed: !!inst,
+        installed_version: inst?.installed_version,
+        enabled: inst?.enabled ?? false,
+      });
+    }
+
+    default:
+      return NextResponse.json({ error: `unknown action: ${action}` }, { status: 400 });
+  }
+}

package/app/api/connectors/route.ts

@@ -1,26 +1,30 @@
 /**
  * Connectors API — discovery surface for the Forge browser extension.
  *
- * The extension hits this endpoint to learn what connector plugins are
- * installed and what tools they advertise. The new declarative-script
- * design (see docs/Connector-DeclarativeExtract-Spec.md) means each tool
- * carries its own `page` (where to navigate) and `script` (function body
- * to run in the user's tab). Forge expands {base_url} / {settings.*}
- * tokens here using the user's saved settings; {args.*} is left literal
- * for the extension's runner to expand at execution time.
+ * Reads from the standalone connector registry (lib/connectors/), no
+ * longer from the plugin registry. The extension hits this endpoint to
+ * learn what connectors are installed and what tools each exposes;
+ * `page` / `host_match` / `login_redirect` are expanded server-side
+ * with the user's saved settings, {args.*} stays literal for the
+ * extension runner.
+ *
+ * Response shape is preserved 1:1 from the old plugin-backed
+ * implementation so the extension and Forge chat keep working without
+ * any client change.
  */
 
 import { NextResponse } from 'next/server';
 import {
-  listPlugins,
-  getPlugin,
-  getInstalledPlugin,
-  getConnectorsForPlugin,
-} from '@/lib/plugins/registry';
+  listInstalledConnectors,
+  getInstalledConnector,
+  getConnectorEntries,
+} from '@/lib/connectors/registry';
 import { expandSettingsTokens } from '@/lib/plugins/templates';
-import type { Connector, ConnectorTool, PluginDefinition } from '@/lib/plugins/types';
-
-// Auth is handled by middleware (session cookie OR X-Forge-Token).
+import type {
+  ConnectorDefinition,
+  ConnectorEntry,
+  ConnectorTool,
+} from '@/lib/connectors/types';
 
 function expandTool(tool: ConnectorTool, settings: Record<string, any> | undefined): ConnectorTool {
   if (!tool.page && !tool.script) return tool;
@@ -35,8 +39,8 @@ function expandTool(tool: ConnectorTool, settings: Record<string, any> | undefin
   return { ...tool, ...(page ? { page } : {}) };
 }
 
-function expandConnector(entry: Connector, settings: Record<string, any> | undefined): Connector {
-  const out: Connector = {
+function expandEntry(entry: ConnectorEntry, settings: Record<string, any> | undefined): ConnectorEntry {
+  const out: ConnectorEntry = {
     ...entry,
     tools: Object.fromEntries(
       Object.entries(entry.tools).map(([name, t]) => [name, expandTool(t, settings)]),
@@ -47,36 +51,33 @@ function expandConnector(entry: Connector, settings: Record<string, any> | undef
   return out;
 }
 
-function toConnectorPayload(def: PluginDefinition) {
-  const inst = getInstalledPlugin(def.id);
-  const settings = inst?.config;
-  const entries = getConnectorsForPlugin(def).map(e => expandConnector(e, settings));
+function toConnectorPayload(def: ConnectorDefinition, config: Record<string, any> | undefined, installed: boolean) {
+  const entries = getConnectorEntries(def).map(e => expandEntry(e, config));
 
-  // Hoist host_match / login_redirect to plugin level for 1:1 cases — the
-  // common case (and what the spec example shows). Pull from the first entry
-  // when those came from the top-level plugin fields.
   const hostMatch = def.host_match
-    ? expandSettingsTokens(def.host_match, settings)
+    ? expandSettingsTokens(def.host_match, config)
     : entries[0]?.host_match;
   const loginRedirect = def.login_redirect
-    ? expandSettingsTokens(def.login_redirect, settings)
+    ? expandSettingsTokens(def.login_redirect, config)
     : entries[0]?.login_redirect;
-  // Default to 'main' so existing connectors (Mantis, GitLab) keep their
-  // current execution path. Plugins on strict-CSP sites opt into 'isolated'.
   const runner = def.runner || entries[0]?.runner || 'main';
 
   return {
-    plugin_id: def.id,
+    plugin_id: def.id,                        // legacy field name preserved
     name: def.name,
     icon: def.icon,
     version: def.version,
     author: def.author || 'forge',
     description: def.description || '',
-    mode: def.mode || 'browser-side',
-    installed: !!inst,
+    mode: 'browser-side',                     // every connector is browser-facing
+    installed,
     ...(hostMatch ? { host_match: hostMatch } : {}),
     ...(loginRedirect ? { login_redirect: loginRedirect } : {}),
     runner,
+    /** Hint for the marketplace UI: a Test button is wired for this connector. */
+    has_test: !!def.test,
+    /** Optional human description of what the test does (no request shape exposed). */
+    test_description: def.test?.description,
     entries,
   };
 }
@@ -85,24 +86,18 @@ export async function GET(req: Request) {
   const url = new URL(req.url);
   const id = url.searchParams.get('id');
 
-  // Single-connector detail
   if (id) {
-    const def = getPlugin(id);
-    if (!def || def.category !== 'connector') {
-      return NextResponse.json({ error: 'connector not found' }, { status: 404 });
-    }
-    return NextResponse.json({ connector: toConnectorPayload(def) });
+    const inst = getInstalledConnector(id);
+    if (!inst) return NextResponse.json({ error: 'connector not found' }, { status: 404 });
+    return NextResponse.json({ connector: toConnectorPayload(inst.definition, inst.config, true) });
   }
 
-  // List all connectors (installed or not — extension surfaces both for marketplace)
-  const onlyInstalled = url.searchParams.get('installed') === 'true';
-  const sources = listPlugins().filter(s => s.category === 'connector');
-
-  const connectors = sources
-    .map(s => getPlugin(s.id))
-    .filter((d): d is PluginDefinition => !!d)
-    .filter(d => !onlyInstalled || !!getInstalledPlugin(d.id))
-    .map(toConnectorPayload);
+  // The extension always wants the installed list — there is no "not
+  // installed but visible" path on this endpoint anymore (that lives
+  // in /api/connectors/marketplace, coming in P1.6).
+  const connectors = listInstalledConnectors()
+    .filter(c => c.enabled)
+    .map(c => toConnectorPayload(c.definition, c.config, true));
 
   return NextResponse.json({ connectors });
 }

package/app/api/jobs/route.ts

@@ -30,6 +30,7 @@ export async function POST(req: Request) {
     dedup_field: String(body.dedup_field),
     dispatch_type: body.dispatch_type,
     dispatch_params: body.dispatch_params,
+    skills: Array.isArray(body.skills) ? body.skills.map((s: unknown) => String(s)) : [],
     mark_existing_as_seen: body.mark_existing_as_seen !== false,
   });
   return NextResponse.json({ job });

package/app/api/skills/install-local/route.ts

@@ -0,0 +1,282 @@
+/**
+ * POST /api/skills/install-local
+ *
+ * Install a Forge skill from a user-supplied bundle. Two flavours:
+ *
+ *   multipart/form-data:
+ *     file: .md      → single-file skill, written as SKILL.md
+ *     file: .zip     → bundle. Must contain SKILL.md (either at the
+ *                      root or inside a single wrapping subdir).
+ *                      Any sibling files (scripts/, references/,
+ *                      assets/) are copied verbatim.
+ *     name           → optional, overrides the frontmatter / wrapper-dir
+ *     project_path   → optional, install into the project's .claude/skills/
+ *                      instead of the global ~/.claude/skills/
+ *
+ *   application/json:
+ *     { name, content }            → quick single-file path for the
+ *     { name, files: { path: content, ... } }   AI / scripts
+ *
+ * The DB row is upserted with `source: 'local'` so the sync layer
+ * won't try to redownload it from the remote registry (and won't mark
+ * it deleted when it's not in registry.json).
+ */
+
+import { NextResponse } from 'next/server';
+import { existsSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { join, dirname, normalize, sep } from 'node:path';
+import AdmZip from 'adm-zip';
+import { homedir } from 'node:os';
+import { getDb } from '@/src/core/db/database';
+import { getDbPath } from '@/src/config';
+
+const MAX_BYTES = 5 * 1024 * 1024;          // 5 MB
+const MAX_FILE_BYTES = 1 * 1024 * 1024;     // 1 MB per file
+const MAX_FILES = 50;
+
+function db() { return getDb(getDbPath()); }
+
+// ─── Helpers ──────────────────────────────────────────────
+
+function getClaudeHome(): string {
+  return process.env.CLAUDE_HOME || join(homedir(), '.claude');
+}
+
+function isValidName(s: string): boolean {
+  return /^[a-z0-9][a-z0-9_-]*$/.test(s);
+}
+
+function safeEntryPath(name: string): string | null {
+  if (!name || name.startsWith('/') || name.includes('..')) return null;
+  const normalized = normalize(name).replace(/^[/\\]+/, '');
+  if (normalized.split(sep).some((seg) => seg === '..' || seg === '')) return null;
+  return normalized;
+}
+
+interface ParsedFrontmatter {
+  name?: string;
+  description?: string;
+  version?: string;
+}
+
+/** Extract `name:` / `description:` / `version:` from a SKILL.md YAML frontmatter block. */
+function parseSkillMd(md: string): ParsedFrontmatter {
+  const fm = md.match(/^---\n([\s\S]*?)\n---/);
+  if (!fm) return {};
+  const out: ParsedFrontmatter = {};
+  for (const line of fm[1].split('\n')) {
+    const m = line.match(/^([a-zA-Z_]+):\s*(.*?)\s*$/);
+    if (!m) continue;
+    const key = m[1].toLowerCase();
+    let val = m[2];
+    if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
+      val = val.slice(1, -1);
+    }
+    if (key === 'name') out.name = val;
+    else if (key === 'description') out.description = val;
+    else if (key === 'version') out.version = val;
+  }
+  return out;
+}
+
+function ensureDir(p: string): void {
+  if (!existsSync(p)) mkdirSync(p, { recursive: true, mode: 0o700 });
+}
+
+interface InstallPayload {
+  name: string;
+  files: Array<{ path: string; data: Buffer }>;
+  description?: string;
+  version?: string;
+}
+
+/**
+ * Pre-validate that the bundle has a SKILL.md and a usable name.
+ * Returns either a usable payload or an error.
+ */
+function buildPayload(
+  files: Array<{ path: string; data: Buffer }>,
+  nameOverride?: string,
+): { ok: true; payload: InstallPayload } | { ok: false; error: string } {
+  // Detect wrapper-dir layout: <name>/SKILL.md
+  const skillMd = files.find((f) => f.path === 'SKILL.md' || f.path.toLowerCase() === 'skill.md');
+  let nestedRoot = '';
+  let actualSkill = skillMd;
+  if (!skillMd) {
+    // Look for <something>/SKILL.md as the only entry of its kind
+    const candidates = files.filter((f) => /(^|\/)SKILL\.md$/i.test(f.path));
+    if (candidates.length !== 1) {
+      return { ok: false, error: 'zip must contain exactly one SKILL.md (at root or inside one wrapper directory)' };
+    }
+    actualSkill = candidates[0];
+    nestedRoot = actualSkill.path.slice(0, -('SKILL.md'.length + 1));
+  }
+  const md = actualSkill!.data.toString('utf-8');
+  const fm = parseSkillMd(md);
+  const inferredName =
+    nameOverride ||
+    fm.name ||
+    (nestedRoot && nestedRoot.split('/')[0]) ||
+    '';
+  if (!inferredName) {
+    return { ok: false, error: 'cannot determine skill name — set frontmatter `name:` or wrap files in a <name>/ directory' };
+  }
+  if (!isValidName(inferredName)) {
+    return { ok: false, error: `skill name "${inferredName}" must be lowercase alphanumerics + hyphens/underscores` };
+  }
+
+  // Strip the wrapper-dir prefix from all file paths
+  const flatFiles = files
+    .map((f) => ({
+      path: nestedRoot && f.path.startsWith(nestedRoot) ? f.path.slice(nestedRoot.length) : f.path,
+      data: f.data,
+    }))
+    .filter((f) => f.path && f.path !== '');
+
+  return {
+    ok: true,
+    payload: { name: inferredName, files: flatFiles, description: fm.description, version: fm.version },
+  };
+}
+
+function writeSkillToDisk(payload: InstallPayload, projectPath?: string): { targetDir: string } {
+  const root = projectPath
+    ? join(projectPath, '.claude', 'skills', payload.name)
+    : join(getClaudeHome(), 'skills', payload.name);
+  // Clean any prior install so removed files don't linger.
+  if (existsSync(root)) {
+    try { rmSync(root, { recursive: true, force: true }); } catch {}
+  }
+  ensureDir(root);
+  for (const f of payload.files) {
+    const target = join(root, f.path);
+    ensureDir(dirname(target));
+    writeFileSync(target, f.data, { mode: 0o600 });
+  }
+  return { targetDir: root };
+}
+
+function upsertSkillRow(payload: InstallPayload, projectPath?: string): void {
+  const existing = db().prepare('SELECT installed_global, installed_projects FROM skills WHERE name = ?')
+    .get(payload.name) as any;
+  const installedProjects: string[] = existing
+    ? (() => { try { return JSON.parse(existing.installed_projects || '[]'); } catch { return []; } })()
+    : [];
+  if (projectPath && !installedProjects.includes(projectPath)) installedProjects.push(projectPath);
+  const installedGlobal = projectPath ? (existing?.installed_global ? 1 : 0) : 1;
+  const version = payload.version || '0.1.0';
+
+  db().prepare(`
+    INSERT INTO skills (name, type, display_name, description, author, version, tags, score, rating,
+                        source_url, archive, synced_at, source,
+                        installed_global, installed_projects, installed_version)
+    VALUES (?, 'skill', ?, ?, '', ?, '[]', 0, 0, '', '', datetime('now'), 'local', ?, ?, ?)
+    ON CONFLICT(name) DO UPDATE SET
+      display_name       = excluded.display_name,
+      description        = excluded.description,
+      version            = excluded.version,
+      synced_at          = datetime('now'),
+      source             = 'local',
+      installed_global   = excluded.installed_global,
+      installed_projects = excluded.installed_projects,
+      installed_version  = excluded.installed_version
+  `).run(
+    payload.name,
+    payload.name,
+    payload.description || '',
+    version,
+    installedGlobal,
+    JSON.stringify(installedProjects),
+    version,
+  );
+}
+
+// ─── POST ────────────────────────────────────────────────
+
+export async function POST(req: Request) {
+  const ct = req.headers.get('content-type') || '';
+
+  // ── JSON path: { name, content } or { name, files: { path: content } }
+  if (ct.includes('application/json')) {
+    let body: any = {};
+    try { body = await req.json(); } catch {}
+    const nameRaw = String(body?.name || '').trim();
+    if (body?.content) {
+      // single SKILL.md
+      const built = buildPayload(
+        [{ path: 'SKILL.md', data: Buffer.from(String(body.content), 'utf-8') }],
+        nameRaw || undefined,
+      );
+      if (!built.ok) return NextResponse.json({ ok: false, error: built.error }, { status: 400 });
+      writeSkillToDisk(built.payload, body.project_path || undefined);
+      upsertSkillRow(built.payload, body.project_path || undefined);
+      return NextResponse.json({ ok: true, name: built.payload.name, version: built.payload.version || '0.1.0' });
+    }
+    if (body?.files && typeof body.files === 'object') {
+      const entries = Object.entries(body.files as Record<string, string>);
+      const built = buildPayload(
+        entries.map(([path, content]) => ({ path, data: Buffer.from(String(content), 'utf-8') })),
+        nameRaw || undefined,
+      );
+      if (!built.ok) return NextResponse.json({ ok: false, error: built.error }, { status: 400 });
+      writeSkillToDisk(built.payload, body.project_path || undefined);
+      upsertSkillRow(built.payload, body.project_path || undefined);
+      return NextResponse.json({ ok: true, name: built.payload.name, version: built.payload.version || '0.1.0' });
+    }
+    return NextResponse.json({ ok: false, error: 'JSON body must have { content } or { files }' }, { status: 400 });
+  }
+
+  // ── multipart: file = .md | .zip
+  if (ct.includes('multipart/form-data')) {
+    let form: FormData;
+    try { form = await req.formData(); }
+    catch { return NextResponse.json({ ok: false, error: 'invalid multipart body' }, { status: 400 }); }
+    const file = form.get('file');
+    if (!(file instanceof File)) {
+      return NextResponse.json({ ok: false, error: 'file field missing' }, { status: 400 });
+    }
+    if (file.size > MAX_BYTES) {
+      return NextResponse.json({ ok: false, error: `file too large (max ${MAX_BYTES} bytes)` }, { status: 413 });
+    }
+    const nameOverride = String(form.get('name') || '').trim() || undefined;
+    const projectPath = String(form.get('project_path') || '').trim() || undefined;
+    const filename = file.name.toLowerCase();
+    const buf = Buffer.from(await file.arrayBuffer());
+
+    let files: Array<{ path: string; data: Buffer }> = [];
+
+    if (filename.endsWith('.md')) {
+      files.push({ path: 'SKILL.md', data: buf });
+    } else if (filename.endsWith('.zip')) {
+      let zip: AdmZip;
+      try { zip = new AdmZip(buf); }
+      catch (e) { return NextResponse.json({ ok: false, error: `invalid zip: ${(e as Error).message}` }, { status: 400 }); }
+      const entries = zip.getEntries().filter((e) => !e.isDirectory);
+      if (entries.length === 0) return NextResponse.json({ ok: false, error: 'zip is empty' }, { status: 400 });
+      if (entries.length > MAX_FILES) return NextResponse.json({ ok: false, error: `zip has too many files (max ${MAX_FILES})` }, { status: 400 });
+      for (const e of entries) {
+        const safe = safeEntryPath(e.entryName);
+        if (!safe) return NextResponse.json({ ok: false, error: `unsafe path in zip: ${e.entryName}` }, { status: 400 });
+        const data = e.getData();
+        if (data.length > MAX_FILE_BYTES) return NextResponse.json({ ok: false, error: `entry too large: ${safe}` }, { status: 413 });
+        files.push({ path: safe, data });
+      }
+    } else {
+      return NextResponse.json({ ok: false, error: 'unsupported file extension (expected .md or .zip)' }, { status: 400 });
+    }
+
+    const built = buildPayload(files, nameOverride);
+    if (!built.ok) return NextResponse.json({ ok: false, error: built.error }, { status: 400 });
+    const { targetDir } = writeSkillToDisk(built.payload, projectPath);
+    upsertSkillRow(built.payload, projectPath);
+    return NextResponse.json({
+      ok: true,
+      name: built.payload.name,
+      version: built.payload.version || '0.1.0',
+      target: targetDir,
+      files_written: built.payload.files.length,
+    });
+  }
+
+  return NextResponse.json({ ok: false, error: 'expected JSON or multipart/form-data' }, { status: 400 });
+}