@aion0/forge 0.6.3 → 0.8.0

package/app/api/jobs/preview/route.ts

@@ -0,0 +1,193 @@
+/**
+ * POST /api/jobs/preview
+ *
+ * Dry-run a Job's source_tool with its source_input and return the first
+ * parsed item. The Jobs editor uses this to auto-build input_template:
+ * "given the bug Mantis just returned, what {{item.X}} expression maps
+ * each pipeline input field?" — eliminating the friction where users
+ * had to know in advance that get_bug returns `id` not `bug_id` etc.
+ *
+ * NO dispatch happens. NO dedup mutation. NO job row required (this works
+ * before the Job is saved, that's the whole point).
+ *
+ * Request body:
+ *   {
+ *     source_connector: string,
+ *     source_tool: string,
+ *     source_input?: object,
+ *     items_path?: string,        // empty / dotted path
+ *     pipeline_inputs?: string[]  // optional: workflow's input field names —
+ *                                 // if given, response includes a suggested
+ *                                 // input_template with fuzzy-matched mappings
+ *   }
+ *
+ * Response:
+ *   {
+ *     ok: true,
+ *     sample_item: object,            // first item from the connector
+ *     item_keys: string[],            // top-level keys on sample_item
+ *     suggested_template?: Record<string,string>  // pipeline_input → "{{item.X}}"
+ *   }
+ *
+ *   or { ok: false, error, ...debug } on failure
+ */
+
+import { NextResponse } from 'next/server';
+import { dispatchTool } from '@/lib/chat/tool-dispatcher';
+
+function pickPath(obj: unknown, path: string): unknown {
+  if (!path) return obj;
+  const parts = path.split('.');
+  let cur: any = obj;
+  for (const p of parts) {
+    if (cur == null || typeof cur !== 'object') return undefined;
+    cur = cur[p];
+  }
+  return cur;
+}
+
+/**
+ * Best-effort match a workflow input name to a key on the sample item.
+ * The mantis case (bug_id ← id, base_branch ← nothing) drove this list:
+ *   - Exact match wins.
+ *   - Strip _id / Id / ID suffixes both sides → check.
+ *   - 'description' fuzz: desc, body, content all work.
+ *   - 'summary' fuzz: title, name.
+ * Returns the matching item-key or null. NEVER guesses across unrelated names.
+ */
+function suggestMapping(inputName: string, itemKeys: string[]): string | null {
+  const lc = inputName.toLowerCase();
+  const lcMap = new Map(itemKeys.map(k => [k.toLowerCase(), k]));
+
+  // 1. Exact match.
+  if (lcMap.has(lc)) return lcMap.get(lc)!;
+
+  // 2. ID-shaped fields. The interesting cases:
+  //    - pipeline input "bug_id" + item has "id"        → match "id"
+  //    - pipeline input "bug_id" + item has "bug"       → match "bug"
+  //    - pipeline input "bug_id" + item has "bugId"     → match "bugId"
+  //    - pipeline input "id"     + item has "issue_id"  → match "issue_id"
+  const endsWithId = /_?id$/i.test(lc);
+  const stripped = lc.replace(/_?id$/i, '');
+  if (endsWithId) {
+    if (lcMap.has('id')) return lcMap.get('id')!;                  // ← the missing case
+    if (stripped && lcMap.has(stripped)) return lcMap.get(stripped)!;
+    if (stripped && lcMap.has(stripped + '_id')) return lcMap.get(stripped + '_id')!;
+  } else if (lc === 'id') {
+    for (const k of itemKeys) {
+      if (/_?id$/i.test(k.toLowerCase())) return k;
+    }
+  }
+
+  // 3. Synonym table — short, targeted. Add cases when a real connector
+  //    surfaces them, not speculatively.
+  const synonyms: Record<string, string[]> = {
+    summary:     ['title', 'name', 'subject'],
+    description: ['body', 'content', 'desc', 'text'],
+    assignee:    ['handler', 'assigned_to', 'owner'],
+    reporter:    ['author', 'created_by', 'submitted_by'],
+    priority:    ['severity'],
+    category:    ['keywords', 'labels', 'tags'],          // mantis: 'keywords' col
+    url:         ['web_url', 'link', 'href'],
+  };
+  for (const alt of synonyms[lc] || []) {
+    if (lcMap.has(alt)) return lcMap.get(alt)!;
+  }
+  return null;
+}
+
+export async function POST(req: Request) {
+  let body: any;
+  try { body = await req.json(); }
+  catch { return NextResponse.json({ ok: false, error: 'invalid JSON body' }, { status: 400 }); }
+
+  const { source_connector, source_tool, source_input, items_path, pipeline_inputs } = body || {};
+  if (!source_connector || !source_tool) {
+    return NextResponse.json({ ok: false, error: 'source_connector and source_tool are required' }, { status: 400 });
+  }
+
+  const callName = `${source_connector}.${source_tool}`;
+  let toolResult;
+  try {
+    toolResult = await dispatchTool({
+      id: `jobs-preview-${Date.now()}`,
+      name: callName,
+      input: source_input || {},
+    });
+  } catch (e) {
+    return NextResponse.json({ ok: false, error: `connector call threw: ${(e as Error).message}` }, { status: 500 });
+  }
+
+  if (toolResult.is_error) {
+    return NextResponse.json({
+      ok: false,
+      error: `connector returned is_error=true`,
+      raw_preview: toolResult.content.slice(0, 800),
+    }, { status: 200 });
+  }
+
+  let parsed: unknown;
+  try { parsed = JSON.parse(toolResult.content); }
+  catch {
+    return NextResponse.json({
+      ok: false,
+      error: 'connector returned non-JSON content',
+      raw_preview: toolResult.content.slice(0, 800),
+    }, { status: 200 });
+  }
+
+  let items = pickPath(parsed, items_path || '');
+  // Same single-object-as-1-item-list logic as the scheduler, so the
+  // preview matches what runtime will see.
+  if (items && typeof items === 'object' && !Array.isArray(items)) {
+    items = [items];
+  }
+  if (!Array.isArray(items) || items.length === 0) {
+    const topKeys = parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+      ? Object.keys(parsed as Record<string, unknown>)
+      : [];
+    return NextResponse.json({
+      ok: false,
+      error: Array.isArray(items)
+        ? `items_path resolved to an empty array — no sample item available`
+        : `items_path "${items_path || '(empty)'}" did not resolve to an array or object`,
+      top_level_keys: topKeys,
+    }, { status: 200 });
+  }
+
+  const sampleItem = items[0] as Record<string, unknown>;
+  // Union of keys across ALL returned items, not just items[0]. Mantis
+  // bug rows occasionally omit fields when blank — e.g. a bug with no
+  // fix_schedule won't have that key on its object — so basing the
+  // template on a single sample misses real-world fields. Scan up to
+  // 20 items (covers >99% of typical job batches).
+  const keySet = new Set<string>();
+  const sampleCount = Math.min(items.length, 20);
+  for (let i = 0; i < sampleCount; i++) {
+    const it = items[i];
+    if (it && typeof it === 'object' && !Array.isArray(it)) {
+      for (const k of Object.keys(it as Record<string, unknown>)) keySet.add(k);
+    }
+  }
+  const itemKeys = Array.from(keySet);
+
+  let suggestedTemplate: Record<string, string> | undefined;
+  if (Array.isArray(pipeline_inputs) && pipeline_inputs.length > 0) {
+    suggestedTemplate = {};
+    for (const inputName of pipeline_inputs) {
+      if (typeof inputName !== 'string') continue;
+      const match = suggestMapping(inputName, itemKeys);
+      // If no fuzzy match found, leave it blank so the user knows it needs
+      // a constant (e.g. base_branch). NEVER guess across unrelated names.
+      suggestedTemplate[inputName] = match ? `{{item.${match}}}` : '';
+    }
+  }
+
+  return NextResponse.json({
+    ok: true,
+    sample_item: sampleItem,
+    item_keys: itemKeys,
+    suggested_template: suggestedTemplate,
+    total_items: items.length,
+  });
+}

package/app/api/jobs/route.ts

@@ -0,0 +1,36 @@
+/**
+ * GET  /api/jobs    list all jobs
+ * POST /api/jobs    create a job
+ */
+
+import { NextResponse } from 'next/server';
+import { listJobs, createJob } from '@/lib/jobs/store';
+
+export async function GET() {
+  return NextResponse.json({ jobs: listJobs() });
+}
+
+export async function POST(req: Request) {
+  let body: any = {};
+  try { body = await req.json(); } catch {}
+  if (!body?.name || !body?.source_connector || !body?.source_tool || !body?.dedup_field || !body?.dispatch_type || !body?.dispatch_params) {
+    return NextResponse.json({ error: 'name, source_connector, source_tool, dedup_field, dispatch_type, dispatch_params are required' }, { status: 400 });
+  }
+  if (body.dispatch_type !== 'pipeline' && body.dispatch_type !== 'chat') {
+    return NextResponse.json({ error: "dispatch_type must be 'pipeline' or 'chat'" }, { status: 400 });
+  }
+  const job = createJob({
+    name: String(body.name),
+    enabled: body.enabled !== false,
+    schedule_interval_minutes: Number(body.schedule_interval_minutes) || 30,
+    source_connector: String(body.source_connector),
+    source_tool: String(body.source_tool),
+    source_input: body.source_input || {},
+    items_path: body.items_path || '',
+    dedup_field: String(body.dedup_field),
+    dispatch_type: body.dispatch_type,
+    dispatch_params: body.dispatch_params,
+    mark_existing_as_seen: body.mark_existing_as_seen !== false,
+  });
+  return NextResponse.json({ job });
+}

package/app/api/notify/test/route.ts

@@ -9,25 +9,57 @@ export async function POST() {
     return NextResponse.json({ ok: false, error: 'Telegram bot token or chat ID not configured' });
   }
 
+  // telegramChatId may be a comma-separated whitelist. The test message
+  // goes to the FIRST id so a misconfigured later id doesn't fail the test.
+  const firstChatId = telegramChatId.split(',').map((s) => s.trim()).filter(Boolean)[0];
+  if (!firstChatId) {
+    return NextResponse.json({ ok: false, error: 'telegramChatId is empty / malformed' });
+  }
+
   try {
     const url = `https://api.telegram.org/bot${telegramBotToken}/sendMessage`;
+    // Plain text — no parse_mode — so an em-dash / asterisk / underscore
+    // can never trip Telegram's Markdown parser ("can't parse entities").
     const res = await fetch(url, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify({
-        chat_id: telegramChatId,
-        text: '✅ *Forge* — Test notification!\n\nTelegram notifications are working.',
-        parse_mode: 'Markdown',
+        chat_id: firstChatId,
+        text: '✅ Forge — Test notification! Telegram notifications are working.',
       }),
     });
 
+    const body = await res.text();
     if (!res.ok) {
-      const body = await res.text();
-      return NextResponse.json({ ok: false, error: body });
+      // Surface Telegram's "description" — covers the common cases like
+      // "chat not found", "Unauthorized" (token revoked), "bot was blocked".
+      try {
+        const j = JSON.parse(body);
+        return NextResponse.json({ ok: false, error: `${res.status} ${j.description || body}` });
+      } catch {
+        return NextResponse.json({ ok: false, error: `${res.status} ${body}` });
+      }
     }
-
     return NextResponse.json({ ok: true });
   } catch (err: any) {
-    return NextResponse.json({ ok: false, error: err.message });
+    // 'fetch failed' from Node is opaque — almost always corp SSL inspection
+    // (Zscaler / corporate firewall) or proxy / DNS / network. Surface
+    // concrete remediation in the alert so the user doesn't have to guess.
+    const msg = String(err?.message || err);
+    if (/fetch failed|ENOTFOUND|ECONNREFUSED|ETIMEDOUT|UNABLE_TO_VERIFY|self.signed/i.test(msg)) {
+      const cause = err?.cause?.code || err?.cause?.message || '';
+      return NextResponse.json({
+        ok: false,
+        error: [
+          `${msg}${cause ? ' (cause: ' + cause + ')' : ''}`,
+          '',
+          'Forge could not reach api.telegram.org. Common causes:',
+          '  • Corporate TLS inspection — set NODE_EXTRA_CA_CERTS=/path/to/corp-ca.pem then `forge server restart`',
+          '  • HTTP proxy required — set HTTPS_PROXY=http://proxy.corp:port then restart',
+          '  • Network / DNS — curl -v https://api.telegram.org/bot<TOKEN>/getMe',
+        ].join('\n'),
+      });
+    }
+    return NextResponse.json({ ok: false, error: msg });
   }
 }

package/app/api/pipelines/[id]/route.ts

@@ -1,5 +1,5 @@
 import { NextResponse } from 'next/server';
-import { getPipeline, cancelPipeline, deletePipeline, injectConversationMessage } from '@/lib/pipeline';
+import { getPipeline, cancelPipeline, deletePipeline, injectConversationMessage, retryNode } from '@/lib/pipeline';
 
 // GET /api/pipelines/:id
 export async function GET(_req: Request, { params }: { params: Promise<{ id: string }> }) {
@@ -25,6 +25,15 @@ export async function POST(req: Request, { params }: { params: Promise<{ id: str
     return NextResponse.json({ ok });
   }
 
+  // Retry a single failed node — see lib/pipeline.ts retryNode for semantics.
+  if (action === 'retry-node') {
+    const { nodeId } = body;
+    if (!nodeId) return NextResponse.json({ error: 'nodeId required' }, { status: 400 });
+    const result = await retryNode(id, String(nodeId));
+    if (!result.ok) return NextResponse.json({ error: result.error }, { status: 400 });
+    return NextResponse.json({ ok: true });
+  }
+
   // Inject a message into a running conversation
   if (action === 'inject') {
     const { agentId, message } = body;

package/app/api/pipelines/route.ts

@@ -1,5 +1,5 @@
 import { NextResponse } from 'next/server';
-import { listPipelines, listWorkflows, startPipeline } from '@/lib/pipeline';
+import { listPipelines, listPipelinesSummary, listWorkflows, startPipeline } from '@/lib/pipeline';
 import { writeFileSync, mkdirSync } from 'node:fs';
 import { join } from 'node:path';
 import YAML from 'yaml';
@@ -40,7 +40,21 @@ export async function GET(req: Request) {
     }
   }
 
-  return NextResponse.json(listPipelines().sort((a, b) => b.createdAt.localeCompare(a.createdAt)));
+  // Light summary by default — strips heavy node outputs / conversation
+  // bodies that the list UI doesn't render. Default limit 100 newest.
+  // Callers can opt into:
+  //   ?full=1         — full Pipeline objects (rare; per-id endpoint is cheaper)
+  //   ?limit=500      — more rows (server caps at 500)
+  //   ?workflow=<name>— scope to one workflow (combine with high limit)
+  //   ?before=<iso>   — cursor for "Load more"
+  const wantFull = searchParams.get('full') === '1';
+  if (wantFull) {
+    return NextResponse.json(listPipelines().sort((a, b) => b.createdAt.localeCompare(a.createdAt)));
+  }
+  const limit = Number(searchParams.get('limit')) || undefined;
+  const workflow = searchParams.get('workflow') || undefined;
+  const before = searchParams.get('before') || undefined;
+  return NextResponse.json(listPipelinesSummary({ limit, workflow, before }));
 }
 
 // POST /api/pipelines — start a pipeline or save a workflow

package/app/api/plugins/route.ts

@@ -1,7 +1,36 @@
 import { NextResponse } from 'next/server';
-import { listPlugins, getPlugin, installPlugin, uninstallPlugin, updatePluginConfig, listInstalledPlugins, getInstalledPlugin } from '@/lib/plugins/registry';
+import { listPlugins, getPlugin, installPlugin, uninstallPlugin, updatePluginConfig, listInstalledPlugins, getInstalledPlugin, getSecretFieldNames } from '@/lib/plugins/registry';
 import { executePluginAction } from '@/lib/plugins/executor';
 
+const SECRET_MASK = '••••••••';
+
+function maskInstalledConfig<T extends { definition?: any; config?: Record<string, any> } | null | undefined>(inst: T): T {
+  if (!inst || !inst.config) return inst;
+  const secrets = getSecretFieldNames(inst.definition || null);
+  if (!secrets.length) return inst;
+  const masked = { ...inst.config };
+  for (const k of secrets) {
+    if (typeof masked[k] === 'string' && masked[k]) masked[k] = SECRET_MASK;
+  }
+  return { ...inst, config: masked } as T;
+}
+
+function restoreInstalledConfig(id: string, incoming: Record<string, any>): Record<string, any> {
+  const existing = getInstalledPlugin(id);
+  if (!existing) return incoming;
+  const secrets = getSecretFieldNames(existing.definition);
+  if (!secrets.length) return incoming;
+  const out = { ...incoming };
+  for (const k of secrets) {
+    if (out[k] === SECRET_MASK) {
+      const prev = existing.config?.[k];
+      if (typeof prev === 'string') out[k] = prev;
+      else delete out[k];
+    }
+  }
+  return out;
+}
+
 // GET: list plugins or get plugin details
 export async function GET(req: Request) {
   const url = new URL(req.url);
@@ -13,17 +42,18 @@ export async function GET(req: Request) {
     const plugin = getPlugin(id);
     const inst = getInstalledPlugin(id);
     if (!plugin && !inst) return NextResponse.json({ error: 'Plugin not found' }, { status: 404 });
+    const masked = maskInstalledConfig(inst);
     return NextResponse.json({
-      plugin: inst?.definition || plugin,
+      plugin: masked?.definition || plugin,
       installed: !!inst,
-      config: inst?.config,
-      instanceName: inst?.instanceName,
-      source: inst?.source,
+      config: masked?.config,
+      instanceName: masked?.instanceName,
+      source: masked?.source,
     });
   }
 
   if (installed === 'true') {
-    return NextResponse.json({ plugins: listInstalledPlugins() });
+    return NextResponse.json({ plugins: listInstalledPlugins().map(p => maskInstalledConfig(p)) });
   }
 
   return NextResponse.json({ plugins: listPlugins() });
@@ -37,7 +67,8 @@ export async function POST(req: Request) {
   switch (action) {
     case 'install': {
       if (!id) return NextResponse.json({ error: 'id required' }, { status: 400 });
-      const ok = installPlugin(id, config || {}, body.source ? { source: body.source, name: body.name } : undefined);
+      const merged = restoreInstalledConfig(id, config || {});
+      const ok = installPlugin(id, merged, body.source ? { source: body.source, name: body.name } : undefined);
       return NextResponse.json({ ok });
     }
     case 'create_instance': {
@@ -59,7 +90,8 @@ export async function POST(req: Request) {
     }
     case 'update_config': {
       if (!id || !config) return NextResponse.json({ error: 'id and config required' }, { status: 400 });
-      const ok = updatePluginConfig(id, config);
+      const merged = restoreInstalledConfig(id, config);
+      const ok = updatePluginConfig(id, merged);
       return NextResponse.json({ ok });
     }
     case 'test': {

package/app/api/project-sessions/route.ts

@@ -1,6 +1,7 @@
 import { NextResponse } from 'next/server';
 import { getFixedSession, setFixedSession, clearFixedSession, getAllFixedSessions } from '@/lib/project-sessions';
-import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { loadSettings } from '@/lib/settings';
+import { existsSync, mkdirSync, writeFileSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
 
 // GET: get fixed session for a project, or all bindings
@@ -35,27 +36,66 @@ export async function POST(req: Request) {
   return NextResponse.json({ ok: true, projectPath, fixedSessionId });
 }
 
-/** Generate .forge/mcp.json in the project directory with workspace context baked in */
+/**
+ * Generate the project's MCP config. Two locations:
+ *   1. <project>/.forge/mcp.json  — Forge-internal copy, referenced by workspace
+ *      orchestrator's --mcp-config flag when it launches agent terminals.
+ *   2. <project>/.mcp.json        — picked up automatically by `claude` (and
+ *      most other Claude Code-compatible CLIs) when launched at the project
+ *      root, so users get Forge MCP tools without needing --mcp-config.
+ * Both files have identical content; (1) is authoritative — (2) is a mirror
+ * that's only written if it doesn't already exist OR is also a Forge-managed
+ * copy (to avoid stomping a user's hand-rolled .mcp.json).
+ */
 function ensureMcpConfig(projectPath: string): void {
   try {
     const forgeDir = join(projectPath, '.forge');
-    const configPath = join(forgeDir, 'mcp.json');
+    const forgeConfigPath = join(forgeDir, 'mcp.json');
+    const rootConfigPath = join(projectPath, '.mcp.json');
     const mcpPort = Number(process.env.MCP_PORT) || 8406;
 
-    // Resolve workspace + primary agent for this project
     let wsParam = '';
     try {
       const { findWorkspaceByProject } = require('@/lib/workspace');
       const ws = findWorkspaceByProject(projectPath);
-      if (ws) {
-        wsParam = `?workspaceId=${ws.id}`;
-      }
+      if (ws) wsParam = `?workspaceId=${ws.id}`;
     } catch {}
 
-    const config = { mcpServers: { forge: { type: 'sse', url: `http://localhost:${mcpPort}/sse${wsParam}` } } };
+    // Built-in Forge MCP server (agent bus, workspace ops)
+    const forgeEntry = { type: 'sse', url: `http://localhost:${mcpPort}/sse${wsParam}` };
+
+    // User-managed external MCP servers — chrome-devtools-mcp etc.
+    let userServers: Record<string, any> = {};
+    try {
+      const settings = loadSettings();
+      userServers = settings.mcpServers || {};
+    } catch {}
+
+    // Forge owns the `forge` key; users can shadow other keys here.
+    const config = {
+      mcpServers: { forge: forgeEntry, ...userServers },
+    };
+    const serialized = JSON.stringify(config, null, 2);
+    const forgeManagedKeys = new Set(Object.keys(config.mcpServers));
 
-    // Always rewrite (workspace context may have changed)
     mkdirSync(forgeDir, { recursive: true });
-    writeFileSync(configPath, JSON.stringify(config, null, 2));
+    writeFileSync(forgeConfigPath, serialized);
+
+    // Mirror to root .mcp.json so plain `claude` in the project picks up
+    // Forge's MCP servers. Two policies:
+    //   - Don't CREATE the root file. Users who delete it want it gone;
+    //     auto-recreating broke that expectation (and pulled global
+    //     mcp servers into pipeline worktrees where they don't belong).
+    //   - DO update an existing root file IF every server in it is one
+    //     we manage — keeps the convenience for users who chose to have it,
+    //     without stomping a hand-rolled one.
+    if (existsSync(rootConfigPath)) {
+      try {
+        const existing = JSON.parse(readFileSync(rootConfigPath, 'utf8'));
+        const existingKeys = Object.keys(existing?.mcpServers ?? {});
+        const allForgeManaged = existingKeys.length > 0 && existingKeys.every(k => forgeManagedKeys.has(k));
+        if (allForgeManaged) writeFileSync(rootConfigPath, serialized);
+      } catch { /* malformed user file — leave alone */ }
+    }
   } catch {}
 }

package/app/api/settings/route.ts

@@ -53,6 +53,19 @@ export async function PUT(req: Request) {
     }
   }
 
+  // Nested agent profile apiKeys are masked by loadSettingsMasked too —
+  // restore them here so a normal settings save doesn't overwrite a real
+  // key with '••••••••'. Empty string means user explicitly cleared it.
+  if (updated.agents) {
+    for (const [id, a] of Object.entries(updated.agents)) {
+      const newKey = (a as any)?.apiKey;
+      if (newKey === '••••••••') {
+        const existing = settings.agents?.[id]?.apiKey;
+        if (existing) (a as any).apiKey = existing;
+      }
+    }
+  }
+
   // Remove internal fields
   delete (updated as any)._secretStatus;