@aion0/forge 0.8.5 → 0.8.7

package/lib/jobs/recipes.ts

@@ -0,0 +1,260 @@
+/**
+ * Recipes — yaml templates for Jobs.
+ *
+ * A recipe bundles "what tool to poll" + "how to dispatch" + "what shape
+ * of input_template to fill" into a single file. The user picks one,
+ * fills 2-4 form fields, and Forge instantiates a complete Job row.
+ *
+ * Storage: <dataDir>/recipes/<name>.yaml (one file per recipe).
+ * Sync (Stage 2B, separate): forge-workflow git repo pulls into the
+ * same dir as `source: 'registry'` rows.
+ *
+ * Schema (v1):
+ *
+ *   name: gitlab-mr-watch
+ *   display_name: "GitLab MR auto-fix"
+ *   description: "..."
+ *   version: "0.1.0"
+ *   author: "forge"
+ *   tags: [gitlab, mr]
+ *
+ *   params:                 # form fields shown to the user
+ *     - { name, label, type, required, default, options, help }
+ *
+ *   job:                    # Job row template; values can use templates
+ *     name_template, enabled, schedule_kind, schedule_interval_minutes,
+ *     source_connector, source_tool, source_input, items_path,
+ *     dedup_field, dispatch_type, dispatch_params { workflow_name,
+ *     project_path, project_name, input_template }
+ *
+ * Templates: {{params.X}} / {{env.X}} / {{recipe.X}}. {{item.X}} is
+ * deliberately left LITERAL — at instantiation time we don't know the
+ * connector items yet; the scheduler later renders {{item.X}} per row.
+ */
+
+import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import YAML from 'yaml';
+import { homedir } from 'node:os';
+import { getDataDir } from '../dirs';
+import { createJob } from './store';
+import type { Job, PipelineDispatchParams, ChatDispatchParams } from './types';
+
+export type ParamType = 'string' | 'number' | 'boolean' | 'select' | 'project_picker' | 'gitlab_mr_url';
+
+export interface RecipeParam {
+  name: string;
+  label: string;
+  type?: ParamType;
+  required?: boolean;
+  default?: string | number | boolean;
+  options?: string[];  // for type=select
+  help?: string;
+}
+
+export interface Recipe {
+  name: string;
+  display_name: string;
+  description?: string;
+  version: string;
+  author?: string;
+  tags?: string[];
+  params: RecipeParam[];
+  job: any;  // free-form — instantiateRecipe walks it
+  /** Filesystem path the recipe was loaded from (set by listRecipes). */
+  _path?: string;
+  /** 'registry' (synced from forge-workflow) or 'local' (uploaded). */
+  source?: 'registry' | 'local';
+}
+
+const RECIPES_DIR = () => join(getDataDir(), 'recipes');
+
+// ─── Load ──────────────────────────────────────────────────
+
+function recipesDir(): string {
+  const d = RECIPES_DIR();
+  if (!existsSync(d)) mkdirSync(d, { recursive: true, mode: 0o700 });
+  return d;
+}
+
+export function listRecipes(): Recipe[] {
+  const dir = recipesDir();
+  const out: Recipe[] = [];
+  for (const name of readdirSync(dir)) {
+    if (!name.endsWith('.yaml') && !name.endsWith('.yml')) continue;
+    const path = join(dir, name);
+    try {
+      const r = YAML.parse(readFileSync(path, 'utf-8')) as Recipe;
+      if (!r || !r.name) continue;
+      r._path = path;
+      // Default source=local for hand-uploaded files; sync-layer (2B)
+      // will rewrite this to 'registry' as part of its install step.
+      r.source = (r as any).source === 'registry' ? 'registry' : 'local';
+      out.push(r);
+    } catch (e) {
+      console.warn(`[recipes] failed to parse ${path}:`, (e as Error).message);
+    }
+  }
+  return out.sort((a, b) => a.display_name.localeCompare(b.display_name));
+}
+
+export function getRecipe(name: string): Recipe | null {
+  return listRecipes().find((r) => r.name === name) || null;
+}
+
+// ─── Save / Delete (local upload path) ─────────────────────
+
+const NAME_RE = /^[a-z0-9][a-z0-9_-]*$/;
+
+export function saveRecipeYaml(yamlText: string): { ok: true; name: string } | { ok: false; error: string } {
+  let parsed: any;
+  try { parsed = YAML.parse(yamlText); }
+  catch (e) { return { ok: false, error: `invalid YAML: ${(e as Error).message}` }; }
+  const name = String(parsed?.name || '').trim();
+  if (!name) return { ok: false, error: 'recipe.name is required' };
+  if (!NAME_RE.test(name)) return { ok: false, error: `recipe.name "${name}" must be lowercase alphanumerics + hyphens/underscores` };
+  if (!Array.isArray(parsed.params)) return { ok: false, error: 'recipe.params (array) is required' };
+  if (!parsed.job || typeof parsed.job !== 'object') return { ok: false, error: 'recipe.job (object) is required' };
+  const path = join(recipesDir(), `${name}.yaml`);
+  writeFileSync(path, yamlText, { mode: 0o600 });
+  return { ok: true, name };
+}
+
+export function deleteRecipe(name: string): boolean {
+  if (!NAME_RE.test(name)) return false;
+  const path = join(recipesDir(), `${name}.yaml`);
+  const altPath = join(recipesDir(), `${name}.yml`);
+  const { unlinkSync } = require('node:fs');
+  if (existsSync(path)) { unlinkSync(path); return true; }
+  if (existsSync(altPath)) { unlinkSync(altPath); return true; }
+  return false;
+}
+
+// ─── Template rendering ────────────────────────────────────
+
+interface RenderContext {
+  params: Record<string, any>;
+  env: Record<string, string>;
+  recipe: { name: string; display_name: string; version: string };
+}
+
+const KNOWN_NAMESPACES = new Set(['params', 'env', 'recipe']);
+
+/**
+ * Expand `{{X.path}}` tokens in a string. Only `params.`, `env.`, and
+ * `recipe.` are resolved; anything else (notably `{{item.X}}`) is left
+ * verbatim so the scheduler can render per-item at runtime.
+ */
+function renderString(input: string, ctx: RenderContext): string {
+  if (!input) return input;
+  return input.replace(/\{\{\s*([a-zA-Z0-9_]+)\.([a-zA-Z0-9_.]+)\s*\}\}/g, (full, ns, path) => {
+    if (!KNOWN_NAMESPACES.has(ns)) return full;
+    const root: any = (ctx as any)[ns];
+    let cur: any = root;
+    for (const seg of path.split('.')) {
+      if (cur == null || typeof cur !== 'object') { cur = undefined; break; }
+      cur = cur[seg];
+    }
+    if (cur == null) return '';
+    if (typeof cur === 'string') return cur;
+    if (typeof cur === 'number' || typeof cur === 'boolean') return String(cur);
+    try { return JSON.stringify(cur); } catch { return full; }
+  });
+}
+
+function renderDeep(value: any, ctx: RenderContext): any {
+  if (value == null) return value;
+  if (typeof value === 'string') return renderString(value, ctx);
+  if (Array.isArray(value)) return value.map((v) => renderDeep(v, ctx));
+  if (typeof value === 'object') {
+    const out: Record<string, any> = {};
+    for (const [k, v] of Object.entries(value)) out[k] = renderDeep(v, ctx);
+    return out;
+  }
+  return value;
+}
+
+// ─── Instantiate ───────────────────────────────────────────
+
+export interface InstantiateResult {
+  ok: boolean;
+  job?: Job;
+  error?: string;
+}
+
+/**
+ * Render a recipe + user params into a complete Job row and persist it.
+ * Returns the created Job (with assigned id). Validates required params
+ * and coerces numeric/boolean fields based on the param type.
+ */
+export function instantiateRecipe(name: string, rawParams: Record<string, any>): InstantiateResult {
+  const recipe = getRecipe(name);
+  if (!recipe) return { ok: false, error: `recipe "${name}" not found` };
+
+  // Coerce + validate params per the recipe's param schema
+  const params: Record<string, any> = {};
+  for (const p of recipe.params) {
+    const incoming = rawParams[p.name];
+    let val: any = incoming != null && incoming !== '' ? incoming : (p.default ?? '');
+    if (p.required && (val === '' || val == null)) {
+      return { ok: false, error: `param "${p.name}" is required` };
+    }
+    if (p.type === 'number' && val !== '') {
+      const n = Number(val);
+      if (!Number.isFinite(n)) return { ok: false, error: `param "${p.name}" must be a number` };
+      val = n;
+    } else if (p.type === 'boolean') {
+      val = !!val && val !== 'false';
+    } else if (p.type === 'gitlab_mr_url' && val !== '') {
+      // Parse a full GitLab MR URL into two derived params:
+      //   <name>__path  →  "namespace/repo" (URL-encoded path-style id)
+      //   <name>__iid   →  numeric MR iid
+      // Lets a recipe ask the user for just ONE field (the MR URL) and
+      // still produce both `project_id` and `mr_iid` for the underlying
+      // tool calls.
+      const m = String(val).match(/^https?:\/\/[^/]+\/(.+?)\/-\/merge_requests\/(\d+)/);
+      if (!m) {
+        return { ok: false, error: `param "${p.name}" must be a GitLab MR URL like https://host/namespace/repo/-/merge_requests/N — got: ${String(val).slice(0, 80)}` };
+      }
+      params[`${p.name}__path`] = m[1];
+      params[`${p.name}__iid`] = parseInt(m[2], 10);
+    }
+    params[p.name] = val;
+  }
+
+  const ctx: RenderContext = {
+    params,
+    env: { HOME: process.env.HOME || homedir(), USER: process.env.USER || '' },
+    recipe: { name: recipe.name, display_name: recipe.display_name, version: recipe.version },
+  };
+
+  const rendered = renderDeep(recipe.job, ctx) as any;
+
+  // Pull out the (rendered) job name; fall back to "<recipe>-<timestamp>".
+  const jobName = String(rendered.name_template || rendered.name || `${recipe.name}-${Date.now()}`).trim();
+  // Remove keys the createJob() signature doesn't accept directly.
+  delete rendered.name_template;
+  delete rendered.name;
+
+  try {
+    const job = createJob({
+      name: jobName,
+      enabled: rendered.enabled !== false,
+      schedule_kind: rendered.schedule_kind || 'period',
+      schedule_interval_minutes: Number(rendered.schedule_interval_minutes) || 30,
+      schedule_at: rendered.schedule_at ?? null,
+      schedule_cron: rendered.schedule_cron ?? null,
+      source_connector: String(rendered.source_connector || ''),
+      source_tool: String(rendered.source_tool || ''),
+      source_input: rendered.source_input || {},
+      items_path: String(rendered.items_path || ''),
+      dedup_field: String(rendered.dedup_field || 'id'),
+      dispatch_type: rendered.dispatch_type || 'pipeline',
+      dispatch_params: (rendered.dispatch_params || {}) as PipelineDispatchParams | ChatDispatchParams,
+      skills: Array.isArray(rendered.skills) ? rendered.skills : [],
+    });
+    return { ok: true, job };
+  } catch (e) {
+    return { ok: false, error: `failed to create job: ${(e as Error).message}` };
+  }
+}

package/lib/jobs/scheduler.ts

@@ -168,7 +168,13 @@ export async function executeRun(job: Job, runId: string): Promise<void> {
     logLine('info', `source input: ${JSON.stringify(sourceInput)}`);
     logLine('info', `calling connector ${callName}…`);
 
-    const toolResult = await dispatchTool({ id: `job-${runId}`, name: callName, input: sourceInput });
+    const toolResult = await dispatchTool(
+      { id: `job-${runId}`, name: callName, input: sourceInput },
+      // We JSON.parse the response — the 8KB LLM-friendly truncation
+      // would break parsing on any moderately large list (Todos, big MR
+      // searches, etc.). Ask for the raw body.
+      { noTruncation: true },
+    );
 
     const respBytes = toolResult.content?.length ?? 0;
     logLine(toolResult.is_error ? 'error' : 'info',
@@ -366,7 +372,16 @@ export async function executeRun(job: Job, runId: string): Promise<void> {
 
 function safeParseJson(content: string): unknown | undefined {
   try { return JSON.parse(content); }
-  catch { return undefined; }
+  catch {}
+  // Strip the http-protocol preamble — `HTTP 200 OK · GET <url>\n[(...)]\n\n`
+  // (and the truncation note line that sometimes follows) — and retry.
+  // The LLM-facing payload bakes in a status line we don't want here.
+  const firstJson = content.search(/[\[{]/);
+  if (firstJson > 0) {
+    try { return JSON.parse(content.slice(firstJson)); }
+    catch {}
+  }
+  return undefined;
 }
 
 function pickPath(obj: unknown, path: string): unknown {