@aion0/forge 0.10.22 → 0.10.25

package/lib/watch/watch-runner.ts

@@ -0,0 +1,176 @@
+/**
+ * Watch runner — the background ticker that drives long-task watches.
+ *
+ * Runs inside chat-standalone (single instance, guarded). Each tick it
+ * polls every due active watch (single-flight, bounded concurrency),
+ * evaluates done/fail/timeout, and on a terminal state runs the watch's
+ * action: feed the result back into the originating chat session
+ * (mode=chat, via the runChat callback), chain a tool (mode=tool), or
+ * just record it (mode=none). Per-poll it emits ambient progress through
+ * onProgress — that lands as a status chip, NOT a chat message.
+ *
+ * Guards (see design doc §3): max_polls, timeout, max_lifetime, single-
+ * flight, consecutive-error cutoff, chain-depth on tool callbacks.
+ */
+
+import { dispatchTool } from '../chat/tool-dispatcher';
+import { resolveTemplate, resolveDeep, getPath } from './template';
+import {
+  listDue, updateWatch, getWatch, type Watch, type WatchState,
+} from './watch-store';
+
+const TICK_MS = 20_000;            // scan cadence; each watch paces itself via next_poll_at
+const MAX_CONCURRENT = 8;          // simultaneous in-flight polls
+const MAX_CONSEC_ERRORS = 10;      // device unreachable (e.g. mid-reboot) tolerated this many polls
+
+export interface WatchRunnerHooks {
+  /** Emit ambient progress (status chip) for a watch's session. */
+  onProgress?: (sessionId: string, payload: Record<string, unknown>) => void;
+  /** Feed a completion message back into a chat session (assistant replies). */
+  runChat?: (sessionId: string, text: string) => void;
+}
+
+function truthy(v: unknown): boolean {
+  if (v == null || v === false || v === 0) return false;
+  if (typeof v === 'string') { const s = v.trim().toLowerCase(); return s !== '' && s !== 'false' && s !== '0' && s !== 'null'; }
+  return true;
+}
+
+function parseResult(content: string): any {
+  try { return JSON.parse(content); } catch { return { _raw: content }; }
+}
+
+const g = globalThis as any;
+
+export function startWatchRunner(hooks: WatchRunnerHooks = {}): void {
+  if (g.__forgeWatchRunner) return;       // single instance per process
+  const running = new Set<string>();      // watch ids currently polling (single-flight)
+
+  const finish = (w: Watch, state: WatchState, obj: unknown, summary: string) => {
+    const now = Date.now();
+    updateWatch(w.id, { state, last_result: obj, last_text: summary }, now);
+    // Terminal watch_status — tells the UI to drop the progress chip
+    // immediately (otherwise it lingers until the 150s prune). The real
+    // completion text goes via on_done below; this is just the chip kill.
+    if (hooks.onProgress && w.session_id) {
+      hooks.onProgress(w.session_id, { watch_id: w.id, state, done: true, text: summary });
+    }
+    const action = state === 'done' ? w.on_done : w.on_fail;
+    const mode = action?.mode || 'chat';
+    if (mode === 'none' || !action) return;
+    const ctx = { poll: obj };
+    if (mode === 'chat') {
+      if (!w.session_id || !hooks.runChat) return;
+      const msg = action.message ? resolveTemplate(action.message, ctx) : summary;
+      const tag = state === 'done' ? '✅' : '⚠️';
+      hooks.runChat(w.session_id, `[background watch ${w.label}] ${tag} ${msg}`);
+    } else if (mode === 'tool' && action.tool) {
+      const input = (resolveDeep(action.args || {}, ctx) as Record<string, unknown>) || {};
+      void dispatchTool(
+        { id: `watch-chain-${w.id}`, name: action.tool, input },
+        { sessionId: w.session_id || undefined, chainDepth: Math.max(0, w.chain_depth - 1) } as any,
+      ).catch((e) => console.error('[watch] on_done tool chain failed', w.id, e));
+    }
+  };
+
+  const emitProgress = (w: Watch, obj: unknown, pollCount: number) => {
+    if (!w.session_id || !hooks.onProgress) return;
+    if (w.progress && w.progress.show === false) return;
+    const tmpl = w.progress?.message || 'Watching {label} … poll {poll_count}/{max_polls}';
+    const text = resolveTemplate(tmpl, { poll: obj, poll_count: pollCount, max_polls: w.max_polls, label: w.label });
+    hooks.onProgress(w.session_id, { watch_id: w.id, state: 'active', poll_count: pollCount, text });
+  };
+
+  const pollOne = async (w: Watch): Promise<void> => {
+    const now = Date.now();
+    // Hard lifetime backstop.
+    if (now - w.created_at > 2 * w.timeout_sec * 1000) {
+      return finish(w, 'timed_out', w.last_result, `${w.label}: watch lifetime exceeded — please verify manually.`);
+    }
+    let res;
+    try {
+      // noTruncation=true so http-protocol polls skip the "HTTP 200 OK · GET …"
+      // preamble — the body comes back as raw JSON so done_path/done_match can
+      // see it. (Without this, every http-protocol watch would silently never
+      // hit its done condition, e.g. jenkins.get_build never resolving.)
+      res = await dispatchTool({ id: `watch-${w.id}-${w.polls}`, name: `${w.connector_id}.${w.poll_tool}`, input: w.poll_args }, { noTruncation: true } as any);
+    } catch (e) {
+      res = { content: String(e), is_error: true };
+    }
+
+    if (res.is_error) {
+      // Poll failed (often expected: device rebooting). Tolerate up to N
+      // consecutive, then give up.
+      const errs = w.err_count + 1;
+      if (errs >= MAX_CONSEC_ERRORS) {
+        return finish(w, 'errored', { error: res.content }, `${w.label}: ${MAX_CONSEC_ERRORS} consecutive poll errors — giving up. Last: ${String(res.content).slice(0, 200)}`);
+      }
+      updateWatch(w.id, { err_count: errs, next_poll_at: now + w.interval_sec * 1000 }, now);
+      emitProgress(w, { _error: String(res.content).slice(0, 120) }, w.polls);
+      return;
+    }
+
+    const obj = parseResult(res.content);
+    const polls = w.polls + 1;
+
+    // fail check
+    if (w.fail_path && truthy(getPath(obj, w.fail_path))) {
+      return finish(w, 'failed', obj, `${w.label}: failure condition met.`);
+    }
+    // done check
+    let done = false;
+    if (w.done_match) {
+      const v = getPath(obj, w.done_match.path);
+      if (w.done_match.equals != null) done = String(v) === String(w.done_match.equals);
+      else if (w.done_match.contains != null) done = String(v ?? '').toLowerCase().includes(String(w.done_match.contains).toLowerCase());
+    } else if (w.done_path) {
+      done = truthy(getPath(obj, w.done_path));
+    }
+    if (done) {
+      return finish(w, 'done', obj, `${w.label}: done.`);
+    }
+    // not done — bound by polls / timeout, else reschedule
+    if (polls >= w.max_polls || now - w.created_at > w.timeout_sec * 1000) {
+      return finish(w, 'timed_out', obj, `${w.label}: not done within ${w.max_polls} polls / ${w.timeout_sec}s — please verify manually.`);
+    }
+    // Persist the latest poll result + a tiny preview text on EVERY poll
+    // (not just terminal) so the Monitor / DB shows what the watch is
+    // actually seeing — crucial for diagnosing "polling forever, no done":
+    // usually means the done condition refers to a field the poll's
+    // result doesn't actually have.
+    const previewKeys = obj && typeof obj === 'object' && !Array.isArray(obj)
+      ? Object.keys(obj as Record<string, unknown>).slice(0, 8).join(', ')
+      : typeof obj;
+    updateWatch(w.id, {
+      polls,
+      err_count: 0,
+      next_poll_at: now + w.interval_sec * 1000,
+      last_result: obj,
+      last_text: `poll ${polls}/${w.max_polls} · keys: ${previewKeys}`,
+    }, now);
+    emitProgress(w, obj, polls);
+  };
+
+  const tick = async () => {
+    try {
+      const slots = MAX_CONCURRENT - running.size;
+      if (slots <= 0) return;
+      const due = listDue(Date.now()).filter((w) => !running.has(w.id)).slice(0, slots);
+      for (const w of due) {
+        running.add(w.id);
+        // Re-read inside to avoid acting on a row cancelled since listDue.
+        void Promise.resolve()
+          .then(() => { const cur = getWatch(w.id); return cur && cur.state === 'active' ? pollOne(cur) : undefined; })
+          .catch((e) => console.error('[watch] poll error', w.id, e))
+          .finally(() => running.delete(w.id));
+      }
+    } catch (e) {
+      console.error('[watch] tick error', e);
+    }
+  };
+
+  const timer = setInterval(() => { void tick(); }, TICK_MS);
+  if (typeof timer.unref === 'function') timer.unref();
+  g.__forgeWatchRunner = timer;
+  console.log('[watch] runner started (tick ' + TICK_MS / 1000 + 's)');
+}

package/lib/watch/watch-store.ts

@@ -0,0 +1,218 @@
+/**
+ * Watch store — sqlite-backed persistence for background "watches".
+ *
+ * A watch is a long-task poller: a tool flagged `async` registers one,
+ * the watch-runner polls it until a terminal state, then feeds the
+ * result back into the originating chat session. Persisted (not in
+ * memory) so a Forge restart resumes in-flight watches. Lives in
+ * <data>/workflow.db alongside sessions.
+ *
+ * Concurrency: the runner (chat-standalone) is the primary writer;
+ * the /api/watches routes (next-server) also write (cancel/delete).
+ * better-sqlite3 + WAL serialises writers via the file lock — fine at
+ * watch frequencies.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+import { getDb } from '../../src/core/db/database';
+import { getDataDir } from '../dirs';
+
+export type WatchState =
+  | 'active' | 'done' | 'failed' | 'timed_out' | 'cancelled' | 'errored';
+
+export interface WatchAction {
+  mode?: 'chat' | 'tool' | 'none';
+  message?: string;
+  tool?: string;
+  args?: Record<string, unknown>;
+}
+
+export interface Watch {
+  id: string;
+  session_id: string | null;
+  label: string;
+  connector_id: string;
+  poll_tool: string;            // bare tool name (poll), dispatched as `${connector_id}.${poll_tool}`
+  poll_args: Record<string, unknown>;
+  done_path: string | null;
+  done_match: { path: string; equals?: string; contains?: string } | null;
+  fail_path: string | null;
+  on_done: WatchAction | null;
+  on_fail: WatchAction | null;
+  progress: { show?: boolean; message?: string } | null;
+  interval_sec: number;
+  timeout_sec: number;
+  max_polls: number;
+  chain_depth: number;
+  state: WatchState;
+  polls: number;
+  err_count: number;
+  created_at: number;
+  next_poll_at: number;
+  updated_at: number;
+  last_text: string | null;
+  last_result: unknown;
+}
+
+interface Row {
+  id: string; session_id: string | null; label: string; connector_id: string;
+  poll_tool: string; poll_args_json: string;
+  done_path: string | null; done_match_json: string | null; fail_path: string | null;
+  on_done_json: string | null; on_fail_json: string | null; progress_json: string | null;
+  interval_sec: number; timeout_sec: number; max_polls: number; chain_depth: number;
+  state: string; polls: number; err_count: number;
+  created_at: number; next_poll_at: number; updated_at: number;
+  last_text: string | null; last_result_json: string | null;
+}
+
+let ready = false;
+function db() {
+  const d = getDb(join(getDataDir(), 'workflow.db'));
+  if (!ready) {
+    d.exec(`
+      CREATE TABLE IF NOT EXISTS connector_watches (
+        id TEXT PRIMARY KEY,
+        session_id TEXT,
+        label TEXT NOT NULL DEFAULT '',
+        connector_id TEXT NOT NULL,
+        poll_tool TEXT NOT NULL,
+        poll_args_json TEXT NOT NULL DEFAULT '{}',
+        done_path TEXT,
+        done_match_json TEXT,
+        fail_path TEXT,
+        on_done_json TEXT,
+        on_fail_json TEXT,
+        progress_json TEXT,
+        interval_sec INTEGER NOT NULL DEFAULT 60,
+        timeout_sec INTEGER NOT NULL DEFAULT 1200,
+        max_polls INTEGER NOT NULL DEFAULT 40,
+        chain_depth INTEGER NOT NULL DEFAULT 0,
+        state TEXT NOT NULL DEFAULT 'active',
+        polls INTEGER NOT NULL DEFAULT 0,
+        err_count INTEGER NOT NULL DEFAULT 0,
+        created_at INTEGER NOT NULL,
+        next_poll_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        last_text TEXT,
+        last_result_json TEXT
+      );
+      CREATE INDEX IF NOT EXISTS idx_watches_state ON connector_watches(state, next_poll_at);
+    `);
+    ready = true;
+  }
+  return d;
+}
+
+function rowToWatch(r: Row): Watch {
+  const j = <T,>(s: string | null, d: T): T => { if (s == null) return d; try { return JSON.parse(s) as T; } catch { return d; } };
+  return {
+    id: r.id, session_id: r.session_id, label: r.label, connector_id: r.connector_id,
+    poll_tool: r.poll_tool, poll_args: j(r.poll_args_json, {}),
+    done_path: r.done_path, done_match: j(r.done_match_json, null), fail_path: r.fail_path,
+    on_done: j(r.on_done_json, null), on_fail: j(r.on_fail_json, null), progress: j(r.progress_json, null),
+    interval_sec: r.interval_sec, timeout_sec: r.timeout_sec, max_polls: r.max_polls, chain_depth: r.chain_depth,
+    state: r.state as WatchState, polls: r.polls, err_count: r.err_count,
+    created_at: r.created_at, next_poll_at: r.next_poll_at, updated_at: r.updated_at,
+    last_text: r.last_text, last_result: j(r.last_result_json, null),
+  };
+}
+
+export interface NewWatch {
+  session_id: string | null;
+  label: string;
+  connector_id: string;
+  poll_tool: string;
+  poll_args: Record<string, unknown>;
+  done_path?: string | null;
+  done_match?: Watch['done_match'];
+  fail_path?: string | null;
+  on_done?: WatchAction | null;
+  on_fail?: WatchAction | null;
+  progress?: Watch['progress'];
+  interval_sec: number;
+  timeout_sec: number;
+  max_polls: number;
+  chain_depth: number;
+  now: number;
+}
+
+export function createWatch(w: NewWatch): Watch {
+  const id = 'w_' + randomUUID().slice(0, 12);
+  const next = w.now + w.interval_sec * 1000;
+  db().prepare(`
+    INSERT INTO connector_watches
+      (id, session_id, label, connector_id, poll_tool, poll_args_json,
+       done_path, done_match_json, fail_path, on_done_json, on_fail_json, progress_json,
+       interval_sec, timeout_sec, max_polls, chain_depth,
+       state, polls, err_count, created_at, next_poll_at, updated_at, last_text, last_result_json)
+    VALUES (@id,@session_id,@label,@connector_id,@poll_tool,@poll_args_json,
+            @done_path,@done_match_json,@fail_path,@on_done_json,@on_fail_json,@progress_json,
+            @interval_sec,@timeout_sec,@max_polls,@chain_depth,
+            'active',0,0,@created_at,@next_poll_at,@updated_at,NULL,NULL)
+  `).run({
+    id, session_id: w.session_id, label: w.label, connector_id: w.connector_id,
+    poll_tool: w.poll_tool, poll_args_json: JSON.stringify(w.poll_args || {}),
+    done_path: w.done_path ?? null, done_match_json: w.done_match ? JSON.stringify(w.done_match) : null,
+    fail_path: w.fail_path ?? null,
+    on_done_json: w.on_done ? JSON.stringify(w.on_done) : null,
+    on_fail_json: w.on_fail ? JSON.stringify(w.on_fail) : null,
+    progress_json: w.progress ? JSON.stringify(w.progress) : null,
+    interval_sec: w.interval_sec, timeout_sec: w.timeout_sec, max_polls: w.max_polls, chain_depth: w.chain_depth,
+    created_at: w.now, next_poll_at: next, updated_at: w.now,
+  });
+  return getWatch(id)!;
+}
+
+export function getWatch(id: string): Watch | null {
+  const r = db().prepare(`SELECT * FROM connector_watches WHERE id = ?`).get(id) as Row | undefined;
+  return r ? rowToWatch(r) : null;
+}
+
+export function countActive(): number {
+  const r = db().prepare(`SELECT COUNT(*) AS n FROM connector_watches WHERE state = 'active'`).get() as { n: number };
+  return r.n;
+}
+
+/** Active watches whose next poll is due (<= now). */
+export function listDue(now: number): Watch[] {
+  const rows = db().prepare(
+    `SELECT * FROM connector_watches WHERE state = 'active' AND next_poll_at <= ? ORDER BY next_poll_at ASC`,
+  ).all(now) as Row[];
+  return rows.map(rowToWatch);
+}
+
+/** All watches (active first, then recent terminal), for the management UI. */
+export function listWatches(limit = 50): Watch[] {
+  const rows = db().prepare(
+    `SELECT * FROM connector_watches ORDER BY (state='active') DESC, updated_at DESC LIMIT ?`,
+  ).all(limit) as Row[];
+  return rows.map(rowToWatch);
+}
+
+export function updateWatch(id: string, patch: Partial<{
+  state: WatchState; polls: number; err_count: number; next_poll_at: number;
+  last_text: string | null; last_result: unknown;
+}>, now: number): void {
+  const sets: string[] = ['updated_at = @updated_at'];
+  const params: Record<string, unknown> = { id, updated_at: now };
+  if (patch.state !== undefined) { sets.push('state = @state'); params.state = patch.state; }
+  if (patch.polls !== undefined) { sets.push('polls = @polls'); params.polls = patch.polls; }
+  if (patch.err_count !== undefined) { sets.push('err_count = @err_count'); params.err_count = patch.err_count; }
+  if (patch.next_poll_at !== undefined) { sets.push('next_poll_at = @next_poll_at'); params.next_poll_at = patch.next_poll_at; }
+  if (patch.last_text !== undefined) { sets.push('last_text = @last_text'); params.last_text = patch.last_text; }
+  if (patch.last_result !== undefined) { sets.push('last_result_json = @last_result_json'); params.last_result_json = patch.last_result == null ? null : JSON.stringify(patch.last_result); }
+  db().prepare(`UPDATE connector_watches SET ${sets.join(', ')} WHERE id = @id`).run(params);
+}
+
+/** Cancel an active watch (terminal). Returns false if not found/active. */
+export function cancelWatch(id: string, now: number): boolean {
+  const r = db().prepare(
+    `UPDATE connector_watches SET state='cancelled', updated_at=? WHERE id=? AND state='active'`,
+  ).run(now, id);
+  return r.changes > 0;
+}
+
+export function deleteWatch(id: string): boolean {
+  return db().prepare(`DELETE FROM connector_watches WHERE id = ?`).run(id).changes > 0;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aion0/forge",
-  "version": "0.10.22",
+  "version": "0.10.25",
   "description": "Unified AI workflow platform — multi-model task orchestration, persistent sessions, web terminal, remote access",
   "type": "module",
   "scripts": {