neuralos 3.9.4 → 3.9.6

package/plugins/web-intel/sidecar.mjs

@@ -1,134 +0,0 @@
-/**
- * sidecar.mjs — manages the wigolo daemon lifecycle for RTerm's web-intel
- * plugin: lazily start `wigolo serve` on first use, keep a stock RTerm install
- * lean (no browser-engine/on-device-model download unless the user opts in),
- * and report status. Pure + injectable: process spawning and health-probing are
- * injected so it's fully unit-testable offline.
- *
- * Lean-by-default: we start the daemon with WIGOLO_NO_WARMUP=1 so the ~1.5 GB
- * browser engine + on-device models are NOT downloaded at init — search/fetch/
- * crawl work keyless without them. The heavier models download in the background
- * on first use that actually needs them. `warmupOnInit: true` opts into the full
- * upfront download (a background `wigolo init` run).
- */
-
-export const DEFAULT_PORT = 3333
-export const DEFAULT_HOST = '127.0.0.1'
-
-/**
- * Build the spawn plan for the wigolo daemon.
- * @param {{ port?: number, host?: string, warmup?: boolean, token?: string }} cfg
- * @returns {{ command: string, args: string[], env: Record<string,string> }}
- */
-export function buildServePlan(cfg = {}) {
-  const port = cfg.port ?? DEFAULT_PORT
-  const host = cfg.host ?? DEFAULT_HOST
-  const env = { ...process.env }
-  // Lean by default: skip the browser-engine/model warmup unless explicitly on.
-  if (cfg.warmup !== true) env.WIGOLO_NO_WARMUP = '1'
-  if (cfg.token) env.WIGOLO_API_TOKEN = cfg.token
-  return {
-    command: 'npx',
-    args: ['-y', 'wigolo', 'serve', '--port', String(port), '--host', host],
-    env,
-  }
-}
-
-/** Build the background `wigolo init` plan (the full ~1.5 GB warmup). */
-export function buildInitPlan(cfg = {}) {
-  const env = { ...process.env }
-  if (cfg.token) env.WIGOLO_API_TOKEN = cfg.token
-  return { command: 'npx', args: ['-y', 'wigolo', 'init'], env }
-}
-
-export class WigoloSidecar {
-  /**
-   * @param {{
-   *   spawnImpl?: (cmd: string, args: string[], opts: object) => any,
-   *   healthImpl?: () => Promise<boolean>,
-   *   log?: (line: string) => void,
-   *   config?: { port?: number, host?: string, warmup?: boolean, token?: string, autoStart?: boolean },
-   *   now?: () => number,
-   * }} deps — all injectable; defaults are real (child_process + a health probe).
-   */
-  constructor(deps = {}) {
-    this.config = deps.config ?? {}
-    this.spawnImpl = deps.spawnImpl
-    this.healthImpl = deps.healthImpl
-    this.log = deps.log ?? (() => {})
-    this.now = deps.now ?? (() => Date.now())
-    this.process = null
-    this.startedAt = 0
-    this.lastError = undefined
-  }
-
-  /** Whether the daemon process is believed to be running. */
-  isRunning() {
-    return this.process != null
-  }
-
-  /** Start the daemon (idempotent). Returns the base URL it's expected on. */
-  async start() {
-    if (this.process) return this.baseUrl()
-    if (typeof this.spawnImpl !== 'function') {
-      this.lastError = 'no spawnImpl (sidecar spawn not available in this runtime)'
-      throw new Error(this.lastError)
-    }
-    const plan = buildServePlan(this.config)
-    this.log(`[web-intel] starting wigolo daemon: ${plan.command} ${plan.args.join(' ')}`)
-    try {
-      this.process = this.spawnImpl(plan.command, plan.args, {
-        env: plan.env,
-        detached: true,
-        stdio: 'ignore',
-      })
-      // Detach so the daemon outlives the plugin turn (it serves many agents).
-      this.process?.unref?.()
-      this.startedAt = this.now()
-      this.lastError = undefined
-    } catch (e) {
-      this.lastError = e?.message ?? String(e)
-      this.process = null
-      throw e
-    }
-    return this.baseUrl()
-  }
-
-  /** Kick off the full ~1.5 GB warmup in the background (opt-in). */
-  async warmupInBackground() {
-    if (typeof this.spawnImpl !== 'function') return false
-    const plan = buildInitPlan(this.config)
-    try {
-      const p = this.spawnImpl(plan.command, plan.args, { env: plan.env, detached: true, stdio: 'ignore' })
-      p?.unref?.()
-      this.log('[web-intel] background wigolo init (browser engine + models) started')
-      return true
-    } catch {
-      return false
-    }
-  }
-
-  /** Stop the daemon. */
-  async stop() {
-    if (!this.process) return
-    try { this.process.kill?.() } catch { /* best-effort */ }
-    this.process = null
-  }
-
-  /** Status snapshot for the health tool / panel. */
-  status() {
-    return {
-      running: this.isRunning(),
-      baseUrl: this.baseUrl(),
-      startedAt: this.startedAt || undefined,
-      lastError: this.lastError,
-      warmup: this.config.warmup === true ? 'full' : 'lean (no warmup)',
-    }
-  }
-
-  baseUrl() {
-    const host = this.config.host ?? DEFAULT_HOST
-    const port = this.config.port ?? DEFAULT_PORT
-    return `http://${host}:${port}`
-  }
-}

package/plugins/web-intel/web-intel.extreme.spec.ts

@@ -1,263 +0,0 @@
-/**
- * web-intel.extreme.spec.ts — exhaustive offline tests for the web-intel plugin:
- * the dependency-free wigolo REST client (URL building, auth header, error
- * mapping, every endpoint), the sidecar lifecycle (lean-by-default spawn plan,
- * start/stop/status), and the plugin glue (config resolution, tool wiring,
- * unreachable-daemon resilience, result normalization, trigger match). No
- * network — fetch is mocked; spawn is injected/mocked.
- */
-import { test } from 'node:test'
-import assert from 'node:assert/strict'
-import { WigoloClient, WigoloApiError, buildHeaders, joinUrl, DEFAULT_BASE_URL } from './wigoloClient.mjs'
-import { WigoloSidecar, buildServePlan, buildInitPlan } from './sidecar.mjs'
-import {
-  register,
-  resolveConfig,
-  buildClient,
-  toResultRows,
-  toPageSummary,
-  toResearchBrief,
-  toWatchRows,
-  isPageChangedEvent,
-} from './index.mjs'
-
-// ─── mock fetch ─────────────────────────────────────────────────────────────
-function mockFetch(respond) {
-  const calls = []
-  const fn = async (url, init) => {
-    calls.push({ url, init })
-    const r = typeof respond === 'function' ? respond(url, init, calls.length) : respond
-    return { ok: r.ok !== false && (r.status ?? 200) < 400, status: r.status ?? 200, text: async () => r.text ?? (r.json !== undefined ? JSON.stringify(r.json) : '') }
-  }
-  fn.calls = calls
-  return fn
-}
-
-// ─── wigoloClient ───────────────────────────────────────────────────────────
-
-test('joinUrl joins base + path with a single slash', () => {
-  assert.equal(joinUrl('http://x:3333/', '/health'), 'http://x:3333/health')
-  assert.equal(joinUrl('http://x:3333', 'health'), 'http://x:3333/health')
-  assert.equal(joinUrl('', '/v1/search'), `${DEFAULT_BASE_URL}/v1/search`)
-})
-
-test('buildHeaders adds bearer token only when set', () => {
-  assert.equal(buildHeaders().authorization, undefined)
-  assert.equal(buildHeaders('tok').authorization, 'Bearer tok')
-})
-
-test('client.health() GETs /health without auth and maps ok/error', async () => {
-  const up = mockFetch({ json: { status: 'ok', searxng: 'not_configured' } })
-  const c = new WigoloClient({ fetchImpl: up })
-  const h = await c.health()
-  assert.equal(h.ok, true)
-  assert.equal(up.calls[0].url, `${DEFAULT_BASE_URL}/health`)
-
-  const down = mockFetch(() => { throw new Error('ECONNREFUSED') })
-  const h2 = await new WigoloClient({ fetchImpl: down }).health()
-  assert.equal(h2.ok, false)
-  assert.match(h2.error, /ECONNREFUSED/)
-})
-
-test('client.search() POSTs query (+ array) to /v1/search with token header', async () => {
-  const f = mockFetch({ json: { results: [] } })
-  const c = new WigoloClient({ token: 'tok', fetchImpl: f })
-  await c.search(['a', 'b'], { time_range: 'week', max_results: 5 })
-  assert.equal(f.calls[0].url, `${DEFAULT_BASE_URL}/v1/search`)
-  assert.equal(f.calls[0].init.method, 'POST')
-  assert.equal(f.calls[0].init.headers.authorization, 'Bearer tok')
-  const body = JSON.parse(f.calls[0].init.body)
-  assert.deepEqual(body.query, ['a', 'b'])
-  assert.equal(body.time_range, 'week')
-  assert.equal(body.max_results, 5)
-})
-
-test('client maps non-2xx to WigoloApiError with status', async () => {
-  const f = mockFetch({ status: 500, text: 'boom' })
-  const c = new WigoloClient({ fetchImpl: f })
-  await assert.rejects(() => c.fetch('https://x'), (e) => {
-    assert.ok(e instanceof WigoloApiError)
-    assert.equal(e.status, 500)
-    return true
-  })
-})
-
-test('client.watch() sends action + url for create/list/remove', async () => {
-  const f = mockFetch({ json: { id: 'w1' } })
-  const c = new WigoloClient({ fetchImpl: f })
-  await c.watch('create', { url: 'https://x', interval: '1h' })
-  assert.equal(JSON.parse(f.calls[0].init.body).action, 'create')
-  await c.watch('list')
-  assert.equal(JSON.parse(f.calls[1].init.body).action, 'list')
-  await c.watch('remove', { id: 'w1' })
-  assert.equal(JSON.parse(f.calls[2].init.body).action, 'remove')
-})
-
-// ─── sidecar ────────────────────────────────────────────────────────────────
-
-test('buildServePlan is lean by default (WIGOLO_NO_WARMUP=1, no token)', () => {
-  const p = buildServePlan({})
-  assert.equal(p.command, 'npx')
-  assert.deepEqual(p.args.slice(0, 3), ['-y', 'wigolo', 'serve'])
-  assert.equal(p.env.WIGOLO_NO_WARMUP, '1')
-  assert.equal(p.env.WIGOLO_API_TOKEN, undefined)
-})
-
-test('buildServePlan honors port/host/token and warmup=true omits the no-warmup flag', () => {
-  const p = buildServePlan({ port: 3477, host: '0.0.0.0', token: 't', warmup: true })
-  assert.ok(p.args.includes('3477'))
-  assert.equal(p.env.WIGOLO_NO_WARMUP, undefined)
-  assert.equal(p.env.WIGOLO_API_TOKEN, 't')
-})
-
-test('sidecar.start() spawns detached + unref and is idempotent', async () => {
-  const spawned = []
-  const spawnImpl = (cmd, args, opts) => { spawned.push({ cmd, args, opts }); return { unref() {}, kill() {} } }
-  const sc = new WigoloSidecar({ spawnImpl, log: () => {} })
-  await sc.start()
-  await sc.start() // idempotent — no second spawn
-  assert.equal(spawned.length, 1)
-  assert.equal(spawned[0].opts.detached, true)
-  assert.equal(sc.isRunning(), true)
-  assert.equal(sc.status().warmup, 'lean (no warmup)')
-})
-
-test('sidecar.stop() kills the process', async () => {
-  let killed = false
-  const spawnImpl = () => ({ unref() {}, kill() { killed = true } })
-  const sc = new WigoloSidecar({ spawnImpl, log: () => {} })
-  await sc.start()
-  await sc.stop()
-  assert.equal(killed, true)
-  assert.equal(sc.isRunning(), false)
-})
-
-// ─── plugin glue ────────────────────────────────────────────────────────────
-
-function makeCtx(fetchImpl, settings = {}, spawnProcess) {
-  const tools = new Map()
-  const triggers = []
-  const panels = []
-  const logs = []
-  const ctx = {
-    settings: { webIntel: settings },
-    fetchImpl,
-    registerTool: (t) => tools.set(t.name, t),
-    registerTrigger: (t) => triggers.push(t),
-    registerPanel: (p) => panels.push(p),
-    log: (l) => logs.push(l),
-    ...(spawnProcess ? { spawnProcess } : {}),
-  }
-  return { tools, triggers, panels, logs, ctx }
-}
-
-test('resolveConfig reads the webIntel block with lean defaults', () => {
-  const c = resolveConfig({ settings: { webIntel: { restUrl: ' http://x:3477/ ', warmupOnInit: true, autoStart: false } } })
-  assert.equal(c.restUrl, 'http://x:3477', 'restUrl trimmed of whitespace + trailing slash')
-  assert.equal(c.warmupOnInit, true)
-  assert.equal(c.autoStart, false)
-  const d = resolveConfig({ settings: {} })
-  assert.equal(d.enabled, true)
-  assert.equal(d.autoStart, true)
-  assert.equal(d.warmupOnInit, false)
-})
-
-test('register wires 9 tools, 1 trigger, 1 panel', () => {
-  const { tools, triggers, panels, ctx } = makeCtx(mockFetch({ json: {} }))
-  register(ctx)
-  assert.equal(tools.size, 9)
-  for (const n of ['webintel_health', 'web_search', 'web_fetch', 'web_crawl', 'web_research', 'web_find_similar', 'web_watch_add', 'web_watch_list', 'web_watch_remove']) assert.ok(tools.has(n), `missing ${n}`)
-  assert.equal(triggers.length, 1)
-  assert.equal(panels.length, 1)
-})
-
-test('web_search returns normalized ranked results', async () => {
-  const f = mockFetch((url) => url.endsWith('/health')
-    ? { json: { status: 'ok' } }
-    : { json: { results: [{ title: 'A', url: 'https://a', excerpt: 'x'.repeat(300), citation_id: 'src-1', evidence_score: { final: 0.9 } }], freshness_signal: { published: '2026-07-01' } } })
-  const { tools, ctx } = makeCtx(f)
-  register(ctx)
-  const r = await tools.get('web_search').handler({ query: 'cisco bgp error' })
-  assert.equal(r.results.length, 1)
-  assert.equal(r.results[0].citation, 'src-1')
-  assert.equal(r.results[0].score, 0.9)
-  assert.ok(r.results[0].excerpt.length <= 240)
-  assert.equal(r.freshness.published, '2026-07-01')
-})
-
-test('web_research returns the evidence brief (synthesis left to RTerm agent)', async () => {
-  const f = mockFetch((url) => url.endsWith('/health')
-    ? { json: { status: 'ok' } }
-    : { json: { question: 'q', evidence: [{ title: 'S', url: 'https://s', snippet: 'ev' }], citations: [{ id: 'src-1', url: 'https://s' }] } })
-  const { tools, ctx } = makeCtx(f)
-  register(ctx)
-  const r = await tools.get('web_research').handler({ question: 'is rijndael128-cbc safe' })
-  assert.equal(r.evidence.length, 1)
-  assert.equal(r.citations.length, 1)
-  assert.match(r.note, /RTerm agent/)
-})
-
-test('web_fetch surfaces a blocked page honestly', async () => {
-  const f = mockFetch((url) => url.endsWith('/health')
-    ? { json: { status: 'ok' } }
-    : { json: { url: 'https://x', blocked_by_challenge: true, markdown: '' } })
-  const { tools, ctx } = makeCtx(f)
-  register(ctx)
-  const r = await tools.get('web_fetch').handler({ url: 'https://x' })
-  assert.equal(r.blocked, true)
-})
-
-test('every web_* tool returns error+hint when daemon is down and no spawn (no throw)', async () => {
-  const down = mockFetch(() => { throw new Error('ECONNREFUSED') })
-  const { tools, ctx } = makeCtx(down, { autoStart: true }) // no spawnProcess → can't start
-  register(ctx)
-  const r = await tools.get('web_search').handler({ query: 'x' })
-  assert.ok(r.error)
-  assert.match(r.hint, /wigolo daemon/)
-})
-
-test('web_search auto-starts the daemon via spawnProcess then serves', async () => {
-  let healthy = false
-  const spawned = []
-  const f = mockFetch((url) => url.endsWith('/health')
-    ? (healthy ? { json: { status: 'ok' } } : (() => { throw new Error('down') })())
-    : { json: { results: [] } })
-  const spawnProcess = (cmd, args, opts) => { spawned.push(true); healthy = true; return { unref() {}, kill() {} } }
-  const { tools, ctx } = makeCtx(f, { autoStart: true }, spawnProcess)
-  register(ctx)
-  const r = await tools.get('web_search').handler({ query: 'x' })
-  assert.equal(spawned.length, 1)
-  assert.ok(Array.isArray(r.results))
-})
-
-test('webintel_page_changed trigger matches only webintel change events', () => {
-  const { triggers, ctx } = makeCtx(mockFetch({ json: {} }))
-  register(ctx)
-  const m = triggers[0].match
-  assert.equal(m({ source: 'webintel', changed: true }), true)
-  assert.equal(m({ source: 'webintel', kind: 'page_changed' }), true)
-  assert.equal(m({ source: 'agentspan', changed: true }), false)
-  assert.equal(m({ source: 'webintel', changed: false }), false)
-})
-
-// ─── normalization helpers ─────────────────────────────────────────────────
-
-test('toResultRows normalizes + truncates excerpts', () => {
-  const rows = toResultRows({ results: [{ url: 'https://a', description: 'y'.repeat(400) }] })
-  assert.equal(rows.length, 1)
-  assert.ok(rows[0].excerpt.length <= 240)
-  assert.equal(toResultRows({}).length, 0)
-})
-
-test('toResearchBrief handles empty payload', () => {
-  assert.deepEqual(toResearchBrief(null).evidence, [])
-  assert.equal(toResearchBrief({ evidence: [] }).evidence.length, 0)
-})
-
-test('toWatchRows normalizes watch entries', () => {
-  const rows = toWatchRows({ watches: [{ id: 'w1', url: 'https://x', changed: true, last_checked: '2026-07-28' }] })
-  assert.equal(rows.length, 1)
-  assert.equal(rows[0].changed, true)
-})
-
-console.log('web-intel: all cases passed')

package/plugins/web-intel/wigoloClient.mjs

@@ -1,145 +0,0 @@
-/**
- * wigoloClient.mjs — a minimal, dependency-free HTTP client for the wigolo web
- * intelligence daemon (`wigolo serve`, default http://127.0.0.1:3333).
- *
- * wigolo exposes one REST route per tool: POST /v1/{search,fetch,crawl,cache,
- * extract,find_similar,research,agent,diff,watch}, GET /health, GET /v1/tools.
- * This client is pure + injectable (a `fetchImpl` is passed in) so it is fully
- * unit-testable offline with a mocked fetch — no runtime network baked in.
- *
- * Auth: when the daemon is started with WIGOLO_API_TOKEN, every /v1 request
- * needs `Authorization: Bearer <token>` (/health stays open). The client sends
- * the header only when a token is provided.
- */
-
-export const DEFAULT_BASE_URL = 'http://127.0.0.1:3333'
-
-/** Build request headers (adds the bearer token only when set). */
-export function buildHeaders(token) {
-  const h = { 'content-type': 'application/json', accept: 'application/json' }
-  if (token) h.authorization = `Bearer ${token}`
-  return h
-}
-
-/** Join a base URL + path safely (single slash). */
-export function joinUrl(base, path) {
-  const b = String(base || DEFAULT_BASE_URL).replace(/\/+$/, '')
-  const p = String(path || '').startsWith('/') ? String(path) : `/${path}`
-  return `${b}${p}`
-}
-
-async function parseBody(res) {
-  const text = await res.text()
-  if (!text) return null
-  try { return JSON.parse(text) } catch { return text }
-}
-
-export class WigoloApiError extends Error {
-  constructor(status, path, body) {
-    super(`wigolo ${status} ${path}: ${typeof body === 'string' ? body.slice(0, 300) : JSON.stringify(body)?.slice(0, 300)}`)
-    this.status = status
-    this.path = path
-    this.body = body
-  }
-}
-
-export class WigoloClient {
-  /**
-   * @param {{ baseUrl?: string, token?: string, fetchImpl: Function }} opts
-   *   fetchImpl(url, {method, headers, body}) -> Promise<{ok,status,text:()=>Promise<string>}>
-   */
-  constructor(opts = {}) {
-    if (typeof opts.fetchImpl !== 'function') throw new Error('WigoloClient needs a fetchImpl')
-    this.baseUrl = opts.baseUrl || DEFAULT_BASE_URL
-    this.token = opts.token
-    this.fetchImpl = opts.fetchImpl
-  }
-
-  async #post(path, payload) {
-    const res = await this.fetchImpl(joinUrl(this.baseUrl, path), {
-      method: 'POST',
-      headers: buildHeaders(this.token),
-      body: JSON.stringify(payload ?? {}),
-    })
-    const body = await parseBody(res)
-    if (!res.ok) throw new WigoloApiError(res.status, path, body)
-    return body
-  }
-
-  async #get(path) {
-    const res = await this.fetchImpl(joinUrl(this.baseUrl, path), {
-      method: 'GET',
-      headers: buildHeaders(this.token),
-    })
-    const body = await parseBody(res)
-    if (!res.ok) throw new WigoloApiError(res.status, path, body)
-    return body
-  }
-
-  /** Liveness + component status. Always open (no token). */
-  async health() {
-    try {
-      const body = await this.#get('/health')
-      return { ok: true, status: body }
-    } catch (e) {
-      return { ok: false, error: e?.message ?? String(e) }
-    }
-  }
-
-  /** List the daemon's tools (descriptions + endpoints). */
-  async tools() {
-    return this.#get('/v1/tools')
-  }
-
-  /** Multi-engine web search. `query` is a string or an array (parallel breadth). */
-  async search(query, opts = {}) {
-    return this.#post('/v1/search', { query, ...opts })
-  }
-
-  /** Fetch one URL as clean markdown (tiered router escalates to the browser engine). */
-  async fetch(url, opts = {}) {
-    return this.#post('/v1/fetch', { url, ...opts })
-  }
-
-  /** Multi-page crawl (BFS/DFS/sitemap/map-only). */
-  async crawl(url, opts = {}) {
-    return this.#post('/v1/crawl', { url, ...opts })
-  }
-
-  /** Structured extraction (tables, metadata, JSON-LD, named/custom schema). */
-  async extract(url, opts = {}) {
-    return this.#post('/v1/extract', { url, ...opts })
-  }
-
-  /** Pages similar to a URL/concept (keyword + semantic + live web fusion). */
-  async findSimilar(input, opts = {}) {
-    return this.#post('/v1/find_similar', typeof input === 'string' ? { url: input, ...opts } : { ...input, ...opts })
-  }
-
-  /** Query the local cache of everything already seen (keyword or hybrid semantic). */
-  async cache(opts = {}) {
-    return this.#post('/v1/cache', opts)
-  }
-
-  /** Decompose → fan out → fetch → return a structured brief + evidence.
-   * (Synthesis is done by the HOST agent, not wigolo's LLM — we pass no LLM key,
-   * so wigolo returns the raw brief + evidence and RTerm's agent writes the answer.) */
-  async research(question, opts = {}) {
-    return this.#post('/v1/research', { question, ...opts })
-  }
-
-  /** Autonomous gather loop (plan → search → fetch → extract) with a step log. */
-  async agent(goal, opts = {}) {
-    return this.#post('/v1/agent', { goal, ...opts })
-  }
-
-  /** Diff two page snapshots (or a page vs its last-seen cached version). */
-  async diff(input, opts = {}) {
-    return this.#post('/v1/diff', typeof input === 'string' ? { url: input, ...opts } : { ...input, ...opts })
-  }
-
-  /** Watch management: action=create|list|remove. */
-  async watch(action, opts = {}) {
-    return this.#post('/v1/watch', { action, ...opts })
-  }
-}