@yemi33/minions 0.1.2448 → 0.1.2449

package/watch-plugins/http.js

@@ -0,0 +1,474 @@
+/**
+ * watch-plugins/http.js — Generic HTTP target type (W-mp7i22mu00191b07)
+ *
+ * The canonical watch target plugin — demonstrates the extension contract
+ * landed in W-mp7hg58e000b5212. Unlocks "watch any REST endpoint / GH Actions
+ * workflow_run / Jenkins job / Slack thread" without writing more bespoke
+ * target types.
+ *
+ * ── Target syntax ────────────────────────────────────────────────────────────
+ *   target: 'https://api.github.com/zen'
+ *     → bare URL, GET, JSON parsed automatically when content-type is JSON.
+ *
+ *   target: { url, method, headers, body, extract, expected, regex, timeoutMs }
+ *     → full config. `extract` is a JSONPath-style accessor (`$.foo[0].bar`)
+ *       or a `/regex/flags` string; the result becomes `state.extracted` and
+ *       is what `value-*` conditions compare. `expected` (for value-equals)
+ *       and `regex` (for value-matches) live alongside on the target.
+ *
+ * Auth: `headers.Authorization: 'Bearer {{env.GH_TOKEN}}'` substitutes from
+ * `process.env` at fetch time. Missing env var = WARN log + header dropped.
+ * Secrets are never persisted in watch records.
+ *
+ * ── Conditions ──────────────────────────────────────────────────────────────
+ *   status-change   — HTTP status code differs from previous poll
+ *   value-change    — `state.extracted` differs (deep equal)
+ *   value-equals    — `state.extracted === target.expected`
+ *   value-matches   — `state.extracted` matches `new RegExp(target.regex)`
+ *   http-error      — HTTP status >= 400 OR fetch threw
+ *
+ * All conditions are change-based — `stopAfter:0` truly runs forever
+ * (absoluteConditions: []).
+ *
+ * ── Templating vars exposed to actions ───────────────────────────────────────
+ *   {{httpStatus}}, {{url}}, {{extracted}}, {{prevExtracted}},
+ *   {{fetchedAt}}, {{durationMs}}, {{rawBody}} (truncated to 4KB).
+ * Plus standard watch vars (target, condition, watchId, message, ...).
+ *
+ * ── Fetch model ─────────────────────────────────────────────────────────────
+ * The engine's checkWatches loop is synchronous (it runs inside a
+ * mutateJsonFileLocked callback), so this plugin maintains its own background
+ * fetch cache. `fetchEntity` is sync — it returns the cached entity and, when
+ * the cached value is older than 30s, kicks off a background refresh. First
+ * tick after a watch is created therefore returns null (no entity yet);
+ * second tick captures the baseline; subsequent ticks detect changes.
+ *
+ * Tests can short-circuit this entirely by passing a pre-populated map at
+ * `state.httpResponses[targetKey]` (see `_targetKey`) — the plugin uses that
+ * shape directly, no async required. Tests can also override the network
+ * fetcher via `setFetcher(fn)` for unit-testing the fetch-and-extract path.
+ *
+ * Out of scope: webhooks, auth schemes beyond `headers`, streaming / chunked
+ * downloads.
+ */
+
+const http = require('http');
+const https = require('https');
+const { URL } = require('url');
+
+// Plugins live in `<MINIONS_DIR>/watch-plugins/` which may not be a sibling of
+// the engine source on installed layouts, so this plugin avoids requiring
+// `engine/core/shared.js`. The two helpers it needs (timestamp + warn-logger) are
+// inlined: `ts()` matches shared.ts(), and `_warn()` falls back to
+// `console.warn` (no dependency on the structured-log file).
+function ts() { return new Date().toISOString(); }
+function _warn(msg) { try { console.warn(`[${ts()}] [warn] ${msg}`); } catch {} }
+
+const DEFAULT_TIMEOUT_MS = 10000;
+const MAX_BODY_BYTES = 4096;
+const CACHE_FRESHNESS_MS = 30000;
+
+// Module-level fetch cache. fetchEntity returns from here synchronously while
+// background refreshes update it as needed. Keyed by _targetKey(target).
+const _cache = new Map();
+const _inFlight = new Set();
+
+// Override-able fetcher — default uses node:http(s). Tests inject a mock.
+let _fetcher = _defaultHttpFetch;
+function setFetcher(fn) { _fetcher = typeof fn === 'function' ? fn : _defaultHttpFetch; }
+function _resetCache() { _cache.clear(); _inFlight.clear(); }
+
+function _targetKey(target) {
+  if (typeof target === 'string') return target;
+  // Stable key — ordered serialization via Object.keys().sort() so cache
+  // hits don't depend on caller's property ordering.
+  if (target && typeof target === 'object') {
+    const ordered = {};
+    for (const k of Object.keys(target).sort()) ordered[k] = target[k];
+    return JSON.stringify(ordered);
+  }
+  return String(target);
+}
+
+function _resolveCfg(target) {
+  if (typeof target === 'string' && target) {
+    return {
+      url: target, method: 'GET', headers: {}, body: null, extract: null,
+      timeoutMs: DEFAULT_TIMEOUT_MS,
+    };
+  }
+  if (target && typeof target === 'object' && typeof target.url === 'string' && target.url) {
+    return {
+      url: target.url,
+      method: String(target.method || 'GET').toUpperCase(),
+      headers: (target.headers && typeof target.headers === 'object') ? target.headers : {},
+      body: target.body ?? null,
+      extract: target.extract || null,
+      timeoutMs: Number.isFinite(Number(target.timeoutMs)) ? Number(target.timeoutMs) : DEFAULT_TIMEOUT_MS,
+    };
+  }
+  return null;
+}
+
+function _validateTarget(target, context = {}) {
+  const cfg = _resolveCfg(target);
+  if (!cfg) return 'target must be an HTTP(S) URL string or an object with a url';
+  let parsed;
+  try {
+    parsed = new URL(cfg.url);
+  } catch {
+    return 'target.url must be a valid HTTP(S) URL';
+  }
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    return 'target.url must use http: or https:';
+  }
+  const targetObject = target && typeof target === 'object' ? target : null;
+  if (targetObject) {
+    if (target.method !== undefined) {
+      if (typeof target.method !== 'string') return 'target.method must be a valid HTTP token';
+      try {
+        http.validateHeaderName(target.method);
+      } catch {
+        return 'target.method must be a valid HTTP token';
+      }
+    }
+    if (target.headers !== undefined) {
+      if (!target.headers || typeof target.headers !== 'object' || Array.isArray(target.headers)) {
+        return 'target.headers must be an object';
+      }
+      for (const [name, value] of Object.entries(target.headers)) {
+        if (typeof value !== 'string') return 'target.headers values must be strings';
+        try {
+          http.validateHeaderName(name);
+          http.validateHeaderValue(name, value);
+        } catch {
+          return 'target.headers contains an invalid HTTP header';
+        }
+      }
+    }
+    if (target.extract !== undefined && target.extract !== null
+        && (typeof target.extract !== 'string' || !target.extract.trim())) {
+      return 'target.extract must be a non-empty string';
+    }
+    if (target.regex !== undefined && target.regex !== null) {
+      if (typeof target.regex !== 'string' || !target.regex) {
+        return 'target.regex must be a non-empty string';
+      }
+      try {
+        new RegExp(target.regex);
+      } catch {
+        return 'target.regex must be a valid regular expression';
+      }
+    }
+    if (target.timeoutMs !== undefined
+        && (typeof target.timeoutMs !== 'number'
+          || !Number.isFinite(target.timeoutMs)
+          || target.timeoutMs <= 0)) {
+      return 'target.timeoutMs must be a positive finite number';
+    }
+  }
+
+  if (['value-change', 'value-equals', 'value-matches'].includes(context.condition)) {
+    if (!targetObject || typeof target.extract !== 'string' || !target.extract.trim()) {
+      return `target.extract is required for ${context.condition}`;
+    }
+  }
+  if (context.condition === 'value-equals'
+      && (!Object.prototype.hasOwnProperty.call(target, 'expected')
+        || target.expected === undefined)) {
+    return 'target.expected is required for value-equals';
+  }
+  if (context.condition === 'value-matches'
+      && (typeof target.regex !== 'string' || !target.regex)) {
+    return 'target.regex is required for value-matches';
+  }
+  return null;
+}
+
+// Substitute {{env.VAR}} in a single header value. Returns the substituted
+// string when every referenced env var resolved, or { _missingEnv } when any
+// referenced var is unset — caller drops the header and emits a WARN.
+function _substituteEnvHeader(value) {
+  const re = /\{\{\s*env\.([A-Z_][A-Z0-9_]*)\s*\}\}/g;
+  let missing = null;
+  const out = String(value).replace(re, (m, k) => {
+    const v = process.env[k];
+    if (v === undefined || v === '') {
+      if (missing === null) missing = k;
+      return m;
+    }
+    return v;
+  });
+  return missing === null ? out : { _missingEnv: missing };
+}
+
+// JSONPath-lite accessor: `$.foo[0].bar` or `$.foo['weird key']`. Tokens are
+// `.field`, `[N]`, or `['key']`. Returns null on parse-failure or any miss.
+function _extractJsonPath(jpath, body) {
+  if (typeof body === 'string') {
+    try { body = JSON.parse(body); } catch { return null; }
+  }
+  if (!jpath || jpath[0] !== '$') return null;
+  const tokens = jpath.slice(1).match(/\.[a-zA-Z_$][\w$]*|\[\d+\]|\['[^']*'\]|\["[^"]*"\]/g) || [];
+  let cursor = body;
+  for (const tok of tokens) {
+    if (cursor === null || cursor === undefined) return null;
+    if (tok[0] === '.') cursor = cursor[tok.slice(1)];
+    else if (tok[1] === "'" || tok[1] === '"') cursor = cursor[tok.slice(2, -2)];
+    else cursor = cursor[parseInt(tok.slice(1, -1), 10)];
+  }
+  return cursor === undefined ? null : cursor;
+}
+
+function _parseRegexLiteral(s) {
+  const m = s.match(/^\/(.+)\/([gimsuy]*)$/);
+  if (!m) return null;
+  return { pattern: m[1], flags: m[2] || '' };
+}
+
+function _extractRegex(rePattern, flags, rawBody) {
+  let re;
+  try { re = new RegExp(rePattern, flags || ''); } catch { return null; }
+  const m = String(rawBody).match(re);
+  if (!m) return null;
+  return m.length > 1 ? m[1] : m[0];
+}
+
+function _extract(extract, rawBody) {
+  if (!extract) return null;
+  if (typeof extract === 'string') {
+    if (extract.startsWith('$')) return _extractJsonPath(extract, rawBody);
+    const lit = _parseRegexLiteral(extract);
+    if (lit) return _extractRegex(lit.pattern, lit.flags, rawBody);
+    // Bare property name → treat as $.name for ergonomics.
+    return _extractJsonPath('$.' + extract, rawBody);
+  }
+  return null;
+}
+
+// Default Node http(s) fetcher. Returns { status, headers, body }.
+function _defaultHttpFetch(cfg) {
+  return new Promise((resolve, reject) => {
+    let parsed;
+    try { parsed = new URL(cfg.url); } catch (err) { reject(err); return; }
+    const lib = parsed.protocol === 'https:' ? https : http;
+    const opts = {
+      method: cfg.method,
+      hostname: parsed.hostname,
+      port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80),
+      path: (parsed.pathname || '/') + (parsed.search || ''),
+      headers: { 'User-Agent': 'minions-watch', ...cfg.headers },
+      timeout: cfg.timeoutMs,
+    };
+    const req = lib.request(opts, (res) => {
+      const chunks = [];
+      let bytes = 0;
+      // Cap captured bytes at 4× truncation budget so we can still parse the
+      // first 4KB of valid JSON without buffering megabyte payloads.
+      const cap = MAX_BODY_BYTES * 4;
+      res.on('data', (c) => {
+        if (bytes >= cap) return;
+        chunks.push(c);
+        bytes += c.length;
+      });
+      res.on('end', () => {
+        resolve({
+          status: res.statusCode,
+          headers: res.headers || {},
+          body: Buffer.concat(chunks).toString('utf8'),
+        });
+      });
+    });
+    req.on('error', reject);
+    req.on('timeout', () => { req.destroy(new Error(`fetch timeout after ${cfg.timeoutMs}ms`)); });
+    if (cfg.body !== null && cfg.body !== undefined) {
+      req.write(typeof cfg.body === 'string' ? cfg.body : JSON.stringify(cfg.body));
+    }
+    req.end();
+  });
+}
+
+async function _doFetch(target) {
+  const cfg = _resolveCfg(target);
+  if (!cfg) {
+    return {
+      url: '', status: 0, body: '', rawBody: '', extracted: null,
+      fetchedAt: ts(), durationMs: 0, error: 'invalid target',
+    };
+  }
+  const headers = {};
+  for (const [k, v] of Object.entries(cfg.headers)) {
+    const subbed = _substituteEnvHeader(v);
+    if (subbed && typeof subbed === 'object' && subbed._missingEnv) {
+      _warn(`watch-plugins/http: header '${k}' dropped — env var ${subbed._missingEnv} not set in process.env`);
+      continue;
+    }
+    headers[k] = subbed;
+  }
+  const start = Date.now();
+  let result, err;
+  try {
+    result = await _fetcher({ ...cfg, headers });
+  } catch (e) {
+    err = e;
+  }
+  const durationMs = Date.now() - start;
+  if (err) {
+    return {
+      url: cfg.url, status: 0, body: '', rawBody: '', extracted: null,
+      fetchedAt: ts(), durationMs, error: err.message || String(err),
+    };
+  }
+  const fullBody = String(result.body || '');
+  const rawBody = fullBody.length > MAX_BODY_BYTES ? fullBody.slice(0, MAX_BODY_BYTES) : fullBody;
+  const extracted = cfg.extract ? _extract(cfg.extract, fullBody) : null;
+  return {
+    url: cfg.url,
+    status: typeof result.status === 'number' ? result.status : 0,
+    body: rawBody,
+    rawBody,
+    extracted,
+    fetchedAt: ts(),
+    durationMs,
+  };
+}
+
+function _scheduleRefresh(target, key) {
+  if (_inFlight.has(key)) return;
+  _inFlight.add(key);
+  Promise.resolve()
+    .then(() => _doFetch(target))
+    .then((entity) => { _cache.set(key, entity); })
+    .catch((e) => {
+      _cache.set(key, {
+        url: '', status: 0, body: '', rawBody: '', extracted: null,
+        fetchedAt: ts(), durationMs: 0, error: e && e.message ? e.message : String(e),
+      });
+    })
+    .then(() => { _inFlight.delete(key); });
+}
+
+function fetchEntity(target, state) {
+  const key = _targetKey(target);
+  // Test/external override path — state.httpResponses[key] short-circuits the
+  // background-fetch cache so unit tests can drive the plugin deterministically.
+  if (state && state.httpResponses && Object.prototype.hasOwnProperty.call(state.httpResponses, key)) {
+    return { ...state.httpResponses[key], _target: target };
+  }
+  const cached = _cache.get(key);
+  const cachedAge = cached && cached.fetchedAt ? Date.now() - new Date(cached.fetchedAt).getTime() : Infinity;
+  if (!cached || cachedAge > CACHE_FRESHNESS_MS) {
+    _scheduleRefresh(target, key);
+  }
+  return cached ? { ...cached, _target: target } : null;
+}
+
+function captureState(entity) {
+  if (!entity) return {};
+  return {
+    httpStatus: entity.status,
+    url: entity.url,
+    extracted: entity.extracted,
+    fetchedAt: entity.fetchedAt,
+    durationMs: entity.durationMs,
+    rawBody: entity.rawBody,
+    error: entity.error || null,
+  };
+}
+
+function _deepEqual(a, b) {
+  if (a === b) return true;
+  if (a === null || b === null || a === undefined || b === undefined) return false;
+  if (typeof a !== typeof b) return false;
+  if (typeof a !== 'object') return a === b;
+  try { return JSON.stringify(a) === JSON.stringify(b); }
+  catch { return false; }
+}
+
+function evaluate(condition, entity, prevState, target) {
+  const cur = captureState(entity);
+  switch (condition) {
+    case 'status-change': {
+      if (prevState.httpStatus === undefined) return { triggered: false, message: '' };
+      const changed = prevState.httpStatus !== cur.httpStatus;
+      return { triggered: changed, message: changed ? `HTTP ${cur.url}: status ${prevState.httpStatus} → ${cur.httpStatus}` : '' };
+    }
+    case 'value-change': {
+      if (!('extracted' in prevState)) return { triggered: false, message: '' };
+      const changed = !_deepEqual(prevState.extracted, cur.extracted);
+      const fmt = (v) => { try { return JSON.stringify(v); } catch { return String(v); } };
+      return { triggered: changed, message: changed ? `HTTP ${cur.url}: extracted changed: ${fmt(prevState.extracted)} → ${fmt(cur.extracted)}` : '' };
+    }
+    case 'value-equals': {
+      const expected = (target && typeof target === 'object' && Object.prototype.hasOwnProperty.call(target, 'expected'))
+        ? target.expected : undefined;
+      if (expected === undefined) return { triggered: false, message: '' };
+      const matched = cur.extracted === expected;
+      return { triggered: matched, message: matched ? `HTTP ${cur.url}: extracted equals expected (${expected})` : '' };
+    }
+    case 'value-matches': {
+      const pattern = (target && typeof target === 'object' && typeof target.regex === 'string') ? target.regex : null;
+      if (!pattern || cur.extracted === null || cur.extracted === undefined) return { triggered: false, message: '' };
+      let re;
+      try { re = new RegExp(pattern); } catch { return { triggered: false, message: '' }; }
+      const matched = re.test(String(cur.extracted));
+      return { triggered: matched, message: matched ? `HTTP ${cur.url}: extracted matches /${pattern}/` : '' };
+    }
+    case 'http-error': {
+      const errored = !!cur.error || (typeof cur.httpStatus === 'number' && cur.httpStatus >= 400);
+      const detail = cur.error ? cur.error : `status ${cur.httpStatus}`;
+      return { triggered: errored, message: errored ? `HTTP ${cur.url}: ${detail}` : '' };
+    }
+    default:
+      return { triggered: false, message: `Unknown condition: ${condition}` };
+  }
+}
+
+// contextVars supplements the action template ctx with plugin-specific aliases.
+// httpStatus/url/extracted/fetchedAt/durationMs/rawBody auto-promote from
+// newState scalars in buildTriggerContext, but prevExtracted is on prevState
+// and an object-shaped extracted value would otherwise be unreachable as a
+// scalar — covering both here keeps the templating contract uniform.
+function contextVars(entity, prevState, newState) {
+  const out = {};
+  if (newState) {
+    if (newState.httpStatus !== undefined) out.httpStatus = newState.httpStatus;
+    if (newState.url !== undefined) out.url = newState.url;
+    if (newState.extracted !== undefined && newState.extracted !== null) out.extracted = newState.extracted;
+    if (newState.fetchedAt !== undefined) out.fetchedAt = newState.fetchedAt;
+    if (newState.durationMs !== undefined) out.durationMs = newState.durationMs;
+    if (newState.rawBody !== undefined) out.rawBody = newState.rawBody;
+  }
+  if (prevState && prevState.extracted !== undefined && prevState.extracted !== null) {
+    out.prevExtracted = prevState.extracted;
+  }
+  return out;
+}
+
+module.exports = {
+  name: 'http',
+  spec: {
+    label: 'HTTP Endpoint',
+    description: 'Poll any URL, extract a value, compare against previous state.',
+    conditions: ['status-change', 'value-change', 'value-equals', 'value-matches', 'http-error'],
+    absoluteConditions: [], // all change-based — stopAfter:0 truly runs forever
+    validateTarget: _validateTarget,
+    fetchEntity,
+    captureState,
+    evaluate,
+    contextVars,
+  },
+  // Test hooks (unit tests only — not part of the plugin contract).
+  setFetcher,
+  _resetCache,
+  _resolveCfg,
+  _targetKey,
+  _extract,
+  _extractJsonPath,
+  _extractRegex,
+  _parseRegexLiteral,
+  _substituteEnvHeader,
+  _doFetch,
+  DEFAULT_TIMEOUT_MS,
+  MAX_BODY_BYTES,
+};