@ludi-uni/ludi-agent-kit 0.1.0

package/lib/observe/differ.mjs

@@ -0,0 +1,185 @@
+// Catalog differ (Phase 4): compare stored observations against model-catalog.json
+// and produce catalog-diff + catalog-proposal documents. Proposals only — the real
+// catalog is never written. removed/deprecated proposals require >=2 independent
+// sources or one high-trust source; a lone availability listing absence is never
+// enough to propose removal.
+import { TRUST_LEVELS } from './observation.mjs';
+
+const TRUST_RANK = Object.fromEntries(TRUST_LEVELS.map((t, i) => [t, TRUST_LEVELS.length - i]));
+const HIGH_TRUST = new Set(['manual_verified', 'provider_api']);
+// Fields whose conflicting values across sources are "major" — never auto-resolved.
+const MAJOR_FIELDS = new Set(['status', 'free', 'inputPricePer1M', 'outputPricePer1M']);
+
+// observation field -> catalog field path. Returns {path, toCatalog(value)} pairs.
+const FIELD_MAP = {
+  status: { get: e => e.status, set: 'status' },
+  free: { get: e => e.cost?.free === true || ((e.cost?.usdPerMInput ?? 1) === 0 && (e.cost?.usdPerMOutput ?? 1) === 0), set: 'cost.free' },
+  inputPricePer1M: { get: e => e.cost?.usdPerMInput ?? null, set: 'cost.usdPerMInput' },
+  outputPricePer1M: { get: e => e.cost?.usdPerMOutput ?? null, set: 'cost.usdPerMOutput' },
+  contextK: { get: e => e.contextK ?? null, set: 'contextK' },
+  toolUse: { get: e => e.toolUse ?? null, set: 'toolUse' },
+  vision: { get: e => e.vision ?? null, set: 'vision' },
+  availability: { get: () => null, set: null }, // availability maps to status inference, not a stored field
+  freeUntil: { get: e => e.freeUntil ?? null, set: 'freeUntil' },
+};
+
+/** Latest observation per (model, field), resolved: newer > trust > confidence. Major ties -> conflict. */
+export function resolveObservations(observations) {
+  const groups = new Map(); // key -> [obs]
+  for (const o of observations) {
+    if (o._corrupt) continue;
+    for (const [field, value] of Object.entries(o.changes ?? {})) {
+      if (value === null || value === undefined) continue;
+      const key = `${o.provider}/${o.model}::${field}`;
+      (groups.get(key) ?? groups.set(key, []).get(key)).push({ o, field, value });
+    }
+  }
+  const resolved = new Map(); // key -> {winner, conflict}
+  for (const [key, items] of groups) {
+    const sorted = items.slice().sort((a, b) =>
+      b.o.observedAt.localeCompare(a.o.observedAt) ||
+      (TRUST_RANK[b.o.source?.trust ?? 'unknown'] - TRUST_RANK[a.o.source?.trust ?? 'unknown']) ||
+      (b.o.confidence ?? 0) - (a.o.confidence ?? 0));
+    const [best, second] = sorted;
+    const conflict = second && best.value !== second.value &&
+      best.o.observedAt === second.o.observedAt &&
+      (TRUST_RANK[best.o.source?.trust] === TRUST_RANK[second.o.source?.trust]) &&
+      MAJOR_FIELDS.has(best.field);
+    resolved.set(key, { winner: best, conflict: conflict ? { rival: second } : null, contenders: items.length });
+  }
+  return resolved;
+}
+
+/** Diff resolved observations against the catalog. Returns a catalog-diff document. */
+export function diffCatalog(catalog, observations) {
+  const resolved = resolveObservations(observations);
+  const catalogByKey = new Map((catalog?.models ?? []).map(m => [`${m.provider}/${m.model}`, m]));
+  const diffs = [];
+  for (const [key, r] of resolved) {
+    const { winner, conflict } = r;
+    const { provider, model } = winner.o;
+    const entry = catalogByKey.get(`${provider}/${model}`);
+    const map = FIELD_MAP[winner.field];
+    const currentValue = entry ? map.get(entry) : null;
+    const base = {
+      model: `${provider}/${model}`, field: winner.field,
+      currentValue, observedValue: winner.value,
+      source: { type: winner.o.source?.type, trust: winner.o.source?.trust, label: winner.o.source?.label },
+      confidence: winner.o.confidence ?? null, observedAt: winner.o.observedAt,
+      knownModel: Boolean(entry),
+    };
+    if (conflict) diffs.push({ ...base, status: 'conflict', conflict: { rival: { value: conflict.rival.value, source: conflict.rival.o.source, observedAt: conflict.rival.o.observedAt } } });
+    else if (currentValue === winner.value) diffs.push({ ...base, status: 'unchanged' });
+    else diffs.push({ ...base, status: 'proposed' });
+  }
+  return { version: 1, kind: 'catalog-diff', generatedAt: new Date().toISOString(), diffs };
+}
+
+/**
+ * Build a catalog proposal from a diff. Rules:
+ *  - 'unchanged' -> ignored listing
+ *  - 'conflict' -> recorded, never applied
+ *  - status removed/deprecated -> requires >=2 independent sources OR one high-trust
+ *    source; availability-only evidence never proposes 'removed'.
+ *  - unknown model + observed status/fields -> addition proposal
+ */
+export function buildCatalogProposal(catalog, diff, observations) {
+  const catalogByKey = new Map((catalog?.models ?? []).map(m => [`${m.provider}/${m.model}`, m]));
+  const proposal = { version: 1, kind: 'catalog-proposal', generatedAt: new Date().toISOString(), additions: [], updates: [], deprecations: [], conflicts: [], ignored: [], freshness: {} };
+
+  const sourcesFor = (modelKey, field) => new Set(observations
+    .filter(o => `${o.provider}/${o.model}` === modelKey && o.changes?.[field] !== null && o.changes?.[field] !== undefined)
+    .map(o => o.source?.label ?? o.source?.type ?? 'unknown'));
+
+  for (const d of diff.diffs) {
+    if (d.status === 'unchanged') { proposal.ignored.push({ model: d.model, field: d.field, reason: 'same value observed' }); continue; }
+    if (d.status === 'conflict') { proposal.conflicts.push(d); continue; }
+    const entry = catalogByKey.get(d.model);
+    if (d.field === 'availability') {
+      // Listing absence is evidence, never a verdict.
+      if (d.observedValue === 'unavailable') proposal.ignored.push({ model: d.model, field: d.field, reason: 'absent from a listing is not removal; recorded as evidence only' });
+      else proposal.ignored.push({ model: d.model, field: d.field, reason: 'availability confirmed; no catalog field' });
+      continue;
+    }
+    if (!entry) {
+      if (d.field === 'status' && d.observedValue === 'active') {
+        proposal.additions.push({ model: d.model, provider: d.model.split('/')[0], modelId: d.model.split('/').slice(1).join('/'), status: 'active', source: d.source, confidence: d.confidence, note: 'new model observed; fill scores/cost before adopting' });
+      } else {
+        proposal.ignored.push({ model: d.model, field: d.field, reason: 'model not in catalog; field-level update skipped until the model is added' });
+      }
+      continue;
+    }
+    if (d.field === 'status' && (d.observedValue === 'removed' || d.observedValue === 'deprecated')) {
+      const srcs = sourcesFor(d.model, 'status');
+      const highTrust = [...srcs].some(s => observations.some(o => `${o.provider}/${o.model}` === d.model && (o.source?.label ?? o.source?.type) === s && HIGH_TRUST.has(o.source?.trust)));
+      if (srcs.size < 2 && !highTrust) {
+        proposal.ignored.push({ model: d.model, field: d.field, observedValue: d.observedValue, reason: `${d.observedValue} requires >=2 independent sources or one high-trust source; got ${srcs.size}` });
+        continue;
+      }
+      proposal.deprecations.push({ model: d.model, field: d.field, currentValue: d.currentValue, observedValue: d.observedValue, source: d.source, evidenceSources: [...srcs], confidence: d.confidence });
+      continue;
+    }
+    proposal.updates.push({ model: d.model, field: FIELD_MAP[d.field]?.set ?? d.field, currentValue: d.currentValue, observedValue: d.observedValue, source: d.source, confidence: d.confidence });
+  }
+
+  const ats = observations.filter(o => o.observedAt).map(o => o.observedAt).sort();
+  proposal.freshness = { observations: observations.length, oldest: ats[0] ?? null, newest: ats[ats.length - 1] ?? null, diffs: diff.diffs.length, proposed: diff.diffs.filter(d => d.status === 'proposed').length };
+  return proposal;
+}
+
+/**
+ * Translate a catalog proposal into maintenance events for the hypothetical preview.
+ * A proposed field change is a *condition change* the engine should re-evaluate —
+ * free->paid becomes 'free-campaign-ended', status changes map directly, price/context
+ * changes become 'price-changed' so the backend is re-scored.
+ */
+export function proposalToEvents(proposal) {
+  const events = [];
+  const push = (type, model, note) => {
+    const [provider, ...rest] = model.split('/');
+    events.push({ type, provider, model: rest.join('/'), asOf: proposal.generatedAt, note });
+  };
+  for (const d of proposal.deprecations ?? []) push(d.observedValue === 'removed' ? 'removed' : 'deprecated', d.model, `observed ${d.observedValue}`);
+  for (const u of proposal.updates ?? []) {
+    if (u.field === 'cost.free' && u.observedValue === false) push('free-campaign-ended', u.model, 'free ended per observation');
+    else if (u.field === 'cost.free' && u.observedValue === true) push('free-campaign-started', u.model, 'free (re)started per observation');
+    else if (u.field === 'status' && (u.observedValue === 'deprecated' || u.observedValue === 'removed')) push(u.observedValue, u.model, 'observed status change');
+    else if (u.field?.startsWith('cost.') || u.field === 'contextK' || u.field === 'toolUse') push('price-changed', u.model, `${u.field} ${JSON.stringify(u.currentValue)} -> ${JSON.stringify(u.observedValue)}`);
+  }
+  for (const a of proposal.additions ?? []) push('model-added', a.model, 'new model observed');
+  return events;
+}
+
+/**
+ * Apply a proposal to a catalog in memory — the "hypothetical catalog" for the
+// maintenance preview. Returns a NEW catalog object; the input is not mutated.
+ */
+export function applyProposalToCatalog(catalog, proposal) {
+  const next = structuredClone(catalog);
+  const byKey = new Map(next.models.map(m => [`${m.provider}/${m.model}`, m]));
+  const setPath = (obj, path, value) => {
+    const parts = path.split('.');
+    let cur = obj;
+    for (let i = 0; i < parts.length - 1; i++) cur = cur[parts[i]] ??= {};
+    cur[parts[parts.length - 1]] = value;
+  };
+  for (const u of proposal.updates ?? []) {
+    const entry = byKey.get(u.model);
+    if (entry) setPath(entry, u.field, u.observedValue);
+  }
+  for (const d of proposal.deprecations ?? []) {
+    const entry = byKey.get(d.model);
+    if (entry) entry.status = d.observedValue;
+  }
+  for (const a of proposal.additions ?? []) {
+    const key = a.model;
+    if (!byKey.has(key)) {
+      const entry = { provider: a.provider, model: a.modelId, status: 'active', cost: null, contextK: null, vision: false, toolUse: 'basic', scores: { coding: null, reasoning: null, speed: null }, notes: 'added by catalog proposal; scores/cost unknown' };
+      next.models.push(entry);
+      byKey.set(key, entry);
+    }
+  }
+  next.updatedAt = proposal.generatedAt;
+  next._hypothetical = true;
+  return next;
+}

package/lib/observe/observation.mjs

@@ -0,0 +1,147 @@
+// Observation layer (Phase 4): normalized provider/model observations from external
+// sources, kept separate from the core maintenance engine. Sources produce
+// observations; this module validates, hashes, dedupes and resolves them — never
+// writes model-catalog.json, routing.json or anything outside out/.
+import { createHash } from 'node:crypto';
+import { readFileSync, appendFileSync, existsSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+
+export const SOURCE_TYPES = new Set(['manual', 'api', 'web', 'cli']);
+export const TRUST_LEVELS = ['manual_verified', 'provider_api', 'provider_local_cache', 'provider_web', 'pi_cli', 'third_party', 'unknown'];
+const TRUST_RANK = Object.fromEntries(TRUST_LEVELS.map((t, i) => [t, TRUST_LEVELS.length - i])); // manual_verified highest
+
+// Fields an observation may report. null = not observed (distinct from false/0).
+export const OBSERVED_FIELDS = ['availability', 'status', 'free', 'freeUntil', 'inputPricePer1M', 'outputPricePer1M', 'contextK', 'toolUse', 'vision'];
+const STATUS_VALUES = new Set(['active', 'free-campaign', 'deprecated', 'removed']);
+const AVAIL_VALUES = new Set(['available', 'unavailable', 'unknown']);
+const TOOL_VALUES = new Set(['good', 'basic', 'poor', 'none']);
+
+export function validateObservation(o) {
+  const errors = [];
+  if (!o || typeof o !== 'object' || Array.isArray(o)) return ['observation: must be an object'];
+  if (typeof o.provider !== 'string' || !o.provider.trim()) errors.push('observation.provider required');
+  if (typeof o.model !== 'string' || !o.model.trim()) errors.push('observation.model required');
+  if (typeof o.observedAt !== 'string' || Number.isNaN(Date.parse(o.observedAt))) errors.push('observation.observedAt must be an ISO timestamp');
+  const src = o.source;
+  if (!src || typeof src !== 'object') errors.push('observation.source required');
+  else {
+    if (!SOURCE_TYPES.has(src.type)) errors.push(`observation.source.type must be one of ${[...SOURCE_TYPES].join('|')}`);
+    if (src.trust !== undefined && !TRUST_LEVELS.includes(src.trust)) errors.push(`observation.source.trust must be one of ${TRUST_LEVELS.join('|')}`);
+  }
+  const ch = o.changes;
+  if (!ch || typeof ch !== 'object' || Array.isArray(ch)) errors.push('observation.changes must be an object');
+  else {
+    for (const k of Object.keys(ch)) if (!OBSERVED_FIELDS.includes(k)) errors.push(`observation.changes.${k} is not a known field`);
+    if (ch.status !== undefined && ch.status !== null && !STATUS_VALUES.has(ch.status)) errors.push('observation.changes.status invalid');
+    if (ch.availability !== undefined && ch.availability !== null && !AVAIL_VALUES.has(ch.availability)) errors.push('observation.changes.availability invalid');
+    if (ch.toolUse !== undefined && ch.toolUse !== null && !TOOL_VALUES.has(ch.toolUse)) errors.push('observation.changes.toolUse invalid');
+    for (const k of ['free', 'vision']) if (ch[k] !== undefined && ch[k] !== null && typeof ch[k] !== 'boolean') errors.push(`observation.changes.${k} must be boolean or null`);
+    for (const k of ['inputPricePer1M', 'outputPricePer1M', 'contextK']) if (ch[k] !== undefined && ch[k] !== null && (typeof ch[k] !== 'number' || ch[k] < 0)) errors.push(`observation.changes.${k} must be a number >= 0 or null`);
+    if (ch.freeUntil !== undefined && ch.freeUntil !== null && typeof ch.freeUntil !== 'string') errors.push('observation.changes.freeUntil must be a string or null');
+  }
+  if (o.confidence !== undefined && (typeof o.confidence !== 'number' || o.confidence < 0 || o.confidence > 1)) errors.push('observation.confidence must be 0-1');
+  if (o.evidence !== undefined && !Array.isArray(o.evidence)) errors.push('observation.evidence must be an array');
+  return errors;
+}
+
+/** Deterministic identity: provider+model+the exact observed change set. Same facts -> same hash. */
+export function observationHash(o) {
+  const stable = {
+    p: o.provider, m: o.model,
+    c: OBSERVED_FIELDS.filter(f => o.changes?.[f] !== undefined && o.changes[f] !== null).sort().map(f => [f, o.changes[f]]),
+  };
+  return createHash('sha256').update(JSON.stringify(stable)).digest('hex').slice(0, 16);
+}
+
+/** Fill defaults and stamp id/hash. Returns a new object. */
+export function normalizeObservation(o, { now } = {}) {
+  const n = structuredClone(o);
+  n.observedAt = n.observedAt ?? now ?? new Date().toISOString();
+  n.confidence = typeof n.confidence === 'number' ? n.confidence : 0.5;
+  n.evidence = Array.isArray(n.evidence) ? n.evidence : [];
+  n.source = { url: null, label: null, trust: 'unknown', ...n.source };
+  n.changes = Object.fromEntries(OBSERVED_FIELDS.map(f => [f, n.changes?.[f] ?? null]));
+  // Environment tag: 'production' (default) vs 'test'. Fixture/test observations are
+  // marked by their source adapter (sourceFixture) or an explicit environment; the
+  // production maintenance pipeline excludes them so a fixture replay can never
+  // contaminate the real catalog/routing evaluation.
+  n.environment = o.environment === 'test' || o.sourceFixture === true || n.source?.fixture === true ? 'test' : 'production';
+  n.sourceFixture = n.environment === 'test';
+  n.id = observationHash(n);
+  return n;
+}
+
+// ---------------------------------------------------------------------------
+// Observation store — append-only JSONL under out/ (gitignored). No credentials,
+// no prompt text: only normalized observations + validation results.
+// ---------------------------------------------------------------------------
+
+export function loadObservationStore(path) {
+  if (!existsSync(path)) return { path, observations: [] };
+  const observations = [];
+  for (const [i, line] of readFileSync(path, 'utf8').split(/\r?\n/).entries()) {
+    if (!line.trim()) continue;
+    try { observations.push(JSON.parse(line)); }
+    catch { observations.push({ _corrupt: true, line: i + 1 }); }
+  }
+  return { path, observations };
+}
+
+/**
+ * Production view: drop test/fixture observations so a fixture replay can never
+ * drive a real catalog/routing proposal. An observation is test when
+ * environment==='test', sourceFixture===true, or source.fixture===true. Anything
+ * without a marker is treated as production (back-compat with pre-tag records).
+ */
+export function productionObservations(observations) {
+  return (observations ?? []).filter(o =>
+    o._corrupt || (o.environment !== 'test' && o.sourceFixture !== true && o.source?.fixture !== true));
+}
+
+/**
+ * Ingest observations into the store. Dedupe rules:
+ *  - identical hash already stored -> 'duplicate'
+ *  - same key+fields with older timestamp than a stored observation -> 'stale'
+ *  - otherwise appended -> 'stored'
+ * Returns { stored, duplicates, stale, invalid }.
+ */
+export function ingestObservations(storePath, rawObservations, { now = () => new Date().toISOString() } = {}) {
+  const store = loadObservationStore(storePath);
+  const byHash = new Map(store.observations.filter(o => o.id).map(o => [o.id, o]));
+  // Newest observedAt per (provider/model, field). Staleness is field-aware: an
+  // observation reporting `free` is not stale just because a newer `availability`
+  // observation exists for the same model — different fields, different facts.
+  const latestByKey = new Map();
+  const fieldsOf = o => OBSERVED_FIELDS.filter(f => o.changes?.[f] !== null && o.changes?.[f] !== undefined);
+  for (const o of store.observations) {
+    if (!o.provider || !o.model || !o.observedAt) continue;
+    for (const f of fieldsOf(o)) {
+      const k = `${o.provider}/${o.model}::${f}`;
+      if (!latestByKey.has(k) || o.observedAt > latestByKey.get(k)) latestByKey.set(k, o.observedAt);
+    }
+  }
+  const result = { stored: [], duplicates: [], stale: [], invalid: [] };
+  mkdirSync(dirname(storePath), { recursive: true });
+  for (const raw of rawObservations) {
+    const errors = validateObservation(raw);
+    if (errors.length) { result.invalid.push({ observation: raw, errors }); continue; }
+    const o = normalizeObservation(raw, { now: now() });
+    if (byHash.has(o.id)) { result.duplicates.push(o); continue; }
+    // stale only if EVERY reported field is older than the newest stored value for it
+    const fields = fieldsOf(o);
+    const allStale = fields.length > 0 && fields.every(f => {
+      const latest = latestByKey.get(`${o.provider}/${o.model}::${f}`);
+      return latest && o.observedAt < latest;
+    });
+    if (allStale) { result.stale.push(o); continue; }
+    const record = { ...o, ingestedAt: now(), validation: 'ok' };
+    appendFileSync(storePath, JSON.stringify(record) + '\n');
+    byHash.set(o.id, record);
+    for (const f of fields) {
+      const k = `${o.provider}/${o.model}::${f}`;
+      if (!latestByKey.has(k) || o.observedAt > latestByKey.get(k)) latestByKey.set(k, o.observedAt);
+    }
+    result.stored.push(record);
+  }
+  return result;
+}

package/lib/observe/observers.mjs

@@ -0,0 +1,134 @@
+// Observer registry (Phase 4b): provider/source observers behind a common
+// interface so the CLI and the scheduled job iterate a requested list instead of
+// stacking per-source `if` branches. Adding a provider = adding one registry entry
+// here — no changes to observe-models.mjs or job.mjs.
+//
+// Observer interface:
+//   async run(context) -> { observations: [], probeFailed: bool, metadata: {} }
+//
+// context carries everything an observer may need (all optional):
+//   { catalog, outDir, now, observedAt, adapterDir, kit,
+//     listing, qoderListing, qoderCachePath, ...overrides }
+// `listing`/`qoderListing`/... are test seams: an explicit value overrides the
+// real probe, `false` simulates a failed probe. Provider specifics stay inside
+// the observer; the runner only sees normalized observations + a failure flag.
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { SOURCES, fromPiCli, fromQoderCache } from './sources.mjs';
+
+// ---------------------------------------------------------------------------
+// Observer registry. Each entry: { id, describe, run(context) }.
+// ---------------------------------------------------------------------------
+export const OBSERVERS = {
+  'pi-cli': {
+    id: 'pi-cli',
+    describe: 'pi --list-models availability (provider presence + model listing)',
+    async run(context) {
+      const { catalog, observedAt, listing } = context;
+      // listing === false simulates a failed probe (tests); undefined = real probe.
+      const lst = listing === false ? null
+        : listing ?? (await import('../../adapters/pi/lib/list-models.mjs')).fetchPiAvailability?.() ?? null;
+      if (!lst) {
+        return { observations: [], probeFailed: true, metadata: { reason: 'pi --list-models unavailable' } };
+      }
+      return { observations: fromPiCli(lst, { catalog, observedAt }), probeFailed: false, metadata: { source: lst.source ?? 'pi --list-models' } };
+    },
+  },
+
+  'qoder-cache': {
+    id: 'qoder-cache',
+    describe: 'qoder-models-cache.json priceFactor -> free/paid campaign state',
+    async run(context) {
+      const { catalog, outDir, observedAt, qoderListing, qoderCachePath } = context;
+      const statePath = join(outDir, 'qoder-observer-state.json');
+      let previous = null;
+      try { previous = existsSync(statePath) ? JSON.parse(readFileSync(statePath, 'utf8')).snapshot ?? null : null; } catch { previous = null; }
+      let r;
+      if (qoderListing === false) r = { state: 'missing', observations: [], snapshot: {} };
+      else if (qoderListing !== undefined) r = qoderListing;
+      else {
+        const { homedir } = await import('node:os');
+        r = fromQoderCache(qoderCachePath ?? join(homedir(), '.pi', 'agent', 'qoder-models-cache.json'), { previous, catalog, observedAt });
+      }
+      if (r.state !== 'ok') {
+        return { observations: [], probeFailed: true, metadata: { reason: `qoder cache probe: ${r.state}` } };
+      }
+      // Persist the new snapshot for next run's transition detection (out/ only).
+      if (outDir) {
+        try {
+          const { writeFileSync, mkdirSync } = await import('node:fs');
+          mkdirSync(outDir, { recursive: true });
+          writeFileSync(statePath, JSON.stringify({ version: 1, snapshot: r.snapshot, observedAt }, null, 2) + '\n');
+        } catch { /* state write must not break the observer */ }
+      }
+      return { observations: r.observations, probeFailed: false, metadata: { state: r.state, snapshot: r.snapshot, noTransition: r.observations.length === 0 } };
+    },
+  },
+};
+
+// ---------------------------------------------------------------------------
+// Requested-observer resolution + execution.
+// ---------------------------------------------------------------------------
+
+/** option property -> observer id. Back-compat aliases live here, not in the runner.
+ * Keys are the JS option names (checkPi / checkQoder); the CLI maps --check-pi ->
+ * checkPi etc. before calling. */
+export const OBSERVER_FLAG_MAP = {
+  checkPi: 'pi-cli',
+  checkQoder: 'qoder-cache',
+};
+
+/**
+ * Resolve the requested observer id list from job/CLI options.
+ *   options.check       — generic list: 'pi-cli,qoder-cache' or ['pi-cli', ...]
+ *   options.checkPi / options.checkQoder — legacy boolean flags (mapped)
+ *   options.observers   — explicit id list (programmatic)
+ * Returns { ids: string[], unknown: string[] }. Order is stable (first-seen),
+ * duplicates removed — the same observer never runs twice.
+ */
+export function resolveRequestedObservers(options = {}) {
+  const ids = [];
+  const seen = new Set();
+  const push = id => { if (!seen.has(id)) { seen.add(id); ids.push(id); } };
+
+  // legacy boolean flags -> ids (declaration order -> deterministic)
+  for (const [flag, id] of Object.entries(OBSERVER_FLAG_MAP)) {
+    if (options[flag] === true) push(id);
+  }
+  // generic --check list (string or array)
+  const generic = options.check;
+  const genericList = Array.isArray(generic) ? generic : (typeof generic === 'string' ? generic.split(',') : []);
+  for (const id of genericList) push(String(id).trim());
+  // explicit programmatic list
+  for (const id of options.observers ?? []) push(String(id).trim());
+
+  const unknown = ids.filter(id => !OBSERVERS[id]);
+  return { ids: ids.filter(id => OBSERVERS[id]), unknown };
+}
+
+/**
+ * Run each requested observer once, in order. One observer's failure is isolated:
+ * it sets probeFailed + records per-source metadata, but never stops the others.
+ * Returns { observations, probeFailed, results: [{id, probeFailed, count, metadata}] }.
+ * `results` is the audit trail of which source produced what / failed.
+ */
+export async function runObservers(requestedIds, context = {}) {
+  const observations = [];
+  const results = [];
+  let probeFailed = false;
+  for (const id of requestedIds) {
+    const observer = OBSERVERS[id];
+    if (!observer) { results.push({ id, probeFailed: true, count: 0, metadata: { reason: 'unknown observer' } }); probeFailed = true; continue; }
+    try {
+      const r = await observer.run(context);
+      const list = r.observations ?? [];
+      observations.push(...list);
+      if (r.probeFailed) probeFailed = true;
+      results.push({ id, probeFailed: r.probeFailed === true, count: list.length, metadata: r.metadata ?? {} });
+    } catch (e) {
+      probeFailed = true;
+      results.push({ id, probeFailed: true, count: 0, metadata: { reason: e.message } });
+    }
+  }
+  return { observations, probeFailed, results };
+}

package/lib/observe/sources.mjs

@@ -0,0 +1,154 @@
+// Observation source adapters (Phase 4). Each adapter turns a raw source into
+// normalized observations. Sources never write; they only emit observation objects.
+// Interface: (input) -> observation[]. A future HTTP/Web adapter plugs in here —
+// the pipeline only consumes normalized output.
+import { readFileSync } from 'node:fs';
+import { parseModelList } from '../../adapters/pi/lib/list-models.mjs';
+
+/**
+ * manual source — canonical fallback. Input is a JSON file (or parsed object) of
+ * { version:1, observations:[...] } or a bare array. Trust defaults to
+ * 'manual_verified' when not stated — a human wrote it on purpose.
+ */
+export function fromManual(input) {
+  const doc = typeof input === 'string' ? JSON.parse(readFileSync(input, 'utf8')) : input;
+  const list = Array.isArray(doc) ? doc : doc?.observations ?? [];
+  return list.map(o => ({
+    ...o,
+    source: { type: 'manual', trust: 'manual_verified', url: null, label: 'manual input', ...o.source, type: 'manual' },
+  }));
+}
+
+/**
+ * pi CLI source — `pi --list-models` text (or a parsed {models,providers} set).
+ * Emits availability observations ONLY for models already in the catalog — a model
+ * absent from the listing is NOT reported 'unavailable' here (that inference is the
+// differ's job, and it never asserts 'removed' from a listing alone).
+ */
+export function fromPiCli(listing, { catalog, observedAt } = {}) {
+  const parsed = typeof listing === 'string' ? parseModelList(listing) : listing;
+  const out = [];
+  for (const m of catalog?.models ?? []) {
+    const key = `${m.provider}/${m.model}`;
+    if (!parsed.providers?.has(m.provider)) continue; // provider unseen -> no observation (unknown, not gone)
+    out.push({
+      provider: m.provider, model: m.model,
+      observedAt,
+      source: { type: 'cli', trust: 'pi_cli', label: 'pi --list-models' },
+      changes: { availability: parsed.models.has(key) ? 'available' : 'unavailable' },
+      confidence: 0.9,
+      evidence: ['pi --list-models'],
+    });
+  }
+  return out;
+}
+
+/**
+ * static/web fixture source — replays provider announcements from a JSON document
+ * without real scraping. Accepts the observation shape directly, or a compact
+ * announcement form:
+ *   { "announcements": [{ "provider":"qoder", "model":"Qwen3.8-Flash",
+ *       "type":"price-changed|free-campaign-started|free-campaign-ended|deprecated|context-changed|model-added",
+ *       "asOf":"...", "inputPricePer1M":0.3, "outputPricePer1M":1.2, "contextK":256,
+ *       "url":"...", "label":"..." }] }
+ * A real HTTP source would implement the same emitter against fetched pages.
+ */
+export function fromFixture(input, { defaultTrust = 'provider_web' } = {}) {
+  const doc = typeof input === 'string' ? JSON.parse(readFileSync(input, 'utf8')) : input;
+  const list = doc?.observations ?? [];
+  const announcements = doc?.announcements ?? [];
+  // Fixture observations are TEST data — mark them so production maintenance excludes them.
+  const tag = { environment: 'test', sourceFixture: true };
+  const out = list.map(o => ({ ...o, ...tag, source: { type: 'web', trust: defaultTrust, fixture: true, ...o.source } }));
+  for (const a of announcements) {
+    const changes = {};
+    switch (a.type) {
+      case 'price-changed':
+        if (a.inputPricePer1M !== undefined) changes.inputPricePer1M = a.inputPricePer1M;
+        if (a.outputPricePer1M !== undefined) changes.outputPricePer1M = a.outputPricePer1M;
+        break;
+      case 'free-campaign-started': changes.free = true; if (a.freeUntil) changes.freeUntil = a.freeUntil; break;
+      case 'free-campaign-ended': changes.free = false; changes.freeUntil = null; break;
+      case 'deprecated': changes.status = 'deprecated'; break;
+      case 'removed': changes.status = 'removed'; break;
+      case 'context-changed': if (a.contextK !== undefined) changes.contextK = a.contextK; break;
+      case 'model-added': changes.status = 'active'; if (a.contextK !== undefined) changes.contextK = a.contextK; if (a.vision !== undefined) changes.vision = a.vision; break;
+      default: break;
+    }
+    out.push({
+      provider: a.provider, model: a.model, observedAt: a.asOf, ...tag,
+      source: { type: 'web', trust: a.trust ?? defaultTrust, url: a.url ?? null, label: a.label ?? a.type, fixture: true },
+      changes, confidence: a.confidence ?? 0.8,
+      evidence: [a.url ?? a.label ?? a.type].filter(Boolean),
+    });
+  }
+  return out;
+}
+
+/**
+ * Qoder provider-metadata source — reads the pi-maintained local cache
+ * `~/.pi/agent/qoder-models-cache.json` (schema v2: `{version, updatedAt, models:[{id, priceFactor, ...}]}`).
+ * Authority is the OBSERVED schema, not an assumption: `priceFactor` is the provider's
+ * subscription cost multiplier; 0 means "currently free". A missing file, unparseable
+ * JSON, absent entry, or absent/invalid priceFactor yields NO observation (unknown),
+ * never a guessed "not free".
+ *
+ * Transition detection vs the previous snapshot (a state file the caller owns):
+ *   0 -> non-zero : free-campaign ended   -> observation {free:false}
+ *   non-zero -> 0 : free-campaign started -> observation {free:true}
+ *   same          : no observation
+ * First observation (no previous snapshot) emits the current free state so the
+ * catalog can be reconciled; a null/unknown previous emits nothing (cannot diff).
+ *
+ * @param {string|object} input cache file path or parsed cache doc
+ * @param {object} opts.models   [{provider, model}] to observe (default: all catalog qoder entries via opts.catalog)
+ * @param {object} opts.previous { "<id>": <priceFactor|null> } prior snapshot, or null
+ * @param {string} opts.observedAt ISO timestamp (defaults to cache updatedAt or now)
+ * @returns {{observations: object[], snapshot: object, state: string}}
+ *   observations:normalized obs; snapshot: new {id:priceFactor} to persist; state:
+ *   'ok'|'missing'|'malformed'|'no-schema'|'entry-missing'|'priceFactor-missing'
+ */
+export function fromQoderCache(input, { models = null, previous = undefined, observedAt = null, catalog = null } = {}) {
+  const out = { observations: [], snapshot: {}, state: 'ok' };
+  let doc;
+  if (typeof input === 'string') {
+    let raw;
+    try { raw = readFileSync(input, 'utf8'); } catch { out.state = 'missing'; return out; }
+    try { doc = JSON.parse(raw); } catch { out.state = 'malformed'; return out; }
+  } else doc = input;
+  const entries = Array.isArray(doc?.models) ? doc.models : null;
+  if (!entries) { out.state = 'no-schema'; return out; }
+  const byId = new Map(entries.filter(e => e && typeof e.id === 'string').map(e => [e.id, e]));
+  // observedAt is the read time (now), not the cache's updatedAt — the observation
+  // records "we saw this state now"; using updatedAt would stale it against fresher
+  // same-model observations from other sources in a combined run.
+  const at = observedAt ?? new Date().toISOString();
+  const want = models ?? (catalog?.models ?? []).filter(m => m.provider === 'qoder').map(m => ({ provider: m.provider, model: m.model }));
+  for (const w of want) {
+    const e = byId.get(w.model);
+    if (!e) { out.state = out.state === 'ok' ? 'entry-missing' : out.state; continue; }
+    const pf = e.priceFactor;
+    const valid = typeof pf === 'number' && Number.isFinite(pf) && pf >= 0;
+    out.snapshot[w.model] = valid ? pf : null;
+    if (!valid) { out.state = out.state === 'ok' ? 'priceFactor-missing' : out.state; continue; } // unknown, not "not free"
+    const prev = previous?.[w.model];
+    const free = pf === 0;
+    const prevFree = prev === 0;
+    const prevKnown = typeof prev === 'number' && Number.isFinite(prev);
+    // Emit on transition, or on first sighting (prev unknown/absent) so the store
+    // holds a baseline. Same value -> nothing.
+    const transitioned = prevKnown && prevFree !== free;
+    const firstSighting = previous === undefined || previous === null || prev === undefined;
+    if (!transitioned && !firstSighting) continue;
+    out.observations.push({
+      provider: w.provider, model: w.model, observedAt: at,
+      source: { type: 'cli', trust: 'provider_local_cache', label: 'qoder-models-cache.json' },
+      changes: { free }, confidence: 0.85,
+      evidence: [`qoder-models-cache.json priceFactor=${pf}${prevKnown ? ` (was ${prev})` : ''}`],
+    });
+  }
+  return out;
+}
+
+/** Registry of source adapters by --source name. */
+export const SOURCES = { manual: fromManual, 'pi-cli': fromPiCli, fixture: fromFixture, 'qoder-cache': fromQoderCache };