@nforma.ai/nforma 0.2.1

package/bin/observe-config.cjs

@@ -0,0 +1,353 @@
+/**
+ * Observe config loader
+ * Loads source configuration from .planning/observe-sources.md (or triage-sources.md fallback)
+ * Parses YAML frontmatter, infers issue_type, applies defaults, validates required fields
+ */
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+// Source types that default to "issue"
+const ISSUE_TYPES = ['github', 'sentry', 'sentry-feedback', 'bash'];
+// Source types that default to "drift"
+const DRIFT_TYPES = ['prometheus', 'grafana', 'logstash'];
+
+/**
+ * Parse a YAML value string into appropriate JS type
+ * @param {string} val - Value string
+ * @returns {*} Parsed value
+ */
+function parseYamlValue(val) {
+  if (val === undefined || val === null || val === '') return '';
+
+  // Remove inline comments (but not inside quotes)
+  if (!val.startsWith('"') && !val.startsWith("'")) {
+    val = val.replace(/\s+#.*$/, '').trim();
+  }
+
+  // Boolean
+  if (val === 'true') return true;
+  if (val === 'false') return false;
+
+  // Null
+  if (val === 'null' || val === '~') return null;
+
+  // Number
+  if (/^-?\d+(\.\d+)?$/.test(val)) return Number(val);
+
+  // Inline array [a, b, c]
+  if (val.startsWith('[') && val.endsWith(']')) {
+    const inner = val.slice(1, -1).trim();
+    if (!inner) return [];
+    return inner.split(',').map(s => parseYamlValue(s.trim()));
+  }
+
+  // Quoted string
+  if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
+    return val.slice(1, -1);
+  }
+
+  return val;
+}
+
+/**
+ * Minimal YAML parser for observe config frontmatter
+ * Handles: key-value pairs, nested objects, arrays of objects (- key: val), inline arrays
+ *
+ * @param {string} yamlStr - YAML string to parse
+ * @returns {object} Parsed object
+ */
+function parseSimpleYaml(yamlStr) {
+  const lines = yamlStr.split('\n');
+
+  // Pre-process: strip comments and track indent + content
+  const entries = [];
+  for (const line of lines) {
+    // Preserve empty lines for structure but skip pure comment lines
+    const stripped = line.replace(/#.*$/, '');
+    const trimmed = stripped.trim();
+    if (!trimmed) continue;
+    const indent = stripped.search(/\S/);
+    entries.push({ indent, raw: trimmed });
+  }
+
+  return parseBlock(entries, 0, -1).result;
+}
+
+/**
+ * Parse a block of YAML entries starting at index `start` with parent indent `parentIndent`.
+ * Returns { result, nextIndex }
+ */
+function parseBlock(entries, start, parentIndent) {
+  const result = {};
+  let i = start;
+
+  while (i < entries.length) {
+    const entry = entries[i];
+
+    // If this line is at or before parent indent, we're done with this block
+    if (entry.indent <= parentIndent) break;
+
+    const raw = entry.raw;
+
+    // Array item: "- key: val" or "- val"
+    if (raw.startsWith('- ')) {
+      // This shouldn't happen at top level without a key — skip
+      i++;
+      continue;
+    }
+
+    // Key: value
+    const colonIdx = raw.indexOf(':');
+    if (colonIdx === -1) {
+      i++;
+      continue;
+    }
+
+    const key = raw.slice(0, colonIdx).trim();
+    const valStr = raw.slice(colonIdx + 1).trim();
+
+    if (valStr === '' || valStr === undefined) {
+      // Check what follows: nested object, array of objects, or empty
+      const nextIdx = i + 1;
+      if (nextIdx < entries.length && entries[nextIdx].indent > entry.indent) {
+        // Check if it's an array (next line starts with "- ")
+        if (entries[nextIdx].raw.startsWith('- ')) {
+          const arrResult = parseArray(entries, nextIdx, entry.indent);
+          result[key] = arrResult.items;
+          i = arrResult.nextIndex;
+        } else {
+          // Nested object
+          const blockResult = parseBlock(entries, nextIdx, entry.indent);
+          result[key] = blockResult.result;
+          i = blockResult.nextIndex;
+        }
+      } else {
+        result[key] = {};
+        i++;
+      }
+    } else {
+      result[key] = parseYamlValue(valStr);
+      i++;
+    }
+  }
+
+  return { result, nextIndex: i };
+}
+
+/**
+ * Parse an array block starting at index `start`.
+ * Each "- key: val" starts a new object; subsequent indented lines add to it.
+ */
+function parseArray(entries, start, parentIndent) {
+  const items = [];
+  let i = start;
+
+  while (i < entries.length) {
+    const entry = entries[i];
+
+    // If we've dedented back to parent level or less, we're done
+    if (entry.indent <= parentIndent) break;
+
+    if (entry.raw.startsWith('- ')) {
+      // Start of a new array item
+      const content = entry.raw.slice(2).trim();
+      const arrayItemIndent = entry.indent;
+
+      if (content.includes(':')) {
+        // Object item
+        const item = {};
+        const colonIdx = content.indexOf(':');
+        const k = content.slice(0, colonIdx).trim();
+        const v = content.slice(colonIdx + 1).trim();
+
+        if (v === '') {
+          // Nested value under this array item key
+          const nextIdx = i + 1;
+          if (nextIdx < entries.length && entries[nextIdx].indent > arrayItemIndent) {
+            if (entries[nextIdx].raw.startsWith('- ')) {
+              const nestedArr = parseArray(entries, nextIdx, arrayItemIndent);
+              item[k] = nestedArr.items;
+              i = nestedArr.nextIndex;
+            } else {
+              const nestedBlock = parseBlock(entries, nextIdx, arrayItemIndent);
+              item[k] = nestedBlock.result;
+              i = nestedBlock.nextIndex;
+            }
+          } else {
+            item[k] = {};
+            i++;
+          }
+        } else {
+          item[k] = parseYamlValue(v);
+          i++;
+        }
+
+        // Collect additional key:val pairs for this array item (indented deeper than "- ")
+        while (i < entries.length && entries[i].indent > arrayItemIndent && !entries[i].raw.startsWith('- ')) {
+          const subEntry = entries[i];
+          const subColonIdx = subEntry.raw.indexOf(':');
+          if (subColonIdx !== -1) {
+            const sk = subEntry.raw.slice(0, subColonIdx).trim();
+            const sv = subEntry.raw.slice(subColonIdx + 1).trim();
+
+            if (sv === '') {
+              // Nested block under this key
+              const nextIdx = i + 1;
+              if (nextIdx < entries.length && entries[nextIdx].indent > subEntry.indent) {
+                if (entries[nextIdx].raw.startsWith('- ')) {
+                  const nestedArr = parseArray(entries, nextIdx, subEntry.indent);
+                  item[sk] = nestedArr.items;
+                  i = nestedArr.nextIndex;
+                } else {
+                  const nestedBlock = parseBlock(entries, nextIdx, subEntry.indent);
+                  item[sk] = nestedBlock.result;
+                  i = nestedBlock.nextIndex;
+                }
+              } else {
+                item[sk] = {};
+                i++;
+              }
+            } else {
+              item[sk] = parseYamlValue(sv);
+              i++;
+            }
+          } else {
+            i++;
+          }
+        }
+
+        items.push(item);
+      } else {
+        // Simple value array item
+        items.push(parseYamlValue(content));
+        i++;
+      }
+    } else {
+      // Not an array item, done with this array
+      break;
+    }
+  }
+
+  return { items, nextIndex: i };
+}
+
+/**
+ * Load observe configuration from YAML frontmatter file
+ *
+ * @param {string} [configPath] - Optional explicit config file path
+ * @param {string} [basePath] - Base directory (default: process.cwd())
+ * @returns {object} { sources, configFile, observeConfig, error? }
+ */
+function loadObserveConfig(configPath, basePath) {
+  const base = basePath || process.cwd();
+
+  // Resolve config file: explicit path > observe-sources.md > triage-sources.md
+  let configFile = null;
+  if (configPath) {
+    const resolved = path.resolve(base, configPath);
+    if (fs.existsSync(resolved)) {
+      configFile = resolved;
+    }
+  }
+
+  if (!configFile) {
+    const observePath = path.resolve(base, '.planning/observe-sources.md');
+    if (fs.existsSync(observePath)) {
+      configFile = observePath;
+    }
+  }
+
+  if (!configFile) {
+    const triagePath = path.resolve(base, '.planning/triage-sources.md');
+    if (fs.existsSync(triagePath)) {
+      configFile = triagePath;
+    }
+  }
+
+  if (!configFile) {
+    return {
+      sources: [],
+      configFile: null,
+      observeConfig: {},
+      error: 'No observe sources configured'
+    };
+  }
+
+  // Read and parse frontmatter
+  const content = fs.readFileSync(configFile, 'utf8');
+  const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
+  if (!fmMatch) {
+    return {
+      sources: [],
+      configFile,
+      observeConfig: {},
+      error: 'No YAML frontmatter found in config file'
+    };
+  }
+
+  const frontmatter = parseSimpleYaml(fmMatch[1]);
+
+  // Extract observe_config global settings
+  const observeConfig = frontmatter.observe_config || {};
+  const defaultTimeout = observeConfig.default_timeout ?? 10;
+  const failOpenDefault = observeConfig.fail_open_default ?? true;
+
+  // Extract sources array
+  let sources = [];
+  if (Array.isArray(frontmatter.sources)) {
+    sources = frontmatter.sources;
+  } else if (observeConfig && Array.isArray(observeConfig.sources)) {
+    sources = observeConfig.sources;
+  }
+
+  // Validate and apply defaults
+  const validationErrors = [];
+  sources = sources.map((source, idx) => {
+    const errors = [];
+
+    // Required fields
+    if (!source.type || typeof source.type !== 'string') {
+      errors.push(`sources[${idx}]: type required (string)`);
+    }
+    if (!source.label || typeof source.label !== 'string') {
+      errors.push(`sources[${idx}]: label required (string)`);
+    }
+
+    if (errors.length > 0) {
+      validationErrors.push(...errors);
+    }
+
+    // Infer issue_type if not specified
+    if (!source.issue_type) {
+      if (ISSUE_TYPES.includes(source.type)) {
+        source.issue_type = 'issue';
+      } else if (DRIFT_TYPES.includes(source.type)) {
+        source.issue_type = 'drift';
+      }
+    }
+
+    // Apply defaults
+    source.timeout = source.timeout ?? defaultTimeout;
+    source.fail_open = source.fail_open ?? failOpenDefault;
+
+    return source;
+  });
+
+  const result = {
+    sources,
+    configFile,
+    observeConfig: {
+      default_timeout: defaultTimeout,
+      fail_open_default: failOpenDefault
+    }
+  };
+
+  if (validationErrors.length > 0) {
+    result.error = validationErrors.join('; ');
+  }
+
+  return result;
+}
+
+module.exports = { loadObserveConfig, parseSimpleYaml, parseYamlValue };

package/bin/observe-debt-writer.cjs

@@ -0,0 +1,140 @@
+/**
+ * Debt ledger write-through for /qgsd:observe
+ * Upserts observations to .planning/formal/debt.json by fingerprint using v0.27-01 functions
+ * Then runs dedup engine and formal reference linker (v0.27-03)
+ *
+ * CRITICAL: Never compute fingerprints inline — always use imported v0.27-01 functions
+ */
+
+const crypto = require('node:crypto');
+const path = require('node:path');
+const { readDebtLedger, writeDebtLedger } = require('./debt-ledger.cjs');
+const { fingerprintIssue } = require('./fingerprint-issue.cjs');
+const { fingerprintDrift } = require('./fingerprint-drift.cjs');
+const { validateDebtEntry } = require('./validate-debt-entry.cjs');
+const { deduplicateEntries } = require('./debt-dedup.cjs');
+const { linkFormalRefs } = require('./formal-ref-linker.cjs');
+
+/**
+ * Write observations from observe command to debt ledger, then dedup and link formal refs.
+ * Pipeline: write/upsert -> dedup (fingerprint + Levenshtein) -> formal-ref link -> save
+ *
+ * @param {object[]} observations - Array of issue/drift objects from handlers (standard schema)
+ * @param {string} [ledgerPath] - Path to debt.json (default: .planning/formal/debt.json)
+ * @param {object} [options] - Options for dedup and linking
+ * @param {number} [options.threshold=0.85] - Levenshtein similarity threshold
+ * @param {boolean} [options.verbose=false] - Include detailed merge/link logs
+ * @param {string} [options.requirementsPath] - Custom requirements.json path
+ * @param {string} [options.specDir] - Custom spec directory path
+ * @returns {object} { written, updated, errors, merged, linked, mergeLog?, linkLog? }
+ */
+function writeObservationsToDebt(observations, ledgerPath, options = {}) {
+  const resolvedPath = ledgerPath || path.resolve(process.cwd(), '.planning/formal/debt.json');
+  const ledger = readDebtLedger(resolvedPath);
+  const now = new Date().toISOString();
+
+  let written = 0;
+  let updated = 0;
+  let errors = 0;
+
+  for (const obs of observations) {
+    try {
+      // Compute fingerprint using v0.27-01 functions (NEVER inline)
+      let fp;
+      if (obs.issue_type === 'drift') {
+        fp = fingerprintDrift({
+          formal_parameter_key: obs.formal_parameter_key || obs.title
+        });
+      } else {
+        fp = fingerprintIssue({
+          exception_type: obs.exception_type || obs.source_type || 'unknown',
+          function_name: obs.function_name || 'unknown',
+          message: obs.title || ''
+        });
+      }
+
+      // Search for existing entry by fingerprint
+      const existingIdx = ledger.debt_entries.findIndex(e => e.fingerprint === fp);
+
+      if (existingIdx >= 0) {
+        // Update existing entry
+        const existing = ledger.debt_entries[existingIdx];
+        existing.occurrences = (existing.occurrences || 1) + 1;
+        existing.last_seen = now;
+        existing.source_entries.push({
+          source_type: obs.source_type || 'unknown',
+          source_id: obs.id || `obs-${Date.now()}`,
+          observed_at: obs.created_at || now
+        });
+        updated++;
+      } else {
+        // Create new debt entry
+        const entry = {
+          id: crypto.randomUUID(),
+          fingerprint: fp,
+          title: (obs.title || 'Unknown observation').slice(0, 256),
+          occurrences: 1,
+          first_seen: now,
+          last_seen: now,
+          environments: ['production'],
+          status: 'open',
+          formal_ref: obs.formal_parameter_key || null,
+          source_entries: [{
+            source_type: obs.source_type || 'unknown',
+            source_id: obs.id || `obs-${Date.now()}`,
+            observed_at: obs.created_at || now
+          }]
+        };
+
+        // Validate before adding
+        const validation = validateDebtEntry(entry);
+        if (validation !== true) {
+          console.warn(`[observe-debt-writer] Skipping invalid entry: ${validation.join('; ')}`);
+          errors++;
+          continue;
+        }
+
+        ledger.debt_entries.push(entry);
+        written++;
+      }
+    } catch (err) {
+      console.warn(`[observe-debt-writer] Error processing observation: ${err.message}`);
+      errors++;
+    }
+  }
+
+  // Phase 2: Dedup (fingerprint exact-match + Levenshtein near-duplicate)
+  const dedupResult = deduplicateEntries(ledger.debt_entries, {
+    threshold: options.threshold ?? 0.85
+  });
+  ledger.debt_entries = dedupResult.entries;
+
+  // Phase 3: Formal reference linking
+  const linkResult = linkFormalRefs(ledger.debt_entries, {
+    requirementsPath: options.requirementsPath,
+    specDir: options.specDir
+  });
+  ledger.debt_entries = linkResult.entries;
+
+  // Write updated ledger only if changes occurred
+  if (written > 0 || updated > 0 || dedupResult.mergeCount > 0 || linkResult.linkedCount > 0) {
+    writeDebtLedger(resolvedPath, ledger);
+  }
+
+  const result = {
+    written,
+    updated,
+    errors,
+    merged: dedupResult.mergeCount,
+    linked: linkResult.linkedCount
+  };
+
+  if (options.verbose) {
+    result.mergeLog = dedupResult.mergeLog;
+    result.linkLog = linkResult.linkLog;
+  }
+
+  return result;
+}
+
+module.exports = { writeObservationsToDebt };

package/bin/observe-handler-grafana.cjs

@@ -0,0 +1,128 @@
+/**
+ * Grafana source handler for /qgsd:observe
+ * Fetches alert rules from Grafana unified alerting API
+ * Returns standard issue schema for the observe registry
+ */
+
+/**
+ * Format age from ISO date to human-readable string
+ * @param {string} isoDate - ISO8601 date string
+ * @returns {string} Human-readable age
+ */
+function formatAge(isoDate) {
+  if (!isoDate) return 'unknown';
+  const diffMs = Date.now() - new Date(isoDate).getTime();
+  if (diffMs < 0) return 'future';
+  const minutes = Math.floor(diffMs / 60000);
+  if (minutes < 60) return `${minutes}m`;
+  const hours = Math.floor(minutes / 60);
+  if (hours < 24) return `${hours}h`;
+  const days = Math.floor(hours / 24);
+  return `${days}d`;
+}
+
+/**
+ * Map Grafana alert state to severity
+ * @param {string} state - Grafana alert state
+ * @returns {string} Severity level
+ */
+function mapStateSeverity(state) {
+  const mapping = {
+    alerting: 'error',
+    firing: 'error',
+    pending: 'warning',
+    nodata: 'warning',
+    normal: 'info',
+    ok: 'info',
+    paused: 'info'
+  };
+  return mapping[state] || 'info';
+}
+
+/**
+ * Grafana source handler
+ * Fetches alert rules from Grafana unified alerting API and maps to standard schema
+ *
+ * @param {object} sourceConfig - { type, label, endpoint, auth_env?, issue_type? }
+ * @param {object} options - { fetchFn? }
+ * @returns {Promise<object>} Standard schema result
+ */
+async function handleGrafana(sourceConfig, options) {
+  const label = sourceConfig.label || 'Grafana';
+  const endpoint = (sourceConfig.endpoint || '').replace(/\/$/, '');
+  const fetchFn = (options && options.fetchFn) || globalThis.fetch;
+
+  try {
+    // Build auth headers
+    const headers = {};
+    if (sourceConfig.auth_env) {
+      const token = process.env[sourceConfig.auth_env];
+      if (token) {
+        headers['Authorization'] = `Bearer ${token}`;
+      }
+    }
+
+    const url = `${endpoint}/api/v1/provisioning/alert-rules`;
+    const response = await fetchFn(url, { headers });
+
+    if (!response.ok) {
+      return {
+        source_label: label,
+        source_type: 'grafana',
+        status: 'error',
+        error: `HTTP ${response.status} from Grafana`,
+        issues: []
+      };
+    }
+
+    const rules = await response.json();
+    const ruleList = Array.isArray(rules) ? rules : [];
+
+    const issues = ruleList.map((rule, idx) => {
+      const labels = rule.labels || {};
+      const annotations = rule.annotations || {};
+      const state = labels.grafana_state || '';
+      const severity = mapStateSeverity(state);
+
+      // Build URL from dashboardUid if available
+      const ruleUrl = rule.dashboardUid
+        ? `${endpoint}/d/${rule.dashboardUid}`
+        : endpoint;
+
+      // Build meta from annotations and context
+      const metaParts = [];
+      if (annotations.summary) metaParts.push(annotations.summary);
+      if (rule.ruleGroup) metaParts.push(`group: ${rule.ruleGroup}`);
+      if (rule.folderUID) metaParts.push(`folder: ${rule.folderUID}`);
+
+      return {
+        id: `grafana-alert-${rule.id || idx}`,
+        title: rule.title || `alert-rule-${idx}`,
+        severity,
+        url: ruleUrl,
+        age: formatAge(rule.updated),
+        created_at: rule.updated || new Date().toISOString(),
+        meta: metaParts.join(' | '),
+        source_type: 'grafana',
+        issue_type: sourceConfig.issue_type || 'drift'
+      };
+    });
+
+    return {
+      source_label: label,
+      source_type: 'grafana',
+      status: 'ok',
+      issues
+    };
+  } catch (err) {
+    return {
+      source_label: label,
+      source_type: 'grafana',
+      status: 'error',
+      error: `Grafana fetch failed: ${err.message}`,
+      issues: []
+    };
+  }
+}
+
+module.exports = { handleGrafana };