@nebulacomponents/citable 1.18.1 → 1.19.0

package/src/shared/domainUtils.js

@@ -0,0 +1,140 @@
+/**
+ * Canonical Domain & Public Suffix Extraction Utility
+ *
+ * Implements bounded registrable domain extraction and authority domain validation
+ * without naive substring matching.
+ */
+
+// Common multi-part public suffixes across gTLDs and ccTLDs
+const MULTI_PART_SUFFIXES = new Set([
+  // UK
+  'co.uk', 'org.uk', 'gov.uk', 'ac.uk', 'ltd.uk', 'plc.uk', 'me.uk', 'net.uk', 'sch.uk',
+  // Australia
+  'com.au', 'net.au', 'org.au', 'edu.au', 'gov.au', 'asn.au', 'id.au',
+  // New Zealand
+  'co.nz', 'org.nz', 'net.nz', 'govt.nz', 'ac.nz', 'school.nz', 'geek.nz',
+  // Japan
+  'co.jp', 'ne.jp', 'or.jp', 'ac.jp', 'go.jp', 'ed.jp', 'gr.jp',
+  // Brazil
+  'com.br', 'org.br', 'gov.br', 'edu.br', 'net.br',
+  // Mexico
+  'com.mx', 'org.mx', 'gob.mx', 'edu.mx', 'net.mx',
+  // India
+  'co.in', 'net.in', 'org.in', 'gen.in', 'firm.in', 'ind.in', 'ac.in', 'edu.in', 'res.in', 'gov.in',
+  // Singapore
+  'com.sg', 'org.sg', 'edu.sg', 'gov.sg', 'net.sg',
+  // Hong Kong
+  'com.hk', 'org.hk', 'edu.hk', 'gov.hk', 'net.hk',
+  // South Africa
+  'co.za', 'org.za', 'gov.za', 'ac.za', 'net.za',
+  // Turkey
+  'com.tr', 'org.tr', 'gov.tr', 'edu.tr', 'net.tr',
+  // South Korea
+  'co.kr', 'ne.kr', 're.kr', 'or.kr', 'go.kr', 'ac.kr',
+  // Canada
+  'gc.ca', 'qc.ca', 'on.ca', 'bc.ca', 'ab.ca',
+  // France / Germany / Israel / etc.
+  'tm.fr', 'asso.fr', 'co.il', 'org.il', 'gov.il', 'muni.il', 'ac.il',
+  // Commercial second-level
+  'us.com', 'eu.com', 'uk.com', 'de.com',
+  // Developer platforms (treated as public suffixes for site identification)
+  'github.io', 'gitlab.io', 'pages.dev', 'vercel.app', 'webflow.io',
+]);
+
+const AUTHORITATIVE_EXACT_DOMAINS = new Set([
+  'w3.org',
+  'rfc-editor.org',
+  'iana.org',
+  'ietf.org',
+  'iso.org',
+  'nist.gov',
+  'arxiv.org',
+  'developer.mozilla.org',
+  'web.dev',
+  'github.com',
+]);
+
+export function extractHostname(urlString) {
+  if (!urlString || typeof urlString !== 'string') return null;
+  const trimmed = urlString.trim();
+  try {
+    const url = new URL(trimmed.includes('://') ? trimmed : `https://${trimmed}`);
+    return url.hostname.toLowerCase();
+  } catch {
+    return null;
+  }
+}
+
+export function extractRegistrableDomain(hostnameOrUrl) {
+  const host = extractHostname(hostnameOrUrl);
+  if (!host) return null;
+
+  // Split into dot-separated parts
+  const parts = host.split('.');
+  if (parts.length <= 1) return host;
+
+  // Check 2-part suffix (e.g. co.uk, com.au)
+  if (parts.length >= 3) {
+    const twoPartSuffix = parts.slice(-2).join('.');
+    if (MULTI_PART_SUFFIXES.has(twoPartSuffix)) {
+      return parts.slice(-3).join('.');
+    }
+  }
+
+  // Check 3-part suffix if any exist
+  if (parts.length >= 4) {
+    const threePartSuffix = parts.slice(-3).join('.');
+    if (MULTI_PART_SUFFIXES.has(threePartSuffix)) {
+      return parts.slice(-4).join('.');
+    }
+  }
+
+  // Default: last 2 parts (e.g. example.com)
+  return parts.slice(-2).join('.');
+}
+
+export function extractPublicSuffix(hostnameOrUrl) {
+  const host = extractHostname(hostnameOrUrl);
+  if (!host) return null;
+
+  const parts = host.split('.');
+  if (parts.length <= 1) return null;
+
+  if (parts.length >= 3) {
+    const twoPart = parts.slice(-2).join('.');
+    if (MULTI_PART_SUFFIXES.has(twoPart)) return twoPart;
+  }
+
+  return parts.at(-1) || null;
+}
+
+export function isAuthoritativeDomain(urlOrHost) {
+  const host = extractHostname(urlOrHost);
+  if (!host) return false;
+
+  const regDomain = extractRegistrableDomain(host);
+
+  // Exact match to known authority domain or its subdomains
+  if (AUTHORITATIVE_EXACT_DOMAINS.has(host) || (regDomain && AUTHORITATIVE_EXACT_DOMAINS.has(regDomain))) {
+    return true;
+  }
+
+  // Check government TLD / suffix: .gov, .gov.uk, .gov.au, etc.
+  if (host.endsWith('.gov') || host.endsWith('.mil') || (regDomain && (regDomain.endsWith('.gov') || regDomain.endsWith('.mil')))) {
+    return true;
+  }
+  const suffix = extractPublicSuffix(host);
+  if (suffix && (suffix.startsWith('gov.') || suffix.endsWith('.gov') || suffix.startsWith('gob.'))) {
+    return true;
+  }
+
+  // Check academic / educational TLD: .edu, .edu.au, .ac.uk, etc.
+  if (host.endsWith('.edu') || (regDomain && regDomain.endsWith('.edu'))) {
+    return true;
+  }
+  if (suffix && (suffix.startsWith('edu.') || suffix.startsWith('ac.') || suffix.endsWith('.edu'))) {
+    return true;
+  }
+
+  return false;
+}

package/src/shared/epistemicStatus.js

@@ -0,0 +1,111 @@
+/**
+ * Epistemic Status & Provenance Envelope
+ *
+ * Core Operating Doctrine:
+ *   OBSERVED stays OBSERVED
+ *   DERIVED stays DERIVED
+ *   MODELED stays MODELED
+ *   HUMAN_REVIEWED stays HUMAN_REVIEWED
+ *   NOT_OBSERVED stays NOT_OBSERVED
+ *   NOT_APPLICABLE stays NOT_APPLICABLE
+ *   UNRESOLVED stays UNRESOLVED
+ *   SYNTHETIC stays SAMPLE-ONLY
+ */
+
+export const EPISTEMIC_STATUS = Object.freeze({
+  OBSERVED: 'OBSERVED',
+  DERIVED: 'DERIVED',
+  MODELED: 'MODELED',
+  HUMAN_REVIEWED: 'HUMAN_REVIEWED',
+  NOT_OBSERVED: 'NOT_OBSERVED',
+  NOT_APPLICABLE: 'NOT_APPLICABLE',
+  UNRESOLVED: 'UNRESOLVED',
+  SYNTHETIC_SAMPLE: 'SYNTHETIC_SAMPLE',
+});
+
+export const DECISION_CATEGORY = Object.freeze({
+  CONFIRMED_OBSERVATION: 'CONFIRMED_OBSERVATION',
+  DERIVED_FINDING: 'DERIVED_FINDING',
+  MODELED_HYPOTHESIS: 'MODELED_HYPOTHESIS',
+  CONTROLLED_CAUSAL_EVIDENCE: 'CONTROLLED_CAUSAL_EVIDENCE',
+  UNRESOLVED: 'UNRESOLVED',
+  REQUIRES_HUMAN_REVIEW: 'REQUIRES_HUMAN_REVIEW',
+  REQUIRES_EXTERNAL_DATA: 'REQUIRES_EXTERNAL_DATA',
+});
+
+export function wrapValue(value, status, options = {}) {
+  if (!Object.values(EPISTEMIC_STATUS).includes(status)) {
+    throw new Error(`Invalid epistemic status: ${status}`);
+  }
+  return {
+    value,
+    status,
+    evidence_refs: options.evidence_refs ? [].concat(options.evidence_refs) : [],
+    method: options.method || null,
+    confidence: options.confidence || null,
+    derivation_id: options.derivation_id || null,
+    model_id: options.model_id || null,
+    required_input: options.required_input || null,
+    note: options.note || null,
+  };
+}
+
+export function observed(value, evidenceRefs = [], method = null, confidence = 'empirical_observation') {
+  return wrapValue(value, EPISTEMIC_STATUS.OBSERVED, {
+    evidence_refs: evidenceRefs,
+    method,
+    confidence,
+  });
+}
+
+export function derived(value, evidenceRefs = [], method = null, derivationId = null) {
+  return wrapValue(value, EPISTEMIC_STATUS.DERIVED, {
+    evidence_refs: evidenceRefs,
+    method,
+    derivation_id: derivationId,
+    confidence: 'reproducible_derivation',
+  });
+}
+
+export function modeled(value, method, modelId = null, confidence = 'modeled_heuristic') {
+  return wrapValue(value, EPISTEMIC_STATUS.MODELED, {
+    method,
+    model_id: modelId,
+    confidence,
+  });
+}
+
+export function humanReviewed(value, reviewerId, reviewTimestamp, note = null) {
+  return wrapValue(value, EPISTEMIC_STATUS.HUMAN_REVIEWED, {
+    method: `human_review:${reviewerId}`,
+    note,
+    confidence: 'attested_human_review',
+  });
+}
+
+export function notObserved(requiredInput, note = null) {
+  return wrapValue(null, EPISTEMIC_STATUS.NOT_OBSERVED, {
+    required_input: requiredInput,
+    note: note || `Metric unavailable: required input "${requiredInput}" was not provided or not observed.`,
+  });
+}
+
+export function notApplicable(reason) {
+  return wrapValue(null, EPISTEMIC_STATUS.NOT_APPLICABLE, {
+    note: reason,
+  });
+}
+
+export function unresolved(reason, requiredInput = null) {
+  return wrapValue(null, EPISTEMIC_STATUS.UNRESOLVED, {
+    required_input: requiredInput,
+    note: reason,
+  });
+}
+
+export function syntheticSample(value, rationale = 'Non-contractual synthetic demonstration sample') {
+  return wrapValue(value, EPISTEMIC_STATUS.SYNTHETIC_SAMPLE, {
+    note: rationale,
+    confidence: 'synthetic_sample_only',
+  });
+}

package/src/shared/evidenceSourceResolver.js

@@ -0,0 +1,286 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { readJson, sha256, sha256File } from './io.js';
+import { verifyRunPackage, RunVerificationError } from './runPackageVerifier.js';
+import { buildContext } from '../commands/context.js';
+import { selectDetectors } from '../detectors/index.js';
+import { runDetectors, indexTargets } from '../detectors/framework.js';
+
+export class SourceResolutionError extends Error {
+  constructor(message, code = 'SOURCE_RESOLUTION_ERROR', details = {}) {
+    super(message);
+    this.name = 'SourceResolutionError';
+    this.code = code;
+    this.details = details;
+  }
+}
+
+export class RunNotFoundError extends SourceResolutionError {
+  constructor(runId) {
+    super(`Specified run not found: ${runId}`, 'RUN_NOT_FOUND', { runId });
+    this.runId = runId;
+  }
+}
+
+export class FindingsMissingError extends SourceResolutionError {
+  constructor(runId, filePath) {
+    super(`Run ${runId} is missing findings.json at ${filePath}`, 'FINDINGS_MISSING', { runId, filePath });
+    this.runId = runId;
+    this.filePath = filePath;
+  }
+}
+
+export class FindingsInvalidError extends SourceResolutionError {
+  constructor(runId, message) {
+    super(`Run ${runId} contains malformed or invalid findings: ${message}`, 'FINDINGS_INVALID', { runId, message });
+    this.runId = runId;
+  }
+}
+
+export class LiveInspectionFailedError extends SourceResolutionError {
+  constructor(target, reason) {
+    super(`Live target inspection failed for ${target}: ${reason}`, 'LIVE_INSPECTION_FAILED', { target, reason });
+    this.target = target;
+  }
+}
+
+export class NoFindingsAvailableError extends SourceResolutionError {
+  constructor(source, mode = 'CONTRACTUAL') {
+    super(
+      `No verified audit findings available from ${source} for ${mode} generation. Production generation requires authoritative findings. (Use --sample or --demo to generate non-contractual sample)`,
+      'NO_FINDINGS',
+      { source, mode }
+    );
+  }
+}
+
+/**
+ * Extract canonical timestamp (epoch ms) from run package
+ * Hierarchy:
+ * 1. manifest.json `timestamp` or `created_at` ISO string
+ * 2. Run ID timestamp prefix (YYYYMMDDTHHmmss)
+ * 3. findings.json mtime
+ * 4. run directory mtime
+ */
+export function getCanonicalRunTimestamp(runDir, runId = '') {
+  // 1. manifest.json
+  const manifestPath = path.join(runDir, 'manifest.json');
+  if (fs.existsSync(manifestPath)) {
+    try {
+      const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+      if (manifest.timestamp) {
+        const ms = Date.parse(manifest.timestamp);
+        if (!Number.isNaN(ms)) return ms;
+      }
+      if (manifest.created_at) {
+        const ms = Date.parse(manifest.created_at);
+        if (!Number.isNaN(ms)) return ms;
+      }
+    } catch {}
+  }
+
+  // 2. Run ID timestamp prefix (YYYYMMDDTHHmmss)
+  const m = String(runId).match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})/);
+  if (m) {
+    const iso = `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z`;
+    const ms = Date.parse(iso);
+    if (!Number.isNaN(ms)) return ms;
+  }
+
+  // 3. findings.json mtime
+  const findPath = path.join(runDir, 'findings.json');
+  if (fs.existsSync(findPath)) {
+    try {
+      return fs.statSync(findPath).mtimeMs;
+    } catch {}
+  }
+
+  // 4. Directory mtime
+  try {
+    return fs.statSync(runDir).mtimeMs;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Sort run candidate directory names by chronological timestamp descending,
+ * with deterministic lexicographical tie-breaking.
+ * Invariant:
+ *   latest run = max(canonical run timestamp)
+ *   tie = deterministic secondary key
+ */
+export function sortRunCandidatesChronologically(runsDir, candidateRunIds = []) {
+  return [...candidateRunIds].sort((a, b) => {
+    const timeA = getCanonicalRunTimestamp(path.join(runsDir, a), a);
+    const timeB = getCanonicalRunTimestamp(path.join(runsDir, b), b);
+    if (timeB !== timeA) {
+      return timeB - timeA; // Descending: latest timestamp first
+    }
+    return b.localeCompare(a); // Deterministic tie-breaker
+  });
+}
+
+/**
+ * Unified Canonical Evidence Source Resolver
+ *
+ * Enforces strict precedence and fail-closed integrity across SOW and Executive Reports:
+ * 1. DIRECT_FINDINGS (in-memory findings array)
+ * 2. EXPLICIT_RUN (--run <id>): verified via runPackageVerifier
+ * 3. LIVE_INSPECTION (--live or target): executed on demand
+ * 4. LATEST_RECORDED_RUN: chronologically latest run in .citable/runs verified via runPackageVerifier
+ * 5. SAMPLE: synthetic demonstration sample
+ */
+export async function resolveEvidenceSource(root, options = {}) {
+  const {
+    findings: inputFindings = null,
+    runId = null,
+    target = null,
+    live = false,
+    baseUrl = null,
+    refDate = null,
+    sample = false,
+    demo = false,
+    draft = false,
+    contractual = false,
+    scopes = ['technical', 'seo', 'aeo', 'geo', 'schema', 'entity', 'cro'],
+  } = options;
+
+  const isSample = Boolean(sample || demo);
+  const isDraft = Boolean(draft);
+  const generationMode = isSample ? 'NON_CONTRACTUAL_SAMPLE' : (isDraft ? 'DRAFT' : 'CONTRACTUAL');
+
+  // 1. Direct findings in memory
+  if (Array.isArray(inputFindings)) {
+    const findingsJson = JSON.stringify(inputFindings);
+    return {
+      source_type: 'DIRECT_FINDINGS',
+      source_identifier: 'in_memory',
+      findings: inputFindings,
+      findings_count: inputFindings.length,
+      integrity_hash: sha256(findingsJson),
+      generation_mode: generationMode,
+      run_metadata: null,
+    };
+  }
+
+  // 2. Explicit run requested via runId
+  if (runId) {
+    const runsDir = path.join(root, '.citable', 'runs');
+    const runPath = path.join(runsDir, runId);
+    if (!fs.existsSync(runPath)) {
+      throw new RunNotFoundError(runId);
+    }
+    const findPath = path.join(runPath, 'findings.json');
+    if (!fs.existsSync(findPath)) {
+      throw new FindingsMissingError(runId, findPath);
+    }
+
+    try {
+      const verification = verifyRunPackage(runPath, {
+        requireFindings: true,
+        requireCompleted: generationMode === 'CONTRACTUAL',
+        requireChecksums: true,
+      });
+
+      return {
+        source_type: 'HISTORICAL_RUN',
+        source_identifier: runId,
+        findings: verification.findings,
+        findings_count: verification.findingsCount,
+        integrity_hash: sha256File(findPath),
+        generation_mode: generationMode,
+        run_metadata: verification.manifest,
+        checksums_verified: verification.checksumsVerified,
+      };
+    } catch (err) {
+      if (err instanceof RunVerificationError) {
+        throw new FindingsInvalidError(runId, err.message);
+      }
+      throw err;
+    }
+  }
+
+  // 3. Live target inspection
+  if (live || target) {
+    if (!target) {
+      throw new SourceResolutionError('Target is required for live inspection (e.g. --target <url|dir>)', 'TARGET_REQUIRED');
+    }
+    try {
+      const ctx = await buildContext(root, { target, baseUrl, refDate });
+      if (!ctx?.site) {
+        throw new Error(`Target ${target} did not produce a valid site context`);
+      }
+      indexTargets(ctx);
+      const detectors = selectDetectors({ scopes });
+      const res = runDetectors(detectors, ctx);
+      const liveFindings = res.findings || [];
+
+      return {
+        source_type: 'LIVE_INSPECTION',
+        source_identifier: target,
+        findings: liveFindings,
+        findings_count: liveFindings.length,
+        integrity_hash: sha256(JSON.stringify(liveFindings)),
+        generation_mode: generationMode,
+        run_metadata: null,
+        context: ctx,
+      };
+    } catch (err) {
+      throw new LiveInspectionFailedError(target, err.message);
+    }
+  }
+
+  // 4. Latest recorded run in .citable/runs
+  const runsDir = path.join(root, '.citable', 'runs');
+  if (fs.existsSync(runsDir)) {
+    const candidateDirs = fs.readdirSync(runsDir).filter((name) => {
+      const p = path.join(runsDir, name);
+      return fs.statSync(p).isDirectory() && fs.existsSync(path.join(p, 'findings.json'));
+    });
+
+    if (candidateDirs.length > 0) {
+      const sorted = sortRunCandidatesChronologically(runsDir, candidateDirs);
+      for (const candidateId of sorted) {
+        const runPath = path.join(runsDir, candidateId);
+        const findPath = path.join(runPath, 'findings.json');
+        try {
+          const verification = verifyRunPackage(runPath, {
+            requireFindings: true,
+            requireCompleted: false, // allow non-completed in discovery if valid
+            requireChecksums: false,
+          });
+
+          return {
+            source_type: 'LATEST_RECORDED_RUN',
+            source_identifier: candidateId,
+            findings: verification.findings,
+            findings_count: verification.findingsCount,
+            integrity_hash: sha256File(findPath),
+            generation_mode: generationMode,
+            run_metadata: verification.manifest,
+          };
+        } catch {
+          // Try next candidate
+          continue;
+        }
+      }
+    }
+  }
+
+  // 5. Sample / Demo mode
+  if (isSample) {
+    return {
+      source_type: 'SAMPLE',
+      source_identifier: 'synthetic_sample_template',
+      findings: [],
+      findings_count: 0,
+      integrity_hash: null,
+      generation_mode: 'NON_CONTRACTUAL_SAMPLE',
+      run_metadata: null,
+    };
+  }
+
+  // If we reach here in contractual mode, fail closed
+  throw new NoFindingsAvailableError('.citable/runs or input target', generationMode);
+}

package/src/shared/htmlEscape.js

@@ -0,0 +1,65 @@
+/**
+ * Canonical HTML and Markdown Escaping Utility
+ *
+ * Prevents HTML/XSS injection vulnerabilities in generated reports, SOWs,
+ * dashboards, and markdown/HTML interchange representations.
+ * Treats all finding summaries, client names, URLs, titles, and detector
+ * outputs as potentially hostile user input.
+ */
+
+const HTML_CHARS = {
+  '&': '&',
+  '<': '&lt;',
+  '>': '&gt;',
+  '"': '&quot;',
+  "'": '&#39;',
+};
+
+const HTML_RX = /[&<>"']/g;
+
+export function escapeHtml(str) {
+  if (str === null || str === undefined) return '';
+  return String(str).replace(HTML_RX, (char) => HTML_CHARS[char]);
+}
+
+export function escapeHtmlAttr(str) {
+  return escapeHtml(str);
+}
+
+const SAFE_PROTOCOLS = new Set(['http:', 'https:', 'mailto:']);
+
+export function sanitizeUrl(urlStr, fallback = '#') {
+  if (!urlStr || typeof urlStr !== 'string') return fallback;
+  const trimmed = urlStr.trim();
+  if (trimmed.startsWith('/') || trimmed.startsWith('#')) {
+    // Relative path or anchor is safe
+    return escapeHtmlAttr(trimmed);
+  }
+  try {
+    const parsed = new URL(trimmed);
+    if (SAFE_PROTOCOLS.has(parsed.protocol)) {
+      return escapeHtmlAttr(trimmed);
+    }
+    return fallback;
+  } catch {
+    return fallback;
+  }
+}
+
+export function escapeMarkdownTableCell(str) {
+  if (str === null || str === undefined) return '';
+  return String(str)
+    .replace(/\r?\n/g, ' ')
+    .replace(/\\/g, '\\\\')
+    .replace(/\|/g, '\\|')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;');
+}
+
+export function sanitizeForMarkdown(str) {
+  if (str === null || str === undefined) return '';
+  // Prevent raw script or malicious HTML tag injections in markdown by encoding brackets
+  return String(str)
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;');
+}