@nebulacomponents/citable 1.13.1 → 1.14.0

package/src/observations/entityGraph.js

@@ -0,0 +1,89 @@
+import { sha256 } from '../shared/io.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+const list = (value) => Array.isArray(value) ? value : value == null ? [] : [value];
+
+function visibleSupport(node, page) {
+  const candidates = [
+    ...list(node.name).map((value) => ['name', value]),
+    ...list(node.headline).map((value) => ['headline', value]),
+    ...list(node.description).map((value) => ['description', value]),
+  ].filter(([, value]) => typeof value === 'string' && value.trim());
+  if (!candidates.length) return { status: 'not_testable', matched_fields: [] };
+  const visible = `${page.title || ''} ${page.text || ''}`.toLocaleLowerCase();
+  const matchedFields = [...new Set(candidates
+    .filter(([, value]) => visible.includes(value.trim().toLocaleLowerCase()))
+    .map(([field]) => field))].sort();
+  return { status: matchedFields.length ? 'supported' : 'not_observed', matched_fields: matchedFields };
+}
+
+function referencedIds(value, relation, out = []) {
+  if (Array.isArray(value)) {
+    for (const item of value) referencedIds(item, relation, out);
+  } else if (value && typeof value === 'object') {
+    if (typeof value['@id'] === 'string') out.push({ relation, target_id: value['@id'] });
+    else for (const [key, nested] of Object.entries(value)) referencedIds(nested, key, out);
+  }
+  return out;
+}
+
+export function buildEntityGraph(pages) {
+  const nodeMap = new Map();
+  const pendingEdges = [];
+  const parseFailures = [];
+  for (const page of pages) {
+    for (const script of page.jsonLd) {
+      if (script.parseError) {
+        parseFailures.push({ url: page.url, error: script.parseError, raw_hash: sha256(script.raw) });
+        continue;
+      }
+      for (const block of script.blocks) {
+        if (!block || typeof block !== 'object' || Array.isArray(block)) continue;
+        const blockHash = sha256(JSON.stringify(block));
+        const nodeId = typeof block['@id'] === 'string' && block['@id'].trim()
+          ? block['@id']
+          : `urn:citable:jsonld:${blockHash}`;
+        const prior = nodeMap.get(nodeId) || {
+          node_id: nodeId, types: [], names: [], source_blocks: [],
+          visible_support: { status: 'not_testable', matched_fields: [] },
+        };
+        prior.types = [...new Set([...prior.types, ...list(block['@type']).filter((value) => typeof value === 'string')])].sort();
+        prior.names = [...new Set([...prior.names, ...list(block.name).filter((value) => typeof value === 'string' && value.trim())])].sort();
+        prior.source_blocks.push({ url: page.url, block_hash: blockHash });
+        const support = visibleSupport(block, page);
+        prior.visible_support = {
+          status: prior.visible_support.status === 'supported' || support.status === 'supported'
+            ? 'supported'
+            : prior.visible_support.status === 'not_observed' || support.status === 'not_observed'
+              ? 'not_observed'
+              : 'not_testable',
+          matched_fields: [...new Set([...prior.visible_support.matched_fields, ...support.matched_fields])].sort(),
+        };
+        nodeMap.set(nodeId, prior);
+        for (const [relation, value] of Object.entries(block)) {
+          if (relation.startsWith('@')) continue;
+          for (const ref of referencedIds(value, relation)) {
+            pendingEdges.push({ source_id: nodeId, relation: ref.relation, target_id: ref.target_id, source_url: page.url });
+          }
+        }
+      }
+    }
+  }
+  const nodes = [...nodeMap.values()].sort((a, b) => a.node_id.localeCompare(b.node_id));
+  const nodeIds = new Set(nodes.map((node) => node.node_id));
+  const allEdges = pendingEdges.map((edge) => ({ ...edge, target_declared: nodeIds.has(edge.target_id) }))
+    .sort((a, b) => `${a.source_id}:${a.relation}:${a.target_id}`.localeCompare(`${b.source_id}:${b.relation}:${b.target_id}`));
+  const graph = {
+    version: 1,
+    nodes,
+    edges: allEdges.filter((edge) => edge.target_declared),
+    unresolved_references: allEdges.filter((edge) => !edge.target_declared),
+    parse_failures: parseFailures,
+    limitations: [
+      'Normalization reports declared JSON-LD structure and literal visible-text matches; it does not establish semantic correctness, eligibility, search-engine recognition, or ranking impact.',
+    ],
+  };
+  const check = validateAgainst('entity-graph.schema.json', graph);
+  if (!check.valid) throw new Error(`entity graph violates contract: ${check.errors.join('; ')}`);
+  return graph;
+}

package/src/observations/renderParity.js

@@ -0,0 +1,49 @@
+import { parse } from 'node-html-parser';
+import { sha256 } from '../shared/io.js';
+
+function profile(html) {
+  const root = parse(String(html || ''));
+  for (const node of root.querySelectorAll('script,style,noscript,nav,footer,header[role=banner]')) node.remove();
+  const text = root.textContent.replace(/\s+/g, ' ').trim();
+  const tokens = new Set(text.toLocaleLowerCase().split(/\s+/).filter(Boolean));
+  const headings = root.querySelectorAll('h1,h2,h3,h4,h5,h6').map((node) => node.text.replace(/\s+/g, ' ').trim()).filter(Boolean).sort();
+  const links = root.querySelectorAll('a[href]').map((node) => node.getAttribute('href')).filter(Boolean).sort();
+  const structuredData = parse(String(html || '')).querySelectorAll('script[type="application/ld+json"]').map((node) => sha256(node.text)).sort();
+  return { text, tokens, headings, links, structuredData };
+}
+
+const missing = (before, after) => before.filter((value) => !after.includes(value));
+
+export function analyzeRenderParity(initialHtml, renderedHtml, policy = null) {
+  const initial = profile(initialHtml);
+  const rendered = profile(renderedHtml);
+  const retained = [...initial.tokens].filter((token) => rendered.tokens.has(token)).length;
+  const textRetentionRatio = initial.tokens.size ? Number((retained / initial.tokens.size).toFixed(3)) : null;
+  const removedHeadings = missing(initial.headings, rendered.headings);
+  const removedLinks = missing(initial.links, rendered.links);
+  const structuredDataMatches = JSON.stringify(initial.structuredData) === JSON.stringify(rendered.structuredData);
+  const violations = [];
+  if (policy) {
+    if (textRetentionRatio != null && textRetentionRatio < policy.min_text_retention_ratio) {
+      violations.push({ check: 'text_retention', observed: textRetentionRatio, required: policy.min_text_retention_ratio });
+    }
+    if (policy.require_heading_parity && removedHeadings.length) violations.push({ check: 'heading_parity', observed: removedHeadings.length, required: 0 });
+    if (policy.require_link_parity && removedLinks.length) violations.push({ check: 'link_parity', observed: removedLinks.length, required: 0 });
+    if (policy.require_structured_data_parity && !structuredDataMatches) violations.push({ check: 'structured_data_parity', observed: false, required: true });
+  }
+  return {
+    policy_id: policy?.policy_id || null,
+    review_owner: policy?.review_owner || null,
+    initial_text_hash: sha256(initial.text),
+    rendered_text_hash: sha256(rendered.text),
+    text_retention_ratio: textRetentionRatio,
+    removed_headings: removedHeadings,
+    added_headings: missing(rendered.headings, initial.headings),
+    removed_links: removedLinks,
+    added_links: missing(rendered.links, initial.links),
+    structured_data_matches: structuredDataMatches,
+    policy_violations: violations,
+    semantic_review_status: policy ? 'review_required' : 'not_evidenced',
+    interpretation_boundary: 'Deterministic parity and policy checks do not establish semantic, retrieval, citation, ranking, or business impact.',
+  };
+}

package/src/observations/sourceIdentity.js

@@ -0,0 +1,62 @@
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+export function buildSourceIdentityChain(registries) {
+  const entities = new Map((registries.entities?.entries || []).map((item) => [item.entity_id, item]));
+  const evidence = new Map((registries.evidence?.entries || []).map((item) => [item.evidence_id, item]));
+  const pages = (registries.pages?.entries || []).map((page) => {
+    const publisherEntity = entities.get(page.publisher_entity);
+    const publisher = publisherEntity ? {
+      entity_id: publisherEntity.entity_id,
+      name: publisherEntity.canonical_name,
+      canonical_url: publisherEntity.canonical_url || null,
+      status: publisherEntity.status,
+    } : null;
+    const authors = (page.author_entities || []).map((id) => {
+      const entity = entities.get(id);
+      return entity ? {
+        entity_id: entity.entity_id,
+        name: entity.canonical_name,
+        canonical_url: entity.canonical_url || null,
+        affiliations: (entity.affiliations || []).map((affiliation) => ({
+          ...affiliation,
+          entity_name: entities.get(affiliation.entity_id)?.canonical_name || null,
+        })),
+        status: entity.status,
+      } : { entity_id: id, name: null, canonical_url: null, affiliations: [], status: 'missing' };
+    });
+    const evidenceOwners = (page.evidence_references || []).map((id) => {
+      const item = evidence.get(id);
+      return { evidence_id: id, owner: item?.source_owner || null, verification_status: item?.verification_status || 'missing' };
+    });
+    const missing = [];
+    if (!publisher) missing.push('publisher_entity');
+    if (!authors.length) missing.push('author_entities');
+    else if (authors.some((author) => !author.affiliations.length)) missing.push('author_affiliations');
+    if (!page.content_owner) missing.push('content_owner');
+    if (!page.factual_reviewer) missing.push('factual_reviewer');
+    if (!page.correction_path) missing.push('correction_path');
+    if (evidenceOwners.some((item) => !item.owner)) missing.push('evidence_owner');
+    return {
+      page_id: page.page_id,
+      url: page.url,
+      publisher,
+      authors,
+      content_owner: page.content_owner || null,
+      factual_reviewer: page.factual_reviewer || null,
+      evidence_owners: evidenceOwners,
+      correction_path: page.correction_path || null,
+      status: missing.length ? 'incomplete' : 'complete',
+      missing_fields: [...new Set(missing)].sort(),
+    };
+  });
+  const chain = {
+    version: 1,
+    pages,
+    limitations: [
+      'Registry linkage records declared identity and governance relationships; it does not establish that disclosures are visible, current, independent, or recognized by external systems.',
+    ],
+  };
+  const check = validateAgainst('source-identity-chain.schema.json', chain);
+  if (!check.valid) throw new Error(`source identity chain violates contract: ${check.errors.join('; ')}`);
+  return chain;
+}

package/src/registries/seed.js

@@ -0,0 +1,95 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { readYaml, nowIso, sha256 } from '../shared/io.js';
+import { loadRegistries, saveRegistry, REGISTRY_SPECS } from './index.js';
+
+const SEEDS_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), 'seeds');
+
+export function listSeeds(seedsDir = SEEDS_DIR) {
+  if (!fs.existsSync(seedsDir)) return [];
+  return fs.readdirSync(seedsDir).filter((name) => fs.existsSync(path.join(seedsDir, name, 'seed.json')));
+}
+
+function loadSeedManifest(name, seedsDir) {
+  const dir = path.join(seedsDir, name);
+  const manifestFile = path.join(dir, 'seed.json');
+  if (!fs.existsSync(manifestFile)) {
+    throw new Error(`unknown seed: ${name}; available seeds: ${listSeeds(seedsDir).join(', ') || 'none bundled'}`);
+  }
+  return { dir, manifest: JSON.parse(fs.readFileSync(manifestFile, 'utf8')) };
+}
+
+/** Deterministic, key-order-independent serialization for content fingerprinting. */
+function stableStringify(value) {
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
+  if (value && typeof value === 'object') {
+    return `{${Object.keys(value).sort().map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`).join(',')}}`;
+  }
+  return JSON.stringify(value);
+}
+
+/** Fingerprint an entry's content, excluding provenance (which carries the fingerprint itself). */
+function contentFingerprint(entry) {
+  const { provenance, ...rest } = entry;
+  return sha256(stableStringify(rest));
+}
+
+/**
+ * Overlay a bundled starter registry seed onto the caller's `.citable/` registries.
+ *
+ * Fail-closed guarantees, enforced in code (never trusted from the seed file itself):
+ *   - every written entry is stamped `status: unverified` unconditionally, regardless of
+ *     what status the seed bundle declares (candidate, active, verified — all forced);
+ *   - an existing entry is only ever refreshed (even with `force`) if its current content
+ *     still matches the fingerprint recorded at seed time — `provenance.seededFrom` being
+ *     present is not by itself proof the entry is unedited, since a user can edit a field
+ *     while leaving provenance untouched; a fingerprint mismatch means the owner changed
+ *     it, and a seed can never clobber that edit.
+ */
+export function applySeed(root, name, { force = false, seedsDir = SEEDS_DIR } = {}) {
+  const { dir, manifest } = loadSeedManifest(name, seedsDir);
+  const { registries } = loadRegistries(root);
+  const appliedAt = nowIso();
+  const result = { seedName: name, seedVersion: manifest.version, appliedAt, registriesTouched: {}, warnings: [] };
+
+  for (const kind of manifest.registries) {
+    const spec = REGISTRY_SPECS.find((s) => s.kind === kind);
+    if (!spec) { result.warnings.push(`seed "${name}" references unknown registry kind "${kind}"; skipped`); continue; }
+    const seedFile = path.join(dir, spec.file);
+    if (!fs.existsSync(seedFile)) continue;
+    const seedRegistry = readYaml(seedFile);
+    const existing = registries[kind];
+    const existingById = new Map(existing.entries.map((e) => [e[spec.idField], e]));
+    let added = 0;
+    let skipped = 0;
+
+    for (const entry of seedRegistry.entries) {
+      const id = entry[spec.idField];
+      const already = existingById.get(id);
+      if (already) {
+        const seededFrom = already.provenance?.seededFrom;
+        const unedited = seededFrom && seededFrom.contentFingerprint === contentFingerprint(already);
+        if (!force || !unedited) { skipped++; continue; }
+      }
+
+      if (entry.status !== 'unverified') {
+        result.warnings.push(`${kind}/${id}: seed attempted status "${entry.status}"; forced to "unverified"`);
+        entry.status = 'unverified';
+      }
+      entry.provenance = {
+        ...(entry.provenance || {}),
+        seededFrom: { seedName: name, seedVersion: manifest.version, importedAt: appliedAt, contentFingerprint: contentFingerprint(entry) },
+      };
+
+      if (already) Object.assign(already, entry);
+      else existing.entries.push(entry);
+      added++;
+    }
+
+    saveRegistry(root, kind, existing);
+    result.registriesTouched[kind] = { added, skipped };
+  }
+
+  return result;
+}

package/src/registries/seeds/saas-pricing/claims.yaml

@@ -0,0 +1,18 @@
+version: 1
+kind: claims
+entries:
+  - claim_id: CLAIM-SEED-PRICING-TRANSPARENCY
+    claim: >
+      Placeholder: replace with a specific, evidenced pricing claim for your
+      product (e.g. published starting price, billing unit, plan boundaries).
+    claim_type: commercial
+    entity: ENT-SEED-PRODUCT
+    publication_surfaces: ["/pricing/"]
+    status: unverified
+  - claim_id: CLAIM-SEED-FEATURE-SCOPE
+    claim: >
+      Placeholder: replace with a specific, evidenced capability claim and its
+      scope/exclusions (what the product does and explicitly does not do).
+    claim_type: capability
+    entity: ENT-SEED-PRODUCT
+    status: unverified

package/src/registries/seeds/saas-pricing/entities.yaml

@@ -0,0 +1,20 @@
+version: 1
+kind: entities
+entries:
+  - entity_id: ENT-SEED-VENDOR
+    canonical_name: (your company name)
+    entity_type: organization
+    category: SaaS vendor
+    definition: >
+      Replace with your organization's own definition. Seeded as a starting
+      shape only — not a fact about your company.
+    status: unverified
+  - entity_id: ENT-SEED-PRODUCT
+    canonical_name: (your product name)
+    entity_type: product
+    owner_entity: ENT-SEED-VENDOR
+    category: SaaS product
+    definition: >
+      Replace with your product's own definition. Seeded as a starting shape
+      only — not a fact about your product.
+    status: unverified

package/src/registries/seeds/saas-pricing/seed.json

@@ -0,0 +1,6 @@
+{
+  "name": "saas-pricing",
+  "version": "1.0.0",
+  "description": "Starter claims/entities for a SaaS pricing page vertical.",
+  "registries": ["entities", "claims"]
+}

package/src/shared/checklist.js

@@ -0,0 +1,19 @@
+/**
+ * Shared required-input checklist item builder used by substantiate, schemaCmd,
+ * and actionPlan so the three no longer maintain independent literal-string lists.
+ * Callers still serialize to array<string> via toStringArray — no consumer-facing
+ * shape changes.
+ */
+export function checklistItem(id, label, note) {
+  return { id, label, satisfied: false, note: note ?? null };
+}
+
+/** Render one item (or an already-plain string) down to its display label. */
+export function toLabel(item) {
+  return typeof item === 'string' ? item : item.label;
+}
+
+/** Serialize a list of items (or plain strings) to array<string>. */
+export function toStringArray(items) {
+  return items.map(toLabel);
+}