@nebulacomponents/citable 1.13.0 → 1.14.0

package/src/commands/planAudit.js

@@ -0,0 +1,151 @@
+import { buildContext } from './context.js';
+import { runtimeCapabilities } from '../installer/index.js';
+
+const PROFILE_RULES = [
+  {
+    id: 'saas',
+    signals: [
+      ['pricing URL discovered', ({ paths }) => paths.some((value) => /\/pricing(?:\/|$)/.test(value))],
+      ['product, feature, integration, or documentation URL discovered', ({ paths }) => paths.some((value) => /\/(?:products?|features?|integrations?|docs)(?:\/|$)/.test(value))],
+      ['software or platform language observed', ({ text }) => /\b(?:software|saas|platform)\b/i.test(text)],
+      ['trial or signup language observed', ({ text }) => /\b(?:free trial|sign up|get started)\b/i.test(text)],
+    ],
+    emphasis: ['pricing-and-capability-claim-support', 'product-entity-consistency'],
+  },
+  {
+    id: 'ecommerce',
+    signals: [
+      ['product or collection URL discovered', ({ paths }) => paths.some((value) => /\/(?:products?|collections?|cart)(?:\/|$)/.test(value))],
+      ['commerce action language observed', ({ text }) => /\b(?:add to cart|buy now|checkout)\b/i.test(text)],
+      ['Product or Offer structured data observed', ({ schemaTypes }) => schemaTypes.has('Product') || schemaTypes.has('Offer')],
+    ],
+    emphasis: ['product-offer-visible-content-support', 'merchant-claim-governance'],
+  },
+  {
+    id: 'local',
+    signals: [
+      ['LocalBusiness structured data observed', ({ schemaTypes }) => schemaTypes.has('LocalBusiness')],
+      ['telephone link observed', ({ hrefs }) => hrefs.some((value) => value.startsWith('tel:'))],
+      ['local-service language observed', ({ text }) => /\b(?:service area|serving [a-z]|visit us|directions)\b/i.test(text)],
+      ['map link or embed observed', ({ rawHtml }) => /(?:google\.[^"']*\/maps|maps\.google)/i.test(rawHtml)],
+    ],
+    emphasis: ['local-entity-and-nap-consistency', 'review-and-rating-evidence'],
+  },
+  {
+    id: 'publisher',
+    signals: [
+      ['article, blog, or topic URL discovered', ({ paths }) => paths.some((value) => /\/(?:blog|articles?|topics?)(?:\/|$)/.test(value))],
+      ['Article-family structured data observed', ({ schemaTypes }) => ['Article', 'BlogPosting', 'NewsArticle'].some((value) => schemaTypes.has(value))],
+      ['author or publication-date language observed', ({ text }) => /\b(?:written by|published|updated on)\b/i.test(text)],
+    ],
+    emphasis: ['authorship-and-source-identity', 'freshness-consensus'],
+  },
+  {
+    id: 'multilingual',
+    signals: [
+      ['multiple document languages observed', ({ languages }) => languages.size > 1],
+      ['hreflang markup observed', ({ rawHtml }) => /hreflang\s*=/i.test(rawHtml)],
+    ],
+    emphasis: ['hreflang-reciprocity-and-language-parity'],
+  },
+  {
+    id: 'api-docs',
+    signals: [
+      ['API or documentation URL discovered', ({ paths }) => paths.some((value) => /\/(?:api|docs?|developers?)(?:\/|$)/.test(value))],
+      ['API or SDK language observed', ({ text }) => /\b(?:api|sdk|webhook|endpoint)\b/i.test(text)],
+      ['machine interface metadata observed', ({ paths }) => paths.some((value) => /\/(?:agent\.json|auth\.md|\.well-known\/)/.test(value))],
+    ],
+    emphasis: ['machine-interface-discoverability', 'capability-and-authentication-claim-support'],
+  },
+];
+
+const COLLECTORS = [
+  ['render', 'browser_render', 'citable observe render --target TARGET'],
+  ['lighthouse', 'lighthouse_lab', 'citable observe performance --target TARGET --lighthouse --repeat 3'],
+  ['crux', 'crux_field', 'citable observe performance --target TARGET'],
+  ['gsc-index', 'gsc_index', 'citable observe index --target TARGET --site-url PROPERTY'],
+  ['ga4-metrics', 'ga4_metrics', 'citable connect sync --connection-id CONNECTION --start-date DATE --end-date DATE'],
+];
+
+function siteSignals(site) {
+  const paths = [];
+  const hrefs = [];
+  const schemaTypes = new Set();
+  const languages = new Set();
+  let text = '';
+  let rawHtml = '';
+  for (const page of site.pages) {
+    paths.push(new URL(page.url).pathname);
+    hrefs.push(...page.links.map((link) => link.href));
+    text += ` ${page.text}`;
+    rawHtml += ` ${page.rawHtml}`;
+    if (page.lang) languages.add(page.lang.toLowerCase());
+    for (const item of page.jsonLd) {
+      for (const block of item.blocks) {
+        const types = Array.isArray(block?.['@type']) ? block['@type'] : [block?.['@type']];
+        for (const type of types.filter(Boolean)) schemaTypes.add(type);
+      }
+    }
+  }
+  return { paths, hrefs, schemaTypes, languages, text, rawHtml };
+}
+
+function classifyProfiles(site) {
+  const signals = siteSignals(site);
+  return PROFILE_RULES.map((rule) => {
+    const evidence = rule.signals.filter(([, matches]) => matches(signals)).map(([label]) => label);
+    return {
+      id: rule.id,
+      confidence: evidence.length >= 3 ? 'high' : evidence.length === 2 ? 'medium' : 'low',
+      evidence,
+      emphasis: rule.emphasis,
+    };
+  }).filter((profile) => profile.evidence.length >= 2);
+}
+
+export async function planAudit(root, { target, baseUrl, refDate, capabilities } = {}) {
+  if (!target) throw new Error('plan-audit requires --target <dir|url>');
+  const ctx = await buildContext(root, { target, baseUrl, refDate });
+  if (!ctx.site) throw new Error('plan-audit could not build a site model');
+  const profiles = classifyProfiles(ctx.site);
+  const runtime = capabilities ?? await runtimeCapabilities();
+  const capabilityById = new Map(runtime.map((item) => [item.id, item]));
+  const collectors = COLLECTORS.map(([id, capabilityId, command]) => {
+    const capability = capabilityById.get(capabilityId);
+    const status = capability?.state === 'ready' ? 'available' : 'blocked';
+    return {
+      id,
+      status,
+      capability: capabilityId,
+      command: command.replace('TARGET', JSON.stringify(target)),
+      required_input: status === 'available' ? [] : capability?.requirements ?? [],
+      limitations: capability?.limitations ?? ['capability status is unavailable'],
+    };
+  });
+  return {
+    target: {
+      kind: ctx.site.mode,
+      location: ctx.site.location,
+      pages_inspected: ctx.site.pages.length,
+    },
+    classification: {
+      statement_type: 'probabilistic_inference',
+      profiles,
+      unresolved: profiles.length ? [] : ['No profile met the two-signal evidence threshold.'],
+    },
+    audit: {
+      scope: 'all',
+      command: `citable audit --target ${JSON.stringify(target)}${baseUrl ? ` --base-url ${JSON.stringify(baseUrl)}` : ''}`,
+      rationale: 'The full audit preserves separate SEO, AEO, GEO, governance, and observation boundaries; profile inference changes emphasis, not detector truth.',
+    },
+    semantic_emphasis: [...new Set(profiles.flatMap((profile) => profile.emphasis))],
+    collectors,
+    warnings: ctx.warnings,
+    boundaries: [
+      'No audit or observation package was created.',
+      'Profile labels are probabilistic inferences from captured page signals, not verified business facts.',
+      'Available collectors establish prerequisite presence only; execution and evidence remain unproven until a collector completes.',
+      'Blocked collectors do not make the proposed audit a success or failure; they identify unavailable follow-up evidence.',
+    ],
+  };
+}

package/src/commands/schemaCmd.js

@@ -1,6 +1,7 @@
 import { buildContext } from './context.js';
 import { selectDetectors } from '../detectors/index.js';
 import { runDetectors, sitePageFor } from '../detectors/framework.js';
+import { checklistItem, toStringArray } from '../shared/checklist.js';
 
 /**
  * `citable schema` — derive JSON-LD proposals from registry data and validate deployed schema.
@@ -28,7 +29,10 @@ export async function schemaCommand(root, { target, baseUrl, refDate } = {}) {
     if (!e.canonical_url) missing.push('canonical_url');
     if (!e.definition && ['proprietary_concept', 'category'].includes(e.entity_type)) missing.push('definition');
     if (missing.length) {
-      blocked.push({ entity_id: e.entity_id, status: 'blocked', reason: 'entity record incomplete; schema would require invented facts', required_input: missing });
+      blocked.push({
+        entity_id: e.entity_id, status: 'blocked', reason: 'entity record incomplete; schema would require invented facts',
+        required_input: toStringArray(missing.map((field) => checklistItem(field, field))),
+      });
       continue;
     }
     const id = `${e.canonical_url}#${e.entity_type}`;

package/src/commands/substantiate.js

@@ -1,5 +1,6 @@
 import { loadRegistries, saveRegistry, diffRegistries } from '../registries/index.js';
 import { isPastDate, nowIso, parseRefDate } from '../shared/io.js';
+import { checklistItem, toStringArray } from '../shared/checklist.js';
 
 const REGULATED = new Set(['legal_regulatory', 'security', 'commercial']);
 
@@ -37,7 +38,7 @@ export function substantiate(root, { write = false, refDate } = {}) {
     } else if (REGULATED.has(c.claim_type) && (!c.legal_status || c.legal_status === 'not_assessed')) {
       a.outcome = 'review_required';
       a.reasons.push(`${c.claim_type} claim requires legal/SME review before any verified status`);
-      a.required_input.push('legal or subject-matter review decision');
+      a.required_input.push(checklistItem('legal-review', 'legal or subject-matter review decision'));
     } else if (['opinion', 'position', 'aspirational'].includes(c.claim_type)) {
       // Opinions and aspirations are positions, not verifiable facts; they never enter the verified track.
       a.outcome = c.status === 'candidate' ? 'candidate' : 'unverified';
@@ -54,19 +55,23 @@ export function substantiate(root, { write = false, refDate } = {}) {
       } else if (live.length === 0) {
         a.outcome = 'insufficient_evidence';
         a.reasons.push('all attached evidence is expired, unverified, revoked, or inaccessible');
-        a.required_input.push('current, verified evidence');
+        a.required_input.push(checklistItem('current-verified-evidence', 'current, verified evidence'));
       } else if (['performance', 'comparative', 'security'].includes(c.claim_type) && live.every((e) => e.primary_or_secondary === 'secondary')) {
         a.outcome = 'insufficient_evidence';
         a.reasons.push(`${c.claim_type} claims require primary evidence; only secondary evidence attached`);
-        a.required_input.push('primary evidence (test result, benchmark, specification, certification)');
+        a.required_input.push(checklistItem('primary-evidence', 'primary evidence (test result, benchmark, specification, certification)'));
       } else if (['performance'].includes(c.claim_type) && live.every((e) => !e.methodology)) {
         a.outcome = 'insufficient_evidence';
         a.reasons.push('performance evidence lacks methodology');
-        a.required_input.push('methodology', 'test conditions', 'measurement period');
+        a.required_input.push(
+          checklistItem('methodology', 'methodology'),
+          checklistItem('test-conditions', 'test conditions'),
+          checklistItem('measurement-period', 'measurement period'),
+        );
       } else if ((c.scope || []).length === 0 && ['capability', 'performance', 'comparative'].includes(c.claim_type)) {
         a.outcome = 'verified_narrowed';
         a.reasons.push('evidence exists but claim declares no scope; treat as verified only with narrowed scope pending scope definition');
-        a.required_input.push('explicit scope and exclusions');
+        a.required_input.push(checklistItem('scope-exclusions', 'explicit scope and exclusions'));
       } else {
         // Deterministic conditions passed. Verified status additionally requires a recorded
         // semantic support assessment (entailment) by a named assessor — evidence linkage
@@ -82,7 +87,7 @@ export function substantiate(root, { write = false, refDate } = {}) {
           } else {
             a.outcome = 'review_required';
             a.reasons.push('verified status held without an adequate semantic support assessment; entailment review required (evidence-strength rubric)');
-            a.required_input.push('support_assessment with named assessor');
+            a.required_input.push(checklistItem('support-assessment', 'support_assessment with named assessor'));
           }
         } else {
           a.outcome = 'review_required';
@@ -90,6 +95,7 @@ export function substantiate(root, { write = false, refDate } = {}) {
         }
       }
     }
+    a.required_input = toStringArray(a.required_input);
     assessments.push(a);
   }
 
@@ -114,8 +120,14 @@ export function substantiate(root, { write = false, refDate } = {}) {
 }
 
 function requiredInputFor(c) {
-  if (c.claim_type === 'performance') return ['baseline', 'measurement period', 'test population/conditions', 'methodology', 'result source'];
-  if (c.claim_type === 'comparative') return ['comparison set', 'comparison criteria', 'evidence for each compared dimension'];
-  if (c.claim_type === 'security') return ['test report, certification, or architecture evidence'];
-  return ['supporting evidence with an accountable owner'];
+  if (c.claim_type === 'performance') {
+    return ['baseline', 'measurement period', 'test population/conditions', 'methodology', 'result source']
+      .map((label) => checklistItem(label.replace(/[^a-z0-9]+/gi, '-').toLowerCase(), label));
+  }
+  if (c.claim_type === 'comparative') {
+    return ['comparison set', 'comparison criteria', 'evidence for each compared dimension']
+      .map((label) => checklistItem(label.replace(/[^a-z0-9]+/gi, '-').toLowerCase(), label));
+  }
+  if (c.claim_type === 'security') return [checklistItem('security-evidence', 'test report, certification, or architecture evidence')];
+  return [checklistItem('owned-evidence', 'supporting evidence with an accountable owner')];
 }

package/src/crawler/fetch.js

@@ -4,9 +4,11 @@ import net from 'node:net';
 const blockedAddresses = new net.BlockList();
 for (const [network, prefix] of [
   ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8],
-  ['169.254.0.0', 16], ['172.16.0.0', 12], ['192.168.0.0', 16], ['224.0.0.0', 4],
+  ['169.254.0.0', 16], ['172.16.0.0', 12], ['192.0.0.0', 24], ['192.0.2.0', 24],
+  ['192.168.0.0', 16], ['198.18.0.0', 15], ['198.51.100.0', 24],
+  ['203.0.113.0', 24], ['224.0.0.0', 4], ['240.0.0.0', 4],
 ]) blockedAddresses.addSubnet(network, prefix, 'ipv4');
-for (const [network, prefix] of [['::', 128], ['::1', 128], ['fc00::', 7], ['fe80::', 10], ['ff00::', 8]]) {
+for (const [network, prefix] of [['::', 128], ['::1', 128], ['2001:db8::', 32], ['fc00::', 7], ['fe80::', 10], ['ff00::', 8]]) {
   blockedAddresses.addSubnet(network, prefix, 'ipv6');
 }
 
@@ -19,7 +21,7 @@ function isBlockedAddress(address) {
   return blockedAddresses.check(address, family === 4 ? 'ipv4' : 'ipv6');
 }
 
-async function validateDestination(url, lookup) {
+export async function validatePublicUrl(url, { lookup = dns.promises.lookup } = {}) {
   const parsed = new URL(url);
   if (!['http:', 'https:'].includes(parsed.protocol)) throw new Error(`unsupported URL protocol: ${parsed.protocol}`);
   if (parsed.username || parsed.password) throw new Error('URL credentials are not permitted');
@@ -34,6 +36,32 @@ async function validateDestination(url, lookup) {
   return parsed;
 }
 
+/** Enforce the public-network policy for each Playwright request. */
+export async function handlePublicBrowserRoute(route, { lookup = dns.promises.lookup } = {}) {
+  const url = route.request().url();
+  let protocol;
+  try {
+    protocol = new URL(url).protocol;
+  } catch {
+    await route.abort('blockedbyclient');
+    return;
+  }
+  if (['data:', 'blob:', 'about:'].includes(protocol)) {
+    await route.continue();
+    return;
+  }
+  if (!['http:', 'https:'].includes(protocol)) {
+    await route.abort('blockedbyclient');
+    return;
+  }
+  try {
+    await validatePublicUrl(url, { lookup });
+    await route.continue();
+  } catch {
+    await route.abort('blockedbyclient');
+  }
+}
+
 async function readBodyLimited(res, maxBodyBytes) {
   const declared = Number(res.headers.get('content-length'));
   if (Number.isFinite(declared) && declared > maxBodyBytes) {
@@ -66,13 +94,13 @@ export async function fetchUrl(url, {
   maxRetries = 3, retryDelayMs = 1000, maxBodyBytes = 5 * 1024 * 1024,
   fetchImpl = globalThis.fetch, lookup = dns.promises.lookup,
 } = {}) {
-  const requested = await validateDestination(url, lookup);
+  const requested = await validatePublicUrl(url, { lookup });
   const allowedOrigin = requested.origin;
   const chain = [];
   let current = requested.href;
 
   for (let redirects = 0; redirects <= maxRedirects; redirects += 1) {
-    await validateDestination(current, lookup);
+    await validatePublicUrl(current, { lookup });
     let res;
     let lastError;
     for (let attempt = 0; attempt < Math.max(1, maxRetries); attempt += 1) {
@@ -108,7 +136,7 @@ export async function fetchUrl(url, {
       if (redirects === maxRedirects) break;
       const next = new URL(headers.location, current);
       if (next.origin !== allowedOrigin) throw new Error(`redirect leaves audited origin: ${next.href}`);
-      await validateDestination(next.href, lookup);
+      await validatePublicUrl(next.href, { lookup });
       chain.push({ url: current, status: res.status, location: headers.location });
       current = next.href;
       await res.body?.cancel();

package/src/extractor/page.js

@@ -57,6 +57,21 @@ export function extractPage({ url, html, status = 200, headers = {}, sourceFile
   }
 
   const body = root.querySelector('body');
+  const rawVisibleText = body ? body.text.replace(/\s+/g, ' ').trim() : '';
+  const structuralRegions = [];
+  const seenRegions = new Set();
+  for (const region of root.querySelectorAll('header,nav,aside,footer,[role=banner],[role=navigation],[role=complementary],[role=contentinfo]')) {
+    if (seenRegions.has(region)) continue;
+    seenRegions.add(region);
+    const text = region.text.replace(/\s+/g, ' ').trim();
+    if (!text) continue;
+    structuralRegions.push({
+      region_type: (region.getAttribute('role') || region.tagName || 'unknown').toLowerCase(),
+      text,
+      html_bytes: Buffer.byteLength(region.outerHTML),
+      token_count: text.split(/\s+/).filter(Boolean).length,
+    });
+  }
   // Strip script/style/nav/footer noise for main-text estimation
   const bodyClone = body ? parse(body.outerHTML) : null;
   if (bodyClone) for (const s of bodyClone.querySelectorAll('script,style,noscript,nav,footer,header[role=banner]')) s.remove();
@@ -115,6 +130,8 @@ export function extractPage({ url, html, status = 200, headers = {}, sourceFile
     images,
     jsonLd,
     text,
+    rawVisibleWordCount: rawVisibleText ? rawVisibleText.split(/\s+/).length : 0,
+    structuralRegions,
     paragraphs,
     wordCount: text ? text.split(/\s+/).length : 0,
     scriptBytes,