@nebulacomponents/citable 1.10.0 → 1.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nebulacomponents/citable",
-  "version": "1.10.0",
+  "version": "1.11.0",
   "description": "Production-grade SEO, AEO, and GEO agent skill with auditable detection, claim governance, evidence validation, and multi-agent installation.",
   "keywords": [
     "seo",
@@ -90,14 +90,19 @@
     "ajv-formats": "^3.0.1",
     "csv-parse": "^7.0.1",
     "js-yaml": "^5.2.1",
-    "node-html-parser": "^9.0.0"
+    "node-html-parser": "^9.0.0",
+    "pdfjs-dist": "^6.1.200"
   },
   "peerDependencies": {
-    "playwright": ">=1.50.0"
+    "playwright": ">=1.50.0",
+    "tesseract.js": ">=7.0.0"
   },
   "peerDependenciesMeta": {
     "playwright": {
       "optional": true
+    },
+    "tesseract.js": {
+      "optional": true
     }
   }
 }

package/schemas/media-manifest.schema.json

@@ -0,0 +1,2 @@
+{
+  "$id":"citable://schemas/media-manifest.schema.json","$schema":"http://json-schema.org/draft-07/schema#","title":"Media evidence import manifest","type":"object","additionalProperties":false,"required":["version","items"],"properties":{"version":{"const":1},"items":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["media_id","type","path","source_url","page_url","claim_ids","permission_status"],"properties":{"media_id":{"type":"string","pattern":"^MEDIA-[A-Z0-9][A-Z0-9_-]*$"},"type":{"type":"string","enum":["pdf","transcript","image"]},"path":{"type":"string","minLength":1},"source_url":{"type":["string","null"]},"page_url":{"type":["string","null"]},"claim_ids":{"type":"array","uniqueItems":true,"items":{"type":"string"}},"permission_status":{"type":"string","enum":["owner_authorized","licensed","public_unknown","internal"]},"image_src":{"type":"string"},"language":{"type":"string"},"published_at":{"type":"string","format":"date-time"}}}}}}

package/schemas/observation.schema.json

@@ -7,7 +7,7 @@
   "required": ["observation_id", "kind", "state", "collected_at", "collection_method", "confidence", "source", "evidence_hash", "data"],
   "properties": {
     "observation_id": { "type": "string", "minLength": 1 },
-    "kind": { "type": "string", "enum": ["render", "index", "citation", "citation_review", "crawler_log", "bing_webmaster", "passage", "canonical_freshness", "performance", "corroboration", "metric"] },
+    "kind": { "type": "string", "enum": ["render", "index", "citation", "citation_review", "crawler_log", "bing_webmaster", "passage", "canonical_freshness", "performance", "corroboration", "metric", "media_pdf", "media_transcript", "media_image"] },
     "state": { "type": "string", "enum": ["observed", "not_observed", "not_evidenced", "incomplete", "review_required", "failed"] },
     "collected_at": { "type": "string", "format": "date-time" },
     "collection_method": { "type": "string", "enum": ["live_api", "browser", "owner_import", "synthetic_fetch", "static_analysis", "human_review"] },

package/skill/SKILL.md

@@ -7,7 +7,7 @@ description: >
   generative engine optimization, structured data governance, claim substantiation,
   crawler policy, entity consistency, content discoverability, or wants a site audited
   for how search and AI systems will retrieve, understand, cite, or recommend it.
-version: 1.10.0
+version: 1.11.0
 ---
 
 # Citable — evidence and change control for search and AI citation readiness
@@ -117,6 +117,7 @@ citable reviews sample <sampling-plan-id> [--write]
 citable reviews evaluate
 citable schedules run <schedule-id> [--ref-date YYYY-MM-DD]
 citable project github <run-id>
+citable observe media --input <manifest.json> [--ocr]
 ```
 
 Audit scopes: `technical seo aeo geo architecture entity claims evidence schema

package/skill/commands/README.md

@@ -30,6 +30,7 @@ intent, not executability, so statuses are strict:
 | /citable governance validate / evaluate | implemented — validates reviewer and exception authority and emits immutable dispositions without changing failed findings |
 | /citable reviews queue / prioritize / plan / sample / evaluate | implemented — materiality queues, reproducible sampling, stale-decision checks, and independent disagreement adjudication |
 | /citable schedules run / project github | implemented — canonical scheduled audits and hash-bound non-authoritative GitHub projections |
+| /citable observe media | implemented — bounded PDF, transcript, image-context, optional OCR, and declared claim-link evidence |
 | /citable ingest, map-site, map-queries, map-prompts, map-entities, map-evidence | specified (ingest.md) |
 | /citable optimize-page | specified (optimize-page.md) — requires source-to-render mapping, claim-preserving rewrites, build execution, rollback; none of that is proven yet |
 | /citable create-page, answer-block, architect, interlink, consolidate, metadata | specified (page-work.md) |

package/skill/commands/media-evidence.md

@@ -0,0 +1,17 @@
+# Media evidence collection
+
+Use `citable observe media --input <manifest.json>` for bounded local-media
+evidence. The manifest declares source URL, owning page, permission status, and
+linked claim IDs for each PDF, transcript, or image-context source.
+
+PDF collection extracts native text, basic metadata, and stable page anchors.
+It does not establish reading order, table structure, footnote linkage,
+signatures, revisions, scan accuracy, accessibility, or claim support.
+Transcript collection preserves cues but does not verify speakers, timing,
+accuracy, or source-media parity. Image collection records alt text, captions,
+and nearby figure text without inferring visual entailment.
+
+OCR is off by default. `--ocr` explicitly requests the optional `tesseract.js`
+path; an unavailable engine or image produces incomplete evidence, never guessed
+text. Every media-to-claim relationship is declarative and still requires
+human-authoritative semantic review.

package/src/cli/index.js

@@ -43,7 +43,7 @@ Commands
   compare-snapshots [a b]   Regression diff between two audit runs
   action-plan [run]         Turn audit findings into ordered remediation work
   observe <mode>            Collect render, index, citation, log, Bing, passage,
-                            consensus, performance, or corroboration evidence
+                            consensus, performance, corroboration, or media evidence
   apply                     Apply a reviewed, hash-locked remediation spec
   monitor [runA runB]       Compare observation runs and emit regression alerts
   metrics import            Import declared metric observations from CSV/JSON
@@ -84,6 +84,7 @@ Options
   --access-token <token>    OAuth token (prefer provider environment variables)
   --endpoint <url>          Controlled citation adapter endpoint
   --repeat <count>          Repetitions per prompt for citation experiments
+  --ocr                     Explicitly request optional OCR for media images
   --write                   Persist registry changes (map-claims, substantiate)
   --json                    Machine-readable output only
 
@@ -96,6 +97,7 @@ function parseArgs(argv) {
     const a = argv[i];
     if (a === '--write') args.write = true;
     else if (a === '--json') args.json = true;
+    else if (a === '--ocr') args.ocr = true;
     else if (a === '--target') args.target = argv[++i];
     else if (a === '--base-url') args.baseUrl = argv[++i];
     else if (a === '--ref-date') args.refDate = argv[++i];

package/src/commands/observe.js

@@ -7,6 +7,7 @@ import path from 'node:path';
 import { parse as parseCsv } from 'csv-parse/sync';
 import { crawlerIdentity } from '../observations/crawlerIdentity.js';
 import { validateAgainst } from '../shared/schemaValidator.js';
+import { observeMedia } from '../observations/media.js';
 
 const originOf = (value) => { try { return new URL(value).origin; } catch { return null; } };
 const words = (text) => String(text || '').trim().split(/\s+/).filter(Boolean);
@@ -289,6 +290,7 @@ export async function observe(root, mode, options = {}) {
     case 'consensus': return observeConsensus(root, options);
     case 'performance': return observePerformance(root, options);
     case 'corroboration': return observeCorroboration(root, options);
-    default: throw new Error('observe mode must be render, index, citations, logs, bing, passages, consensus, performance, or corroboration');
+    case 'media': return observeMedia(root, options);
+    default: throw new Error('observe mode must be render, index, citations, logs, bing, passages, consensus, performance, corroboration, or media');
   }
 }

package/src/observations/media.js

@@ -0,0 +1,75 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { parse } from 'node-html-parser';
+import { envelope, observationRun, readInput } from './common.js';
+import { loadRegistries } from '../registries/index.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+function resolveMediaPath(manifestFile, value) {
+  const file = path.resolve(path.dirname(manifestFile), value);
+  if (!fs.existsSync(file) || !fs.statSync(file).isFile()) throw new Error(`media file not found: ${value}`);
+  return file;
+}
+
+async function pdfEvidence(item, file, artifacts) {
+  const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
+  const bytes = new Uint8Array(fs.readFileSync(file));
+  const document = await pdfjs.getDocument({ data: bytes, disableWorker: true, useSystemFonts: true }).promise;
+  const metadata = await document.getMetadata().catch(() => ({ info: {}, metadata: null }));
+  const pages = [];
+  for (let number = 1; number <= document.numPages; number++) {
+    const page = await document.getPage(number);
+    const content = await page.getTextContent();
+    const text = content.items.map((entry) => entry.str).join(' ').replace(/\s+/g, ' ').trim();
+    pages.push({ page: number, text, word_count: text ? text.split(/\s+/).length : 0, anchor: `${item.source_url || file}#page=${number}` });
+  }
+  artifacts[`media/${item.media_id}/extracted.txt`] = pages.map((page) => `--- page ${page.page} ---\n${page.text}`).join('\n');
+  return envelope('media_pdf', { media_id: item.media_id, source_url: item.source_url, page_url: item.page_url, claim_ids: item.claim_ids, permission_status: item.permission_status, page_count: document.numPages, pages, metadata: metadata.info || {}, tagged: Boolean(metadata.info?.IsTagged) }, { method: 'static_analysis', source: file, raw: bytes, confidence: 'high', limitations: ['Reading order, tables, footnotes, signatures, revisions, accessibility tags, and scanned text are not fully validated.', 'Page text extraction does not establish that linked claims are supported.'] });
+}
+
+function transcriptEvidence(item, file, artifacts) {
+  const raw = fs.readFileSync(file, 'utf8');
+  const cues = raw.split(/\r?\n\r?\n/).map((block) => { const lines = block.split(/\r?\n/).filter(Boolean); const timing = lines.find((line) => line.includes('-->')) || null; const text = lines.filter((line) => line !== 'WEBVTT' && line !== timing && !/^\d+$/.test(line)).join(' ').replace(/<[^>]+>/g, '').trim(); return text ? { timing, text } : null; }).filter(Boolean);
+  const text = cues.map((cue) => cue.text).join(' ');
+  artifacts[`media/${item.media_id}/transcript.txt`] = text;
+  return envelope('media_transcript', { media_id: item.media_id, source_url: item.source_url, page_url: item.page_url, claim_ids: item.claim_ids, permission_status: item.permission_status, language: item.language || null, published_at: item.published_at || null, cues, word_count: text ? text.split(/\s+/).length : 0 }, { method: 'owner_import', source: file, raw, limitations: ['Speaker identity, transcription accuracy, timing accuracy, and media parity require human verification.', 'Transcript ingestion does not establish that linked claims are supported.'] });
+}
+
+async function imageEvidence(item, file, options) {
+  const html = fs.readFileSync(file, 'utf8');
+  const root = parse(html);
+  const images = root.querySelectorAll('img');
+  const image = item.image_src ? images.find((node) => node.getAttribute('src') === item.image_src) : images[0];
+  if (!image) return envelope('media_image', { media_id: item.media_id, image_src: item.image_src || null, page_url: item.page_url, claim_ids: item.claim_ids }, { method: 'static_analysis', source: file, raw: html, state: 'not_observed', limitations: ['No matching image was found in the supplied HTML.'] });
+  const figure = image.closest('figure');
+  const caption = figure?.querySelector('figcaption')?.text.replace(/\s+/g, ' ').trim() || null;
+  const alt = image.getAttribute('alt');
+  const nearby_text = (figure || image.parentNode)?.text?.replace(/\s+/g, ' ').trim() || null;
+  let ocr = { state: 'not_requested', text: null };
+  if (options.ocr) {
+    try { const tesseract = await import('tesseract.js'); const result = await tesseract.recognize(path.resolve(path.dirname(file), image.getAttribute('src')), item.language || 'eng'); ocr = { state: 'observed', text: result.data.text }; }
+    catch { ocr = { state: 'not_evidenced', text: null }; }
+  }
+  return envelope('media_image', { media_id: item.media_id, image_src: image.getAttribute('src'), source_url: item.source_url, page_url: item.page_url, claim_ids: item.claim_ids, permission_status: item.permission_status, alt: alt ?? null, caption, nearby_text, ocr }, { method: 'static_analysis', source: file, raw: html, state: options.ocr && ocr.state === 'not_evidenced' ? 'incomplete' : 'observed', confidence: 'high', limitations: [...(!alt && !caption ? ['Neither alt text nor a figure caption supplies textual context.'] : []), ...(ocr.state === 'not_evidenced' ? ['OCR was explicitly requested but the optional tesseract.js dependency or image resource was unavailable.'] : []), 'Alt text, captions, nearby text, and OCR do not establish semantic equivalence or claim support.'] });
+}
+
+export async function observeMedia(root, options) {
+  const input = readInput(options.input);
+  const check = validateAgainst('media-manifest.schema.json', input.value);
+  if (!check.valid) throw new Error(`media manifest violates contract: ${check.errors.join('; ')}`);
+  const { registries, problems } = loadRegistries(root);
+  if (problems.length) throw new Error(`registry validation failed: ${problems.join('; ')}`);
+  const claimIds = new Set(registries.claims.entries.map((claim) => claim.claim_id));
+  const observations = [], artifacts = {}, rawInputs = { media_manifest: input.raw };
+  for (const item of input.value.items) {
+    const unknown = item.claim_ids.filter((id) => !claimIds.has(id));
+    if (unknown.length) throw new Error(`${item.media_id} references unknown claim ids: ${unknown.join(', ')}`);
+    const file = resolveMediaPath(input.file, item.path);
+    rawInputs[`media_${item.media_id}`] = fs.readFileSync(file);
+    if (item.type === 'pdf') observations.push(await pdfEvidence(item, file, artifacts));
+    else if (item.type === 'transcript') observations.push(transcriptEvidence(item, file, artifacts));
+    else observations.push(await imageEvidence(item, file, options));
+  }
+  const incomplete = observations.filter((item) => item.state === 'incomplete').map((item) => `${item.data.media_id}: requested extraction is incomplete`);
+  return observationRun(root, 'observe media', input.file, observations, { rawInputs, artifacts, incomplete });
+}