@nebulacomponents/citable 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +7 -2
- package/dist/universal/.agents/skills/citable/SKILL.md +6 -2
- package/dist/universal/.agents/skills/citable/VERSION +1 -1
- package/dist/universal/.agents/skills/citable/commands/README.md +2 -0
- package/dist/universal/.agents/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.agents/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.agents/skills/citable/manifest.json +11 -7
- package/dist/universal/.agents/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.agents/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.agents/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.claude/skills/citable/SKILL.md +6 -2
- package/dist/universal/.claude/skills/citable/VERSION +1 -1
- package/dist/universal/.claude/skills/citable/commands/README.md +2 -0
- package/dist/universal/.claude/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.claude/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.claude/skills/citable/manifest.json +11 -7
- package/dist/universal/.claude/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.claude/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.claude/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.cursor/skills/citable/SKILL.md +6 -2
- package/dist/universal/.cursor/skills/citable/VERSION +1 -1
- package/dist/universal/.cursor/skills/citable/commands/README.md +2 -0
- package/dist/universal/.cursor/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.cursor/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.cursor/skills/citable/manifest.json +11 -7
- package/dist/universal/.cursor/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.cursor/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.cursor/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.gemini/skills/citable/SKILL.md +6 -2
- package/dist/universal/.gemini/skills/citable/VERSION +1 -1
- package/dist/universal/.gemini/skills/citable/commands/README.md +2 -0
- package/dist/universal/.gemini/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.gemini/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.gemini/skills/citable/manifest.json +11 -7
- package/dist/universal/.gemini/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.gemini/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.gemini/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.github/skills/citable/SKILL.md +6 -2
- package/dist/universal/.github/skills/citable/VERSION +1 -1
- package/dist/universal/.github/skills/citable/commands/README.md +2 -0
- package/dist/universal/.github/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.github/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.github/skills/citable/manifest.json +11 -7
- package/dist/universal/.github/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.github/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.github/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.kiro/skills/citable/SKILL.md +6 -2
- package/dist/universal/.kiro/skills/citable/VERSION +1 -1
- package/dist/universal/.kiro/skills/citable/commands/README.md +2 -0
- package/dist/universal/.kiro/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.kiro/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.kiro/skills/citable/manifest.json +11 -7
- package/dist/universal/.kiro/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.kiro/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.kiro/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.opencode/skills/citable/SKILL.md +6 -2
- package/dist/universal/.opencode/skills/citable/VERSION +1 -1
- package/dist/universal/.opencode/skills/citable/commands/README.md +2 -0
- package/dist/universal/.opencode/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.opencode/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.opencode/skills/citable/manifest.json +11 -7
- package/dist/universal/.opencode/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.opencode/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.opencode/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.pi/agent/skills/citable/SKILL.md +6 -2
- package/dist/universal/.pi/agent/skills/citable/VERSION +1 -1
- package/dist/universal/.pi/agent/skills/citable/commands/README.md +2 -0
- package/dist/universal/.pi/agent/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.pi/agent/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.pi/agent/skills/citable/manifest.json +11 -7
- package/dist/universal/.pi/agent/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.pi/agent/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.pi/agent/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.qoder/skills/citable/SKILL.md +6 -2
- package/dist/universal/.qoder/skills/citable/VERSION +1 -1
- package/dist/universal/.qoder/skills/citable/commands/README.md +2 -0
- package/dist/universal/.qoder/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.qoder/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.qoder/skills/citable/manifest.json +11 -7
- package/dist/universal/.qoder/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.qoder/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.qoder/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.rovodev/skills/citable/SKILL.md +6 -2
- package/dist/universal/.rovodev/skills/citable/VERSION +1 -1
- package/dist/universal/.rovodev/skills/citable/commands/README.md +2 -0
- package/dist/universal/.rovodev/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.rovodev/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.rovodev/skills/citable/manifest.json +11 -7
- package/dist/universal/.rovodev/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.rovodev/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.rovodev/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.trae/skills/citable/SKILL.md +6 -2
- package/dist/universal/.trae/skills/citable/VERSION +1 -1
- package/dist/universal/.trae/skills/citable/commands/README.md +2 -0
- package/dist/universal/.trae/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.trae/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.trae/skills/citable/manifest.json +11 -7
- package/dist/universal/.trae/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.trae/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.trae/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/.trae-cn/skills/citable/SKILL.md +6 -2
- package/dist/universal/.trae-cn/skills/citable/VERSION +1 -1
- package/dist/universal/.trae-cn/skills/citable/commands/README.md +2 -0
- package/dist/universal/.trae-cn/skills/citable/commands/delivery.md +18 -0
- package/dist/universal/.trae-cn/skills/citable/commands/media-evidence.md +17 -0
- package/dist/universal/.trae-cn/skills/citable/manifest.json +11 -7
- package/dist/universal/.trae-cn/skills/citable/schemas/media-manifest.schema.json +2 -0
- package/dist/universal/.trae-cn/skills/citable/schemas/observation.schema.json +1 -1
- package/dist/universal/.trae-cn/skills/citable/schemas/schedule.schema.json +4 -0
- package/dist/universal/manifest.json +135 -87
- package/package.json +8 -3
- package/schemas/media-manifest.schema.json +2 -0
- package/schemas/observation.schema.json +1 -1
- package/schemas/schedule.schema.json +4 -0
- package/skill/SKILL.md +6 -2
- package/skill/commands/README.md +2 -0
- package/skill/commands/delivery.md +18 -0
- package/skill/commands/media-evidence.md +17 -0
- package/src/cli/index.js +18 -1
- package/src/commands/compareSnapshots.js +15 -0
- package/src/commands/delivery.js +50 -0
- package/src/commands/observe.js +3 -1
- package/src/observations/media.js +75 -0
- package/src/registries/index.js +1 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nebulacomponents/citable",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.11.0",
|
|
4
4
|
"description": "Production-grade SEO, AEO, and GEO agent skill with auditable detection, claim governance, evidence validation, and multi-agent installation.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"seo",
|
|
@@ -90,14 +90,19 @@
|
|
|
90
90
|
"ajv-formats": "^3.0.1",
|
|
91
91
|
"csv-parse": "^7.0.1",
|
|
92
92
|
"js-yaml": "^5.2.1",
|
|
93
|
-
"node-html-parser": "^9.0.0"
|
|
93
|
+
"node-html-parser": "^9.0.0",
|
|
94
|
+
"pdfjs-dist": "^6.1.200"
|
|
94
95
|
},
|
|
95
96
|
"peerDependencies": {
|
|
96
|
-
"playwright": ">=1.50.0"
|
|
97
|
+
"playwright": ">=1.50.0",
|
|
98
|
+
"tesseract.js": ">=7.0.0"
|
|
97
99
|
},
|
|
98
100
|
"peerDependenciesMeta": {
|
|
99
101
|
"playwright": {
|
|
100
102
|
"optional": true
|
|
103
|
+
},
|
|
104
|
+
"tesseract.js": {
|
|
105
|
+
"optional": true
|
|
101
106
|
}
|
|
102
107
|
}
|
|
103
108
|
}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$id":"citable://schemas/media-manifest.schema.json","$schema":"http://json-schema.org/draft-07/schema#","title":"Media evidence import manifest","type":"object","additionalProperties":false,"required":["version","items"],"properties":{"version":{"const":1},"items":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["media_id","type","path","source_url","page_url","claim_ids","permission_status"],"properties":{"media_id":{"type":"string","pattern":"^MEDIA-[A-Z0-9][A-Z0-9_-]*$"},"type":{"type":"string","enum":["pdf","transcript","image"]},"path":{"type":"string","minLength":1},"source_url":{"type":["string","null"]},"page_url":{"type":["string","null"]},"claim_ids":{"type":"array","uniqueItems":true,"items":{"type":"string"}},"permission_status":{"type":"string","enum":["owner_authorized","licensed","public_unknown","internal"]},"image_src":{"type":"string"},"language":{"type":"string"},"published_at":{"type":"string","format":"date-time"}}}}}}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
"required": ["observation_id", "kind", "state", "collected_at", "collection_method", "confidence", "source", "evidence_hash", "data"],
|
|
8
8
|
"properties": {
|
|
9
9
|
"observation_id": { "type": "string", "minLength": 1 },
|
|
10
|
-
"kind": { "type": "string", "enum": ["render", "index", "citation", "citation_review", "crawler_log", "bing_webmaster", "passage", "canonical_freshness", "performance", "corroboration", "metric"] },
|
|
10
|
+
"kind": { "type": "string", "enum": ["render", "index", "citation", "citation_review", "crawler_log", "bing_webmaster", "passage", "canonical_freshness", "performance", "corroboration", "metric", "media_pdf", "media_transcript", "media_image"] },
|
|
11
11
|
"state": { "type": "string", "enum": ["observed", "not_observed", "not_evidenced", "incomplete", "review_required", "failed"] },
|
|
12
12
|
"collected_at": { "type": "string", "format": "date-time" },
|
|
13
13
|
"collection_method": { "type": "string", "enum": ["live_api", "browser", "owner_import", "synthetic_fetch", "static_analysis", "human_review"] },
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$id":"citable://schemas/schedule.schema.json","$schema":"http://json-schema.org/draft-07/schema#","title":"Audit schedule registry","type":"object","additionalProperties":false,"required":["version","kind","updated","entries"],
|
|
3
|
+
"properties":{"version":{"const":1},"kind":{"const":"schedules"},"updated":{"type":"string","format":"date-time"},"entries":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["schedule_id","name","status","cron","owner","expected_tool_version","audit","retention_days","limitations"],"properties":{"schedule_id":{"type":"string","pattern":"^SCHEDULE-[A-Z0-9][A-Z0-9_-]*$"},"name":{"type":"string","minLength":1},"status":{"type":"string","enum":["active","paused","retired"]},"cron":{"type":"string","minLength":5},"owner":{"type":"string","minLength":1},"expected_tool_version":{"type":"string","pattern":"^\\d+\\.\\d+\\.\\d+$"},"audit":{"type":"object","additionalProperties":false,"required":["target","scope"],"properties":{"target":{"type":"string","minLength":1},"scope":{"type":["string","null"]},"base_url":{"type":["string","null"]}}},"retention_days":{"type":"integer","minimum":1},"limitations":{"type":"array","minItems":1,"items":{"type":"string"}}}}}}
|
|
4
|
+
}
|
package/skill/SKILL.md
CHANGED
|
@@ -7,7 +7,7 @@ description: >
|
|
|
7
7
|
generative engine optimization, structured data governance, claim substantiation,
|
|
8
8
|
crawler policy, entity consistency, content discoverability, or wants a site audited
|
|
9
9
|
for how search and AI systems will retrieve, understand, cite, or recommend it.
|
|
10
|
-
version: 1.
|
|
10
|
+
version: 1.11.0
|
|
11
11
|
---
|
|
12
12
|
|
|
13
13
|
# Citable — evidence and change control for search and AI citation readiness
|
|
@@ -115,6 +115,9 @@ citable reviews prioritize [--write]
|
|
|
115
115
|
citable reviews plan --input <json|yaml> [--write]
|
|
116
116
|
citable reviews sample <sampling-plan-id> [--write]
|
|
117
117
|
citable reviews evaluate
|
|
118
|
+
citable schedules run <schedule-id> [--ref-date YYYY-MM-DD]
|
|
119
|
+
citable project github <run-id>
|
|
120
|
+
citable observe media --input <manifest.json> [--ocr]
|
|
118
121
|
```
|
|
119
122
|
|
|
120
123
|
Audit scopes: `technical seo aeo geo architecture entity claims evidence schema
|
|
@@ -174,7 +177,8 @@ mandatory.
|
|
|
174
177
|
`.citable/` holds YAML registries for queries, prompts, entities, claims,
|
|
175
178
|
evidence, pages, crawlers, competitors, experiments, metrics, objectives,
|
|
176
179
|
interventions, optional connection state, reviewers, review policies, and
|
|
177
|
-
governed exceptions, semantic review items,
|
|
180
|
+
governed exceptions, semantic review items, sampling plans, and audit schedules,
|
|
181
|
+
all schema-validated
|
|
178
182
|
(`schemas/*.schema.json`) with referential integrity checks. Rules:
|
|
179
183
|
|
|
180
184
|
- Never overwrite registry content without history — use the loader/saver in
|
package/skill/commands/README.md
CHANGED
|
@@ -29,6 +29,8 @@ intent, not executability, so statuses are strict:
|
|
|
29
29
|
| /citable evaluate | implemented — independent baseline/evaluation comparisons with inconclusive handling |
|
|
30
30
|
| /citable governance validate / evaluate | implemented — validates reviewer and exception authority and emits immutable dispositions without changing failed findings |
|
|
31
31
|
| /citable reviews queue / prioritize / plan / sample / evaluate | implemented — materiality queues, reproducible sampling, stale-decision checks, and independent disagreement adjudication |
|
|
32
|
+
| /citable schedules run / project github | implemented — canonical scheduled audits and hash-bound non-authoritative GitHub projections |
|
|
33
|
+
| /citable observe media | implemented — bounded PDF, transcript, image-context, optional OCR, and declared claim-link evidence |
|
|
32
34
|
| /citable ingest, map-site, map-queries, map-prompts, map-entities, map-evidence | specified (ingest.md) |
|
|
33
35
|
| /citable optimize-page | specified (optimize-page.md) — requires source-to-render mapping, claim-preserving rewrites, build execution, rollback; none of that is proven yet |
|
|
34
36
|
| /citable create-page, answer-block, architect, interlink, consolidate, metadata | specified (page-work.md) |
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Scheduling, projections, and differential delivery
|
|
2
|
+
|
|
3
|
+
`citable schedules run <schedule-id>` executes the same canonical audit function
|
|
4
|
+
used by direct CLI calls. Schedules are version-pinned and fail closed when
|
|
5
|
+
paused, missing, invalid, or bound to another Citable version. The external
|
|
6
|
+
cron service remains responsible for triggering the command; a declared cron
|
|
7
|
+
does not prove execution. Schedule metadata is stored outside the finalized
|
|
8
|
+
audit package and binds back to its manifest checksum.
|
|
9
|
+
|
|
10
|
+
`citable project github <run-id>` creates GitHub annotation JSON as a
|
|
11
|
+
non-authoritative projection. It records hashes of the source manifest and
|
|
12
|
+
findings and never edits the run. Dashboards and check annotations must link to
|
|
13
|
+
these artifacts rather than maintain separate finding state.
|
|
14
|
+
|
|
15
|
+
`compare-snapshots` reports observed comparability dimensions: resource input,
|
|
16
|
+
evidence artifact, detector set, configuration, observation method, tool
|
|
17
|
+
version, and possible external-system change. These dimensions identify why
|
|
18
|
+
runs may not be directly comparable; they do not establish causal attribution.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Media evidence collection
|
|
2
|
+
|
|
3
|
+
Use `citable observe media --input <manifest.json>` for bounded local-media
|
|
4
|
+
evidence. The manifest declares source URL, owning page, permission status, and
|
|
5
|
+
linked claim IDs for each PDF, transcript, or image-context source.
|
|
6
|
+
|
|
7
|
+
PDF collection extracts native text, basic metadata, and stable page anchors.
|
|
8
|
+
It does not establish reading order, table structure, footnote linkage,
|
|
9
|
+
signatures, revisions, scan accuracy, accessibility, or claim support.
|
|
10
|
+
Transcript collection preserves cues but does not verify speakers, timing,
|
|
11
|
+
accuracy, or source-media parity. Image collection records alt text, captions,
|
|
12
|
+
and nearby figure text without inferring visual entailment.
|
|
13
|
+
|
|
14
|
+
OCR is off by default. `--ocr` explicitly requests the optional `tesseract.js`
|
|
15
|
+
path; an unavailable engine or image produces incomplete evidence, never guessed
|
|
16
|
+
text. Every media-to-claim relationship is declarative and still requires
|
|
17
|
+
human-authoritative semantic review.
|
package/src/cli/index.js
CHANGED
|
@@ -19,6 +19,7 @@ import { evaluateObjective, importMetrics, initializeObjective, validateObjectiv
|
|
|
19
19
|
import { configureConnection, connectionStatus, discoverConnections, disconnectConnection, syncConnection, validateConnection } from '../commands/connect.js';
|
|
20
20
|
import { evaluateDispositions, validateGovernance } from '../commands/governance.js';
|
|
21
21
|
import { evaluateReviews, initializeSamplingPlan, prioritizeReviews, queueReviews, selectSample } from '../commands/reviews.js';
|
|
22
|
+
import { projectGithub, runSchedule } from '../commands/delivery.js';
|
|
22
23
|
|
|
23
24
|
const HELP = `citable — SEO / AEO / GEO audit, remediation, validation, and governance
|
|
24
25
|
|
|
@@ -42,7 +43,7 @@ Commands
|
|
|
42
43
|
compare-snapshots [a b] Regression diff between two audit runs
|
|
43
44
|
action-plan [run] Turn audit findings into ordered remediation work
|
|
44
45
|
observe <mode> Collect render, index, citation, log, Bing, passage,
|
|
45
|
-
consensus, performance, or
|
|
46
|
+
consensus, performance, corroboration, or media evidence
|
|
46
47
|
apply Apply a reviewed, hash-locked remediation spec
|
|
47
48
|
monitor [runA runB] Compare observation runs and emit regression alerts
|
|
48
49
|
metrics import Import declared metric observations from CSV/JSON
|
|
@@ -62,6 +63,8 @@ Commands
|
|
|
62
63
|
reviews plan Validate/add a sampling plan from --input
|
|
63
64
|
reviews sample [plan] Select a reproducible census or seeded random sample
|
|
64
65
|
reviews evaluate Detect stale decisions and require disagreement adjudication
|
|
66
|
+
schedules run [id] Execute an active version-pinned audit schedule
|
|
67
|
+
project github [run] Render non-authoritative GitHub annotations from a run
|
|
65
68
|
self-upgrade Check for a newer version and upgrade the npx cache
|
|
66
69
|
|
|
67
70
|
Options
|
|
@@ -81,6 +84,7 @@ Options
|
|
|
81
84
|
--access-token <token> OAuth token (prefer provider environment variables)
|
|
82
85
|
--endpoint <url> Controlled citation adapter endpoint
|
|
83
86
|
--repeat <count> Repetitions per prompt for citation experiments
|
|
87
|
+
--ocr Explicitly request optional OCR for media images
|
|
84
88
|
--write Persist registry changes (map-claims, substantiate)
|
|
85
89
|
--json Machine-readable output only
|
|
86
90
|
|
|
@@ -93,6 +97,7 @@ function parseArgs(argv) {
|
|
|
93
97
|
const a = argv[i];
|
|
94
98
|
if (a === '--write') args.write = true;
|
|
95
99
|
else if (a === '--json') args.json = true;
|
|
100
|
+
else if (a === '--ocr') args.ocr = true;
|
|
96
101
|
else if (a === '--target') args.target = argv[++i];
|
|
97
102
|
else if (a === '--base-url') args.baseUrl = argv[++i];
|
|
98
103
|
else if (a === '--ref-date') args.refDate = argv[++i];
|
|
@@ -272,6 +277,18 @@ export async function main(argv = process.argv.slice(2), options = {}) {
|
|
|
272
277
|
out(args,`reviews ${mode}: ${r.created?.length ?? r.items?.length ?? r.selected_item_ids?.length ?? r.results?.length ?? 1} item(s)${args.write?' written':' (dry run)'}`,r);
|
|
273
278
|
break;
|
|
274
279
|
}
|
|
280
|
+
case 'schedules': {
|
|
281
|
+
if(args._[0]!=='run') throw new Error('usage: citable schedules run <schedule-id> [--ref-date YYYY-MM-DD]');
|
|
282
|
+
const r=await runSchedule(root,{scheduleId:args._[1],refDate:args.refDate});
|
|
283
|
+
out(args,`schedule ${r.schedule_execution.schedule_id}: audit ${r.runId}\nEvidence package: ${r.dir}\nExecution record: ${r.execution_file}`,r);
|
|
284
|
+
break;
|
|
285
|
+
}
|
|
286
|
+
case 'project': {
|
|
287
|
+
if(args._[0]!=='github') throw new Error('usage: citable project github <run-id>');
|
|
288
|
+
const r=projectGithub(root,{runId:args._[1]});
|
|
289
|
+
out(args,`project github ${r.source_run_id}: ${r.annotations.length} annotation(s)\nProjection: ${path.join(r.dir,'annotations.json')}`,r);
|
|
290
|
+
break;
|
|
291
|
+
}
|
|
275
292
|
case 'self-upgrade': {
|
|
276
293
|
const output = await selfUpgradeCommand(argv.slice(1));
|
|
277
294
|
console.log(output);
|
|
@@ -27,6 +27,16 @@ export function compareSnapshots(root, { runA, runB } = {}) {
|
|
|
27
27
|
};
|
|
28
28
|
const a = load(runA);
|
|
29
29
|
const b = load(runB);
|
|
30
|
+
const changedKeys = (left = {}, right = {}) => [...new Set([...Object.keys(left), ...Object.keys(right)])].filter((key) => left[key] !== right[key]);
|
|
31
|
+
const changeDimensions = {
|
|
32
|
+
resource_changed: changedKeys(a.manifest.input_hashes, b.manifest.input_hashes),
|
|
33
|
+
evidence_artifacts_changed: changedKeys(a.manifest.output_hashes, b.manifest.output_hashes),
|
|
34
|
+
detector_set_changed: JSON.stringify(a.manifest.detectors_run) !== JSON.stringify(b.manifest.detectors_run),
|
|
35
|
+
configuration_changed: a.manifest.configuration_hash !== b.manifest.configuration_hash,
|
|
36
|
+
observation_method_changed: a.manifest.command !== b.manifest.command || JSON.stringify(a.manifest.argv) !== JSON.stringify(b.manifest.argv) || a.manifest.target?.kind !== b.manifest.target?.kind,
|
|
37
|
+
tool_changed: a.manifest.tool_version !== b.manifest.tool_version,
|
|
38
|
+
external_system_may_have_changed: a.manifest.target?.kind === 'url' || b.manifest.target?.kind === 'url',
|
|
39
|
+
};
|
|
30
40
|
const key = (f) => `${f.detector_id}|${f.subject.identifier}|${f.observation.summary}`;
|
|
31
41
|
const aKeys = new Map(a.findings.map((f) => [key(f), f]));
|
|
32
42
|
const bKeys = new Map(b.findings.map((f) => [key(f), f]));
|
|
@@ -38,6 +48,11 @@ export function compareSnapshots(root, { runA, runB } = {}) {
|
|
|
38
48
|
baseline_timestamp: a.manifest.timestamp,
|
|
39
49
|
comparison_timestamp: b.manifest.timestamp,
|
|
40
50
|
regressions, resolved, persisting,
|
|
51
|
+
comparability: {
|
|
52
|
+
comparable: !changeDimensions.detector_set_changed && !changeDimensions.configuration_changed && !changeDimensions.observation_method_changed && !changeDimensions.tool_changed,
|
|
53
|
+
change_dimensions: changeDimensions,
|
|
54
|
+
limitation: 'These dimensions identify observed differences between run envelopes; they do not establish what caused a finding change.',
|
|
55
|
+
},
|
|
41
56
|
summary: {
|
|
42
57
|
new_findings: regressions.length,
|
|
43
58
|
resolved_findings: resolved.length,
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { audit } from './audit.js';
|
|
5
|
+
import { loadRegistries } from '../registries/index.js';
|
|
6
|
+
import { readJson, sha256, writeJson, nowIso } from '../shared/io.js';
|
|
7
|
+
|
|
8
|
+
const PKG = readJson(path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../package.json'));
|
|
9
|
+
|
|
10
|
+
export async function runSchedule(root, { scheduleId, refDate } = {}) {
|
|
11
|
+
const { registries, problems } = loadRegistries(root);
|
|
12
|
+
if (problems.length) throw new Error(`registry validation failed: ${problems.join('; ')}`);
|
|
13
|
+
const schedule = registries.schedules.entries.find((item) => item.schedule_id === scheduleId);
|
|
14
|
+
if (!schedule) throw new Error(`schedule not found: ${scheduleId}`);
|
|
15
|
+
if (schedule.status !== 'active') throw new Error(`schedule ${scheduleId} is ${schedule.status}`);
|
|
16
|
+
if (schedule.expected_tool_version !== PKG.version) throw new Error(`schedule ${scheduleId} expects Citable ${schedule.expected_tool_version}, current version is ${PKG.version}`);
|
|
17
|
+
const result = await audit(root, { target: schedule.audit.target, scope: schedule.audit.scope, baseUrl: schedule.audit.base_url, refDate });
|
|
18
|
+
const execution = { schedule_id: scheduleId, cron: schedule.cron, owner: schedule.owner, executed_at: nowIso(), run_id: result.runId, manifest_hash: sha256(fs.readFileSync(path.join(result.dir, 'manifest.json'))), limitations: schedule.limitations };
|
|
19
|
+
const executionFile = path.join(root, '.citable', 'schedule-executions', `${result.runId}.json`);
|
|
20
|
+
writeJson(executionFile, execution);
|
|
21
|
+
return { ...result, schedule_execution: execution, execution_file: executionFile };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function annotationLevel(severity) {
|
|
25
|
+
if (['critical', 'high'].includes(severity)) return 'failure';
|
|
26
|
+
if (severity === 'medium') return 'warning';
|
|
27
|
+
return 'notice';
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function projectGithub(root, { runId } = {}) {
|
|
31
|
+
if (!runId) throw new Error('projection requires a source run id');
|
|
32
|
+
const runDir = path.join(root, '.citable', 'runs', runId);
|
|
33
|
+
const findingsFile = path.join(runDir, 'findings.json');
|
|
34
|
+
const manifestFile = path.join(runDir, 'manifest.json');
|
|
35
|
+
if (!fs.existsSync(findingsFile) || !fs.existsSync(manifestFile)) throw new Error(`run ${runId} is missing findings or manifest evidence`);
|
|
36
|
+
const findingsBytes = fs.readFileSync(findingsFile);
|
|
37
|
+
const manifestBytes = fs.readFileSync(manifestFile);
|
|
38
|
+
const annotations = JSON.parse(findingsBytes).map((finding) => ({
|
|
39
|
+
level: annotationLevel(finding.classification.severity),
|
|
40
|
+
title: `${finding.detector_id}: ${finding.classification.severity}`,
|
|
41
|
+
message: finding.observation.summary,
|
|
42
|
+
path: finding.subject.source_file || null,
|
|
43
|
+
line: finding.subject.source_location ? Number(String(finding.subject.source_location).match(/\d+/)?.[0]) || null : null,
|
|
44
|
+
finding_id: finding.finding_id,
|
|
45
|
+
}));
|
|
46
|
+
const projection = { source_run_id: runId, source_manifest_hash: sha256(manifestBytes), source_findings_hash: sha256(findingsBytes), generated_at: nowIso(), projection: 'github-check-annotations', authoritative: false, annotations };
|
|
47
|
+
const dir = path.join(root, '.citable', 'projections', 'github', runId);
|
|
48
|
+
writeJson(path.join(dir, 'annotations.json'), projection);
|
|
49
|
+
return { ...projection, dir };
|
|
50
|
+
}
|
package/src/commands/observe.js
CHANGED
|
@@ -7,6 +7,7 @@ import path from 'node:path';
|
|
|
7
7
|
import { parse as parseCsv } from 'csv-parse/sync';
|
|
8
8
|
import { crawlerIdentity } from '../observations/crawlerIdentity.js';
|
|
9
9
|
import { validateAgainst } from '../shared/schemaValidator.js';
|
|
10
|
+
import { observeMedia } from '../observations/media.js';
|
|
10
11
|
|
|
11
12
|
const originOf = (value) => { try { return new URL(value).origin; } catch { return null; } };
|
|
12
13
|
const words = (text) => String(text || '').trim().split(/\s+/).filter(Boolean);
|
|
@@ -289,6 +290,7 @@ export async function observe(root, mode, options = {}) {
|
|
|
289
290
|
case 'consensus': return observeConsensus(root, options);
|
|
290
291
|
case 'performance': return observePerformance(root, options);
|
|
291
292
|
case 'corroboration': return observeCorroboration(root, options);
|
|
292
|
-
|
|
293
|
+
case 'media': return observeMedia(root, options);
|
|
294
|
+
default: throw new Error('observe mode must be render, index, citations, logs, bing, passages, consensus, performance, corroboration, or media');
|
|
293
295
|
}
|
|
294
296
|
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { parse } from 'node-html-parser';
|
|
4
|
+
import { envelope, observationRun, readInput } from './common.js';
|
|
5
|
+
import { loadRegistries } from '../registries/index.js';
|
|
6
|
+
import { validateAgainst } from '../shared/schemaValidator.js';
|
|
7
|
+
|
|
8
|
+
function resolveMediaPath(manifestFile, value) {
|
|
9
|
+
const file = path.resolve(path.dirname(manifestFile), value);
|
|
10
|
+
if (!fs.existsSync(file) || !fs.statSync(file).isFile()) throw new Error(`media file not found: ${value}`);
|
|
11
|
+
return file;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
async function pdfEvidence(item, file, artifacts) {
|
|
15
|
+
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
|
16
|
+
const bytes = new Uint8Array(fs.readFileSync(file));
|
|
17
|
+
const document = await pdfjs.getDocument({ data: bytes, disableWorker: true, useSystemFonts: true }).promise;
|
|
18
|
+
const metadata = await document.getMetadata().catch(() => ({ info: {}, metadata: null }));
|
|
19
|
+
const pages = [];
|
|
20
|
+
for (let number = 1; number <= document.numPages; number++) {
|
|
21
|
+
const page = await document.getPage(number);
|
|
22
|
+
const content = await page.getTextContent();
|
|
23
|
+
const text = content.items.map((entry) => entry.str).join(' ').replace(/\s+/g, ' ').trim();
|
|
24
|
+
pages.push({ page: number, text, word_count: text ? text.split(/\s+/).length : 0, anchor: `${item.source_url || file}#page=${number}` });
|
|
25
|
+
}
|
|
26
|
+
artifacts[`media/${item.media_id}/extracted.txt`] = pages.map((page) => `--- page ${page.page} ---\n${page.text}`).join('\n');
|
|
27
|
+
return envelope('media_pdf', { media_id: item.media_id, source_url: item.source_url, page_url: item.page_url, claim_ids: item.claim_ids, permission_status: item.permission_status, page_count: document.numPages, pages, metadata: metadata.info || {}, tagged: Boolean(metadata.info?.IsTagged) }, { method: 'static_analysis', source: file, raw: bytes, confidence: 'high', limitations: ['Reading order, tables, footnotes, signatures, revisions, accessibility tags, and scanned text are not fully validated.', 'Page text extraction does not establish that linked claims are supported.'] });
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function transcriptEvidence(item, file, artifacts) {
|
|
31
|
+
const raw = fs.readFileSync(file, 'utf8');
|
|
32
|
+
const cues = raw.split(/\r?\n\r?\n/).map((block) => { const lines = block.split(/\r?\n/).filter(Boolean); const timing = lines.find((line) => line.includes('-->')) || null; const text = lines.filter((line) => line !== 'WEBVTT' && line !== timing && !/^\d+$/.test(line)).join(' ').replace(/<[^>]+>/g, '').trim(); return text ? { timing, text } : null; }).filter(Boolean);
|
|
33
|
+
const text = cues.map((cue) => cue.text).join(' ');
|
|
34
|
+
artifacts[`media/${item.media_id}/transcript.txt`] = text;
|
|
35
|
+
return envelope('media_transcript', { media_id: item.media_id, source_url: item.source_url, page_url: item.page_url, claim_ids: item.claim_ids, permission_status: item.permission_status, language: item.language || null, published_at: item.published_at || null, cues, word_count: text ? text.split(/\s+/).length : 0 }, { method: 'owner_import', source: file, raw, limitations: ['Speaker identity, transcription accuracy, timing accuracy, and media parity require human verification.', 'Transcript ingestion does not establish that linked claims are supported.'] });
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function imageEvidence(item, file, options) {
|
|
39
|
+
const html = fs.readFileSync(file, 'utf8');
|
|
40
|
+
const root = parse(html);
|
|
41
|
+
const images = root.querySelectorAll('img');
|
|
42
|
+
const image = item.image_src ? images.find((node) => node.getAttribute('src') === item.image_src) : images[0];
|
|
43
|
+
if (!image) return envelope('media_image', { media_id: item.media_id, image_src: item.image_src || null, page_url: item.page_url, claim_ids: item.claim_ids }, { method: 'static_analysis', source: file, raw: html, state: 'not_observed', limitations: ['No matching image was found in the supplied HTML.'] });
|
|
44
|
+
const figure = image.closest('figure');
|
|
45
|
+
const caption = figure?.querySelector('figcaption')?.text.replace(/\s+/g, ' ').trim() || null;
|
|
46
|
+
const alt = image.getAttribute('alt');
|
|
47
|
+
const nearby_text = (figure || image.parentNode)?.text?.replace(/\s+/g, ' ').trim() || null;
|
|
48
|
+
let ocr = { state: 'not_requested', text: null };
|
|
49
|
+
if (options.ocr) {
|
|
50
|
+
try { const tesseract = await import('tesseract.js'); const result = await tesseract.recognize(path.resolve(path.dirname(file), image.getAttribute('src')), item.language || 'eng'); ocr = { state: 'observed', text: result.data.text }; }
|
|
51
|
+
catch { ocr = { state: 'not_evidenced', text: null }; }
|
|
52
|
+
}
|
|
53
|
+
return envelope('media_image', { media_id: item.media_id, image_src: image.getAttribute('src'), source_url: item.source_url, page_url: item.page_url, claim_ids: item.claim_ids, permission_status: item.permission_status, alt: alt ?? null, caption, nearby_text, ocr }, { method: 'static_analysis', source: file, raw: html, state: options.ocr && ocr.state === 'not_evidenced' ? 'incomplete' : 'observed', confidence: 'high', limitations: [...(!alt && !caption ? ['Neither alt text nor a figure caption supplies textual context.'] : []), ...(ocr.state === 'not_evidenced' ? ['OCR was explicitly requested but the optional tesseract.js dependency or image resource was unavailable.'] : []), 'Alt text, captions, nearby text, and OCR do not establish semantic equivalence or claim support.'] });
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export async function observeMedia(root, options) {
|
|
57
|
+
const input = readInput(options.input);
|
|
58
|
+
const check = validateAgainst('media-manifest.schema.json', input.value);
|
|
59
|
+
if (!check.valid) throw new Error(`media manifest violates contract: ${check.errors.join('; ')}`);
|
|
60
|
+
const { registries, problems } = loadRegistries(root);
|
|
61
|
+
if (problems.length) throw new Error(`registry validation failed: ${problems.join('; ')}`);
|
|
62
|
+
const claimIds = new Set(registries.claims.entries.map((claim) => claim.claim_id));
|
|
63
|
+
const observations = [], artifacts = {}, rawInputs = { media_manifest: input.raw };
|
|
64
|
+
for (const item of input.value.items) {
|
|
65
|
+
const unknown = item.claim_ids.filter((id) => !claimIds.has(id));
|
|
66
|
+
if (unknown.length) throw new Error(`${item.media_id} references unknown claim ids: ${unknown.join(', ')}`);
|
|
67
|
+
const file = resolveMediaPath(input.file, item.path);
|
|
68
|
+
rawInputs[`media_${item.media_id}`] = fs.readFileSync(file);
|
|
69
|
+
if (item.type === 'pdf') observations.push(await pdfEvidence(item, file, artifacts));
|
|
70
|
+
else if (item.type === 'transcript') observations.push(transcriptEvidence(item, file, artifacts));
|
|
71
|
+
else observations.push(await imageEvidence(item, file, options));
|
|
72
|
+
}
|
|
73
|
+
const incomplete = observations.filter((item) => item.state === 'incomplete').map((item) => `${item.data.media_id}: requested extraction is incomplete`);
|
|
74
|
+
return observationRun(root, 'observe media', input.file, observations, { rawInputs, artifacts, incomplete });
|
|
75
|
+
}
|
package/src/registries/index.js
CHANGED
|
@@ -22,6 +22,7 @@ export const REGISTRY_SPECS = [
|
|
|
22
22
|
{ file: 'exceptions.yaml', kind: 'exceptions', schema: 'exception.schema.json', idField: 'exception_id' },
|
|
23
23
|
{ file: 'review-items.yaml', kind: 'review_items', schema: 'review-item.schema.json', idField: 'review_item_id' },
|
|
24
24
|
{ file: 'sampling-plans.yaml', kind: 'sampling_plans', schema: 'sampling-plan.schema.json', idField: 'sampling_plan_id' },
|
|
25
|
+
{ file: 'schedules.yaml', kind: 'schedules', schema: 'schedule.schema.json', idField: 'schedule_id' },
|
|
25
26
|
];
|
|
26
27
|
|
|
27
28
|
export function contextDir(root) {
|