@nebulacomponents/citable 1.16.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +16 -3
- package/dist/universal/.agents/skills/citable/SKILL.md +66 -1
- package/dist/universal/.agents/skills/citable/VERSION +1 -1
- package/dist/universal/.agents/skills/citable/manifest.json +9 -6
- package/dist/universal/.agents/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.agents/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.agents/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.agents/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.claude/agents/citable/manifest.json +2 -2
- package/dist/universal/.claude/skills/citable/SKILL.md +66 -1
- package/dist/universal/.claude/skills/citable/VERSION +1 -1
- package/dist/universal/.claude/skills/citable/manifest.json +9 -6
- package/dist/universal/.claude/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.claude/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.claude/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.claude/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.cursor/skills/citable/SKILL.md +66 -1
- package/dist/universal/.cursor/skills/citable/VERSION +1 -1
- package/dist/universal/.cursor/skills/citable/manifest.json +9 -6
- package/dist/universal/.cursor/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.cursor/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.cursor/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.cursor/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.gemini/skills/citable/SKILL.md +66 -1
- package/dist/universal/.gemini/skills/citable/VERSION +1 -1
- package/dist/universal/.gemini/skills/citable/manifest.json +9 -6
- package/dist/universal/.gemini/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.gemini/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.gemini/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.gemini/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.github/skills/citable/SKILL.md +66 -1
- package/dist/universal/.github/skills/citable/VERSION +1 -1
- package/dist/universal/.github/skills/citable/manifest.json +9 -6
- package/dist/universal/.github/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.github/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.github/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.github/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.kiro/skills/citable/SKILL.md +66 -1
- package/dist/universal/.kiro/skills/citable/VERSION +1 -1
- package/dist/universal/.kiro/skills/citable/manifest.json +9 -6
- package/dist/universal/.kiro/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.kiro/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.kiro/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.kiro/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.opencode/skills/citable/SKILL.md +66 -1
- package/dist/universal/.opencode/skills/citable/VERSION +1 -1
- package/dist/universal/.opencode/skills/citable/manifest.json +9 -6
- package/dist/universal/.opencode/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.opencode/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.opencode/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.opencode/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.pi/agent/skills/citable/SKILL.md +66 -1
- package/dist/universal/.pi/agent/skills/citable/VERSION +1 -1
- package/dist/universal/.pi/agent/skills/citable/manifest.json +9 -6
- package/dist/universal/.pi/agent/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.pi/agent/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.pi/agent/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.pi/agent/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.qoder/skills/citable/SKILL.md +66 -1
- package/dist/universal/.qoder/skills/citable/VERSION +1 -1
- package/dist/universal/.qoder/skills/citable/manifest.json +9 -6
- package/dist/universal/.qoder/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.qoder/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.qoder/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.qoder/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.rovodev/skills/citable/SKILL.md +66 -1
- package/dist/universal/.rovodev/skills/citable/VERSION +1 -1
- package/dist/universal/.rovodev/skills/citable/manifest.json +9 -6
- package/dist/universal/.rovodev/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.rovodev/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.rovodev/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.rovodev/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.trae/skills/citable/SKILL.md +66 -1
- package/dist/universal/.trae/skills/citable/VERSION +1 -1
- package/dist/universal/.trae/skills/citable/manifest.json +9 -6
- package/dist/universal/.trae/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.trae/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.trae/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.trae/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.trae-cn/skills/citable/SKILL.md +66 -1
- package/dist/universal/.trae-cn/skills/citable/VERSION +1 -1
- package/dist/universal/.trae-cn/skills/citable/manifest.json +9 -6
- package/dist/universal/.trae-cn/skills/citable/schemas/cli-output-envelope.schema.json +30 -0
- package/dist/universal/.trae-cn/skills/citable/schemas/finding.schema.json +69 -0
- package/dist/universal/.trae-cn/skills/citable/schemas/remediation-verification.schema.json +98 -0
- package/dist/universal/.trae-cn/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/manifest.json +112 -76
- package/package.json +1 -1
- package/schemas/cli-output-envelope.schema.json +30 -0
- package/schemas/finding.schema.json +69 -0
- package/schemas/remediation-verification.schema.json +98 -0
- package/schemas/sow.schema.json +92 -0
- package/skill/SKILL.md +66 -1
- package/src/analysis/behavioral.js +126 -0
- package/src/analysis/croAudit.js +302 -0
- package/src/analysis/croRoadmap.js +121 -0
- package/src/analysis/eeat.js +300 -0
- package/src/analysis/funnelAnalysis.js +171 -0
- package/src/analysis/iceMatrix.js +205 -0
- package/src/analysis/offpage.js +228 -0
- package/src/analysis/readiness.js +253 -0
- package/src/analysis/saliency.js +119 -0
- package/src/analysis/schemaRecommendations.js +173 -0
- package/src/analysis/strategicRoadmap.js +148 -0
- package/src/cli/index.js +418 -9
- package/src/commands/answerEngineReadiness.js +48 -0
- package/src/commands/audit.js +2 -2
- package/src/commands/auditBacklinks.js +79 -0
- package/src/commands/ciWorkflow.js +63 -0
- package/src/commands/compatibility.js +205 -0
- package/src/commands/context.js +2 -1
- package/src/commands/croBacklog.js +182 -0
- package/src/commands/croSuite.js +156 -0
- package/src/commands/edgeRules.js +80 -4
- package/src/commands/edgeSecurity.js +192 -0
- package/src/commands/executive.js +6 -0
- package/src/commands/executiveCro.js +30 -0
- package/src/commands/executiveSearch.js +28 -0
- package/src/commands/experimentGuardrails.js +213 -0
- package/src/commands/goldenCorpus.js +155 -0
- package/src/commands/implementationKit.js +202 -0
- package/src/commands/inspectCro.js +67 -0
- package/src/commands/inspectEeat.js +53 -0
- package/src/commands/planExperiment.js +52 -0
- package/src/commands/previewCro.js +241 -0
- package/src/commands/prioritize.js +42 -6
- package/src/commands/remediate.js +461 -0
- package/src/commands/roadmapCmd.js +69 -0
- package/src/commands/schemaCmd.js +10 -2
- package/src/commands/sowCmd.js +67 -0
- package/src/commands/sweep.js +357 -0
- package/src/commands/verifyRemediation.js +204 -0
- package/src/commands/visualRegression.js +165 -0
- package/src/components/index.js +556 -0
- package/src/detectors/cro.js +48 -0
- package/src/detectors/framework.js +31 -1
- package/src/reporting/executiveCroReport.js +731 -0
- package/src/reporting/executiveExport.js +175 -1
- package/src/reporting/executiveSearchReport.js +704 -0
- package/src/sow/admissibilityGate.js +194 -0
- package/src/sow/generateSow.js +762 -0
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import { loadRegistries } from '../registries/index.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Experiment safety guardrails.
|
|
5
|
+
*
|
|
6
|
+
* Lifecycle statuses are mutually exclusive and explicitly defined:
|
|
7
|
+
* planned — declared, not yet receiving traffic decisions
|
|
8
|
+
* running — receiving traffic, sample below the computed requirement and
|
|
9
|
+
* window not elapsed; no reading may be taken
|
|
10
|
+
* inconclusive— window and sample satisfied, effect not statistically
|
|
11
|
+
* significant; absence of evidence is not evidence of absence
|
|
12
|
+
* validated — full window, full sample, no SRM, effect significant at the
|
|
13
|
+
* declared alpha for the primary metric ONLY
|
|
14
|
+
*
|
|
15
|
+
* Guardrails fail closed: missing observations, missing windows, or missing
|
|
16
|
+
* registry context produce `blocked` findings, never a silent pass.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const MIN_WINDOW_DAYS = 7;
|
|
20
|
+
const SRM_ALPHA = 0.001; // standard SRM alert threshold
|
|
21
|
+
|
|
22
|
+
/** Abramowitz–Stegun normal CDF. */
|
|
23
|
+
function normalCdf(z) {
|
|
24
|
+
const t = 1 / (1 + 0.2316419 * Math.abs(z));
|
|
25
|
+
const d = 0.3989423 * Math.exp((-z * z) / 2);
|
|
26
|
+
const p = d * t * (0.3193815 + t * (-0.3565638 + t * (1.781478 + t * (-1.821256 + t * 1.330274))));
|
|
27
|
+
return z >= 0 ? 1 - p : p;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function chiSquare1dfP(chi2) {
|
|
31
|
+
return 2 * (1 - normalCdf(Math.sqrt(chi2)));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function requiredSamplePerVariant(baselineRate, mde, alpha = 0.05, power = 0.8) {
|
|
35
|
+
const p1 = baselineRate;
|
|
36
|
+
const p2 = p1 * (1 + mde);
|
|
37
|
+
const pBar = (p1 + p2) / 2;
|
|
38
|
+
const zAlpha = alpha === 0.01 ? 2.576 : alpha === 0.10 ? 1.645 : 1.96;
|
|
39
|
+
const zBeta = power === 0.9 ? 1.282 : power === 0.95 ? 1.645 : 0.8416;
|
|
40
|
+
const numerator = Math.pow(zAlpha * Math.sqrt(2 * pBar * (1 - pBar)) + zBeta * Math.sqrt(p1 * (1 - p1) + p2 * (1 - p2)), 2);
|
|
41
|
+
const denominator = Math.pow(p2 - p1, 2);
|
|
42
|
+
return Math.ceil(numerator / denominator);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function windowDays(str) {
|
|
46
|
+
const m = String(str || '').match(/(\d+)\s*d/i);
|
|
47
|
+
return m ? Number(m[1]) : null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Core guardrail evaluation for one experiment record plus optional observed
|
|
52
|
+
* data: { control, variant, baseline_rate, mde, days_running, revenue_claimed,
|
|
53
|
+
* other_active_experiments: [{experiment_id, target_page}] }.
|
|
54
|
+
*/
|
|
55
|
+
export function evaluateExperiment(experiment, observed = {}) {
|
|
56
|
+
const findings = [];
|
|
57
|
+
const add = (guardrail_id, severity, summary, required_input = null) =>
|
|
58
|
+
findings.push({ guardrail_id, severity, summary, ...(required_input ? { required_input } : {}) });
|
|
59
|
+
|
|
60
|
+
const status = experiment.status || 'draft';
|
|
61
|
+
const declaredSplit = (experiment.variants || []).length
|
|
62
|
+
? (experiment.variants.find((v) => v.traffic_pct) || {}).traffic_pct || 50
|
|
63
|
+
: 50;
|
|
64
|
+
const window = windowDays(experiment.evaluation_window || experiment.minimum_observation_window);
|
|
65
|
+
const baseline = observed.baseline_rate ?? experiment.baseline_rate ?? null;
|
|
66
|
+
const mde = observed.mde ?? experiment.mde ?? null;
|
|
67
|
+
|
|
68
|
+
// Underpowered / required sample
|
|
69
|
+
let requiredN = null;
|
|
70
|
+
if (baseline > 0 && baseline < 1 && mde > 0) {
|
|
71
|
+
requiredN = requiredSamplePerVariant(baseline, mde);
|
|
72
|
+
if (window && window < MIN_WINDOW_DAYS) {
|
|
73
|
+
add('EXP-WINDOW-TOO-SHORT', 'high',
|
|
74
|
+
`Evaluation window ${window}d is shorter than the ${MIN_WINDOW_DAYS}d minimum; day-of-week seasonality produces high false-positive risk.`);
|
|
75
|
+
}
|
|
76
|
+
} else {
|
|
77
|
+
add('EXP-BASELINE-MISSING', 'blocked',
|
|
78
|
+
'No usable baseline_rate/mde declared; sample-size and power requirements cannot be established (fail closed).',
|
|
79
|
+
'required_input: baseline_rate (0-1) and mde (relative)');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Stopping early
|
|
83
|
+
if (status === 'active' || status === 'observing') {
|
|
84
|
+
const daysRunning = observed.days_running ?? null;
|
|
85
|
+
if (window && daysRunning !== null && daysRunning < window) {
|
|
86
|
+
add('EXP-STOPPING-EARLY', 'high',
|
|
87
|
+
`Experiment has run ${daysRunning}d of the declared ${window}d window; stopping now invalidates the design. No interim reading is decision-grade.`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Sample-ratio mismatch
|
|
92
|
+
if (observed.control !== undefined && observed.variant !== undefined) {
|
|
93
|
+
const c = Number(observed.control);
|
|
94
|
+
const v = Number(observed.variant);
|
|
95
|
+
if (!Number.isFinite(c) || !Number.isFinite(v) || c < 0 || v < 0) {
|
|
96
|
+
add('EXP-OBSERVATIONS-INVALID', 'blocked', 'Observed counts must be non-negative numbers (fail closed).');
|
|
97
|
+
} else if (c + v >= 100) {
|
|
98
|
+
const total = c + v;
|
|
99
|
+
const share = declaredSplit / 100;
|
|
100
|
+
const expectedC = total * share;
|
|
101
|
+
const expectedV = total * (1 - share);
|
|
102
|
+
const chi2 = ((c - expectedC) ** 2) / expectedC + ((v - expectedV) ** 2) / expectedV;
|
|
103
|
+
const p = chiSquare1dfP(chi2);
|
|
104
|
+
if (p < SRM_ALPHA) {
|
|
105
|
+
add('EXP-SRM-DETECTED', 'critical',
|
|
106
|
+
`Sample-ratio mismatch: observed ${c}/${v} vs declared ${declaredSplit}/${100 - declaredSplit} split (chi-square p=${p.toExponential(2)}). Instrumentation or routing is broken; all readings from this experiment are untrustworthy until fixed.`);
|
|
107
|
+
}
|
|
108
|
+
} else {
|
|
109
|
+
add('EXP-SAMPLE-TOO-SMALL-FOR-SRM', 'blocked',
|
|
110
|
+
'Fewer than 100 total observations; SRM cannot be assessed yet (fail closed).');
|
|
111
|
+
}
|
|
112
|
+
} else if (status === 'active' || status === 'observing') {
|
|
113
|
+
add('EXP-OBSERVATIONS-MISSING', 'blocked',
|
|
114
|
+
'Active experiment without observed variant counts; sample ratio, power, and stopping state cannot be verified (fail closed).',
|
|
115
|
+
'required_input: observed control and variant assignment counts');
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Power status against observed sample
|
|
119
|
+
if (requiredN !== null && observed.control !== undefined && observed.variant !== undefined) {
|
|
120
|
+
const perVariant = Math.min(Number(observed.control), Number(observed.variant));
|
|
121
|
+
if (perVariant < requiredN) {
|
|
122
|
+
add('EXP-UNDERPOWERED', 'medium',
|
|
123
|
+
`Current sample (${perVariant}/variant) is below the ${requiredN}/variant requirement for ${Math.round((mde ?? 0) * 100)}% MDE at 80% power; any observed difference is not decision-grade.`);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Revenue attribution without sufficient evidence
|
|
128
|
+
if (observed.revenue_claimed || /revenue|aov|arp/i.test(experiment.primary_metric || '')) {
|
|
129
|
+
const hasConcluded = status === 'concluded';
|
|
130
|
+
const hasSrm = findings.some((f) => f.guardrail_id === 'EXP-SRM-DETECTED');
|
|
131
|
+
if (hasSrm || !hasConcluded) {
|
|
132
|
+
add('EXP-REVENUE-UNSUPPORTED', 'high',
|
|
133
|
+
'Revenue attribution asserted without a concluded, SRM-clean experiment and verified conversion instrumentation; this remains a modeled estimate, not a measured outcome.');
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Contamination across pages / incompatible variants
|
|
138
|
+
const others = observed.other_active_experiments || [];
|
|
139
|
+
for (const other of others) {
|
|
140
|
+
if (other.target_page && experiment.target_page && other.target_page === experiment.target_page) {
|
|
141
|
+
add('EXP-CONTAMINATION', 'high',
|
|
142
|
+
`Experiment ${other.experiment_id} is also active on ${other.target_page}; overlapping experiments contaminate each other's readings.`);
|
|
143
|
+
}
|
|
144
|
+
const sharedVariants = (other.variant_ids || []).filter((id) => (experiment.variant_ids || []).includes(id));
|
|
145
|
+
if (sharedVariants.length) {
|
|
146
|
+
add('EXP-VARIANT-COLLISION', 'high',
|
|
147
|
+
`Variant id(s) ${sharedVariants.join(', ')} collide with active experiment ${other.experiment_id}; assignment is ambiguous.`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Lifecycle status (mutually exclusive, explicitly derived)
|
|
152
|
+
let lifecycle;
|
|
153
|
+
const blocking = findings.some((f) => f.severity === 'blocked' || f.severity === 'critical');
|
|
154
|
+
if (status === 'draft' || status === 'observing' || (!observed.control && !observed.variant && status !== 'active')) {
|
|
155
|
+
lifecycle = 'planned';
|
|
156
|
+
} else if (blocking) {
|
|
157
|
+
lifecycle = 'inconclusive';
|
|
158
|
+
} else {
|
|
159
|
+
const perVariant = Math.min(Number(observed.control ?? 0), Number(observed.variant ?? 0));
|
|
160
|
+
const withinWindow = window ? (observed.days_running ?? Infinity) >= window : false;
|
|
161
|
+
const sampleMet = requiredN !== null ? perVariant >= requiredN : false;
|
|
162
|
+
const significant = typeof observed.p_value === 'number' ? observed.p_value < (observed.alpha ?? 0.05) : false;
|
|
163
|
+
if (withinWindow && sampleMet && significant) lifecycle = 'validated';
|
|
164
|
+
else if (withinWindow && sampleMet) lifecycle = 'inconclusive';
|
|
165
|
+
else lifecycle = 'running';
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const definitions = {
|
|
169
|
+
planned: 'declared, not yet receiving traffic; no reading exists',
|
|
170
|
+
running: 'receiving traffic; sample/window incomplete; interim readings are not decision-grade',
|
|
171
|
+
inconclusive: 'window and sample satisfied but the effect is not statistically significant, or a blocking guardrail fired; absence of evidence is not evidence of absence',
|
|
172
|
+
validated: 'full window, sufficient sample, no SRM, primary metric significant at declared alpha — statistically validated for the primary metric only, never a conversion or revenue guarantee',
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
return {
|
|
176
|
+
experiment_id: experiment.experiment_id,
|
|
177
|
+
registry_status: status,
|
|
178
|
+
lifecycle,
|
|
179
|
+
lifecycle_definition: definitions[lifecycle],
|
|
180
|
+
required_sample_per_variant: requiredN,
|
|
181
|
+
declared_traffic_split: `${declaredSplit}/${100 - declaredSplit}`,
|
|
182
|
+
guardrail_count: findings.length,
|
|
183
|
+
findings,
|
|
184
|
+
lifecycle_definitions: definitions,
|
|
185
|
+
note: 'lifecycle labels describe statistical state only; Citable does not guarantee conversion, ranking, or citation outcomes',
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* `citable check experiment <id> [--observed-control N --observed-variant N]`
|
|
191
|
+
*/
|
|
192
|
+
export async function checkExperiment(root, options = {}) {
|
|
193
|
+
const { registries, problems } = loadRegistries(root);
|
|
194
|
+
const entries = registries?.experiments?.entries || [];
|
|
195
|
+
const id = options.experimentId || options.experiment_id;
|
|
196
|
+
if (!id) throw new Error('usage: citable check experiment <experiment-id> [--observed-control N --observed-variant N --days-running N]');
|
|
197
|
+
const experiment = entries.find((e) => e.experiment_id === id);
|
|
198
|
+
if (!experiment) {
|
|
199
|
+
throw new Error(`experiment "${id}" not found in experiments.yaml. Available: ${entries.map((e) => e.experiment_id).join(', ') || 'none'}`);
|
|
200
|
+
}
|
|
201
|
+
const observed = {
|
|
202
|
+
control: options.observedControl ?? options.observed_control,
|
|
203
|
+
variant: options.observedVariant ?? options.observed_variant,
|
|
204
|
+
days_running: options.daysRunning ?? options.days_running,
|
|
205
|
+
baseline_rate: options.baselineRate ? Number(options.baselineRate) : undefined,
|
|
206
|
+
mde: options.mde ? Number(options.mde) : undefined,
|
|
207
|
+
p_value: options.pValue ? Number(options.pValue) : undefined,
|
|
208
|
+
revenue_claimed: options.revenueClaimed ?? options.revenue_claimed ?? false,
|
|
209
|
+
other_active_experiments: entries.filter((e) => e.status === 'active' && e.experiment_id !== id),
|
|
210
|
+
};
|
|
211
|
+
const result = evaluateExperiment(experiment, observed);
|
|
212
|
+
return { ...result, registry_problems: problems };
|
|
213
|
+
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { readYaml, nowIso } from '../shared/io.js';
|
|
5
|
+
import { buildSiteFromDir } from '../extractor/site.js';
|
|
6
|
+
import { runDetectors } from '../detectors/framework.js';
|
|
7
|
+
import { selectDetectors } from '../detectors/index.js';
|
|
8
|
+
import { calculateVisualSaliency } from '../analysis/saliency.js';
|
|
9
|
+
|
|
10
|
+
const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* `citable corpus benchmark` — golden-benchmark evaluation of deterministic
|
|
14
|
+
* CRO detectors over a versioned, anonymized, labeled page corpus.
|
|
15
|
+
*
|
|
16
|
+
* Labels are page-level: expect_findings (must fire), expect_clean (must not
|
|
17
|
+
* fire), saliency bounds for the modeled conspicuity index. Output carries
|
|
18
|
+
* per-detector true/false positive/negative counts, precision, recall, and
|
|
19
|
+
* F1, plus an explicit pass/fail gate against the manifest thresholds.
|
|
20
|
+
*
|
|
21
|
+
* The benchmark evaluates detectors against synthetic fixtures; it is not
|
|
22
|
+
* evidence about real-world accuracy, and it never becomes a visibility or
|
|
23
|
+
* conversion claim.
|
|
24
|
+
*/
|
|
25
|
+
export async function runGoldenBenchmark(root, { corpusDir = null, writeReport = null } = {}) {
|
|
26
|
+
const resolvedCorpus = corpusDir
|
|
27
|
+
? path.resolve(root, corpusDir)
|
|
28
|
+
: path.join(MODULE_DIR, '..', '..', 'tests', 'fixtures', 'golden-corpus');
|
|
29
|
+
const manifestPath = path.join(resolvedCorpus, 'MANIFEST.yaml');
|
|
30
|
+
if (!fs.existsSync(manifestPath)) {
|
|
31
|
+
throw new Error(`golden corpus manifest not found: ${manifestPath}`);
|
|
32
|
+
}
|
|
33
|
+
const manifest = readYaml(manifestPath);
|
|
34
|
+
const namespaces = manifest.namespaces || ['CRO'];
|
|
35
|
+
const detectors = selectDetectors({ namespaces });
|
|
36
|
+
const thresholds = manifest.thresholds || { recall_min: 1.0, precision_min: 1.0 };
|
|
37
|
+
|
|
38
|
+
// per-detector confusion counters
|
|
39
|
+
const tally = new Map();
|
|
40
|
+
for (const d of detectors) {
|
|
41
|
+
tally.set(d.id, { detector_id: d.id, deterministic: d.deterministic, true_positives: 0, false_positives: 0, false_negatives: 0, true_negatives: 0 });
|
|
42
|
+
}
|
|
43
|
+
const pageResults = [];
|
|
44
|
+
const saliencyResults = [];
|
|
45
|
+
|
|
46
|
+
for (const [siteName, site] of Object.entries(manifest.sites || {})) {
|
|
47
|
+
const siteDir = path.join(resolvedCorpus, siteName);
|
|
48
|
+
if (!fs.existsSync(siteDir)) throw new Error(`corpus site directory missing: ${siteDir}`);
|
|
49
|
+
const built = buildSiteFromDir(siteDir, { baseUrl: `https://${siteName}.corpus.test` });
|
|
50
|
+
for (const page of built.pages) {
|
|
51
|
+
const relPage = path.relative(siteDir, page.sourceFile).split(path.sep).join('/');
|
|
52
|
+
const labels = (site.pages || {})[relPage] || {};
|
|
53
|
+
const expected = new Set(labels.expect_findings || []);
|
|
54
|
+
const expectClean = new Set(labels.expect_clean || []);
|
|
55
|
+
|
|
56
|
+
const scoped = { site: { ...built, pages: [page] }, registries: null, observations: null };
|
|
57
|
+
const { findings } = runDetectors(detectors, scoped);
|
|
58
|
+
const fired = new Set(findings.map((f) => f.detector_id));
|
|
59
|
+
|
|
60
|
+
for (const d of detectors) {
|
|
61
|
+
const t = tally.get(d.id);
|
|
62
|
+
const isExpected = expected.has(d.id);
|
|
63
|
+
const didFire = fired.has(d.id);
|
|
64
|
+
if (isExpected && didFire) t.true_positives++;
|
|
65
|
+
else if (isExpected && !didFire) t.false_negatives++;
|
|
66
|
+
else if (!isExpected && didFire) t.false_positives++;
|
|
67
|
+
else t.true_negatives++;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const violations = [];
|
|
71
|
+
for (const id of expected) {
|
|
72
|
+
if (!fired.has(id)) violations.push({ kind: 'false_negative', detector_id: id });
|
|
73
|
+
}
|
|
74
|
+
for (const id of fired) {
|
|
75
|
+
if (!expected.has(id)) {
|
|
76
|
+
violations.push({ kind: expectClean.has(id) ? 'labeled_clean_violation' : 'unexpected_finding', detector_id: id, summary: findings.find((f) => f.detector_id === id)?.observation?.summary });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
pageResults.push({
|
|
81
|
+
site: siteName,
|
|
82
|
+
kind: site.kind,
|
|
83
|
+
page: relPage,
|
|
84
|
+
url: page.url,
|
|
85
|
+
expected: [...expected],
|
|
86
|
+
fired: [...fired],
|
|
87
|
+
violations,
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
if (labels.saliency) {
|
|
91
|
+
const sal = calculateVisualSaliency({
|
|
92
|
+
ctas: page.ctas.map((c) => ({ ...c })),
|
|
93
|
+
headings: (page.h1s || []).map((h) => ({ level: 1, text: h.text })),
|
|
94
|
+
forms: page.forms.map((f) => ({ fieldCount: f.fieldCount })),
|
|
95
|
+
});
|
|
96
|
+
const pciMin = labels.saliency.pci_min;
|
|
97
|
+
const pass = pciMin === undefined ? true : sal.primary_cta_conspicuity_index >= pciMin;
|
|
98
|
+
saliencyResults.push({
|
|
99
|
+
site: siteName,
|
|
100
|
+
page: relPage,
|
|
101
|
+
primary_cta_conspicuity_index: sal.primary_cta_conspicuity_index,
|
|
102
|
+
pci_min_label: pciMin ?? null,
|
|
103
|
+
passed: pass,
|
|
104
|
+
methodology: 'PCI is a modeled heuristic index over DOM geometry labels, not observed user attention',
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const perDetector = [...tally.values()].map((t) => {
|
|
111
|
+
const precision = (t.true_positives + t.false_positives) > 0 ? t.true_positives / (t.true_positives + t.false_positives) : null;
|
|
112
|
+
const recall = (t.true_positives + t.false_negatives) > 0 ? t.true_positives / (t.true_positives + t.false_negatives) : null;
|
|
113
|
+
const f1 = precision !== null && recall !== null && (precision + recall) > 0 ? (2 * precision * recall) / (precision + recall) : null;
|
|
114
|
+
return { ...t, precision, recall, f1 };
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
const exercised = perDetector.filter((d) => d.true_positives + d.false_negatives > 0);
|
|
118
|
+
const violations = pageResults.flatMap((p) => p.violations.map((v) => ({ site: p.site, page: p.page, ...v })));
|
|
119
|
+
const recallFailures = exercised.filter((d) => d.recall < thresholds.recall_min);
|
|
120
|
+
const precisionFailures = perDetector.filter((d) => d.precision !== null && d.precision < thresholds.precision_min);
|
|
121
|
+
|
|
122
|
+
const result = {
|
|
123
|
+
benchmark: 'golden-corpus',
|
|
124
|
+
corpus_version: manifest.corpus_version,
|
|
125
|
+
corpus_dir: path.relative(root, resolvedCorpus) || resolvedCorpus,
|
|
126
|
+
anonymized: Boolean(manifest.anonymized),
|
|
127
|
+
timestamp: nowIso(),
|
|
128
|
+
tool_version: JSON.parse(fs.readFileSync(new URL('../../package.json', import.meta.url), 'utf8')).version,
|
|
129
|
+
thresholds,
|
|
130
|
+
sites_evaluated: Object.keys(manifest.sites || {}).length,
|
|
131
|
+
pages_evaluated: pageResults.length,
|
|
132
|
+
per_detector: perDetector,
|
|
133
|
+
saliency: saliencyResults,
|
|
134
|
+
violations,
|
|
135
|
+
gate: {
|
|
136
|
+
ok: recallFailures.length === 0 && precisionFailures.length === 0 && violations.length === 0,
|
|
137
|
+
recall_failures: recallFailures.map((d) => d.detector_id),
|
|
138
|
+
precision_failures: precisionFailures.map((d) => d.detector_id),
|
|
139
|
+
violation_count: violations.length,
|
|
140
|
+
},
|
|
141
|
+
limitations: [
|
|
142
|
+
'Synthetic fixture benchmark: measures consistency with labels, not real-world accuracy.',
|
|
143
|
+
'Registry-dependent detectors (funnels, experiments) are not exercised by this corpus.',
|
|
144
|
+
'Saliency bounds evaluate a modeled heuristic index, not observed user attention.',
|
|
145
|
+
],
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
if (writeReport) {
|
|
149
|
+
const outFull = path.resolve(root, writeReport);
|
|
150
|
+
fs.mkdirSync(path.dirname(outFull), { recursive: true });
|
|
151
|
+
fs.writeFileSync(outFull, JSON.stringify(result, null, 2) + '\n', 'utf8');
|
|
152
|
+
result.report_path = outFull;
|
|
153
|
+
}
|
|
154
|
+
return result;
|
|
155
|
+
}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { readJson, nowIso } from '../shared/io.js';
|
|
4
|
+
import { remediateCommand } from './remediate.js';
|
|
5
|
+
|
|
6
|
+
const PKG_VERSION = JSON.parse(
|
|
7
|
+
fs.readFileSync(new URL('../../package.json', import.meta.url), 'utf8')
|
|
8
|
+
).version;
|
|
9
|
+
|
|
10
|
+
const IMAGE_EXTS = new Set(['.png', '.jpg', '.jpeg', '.webp', '.svg']);
|
|
11
|
+
|
|
12
|
+
function findRenderEvidence(runDir) {
|
|
13
|
+
const shots = [];
|
|
14
|
+
const walk = (d) => {
|
|
15
|
+
for (const name of fs.readdirSync(d)) {
|
|
16
|
+
const p = path.join(d, name);
|
|
17
|
+
if (fs.statSync(p).isDirectory()) walk(p);
|
|
18
|
+
else if (IMAGE_EXTS.has(path.extname(name).toLowerCase())) shots.push(path.relative(runDir, p));
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
if (fs.existsSync(runDir)) walk(runDir);
|
|
22
|
+
return shots;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function acceptanceTestsFor(finding, rem) {
|
|
26
|
+
const lines = [
|
|
27
|
+
'## Acceptance tests',
|
|
28
|
+
'',
|
|
29
|
+
'### 1. Deterministic detector re-check (authoritative)',
|
|
30
|
+
'```',
|
|
31
|
+
`citable verify remediation --run <source-run-id> --finding ${finding.detector_id} --target <file> --apply`,
|
|
32
|
+
'```',
|
|
33
|
+
'Pass criterion: status `verified` — the detector no longer reports this finding for the same subject. Detector absence is not an outcome guarantee.',
|
|
34
|
+
'',
|
|
35
|
+
];
|
|
36
|
+
if (rem?.scaffold_command) {
|
|
37
|
+
lines.push(`### 2. Component integration`, '```', rem.scaffold_command, '```', 'After scaffolding, confirm the component renders without console errors in the consuming page.');
|
|
38
|
+
lines.push('');
|
|
39
|
+
}
|
|
40
|
+
lines.push(
|
|
41
|
+
'### 3. Manual review gates',
|
|
42
|
+
'- Confirm the change preserves factual accuracy, human comprehension, accessibility, and conversion function.',
|
|
43
|
+
'- Run the repository build/test suite. A failed build makes "validated" an unavailable conclusion.',
|
|
44
|
+
'- Capture after-change rendering evidence (screenshot) for the customer record.'
|
|
45
|
+
);
|
|
46
|
+
return lines.join('\n');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function deploymentFor(rem) {
|
|
50
|
+
const fw = rem?.framework?.framework || 'unknown';
|
|
51
|
+
const lines = [
|
|
52
|
+
'## Deployment instructions',
|
|
53
|
+
'',
|
|
54
|
+
`Detected framework: **${fw}**${rem?.framework ? ` (detection confidence ${rem.framework.confidence}; signals: ${rem.framework.signals.join(', ') || 'none'})` : ''}`,
|
|
55
|
+
'',
|
|
56
|
+
'1. Review `patch.diff`. Apply with `citable remediate --finding <id> --target <file> --write` or by hand from the diff.',
|
|
57
|
+
'2. Run your project build and test commands. The Citable static check does not replace your compiler.',
|
|
58
|
+
'3. Deploy through your normal pipeline; commit the rollback snapshot path noted in the verification record.',
|
|
59
|
+
'4. Frozen-CMS alternative: regenerate the edge adapter with `citable export edge --format cloudflare-cro` (Cloudflare Workers) and verify with `citable test edge`. Edge changes ship without a code deploy but remain subject to the same detector verification.',
|
|
60
|
+
'',
|
|
61
|
+
'Rollback: restore the snapshot recorded in `.citable/remediation/snapshots/` (see verification record `rollback_snapshot`).',
|
|
62
|
+
];
|
|
63
|
+
return lines.join('\n');
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* `citable kit export` — customer-ready implementation kit connecting audit
|
|
68
|
+
* findings to Nebula's fulfillment artifact: explanation, exact changes,
|
|
69
|
+
* rendering evidence, acceptance tests, deployment steps, re-audit command,
|
|
70
|
+
* and explicit limitations. Screenshots are included only when real rendering
|
|
71
|
+
* evidence exists in the source run; Citable never fabricates imagery.
|
|
72
|
+
*/
|
|
73
|
+
export async function exportImplementationKit(root, options = {}) {
|
|
74
|
+
const { run: runId, finding, subject = null, target = null, output = null } = options;
|
|
75
|
+
const runDir = path.join(root, '.citable', 'runs', runId || '');
|
|
76
|
+
if (!runId || !fs.existsSync(path.join(runDir, 'findings.json'))) {
|
|
77
|
+
throw new Error(`source run not found: .citable/runs/${runId || '(none)'}`);
|
|
78
|
+
}
|
|
79
|
+
const findings = readJson(path.join(runDir, 'findings.json'));
|
|
80
|
+
const matches = findings.filter((f) =>
|
|
81
|
+
(f.detector_id || '').toUpperCase() === String(finding || '').toUpperCase()
|
|
82
|
+
&& (!subject || (f.subject?.identifier || f.subject?.url) === subject));
|
|
83
|
+
if (matches.length === 0) {
|
|
84
|
+
throw new Error(`finding ${finding} not present in run ${runId}`);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const f = matches[0];
|
|
88
|
+
const patch = target
|
|
89
|
+
? await remediateCommand(root, { finding: f.detector_id, target, format: options.format || 'react' })
|
|
90
|
+
: await remediateCommand(root, { finding: f.detector_id, format: options.format || 'react' });
|
|
91
|
+
|
|
92
|
+
const slug = `${f.detector_id}-${String(subject || (f.subject?.identifier || f.subject?.url || 'subject'))
|
|
93
|
+
.replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0, 48)}`;
|
|
94
|
+
const kitDir = output ? path.resolve(root, output) : path.join(root, '.citable', 'kits', `${nowIso().replace(/\D/g, '').slice(0, 14)}-${slug}`);
|
|
95
|
+
fs.mkdirSync(path.join(kitDir, 'evidence'), { recursive: true });
|
|
96
|
+
|
|
97
|
+
// finding explanation (verbatim from the immutable run)
|
|
98
|
+
fs.writeFileSync(path.join(kitDir, 'finding.json'), JSON.stringify(matches, null, 2) + '\n');
|
|
99
|
+
|
|
100
|
+
// exact change: unified diff when a target file is known, else component code
|
|
101
|
+
if (target && patch.diff) {
|
|
102
|
+
fs.writeFileSync(path.join(kitDir, 'patch.diff'), patch.diff);
|
|
103
|
+
fs.writeFileSync(path.join(kitDir, 'patch-metadata.json'), JSON.stringify({
|
|
104
|
+
target_file: target,
|
|
105
|
+
framework: patch.framework,
|
|
106
|
+
patcher_class: patch.patcher_class,
|
|
107
|
+
confidence: patch.confidence?.score ?? null,
|
|
108
|
+
validation: patch.validation,
|
|
109
|
+
write_policy: patch.write_policy,
|
|
110
|
+
refusal: patch.write_refused ? patch.refusal_reason : null,
|
|
111
|
+
}, null, 2) + '\n');
|
|
112
|
+
}
|
|
113
|
+
if (patch.code) {
|
|
114
|
+
fs.writeFileSync(path.join(kitDir, `component-${patch.format || 'react'}.${patch.format === 'vue' ? 'vue' : patch.format === 'html' ? 'html' : 'jsx'}`), patch.code + '\n');
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// before/after screenshots: only real rendering evidence from the run
|
|
118
|
+
const shots = findRenderEvidence(runDir);
|
|
119
|
+
for (const rel of shots) {
|
|
120
|
+
fs.copyFileSync(path.join(runDir, rel), path.join(kitDir, 'evidence', path.basename(rel)));
|
|
121
|
+
}
|
|
122
|
+
fs.writeFileSync(path.join(kitDir, 'evidence', 'MANIFEST.md'), shots.length
|
|
123
|
+
? `Rendering evidence files copied from run ${runId}:\n${shots.map((s) => `- ${s}`).join('\n')}\nThese are before-state observations unless a paired post-remediation run is referenced. Citable does not fabricate imagery.\n`
|
|
124
|
+
: `No rendering evidence (screenshots) was present in run ${runId}. Collect after-state evidence with \`citable observe render --target <url>\` before and after the change. No placeholder imagery is provided because Citable does not fabricate evidence.\n`);
|
|
125
|
+
|
|
126
|
+
fs.writeFileSync(path.join(kitDir, 'acceptance-tests.md'), acceptanceTestsFor(f, patch) + '\n');
|
|
127
|
+
fs.writeFileSync(path.join(kitDir, 'deployment.md'), deploymentFor(patch) + '\n');
|
|
128
|
+
|
|
129
|
+
const ns = f.detector_id.split('-')[0].toLowerCase();
|
|
130
|
+
fs.writeFileSync(path.join(kitDir, 're-audit.sh'), `#!/bin/sh
|
|
131
|
+
# Re-audit the affected scope after deployment and compare against the source run.
|
|
132
|
+
set -eu
|
|
133
|
+
citable audit --scope ${ns} --target <deployed-target> --base-url <base-url>
|
|
134
|
+
citable verify remediation --run ${runId} --finding ${f.detector_id} --target <file> --apply
|
|
135
|
+
citable compare-snapshots ${runId} <new-run-id>
|
|
136
|
+
` + '\n');
|
|
137
|
+
fs.chmodSync(path.join(kitDir, 're-audit.sh'), 0o755);
|
|
138
|
+
|
|
139
|
+
const limitations = [
|
|
140
|
+
...(f.reasoning?.limitations || []),
|
|
141
|
+
'This kit contains no modeled revenue or conversion-lift estimates; impact projections are excluded by policy.',
|
|
142
|
+
'Detection was performed against a snapshot; the deployed property may differ by the time this kit is applied.',
|
|
143
|
+
'A verified patch resolves the detector condition only; it is not a guarantee of conversion, ranking, or citation outcomes.',
|
|
144
|
+
];
|
|
145
|
+
const assumptions = f.reasoning?.assumptions || [];
|
|
146
|
+
fs.writeFileSync(path.join(kitDir, 'limitations.md'), [
|
|
147
|
+
'## Limitations', '', ...limitations.map((l) => `- ${l}`), '',
|
|
148
|
+
'## Assumptions', '',
|
|
149
|
+
...(assumptions.length ? assumptions.map((a) => `- ${a}`) : ['- No additional assumptions recorded by the detector.']),
|
|
150
|
+
].join('\n') + '\n');
|
|
151
|
+
|
|
152
|
+
fs.writeFileSync(path.join(kitDir, 'README.md'), [
|
|
153
|
+
`# Implementation Kit — ${f.detector_id}`,
|
|
154
|
+
'',
|
|
155
|
+
`**Finding:** ${f.observation?.summary}`,
|
|
156
|
+
`**Subject:** ${f.subject?.identifier || f.subject?.url || 'unknown'}`,
|
|
157
|
+
`**Severity:** ${f.classification?.severity} | **Confidence:** ${f.classification?.confidence}`,
|
|
158
|
+
`**Source run:** ${runId} | **Kit generated:** ${nowIso()} by Citable ${PKG_VERSION}`,
|
|
159
|
+
'',
|
|
160
|
+
'## Contents',
|
|
161
|
+
'',
|
|
162
|
+
'- `finding.json` — verbatim finding record from the immutable source run',
|
|
163
|
+
target && patch.diff ? '- `patch.diff` — unified diff of the exact proposed change (dry-run; nothing was written)' : '- (no patch.diff: no target file supplied; component template below)',
|
|
164
|
+
patch.code ? '- `component-*.jsx|vue|html` — recommended Nebula Component template' : null,
|
|
165
|
+
'- `patch-metadata.json` — framework detection, confidence, validation checks (when target supplied)',
|
|
166
|
+
'- `evidence/` — rendering evidence from the source run (see evidence/MANIFEST.md)',
|
|
167
|
+
'- `acceptance-tests.md` — detector re-check plus manual review gates',
|
|
168
|
+
'- `deployment.md` — deployment and rollback instructions',
|
|
169
|
+
'- `re-audit.sh` — re-audit and verification commands',
|
|
170
|
+
'- `limitations.md` — limitations and assumptions (fact vs inference separation)',
|
|
171
|
+
'',
|
|
172
|
+
'## Order of operations',
|
|
173
|
+
'',
|
|
174
|
+
'1. Review finding and diff; assign an accountable owner.',
|
|
175
|
+
'2. Apply the smallest change (patch or component).',
|
|
176
|
+
'3. Run acceptance tests, then deployment.',
|
|
177
|
+
'4. Run `re-audit.sh`; keep the verification bundle with the customer record.',
|
|
178
|
+
'',
|
|
179
|
+
'Citable does not guarantee crawling, indexing, ranking, citation, recommendation, or conversion.',
|
|
180
|
+
].filter(Boolean).join('\n') + '\n');
|
|
181
|
+
|
|
182
|
+
const files = [];
|
|
183
|
+
const walk = (d) => {
|
|
184
|
+
for (const name of fs.readdirSync(d)) {
|
|
185
|
+
const p = path.join(d, name);
|
|
186
|
+
if (fs.statSync(p).isDirectory()) walk(p);
|
|
187
|
+
else files.push(path.relative(kitDir, p));
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
walk(kitDir);
|
|
191
|
+
|
|
192
|
+
return {
|
|
193
|
+
ok: true,
|
|
194
|
+
finding_id: f.finding_id,
|
|
195
|
+
detector_id: f.detector_id,
|
|
196
|
+
source_run_id: runId,
|
|
197
|
+
kit_dir: path.relative(root, kitDir),
|
|
198
|
+
files: files.sort(),
|
|
199
|
+
rendering_evidence_count: shots.length,
|
|
200
|
+
has_patch_diff: Boolean(target && patch.diff),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { auditPageCro } from "../analysis/croAudit.js";
|
|
1
2
|
import { buildContext } from './context.js';
|
|
2
3
|
import { registryPageFor, safePath } from '../detectors/framework.js';
|
|
3
4
|
import { selectDetectors } from '../detectors/index.js';
|
|
4
5
|
import { runDetectors } from '../detectors/framework.js';
|
|
6
|
+
import { calculateVisualSaliency } from '../analysis/saliency.js';
|
|
5
7
|
|
|
6
8
|
/**
|
|
7
9
|
* `citable inspect cro <page>` — evaluate page conversion readiness, CTA visibility, form friction, and trust badges.
|
|
@@ -37,14 +39,58 @@ export async function inspectCro(root, pageRef, { target, baseUrl, refDate } = {
|
|
|
37
39
|
const titleStem = page.title ? page.title.split(/[|—–-]/)[0].trim() : '';
|
|
38
40
|
const h1Text = page.h1s?.[0]?.text || '';
|
|
39
41
|
|
|
42
|
+
// Visual attention saliency modeling
|
|
43
|
+
const saliency = calculateVisualSaliency({
|
|
44
|
+
ctas: ctas.map((c) => ({ ...c, isPrimary: c.isPrimary })),
|
|
45
|
+
headings: (page.h1s || []).map((h) => ({ level: 1, text: h.text })),
|
|
46
|
+
forms: forms.map((f) => ({ fieldCount: f.fieldCount })),
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// Keystroke Effort Index (KEI) across all forms
|
|
50
|
+
const totalInputs = forms.reduce((sum, f) => sum + (f.fieldCount || 0), 0);
|
|
51
|
+
const inputsWithAutocomplete = forms.reduce((sum, f) => sum + f.inputs.filter((inp) => Boolean(inp.autocomplete)).length, 0);
|
|
52
|
+
const manualKeystrokes = totalInputs * 14; // Average 14 keystrokes + focus taps per field
|
|
53
|
+
const autofillKeystrokes = totalInputs > 0 && inputsWithAutocomplete > 0
|
|
54
|
+
? (totalInputs - inputsWithAutocomplete) * 14 + 1
|
|
55
|
+
: manualKeystrokes;
|
|
56
|
+
const keystrokeReductionPct = manualKeystrokes > 0
|
|
57
|
+
? Math.round(((manualKeystrokes - autofillKeystrokes) / manualKeystrokes) * 100)
|
|
58
|
+
: 0;
|
|
59
|
+
|
|
60
|
+
// Express checkout readiness: payment wallets and authentication tracked separately.
|
|
61
|
+
// Apple Pay / Google Pay / PayPal are payment methods; WebAuthn/passkeys are account
|
|
62
|
+
// authentication. Conflating them produces misleading "biometric readiness" claims.
|
|
63
|
+
const walletRx = /apple\s*pay|google\s*pay|paypal|one-click/i;
|
|
64
|
+
const authRx = /passkey|webauthn/i;
|
|
65
|
+
const walletCtas = ctas.filter((c) => walletRx.test(c.text || ''));
|
|
66
|
+
const authCtas = ctas.filter((c) => authRx.test(c.text || ''));
|
|
67
|
+
const walletViaBadgeOrHtml = trustBadges.some((t) => walletRx.test(t.signal || '')) ||
|
|
68
|
+
Boolean(page.rawHtml && /data-express-payment|apple-pay|google-pay|paypal-button/i.test(page.rawHtml));
|
|
69
|
+
const authViaHtml = Boolean(page.rawHtml && /passkey|webauthn|publickey-credentials/i.test(page.rawHtml));
|
|
70
|
+
|
|
71
|
+
// Friction Surface Area (FSA) — a modeled heuristic index, not a measured
|
|
72
|
+
// revenue or conversion outcome.
|
|
73
|
+
const fsaScore = croFindings.reduce((acc, f) => {
|
|
74
|
+
const w = f.classification.severity === 'critical' ? 15 : f.classification.severity === 'high' ? 10 : f.classification.severity === 'medium' ? 5 : 2;
|
|
75
|
+
return acc + w;
|
|
76
|
+
}, 0);
|
|
77
|
+
|
|
78
|
+
const pageCroAudit = auditPageCro(page, ctx);
|
|
79
|
+
|
|
40
80
|
return {
|
|
41
81
|
url: page.url,
|
|
82
|
+
conversion_readiness_score: pageCroAudit.conversion_readiness_score,
|
|
83
|
+
atf_clarity: pageCroAudit.atf_clarity,
|
|
84
|
+
trust_and_credibility: pageCroAudit.trust_and_credibility,
|
|
85
|
+
offer_architecture: pageCroAudit.offer_architecture,
|
|
42
86
|
sourceFile: page.sourceFile,
|
|
43
87
|
status: page.status,
|
|
44
88
|
declared_conversion_action: declaredAction,
|
|
45
89
|
primary_intent: reg?.primary_intent ?? null,
|
|
46
90
|
page_type: reg?.page_type ?? null,
|
|
47
91
|
conversion_status: status,
|
|
92
|
+
friction_surface_area: fsaScore,
|
|
93
|
+
friction_surface_area_note: 'modeled heuristic index (severity-weighted mechanical defect count); not a measured conversion or revenue outcome',
|
|
48
94
|
title_to_h1_alignment: {
|
|
49
95
|
titleStem,
|
|
50
96
|
h1Text,
|
|
@@ -55,6 +101,27 @@ export async function inspectCro(root, pageRef, { target, baseUrl, refDate } = {
|
|
|
55
101
|
analytics_installed: (page.analyticsTags || []).length > 0,
|
|
56
102
|
analytics_tags: page.analyticsTags || [],
|
|
57
103
|
nav_links_count: page.navLinksCount || 0,
|
|
104
|
+
saliency,
|
|
105
|
+
keystroke_effort: {
|
|
106
|
+
total_inputs: totalInputs,
|
|
107
|
+
inputs_with_autocomplete: inputsWithAutocomplete,
|
|
108
|
+
autofill_coverage_pct: totalInputs > 0 ? Math.round((inputsWithAutocomplete / totalInputs) * 100) : 100,
|
|
109
|
+
manual_keystrokes_required: manualKeystrokes,
|
|
110
|
+
autofill_keystrokes_required: autofillKeystrokes,
|
|
111
|
+
keystroke_reduction_pct: keystrokeReductionPct,
|
|
112
|
+
},
|
|
113
|
+
express_checkout_readiness: {
|
|
114
|
+
payment_wallet_readiness: {
|
|
115
|
+
supported: walletCtas.length > 0 || walletViaBadgeOrHtml,
|
|
116
|
+
detected_triggers: walletCtas.map((c) => c.text),
|
|
117
|
+
note: 'payment wallets are payment methods (Apple Pay, Google Pay, PayPal)',
|
|
118
|
+
},
|
|
119
|
+
authentication_readiness: {
|
|
120
|
+
supported: authCtas.length > 0 || authViaHtml,
|
|
121
|
+
detected_triggers: authCtas.map((c) => c.text),
|
|
122
|
+
note: 'WebAuthn/passkeys are account authentication, not a payment method; this is an informational readiness index, not observed usage',
|
|
123
|
+
},
|
|
124
|
+
},
|
|
58
125
|
ctas: ctas.map((c) => ({
|
|
59
126
|
text: c.text,
|
|
60
127
|
tag: c.tag,
|