@odavl/guardian 0.2.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +86 -2
- package/README.md +155 -97
- package/bin/guardian.js +1345 -60
- package/config/README.md +59 -0
- package/config/profiles/landing-demo.yaml +16 -0
- package/package.json +21 -11
- package/policies/landing-demo.json +22 -0
- package/src/enterprise/audit-logger.js +166 -0
- package/src/enterprise/pdf-exporter.js +267 -0
- package/src/enterprise/rbac-gate.js +142 -0
- package/src/enterprise/rbac.js +239 -0
- package/src/enterprise/site-manager.js +180 -0
- package/src/founder/feedback-system.js +156 -0
- package/src/founder/founder-tracker.js +213 -0
- package/src/founder/usage-signals.js +141 -0
- package/src/guardian/alert-ledger.js +121 -0
- package/src/guardian/attempt-engine.js +568 -7
- package/src/guardian/attempt-registry.js +42 -1
- package/src/guardian/attempt-relevance.js +106 -0
- package/src/guardian/attempt.js +24 -0
- package/src/guardian/baseline.js +12 -4
- package/src/guardian/breakage-intelligence.js +1 -0
- package/src/guardian/ci-cli.js +121 -0
- package/src/guardian/ci-output.js +4 -3
- package/src/guardian/cli-summary.js +79 -92
- package/src/guardian/config-loader.js +162 -0
- package/src/guardian/drift-detector.js +100 -0
- package/src/guardian/enhanced-html-reporter.js +221 -4
- package/src/guardian/env-guard.js +127 -0
- package/src/guardian/failure-intelligence.js +173 -0
- package/src/guardian/first-run-profile.js +89 -0
- package/src/guardian/first-run.js +6 -1
- package/src/guardian/flag-validator.js +17 -3
- package/src/guardian/html-reporter.js +2 -0
- package/src/guardian/human-reporter.js +431 -0
- package/src/guardian/index.js +22 -19
- package/src/guardian/init-command.js +9 -5
- package/src/guardian/intent-detector.js +146 -0
- package/src/guardian/journey-definitions.js +132 -0
- package/src/guardian/journey-scan-cli.js +145 -0
- package/src/guardian/journey-scanner.js +583 -0
- package/src/guardian/junit-reporter.js +18 -1
- package/src/guardian/live-cli.js +95 -0
- package/src/guardian/live-scheduler-runner.js +137 -0
- package/src/guardian/live-scheduler.js +146 -0
- package/src/guardian/market-reporter.js +341 -81
- package/src/guardian/pattern-analyzer.js +348 -0
- package/src/guardian/policy.js +80 -3
- package/src/guardian/preset-loader.js +9 -6
- package/src/guardian/reality.js +1278 -117
- package/src/guardian/reporter.js +27 -41
- package/src/guardian/run-artifacts.js +212 -0
- package/src/guardian/run-cleanup.js +207 -0
- package/src/guardian/run-latest.js +90 -0
- package/src/guardian/run-list.js +211 -0
- package/src/guardian/scan-presets.js +100 -11
- package/src/guardian/selector-fallbacks.js +394 -0
- package/src/guardian/semantic-contact-finder.js +2 -1
- package/src/guardian/site-introspection.js +257 -0
- package/src/guardian/smoke.js +2 -2
- package/src/guardian/snapshot-schema.js +25 -1
- package/src/guardian/snapshot.js +46 -2
- package/src/guardian/stability-scorer.js +169 -0
- package/src/guardian/template-command.js +184 -0
- package/src/guardian/text-formatters.js +426 -0
- package/src/guardian/verdict.js +320 -0
- package/src/guardian/verdicts.js +74 -0
- package/src/guardian/watch-runner.js +3 -7
- package/src/payments/stripe-checkout.js +169 -0
- package/src/plans/plan-definitions.js +148 -0
- package/src/plans/plan-manager.js +211 -0
- package/src/plans/usage-tracker.js +210 -0
- package/src/recipes/recipe-engine.js +188 -0
- package/src/recipes/recipe-failure-analysis.js +159 -0
- package/src/recipes/recipe-registry.js +134 -0
- package/src/recipes/recipe-runtime.js +507 -0
- package/src/recipes/recipe-store.js +410 -0
- package/guardian-contract-v1.md +0 -149
- /package/{guardian.config.json → config/guardian.config.json} +0 -0
- /package/{guardian.policy.json → config/guardian.policy.json} +0 -0
- /package/{guardian.profile.docs.yaml → config/profiles/docs.yaml} +0 -0
- /package/{guardian.profile.ecommerce.yaml → config/profiles/ecommerce.yaml} +0 -0
- /package/{guardian.profile.marketing.yaml → config/profiles/marketing.yaml} +0 -0
- /package/{guardian.profile.saas.yaml → config/profiles/saas.yaml} +0 -0
package/src/guardian/reality.js
CHANGED
|
@@ -13,15 +13,34 @@ const { buildAutoAttempts } = require('./auto-attempt-builder');
|
|
|
13
13
|
const { baselineExists, loadBaseline, saveBaselineAtomic, createBaselineFromSnapshot, compareSnapshots } = require('./baseline-storage');
|
|
14
14
|
const { analyzeMarketImpact, determineExitCodeFromEscalation } = require('./market-criticality');
|
|
15
15
|
const { parsePolicyOption } = require('./preset-loader');
|
|
16
|
-
const { evaluatePolicy } = require('./policy');
|
|
16
|
+
const { evaluatePolicy, loadPolicy } = require('./policy');
|
|
17
17
|
const { aggregateIntelligence } = require('./breakage-intelligence');
|
|
18
18
|
const { writeEnhancedHtml } = require('./enhanced-html-reporter');
|
|
19
|
-
const {
|
|
19
|
+
const { analyzePatterns, loadRecentRunsForSite } = require('./pattern-analyzer');
|
|
20
|
+
const crypto = require('crypto');
|
|
21
|
+
const {
|
|
22
|
+
formatVerdictStatus,
|
|
23
|
+
formatConfidence,
|
|
24
|
+
formatVerdictWhy,
|
|
25
|
+
formatKeyFindings,
|
|
26
|
+
formatLimits,
|
|
27
|
+
formatPatternSummary,
|
|
28
|
+
formatPatternFocus,
|
|
29
|
+
formatConfidenceDrivers,
|
|
30
|
+
formatFocusSummary,
|
|
31
|
+
formatDeltaInsight,
|
|
32
|
+
// Stage V / Step 5.2: Silence Discipline helpers
|
|
33
|
+
shouldRenderFocusSummary,
|
|
34
|
+
shouldRenderDeltaInsight,
|
|
35
|
+
shouldRenderPatterns,
|
|
36
|
+
shouldRenderConfidenceDrivers
|
|
37
|
+
} = require('./text-formatters');
|
|
20
38
|
const { sendWebhooks, getWebhookUrl, buildWebhookPayload } = require('./webhook');
|
|
21
39
|
const { findContactOnPage, formatDetectionForReport } = require('./semantic-contact-finder');
|
|
22
40
|
const { formatRunSummary } = require('./run-summary');
|
|
23
41
|
const { isCiMode } = require('./ci-mode');
|
|
24
42
|
const { formatCiSummary, deriveBaselineVerdict } = require('./ci-output');
|
|
43
|
+
const { toCanonicalVerdict, mapExitCodeFromCanonical } = require('./verdicts');
|
|
25
44
|
// Phase 7.1: Performance modes
|
|
26
45
|
const { getTimeoutProfile } = require('./timeout-profiles');
|
|
27
46
|
const { validateAttemptFilter, filterAttempts, filterFlows } = require('./attempts-filter');
|
|
@@ -31,12 +50,16 @@ const { executeParallel, validateParallel } = require('./parallel-executor');
|
|
|
31
50
|
const { BrowserPool } = require('./browser-pool');
|
|
32
51
|
// Phase 7.4: Smart skips
|
|
33
52
|
const { checkPrerequisites } = require('./prerequisite-checker');
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
}
|
|
53
|
+
// Wave 1: Run artifacts naming and metadata
|
|
54
|
+
const { makeRunDirName, makeSiteSlug, writeMetaJson, readMetaJson } = require('./run-artifacts');
|
|
55
|
+
// Wave 2: Latest pointers
|
|
56
|
+
const { updateLatestGlobal, updateLatestBySite } = require('./run-latest');
|
|
57
|
+
// Wave 3: Smart attempt selection
|
|
58
|
+
const { inspectSite, detectProfile } = require('./site-introspection');
|
|
59
|
+
const { filterAttempts: filterAttemptsByRelevance, summarizeIntrospection } = require('./attempt-relevance');
|
|
60
|
+
// Stage II: Golden path
|
|
61
|
+
const { isFirstRun, markFirstRunComplete, applyFirstRunProfile } = require('./first-run-profile');
|
|
62
|
+
const { applyLocalConfig } = require('./config-loader');
|
|
40
63
|
|
|
41
64
|
function applySafeDefaults(config, warn) {
|
|
42
65
|
const updated = { ...config };
|
|
@@ -51,14 +74,30 @@ function applySafeDefaults(config, warn) {
|
|
|
51
74
|
return updated;
|
|
52
75
|
}
|
|
53
76
|
|
|
77
|
+
const EXECUTED_OUTCOMES = new Set(['SUCCESS', 'FAILURE', 'FRICTION', 'DISCOVERY_FAILED']);
|
|
78
|
+
const isExecutedAttempt = (attemptResult) => attemptResult && EXECUTED_OUTCOMES.has(attemptResult.outcome);
|
|
79
|
+
const SKIP_CODES = {
|
|
80
|
+
DISABLED_BY_PRESET: 'DISABLED_BY_PRESET',
|
|
81
|
+
NOT_APPLICABLE: 'NOT_APPLICABLE',
|
|
82
|
+
ENGINE_MISSING: 'ENGINE_MISSING',
|
|
83
|
+
USER_FILTERED: 'USER_FILTERED',
|
|
84
|
+
PREREQ: 'PREREQUISITE_FAILED'
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
// Removed compact decision packet writer for Level 1 singleton decision schema
|
|
88
|
+
|
|
54
89
|
async function executeReality(config) {
|
|
55
90
|
const baseWarn = (...args) => console.warn(...args);
|
|
56
|
-
const
|
|
91
|
+
const firstRunMode = isFirstRun();
|
|
92
|
+
// Apply first-run profile if needed (conservative defaults)
|
|
93
|
+
const profiledConfig = firstRunMode ? applyFirstRunProfile(config) : config;
|
|
94
|
+
const safeConfig = applySafeDefaults(profiledConfig, baseWarn);
|
|
95
|
+
const runSignals = [];
|
|
57
96
|
|
|
58
97
|
const {
|
|
59
98
|
baseUrl,
|
|
60
99
|
attempts = getDefaultAttemptIds(),
|
|
61
|
-
artifactsDir = '
|
|
100
|
+
artifactsDir = './.odavlguardian',
|
|
62
101
|
headful = false,
|
|
63
102
|
enableTrace = true,
|
|
64
103
|
enableScreenshots = true,
|
|
@@ -110,8 +149,11 @@ async function executeReality(config) {
|
|
|
110
149
|
let filteredAttempts = attempts;
|
|
111
150
|
let filteredFlows = flows;
|
|
112
151
|
if (attemptsFilter && validation && validation.valid && validation.ids.length > 0) {
|
|
152
|
+
const beforeFilter = Array.isArray(filteredAttempts) ? filteredAttempts.slice() : [];
|
|
113
153
|
filteredAttempts = filterAttempts(attempts, validation.ids);
|
|
114
154
|
filteredFlows = filterFlows(flows, validation.ids);
|
|
155
|
+
const removed = beforeFilter.filter(id => !filteredAttempts.includes(id));
|
|
156
|
+
userFilteredAttempts.push(...removed.map(id => ({ attemptId: id, reason: 'Filtered by --attempts' })));
|
|
115
157
|
if (filteredAttempts.length === 0 && filteredFlows.length === 0) {
|
|
116
158
|
console.error('Error: No matching attempts or flows found after filtering');
|
|
117
159
|
console.error(`Hint: Check your --attempts filter: ${attemptsFilter}`);
|
|
@@ -119,6 +161,13 @@ async function executeReality(config) {
|
|
|
119
161
|
}
|
|
120
162
|
}
|
|
121
163
|
|
|
164
|
+
const requestedAttempts = Array.isArray(filteredAttempts) ? filteredAttempts.slice() : [];
|
|
165
|
+
const disabledByPreset = new Set((config.disabledAttempts || []).map(id => String(id)));
|
|
166
|
+
const enabledRequestedAttempts = requestedAttempts.filter(id => !disabledByPreset.has(String(id)));
|
|
167
|
+
const presetDisabledAttempts = requestedAttempts.filter(id => disabledByPreset.has(String(id)));
|
|
168
|
+
const userFilteredAttempts = [];
|
|
169
|
+
const missingAttempts = [];
|
|
170
|
+
|
|
122
171
|
// Phase 7.1: Resolve timeout profile
|
|
123
172
|
const timeoutProfileConfig = getTimeoutProfile(timeoutProfile);
|
|
124
173
|
const resolvedTimeout = timeout || timeoutProfileConfig.default;
|
|
@@ -130,11 +179,46 @@ async function executeReality(config) {
|
|
|
130
179
|
throw new Error(`Invalid URL: ${baseUrl}`);
|
|
131
180
|
}
|
|
132
181
|
|
|
133
|
-
|
|
134
|
-
const
|
|
182
|
+
// Wave 1: Generate human-readable run directory name
|
|
183
|
+
const startTime = new Date();
|
|
184
|
+
const siteSlug = makeSiteSlug(baseUrl);
|
|
185
|
+
// Use 'default' if no policy specified, otherwise extract preset name
|
|
186
|
+
let policyName = (() => {
|
|
187
|
+
if (!policy) return 'default';
|
|
188
|
+
if (typeof policy === 'string') {
|
|
189
|
+
return policy.startsWith('preset:') ? policy.replace('preset:', '') : policy;
|
|
190
|
+
}
|
|
191
|
+
if (typeof policy === 'object' && policy.id) return policy.id;
|
|
192
|
+
return 'custom';
|
|
193
|
+
})();
|
|
194
|
+
// Result will be determined at the end; use placeholder for now
|
|
195
|
+
let runDirName = makeRunDirName({
|
|
196
|
+
timestamp: startTime,
|
|
197
|
+
url: baseUrl,
|
|
198
|
+
policy: policyName,
|
|
199
|
+
result: 'PENDING'
|
|
200
|
+
});
|
|
201
|
+
let runDir = path.join(artifactsDir, runDirName);
|
|
135
202
|
fs.mkdirSync(runDir, { recursive: true });
|
|
203
|
+
const runId = runDirName;
|
|
136
204
|
const ciMode = isCiMode();
|
|
137
205
|
|
|
206
|
+
// Print positioning message based on policy
|
|
207
|
+
const isPolicyProtect = policy && ((typeof policy === 'string' && (policy === 'preset:startup' || policy.includes('startup'))) || (typeof policy === 'object' && policy.id === 'startup'));
|
|
208
|
+
if (!ciMode) {
|
|
209
|
+
if (isPolicyProtect) {
|
|
210
|
+
console.log('\nPROTECT MODE: Full market reality test (slower, deeper)');
|
|
211
|
+
} else {
|
|
212
|
+
console.log('\nREALITY MODE: Full market reality snapshot');
|
|
213
|
+
}
|
|
214
|
+
} else {
|
|
215
|
+
if (isPolicyProtect) {
|
|
216
|
+
console.log('PROTECT MODE: Full market reality test');
|
|
217
|
+
} else {
|
|
218
|
+
console.log('REALITY MODE: Full market reality snapshot');
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
138
222
|
// Phase 7.1: Print mode info
|
|
139
223
|
if (!ciMode) {
|
|
140
224
|
const modeLines = [];
|
|
@@ -152,6 +236,11 @@ async function executeReality(config) {
|
|
|
152
236
|
console.log(`Base URL: ${baseUrl}`);
|
|
153
237
|
console.log(`Attempts: ${filteredAttempts.join(', ')}`);
|
|
154
238
|
console.log(`Run Dir: ${runDir}`);
|
|
239
|
+
} else if (firstRunMode) {
|
|
240
|
+
// Simplified output for first run
|
|
241
|
+
console.log(`\n🚀 Guardian First Run`);
|
|
242
|
+
console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`);
|
|
243
|
+
console.log(`🧪 Scanning: ${baseUrl}`);
|
|
155
244
|
} else {
|
|
156
245
|
console.log(`\n🧪 Market Reality Snapshot v1`);
|
|
157
246
|
console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`);
|
|
@@ -168,6 +257,8 @@ async function executeReality(config) {
|
|
|
168
257
|
let discoveryResult = null;
|
|
169
258
|
let pageLanguage = 'unknown';
|
|
170
259
|
let contactDetectionResult = null;
|
|
260
|
+
let siteIntrospection = null;
|
|
261
|
+
let siteProfile = 'unknown';
|
|
171
262
|
|
|
172
263
|
// Optional: Crawl to discover URLs (lightweight, first N pages)
|
|
173
264
|
if (enableCrawl) {
|
|
@@ -177,6 +268,28 @@ async function executeReality(config) {
|
|
|
177
268
|
await browser.launch(resolvedTimeout);
|
|
178
269
|
await browser.page.goto(baseUrl, { waitUntil: 'domcontentloaded', timeout: resolvedTimeout });
|
|
179
270
|
|
|
271
|
+
// Wave 3: Site introspection for smart attempt selection
|
|
272
|
+
try {
|
|
273
|
+
console.log(`\n🔬 Inspecting site capabilities...`);
|
|
274
|
+
siteIntrospection = await inspectSite(browser.page);
|
|
275
|
+
siteProfile = detectProfile(siteIntrospection);
|
|
276
|
+
const summary = summarizeIntrospection(siteIntrospection);
|
|
277
|
+
console.log(`✅ Site profile: ${siteProfile}`);
|
|
278
|
+
console.log(` ${summary}`);
|
|
279
|
+
} catch (introspectionErr) {
|
|
280
|
+
console.warn(`⚠️ Site introspection failed (non-critical): ${introspectionErr.message}`);
|
|
281
|
+
// Default to empty introspection if it fails
|
|
282
|
+
siteIntrospection = {
|
|
283
|
+
hasLogin: false,
|
|
284
|
+
hasSignup: false,
|
|
285
|
+
hasCheckout: false,
|
|
286
|
+
hasNewsletter: false,
|
|
287
|
+
hasContactForm: false,
|
|
288
|
+
hasLanguageSwitch: false
|
|
289
|
+
};
|
|
290
|
+
siteProfile = 'unknown';
|
|
291
|
+
}
|
|
292
|
+
|
|
180
293
|
// Wave 1.1: Detect page language and contact
|
|
181
294
|
try {
|
|
182
295
|
contactDetectionResult = await findContactOnPage(browser.page, baseUrl);
|
|
@@ -194,10 +307,40 @@ async function executeReality(config) {
|
|
|
194
307
|
await browser.close();
|
|
195
308
|
} catch (crawlErr) {
|
|
196
309
|
console.log(`⚠️ Crawl failed (non-critical): ${crawlErr.message}`);
|
|
310
|
+
runSignals.push({ id: 'crawl_failed', severity: 'high', type: 'coverage', description: `Crawl failed: ${crawlErr.message}` });
|
|
197
311
|
// Continue anyway - crawl is optional
|
|
198
312
|
}
|
|
199
313
|
}
|
|
200
314
|
|
|
315
|
+
// Wave 3: If crawl was disabled but introspection wasn't done, do it now
|
|
316
|
+
if (!enableCrawl && !siteIntrospection) {
|
|
317
|
+
console.log(`\n🔬 Inspecting site capabilities...`);
|
|
318
|
+
const browser = new GuardianBrowser();
|
|
319
|
+
try {
|
|
320
|
+
await browser.launch(resolvedTimeout);
|
|
321
|
+
await browser.page.goto(baseUrl, { waitUntil: 'domcontentloaded', timeout: resolvedTimeout });
|
|
322
|
+
|
|
323
|
+
siteIntrospection = await inspectSite(browser.page);
|
|
324
|
+
siteProfile = detectProfile(siteIntrospection);
|
|
325
|
+
const summary = summarizeIntrospection(siteIntrospection);
|
|
326
|
+
console.log(`✅ Site profile: ${siteProfile}`);
|
|
327
|
+
console.log(` ${summary}`);
|
|
328
|
+
|
|
329
|
+
await browser.close();
|
|
330
|
+
} catch (introspectionErr) {
|
|
331
|
+
console.warn(`⚠️ Site introspection failed (non-critical): ${introspectionErr.message}`);
|
|
332
|
+
siteIntrospection = {
|
|
333
|
+
hasLogin: false,
|
|
334
|
+
hasSignup: false,
|
|
335
|
+
hasCheckout: false,
|
|
336
|
+
hasNewsletter: false,
|
|
337
|
+
hasContactForm: false,
|
|
338
|
+
hasLanguageSwitch: false
|
|
339
|
+
};
|
|
340
|
+
siteProfile = 'unknown';
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
201
344
|
// Optional: Discovery Engine (Phase 4) — deterministic safe exploration
|
|
202
345
|
if (enableDiscovery) {
|
|
203
346
|
console.log(`\n🔎 Running discovery engine...`);
|
|
@@ -217,6 +360,7 @@ async function executeReality(config) {
|
|
|
217
360
|
await browser.close();
|
|
218
361
|
} catch (discErr) {
|
|
219
362
|
console.log(`⚠️ Discovery failed (non-critical): ${discErr.message}`);
|
|
363
|
+
runSignals.push({ id: 'discovery_failed', severity: 'high', type: 'discovery', description: `Discovery failed: ${discErr.message}` });
|
|
220
364
|
}
|
|
221
365
|
}
|
|
222
366
|
|
|
@@ -253,7 +397,7 @@ async function executeReality(config) {
|
|
|
253
397
|
const flowResults = [];
|
|
254
398
|
|
|
255
399
|
// Determine attempts to run (manual + auto-generated)
|
|
256
|
-
let attemptsToRun =
|
|
400
|
+
let attemptsToRun = enabledRequestedAttempts.slice();
|
|
257
401
|
|
|
258
402
|
// Phase 2: Add auto-generated attempts
|
|
259
403
|
if (enableAutoAttempts && autoAttempts.length > 0) {
|
|
@@ -262,11 +406,11 @@ async function executeReality(config) {
|
|
|
262
406
|
console.log(`➕ Added ${autoAttemptIds.length} auto-generated attempts`);
|
|
263
407
|
}
|
|
264
408
|
|
|
265
|
-
if (includeUniversal && !attemptsToRun.includes('universal_reality')) {
|
|
409
|
+
if (includeUniversal && !attemptsToRun.includes('universal_reality') && !disabledByPreset.has('universal_reality')) {
|
|
266
410
|
attemptsToRun.push('universal_reality');
|
|
267
411
|
}
|
|
268
412
|
// If discovery enabled and site is simple (few interactions), add universal pack
|
|
269
|
-
if (enableDiscovery && discoveryResult && !attemptsToRun.includes('universal_reality')) {
|
|
413
|
+
if (enableDiscovery && discoveryResult && !attemptsToRun.includes('universal_reality') && !disabledByPreset.has('universal_reality')) {
|
|
270
414
|
const simpleSite = (discoveryResult.interactionsDiscovered || 0) === 0 || (discoveryResult.pagesVisitedCount || 0) <= 1;
|
|
271
415
|
if (simpleSite) {
|
|
272
416
|
attemptsToRun.push('universal_reality');
|
|
@@ -279,6 +423,22 @@ async function executeReality(config) {
|
|
|
279
423
|
attemptsToRun = filterAttempts(attemptsToRun, validation.ids);
|
|
280
424
|
}
|
|
281
425
|
|
|
426
|
+
// Wave 3: Apply smart attempt selection based on introspection
|
|
427
|
+
let attemptsSkipped = [];
|
|
428
|
+
if (siteIntrospection) {
|
|
429
|
+
const attemptObjects = attemptsToRun.map(id => ({ id }));
|
|
430
|
+
const relevanceResult = filterAttemptsByRelevance(attemptObjects, siteIntrospection);
|
|
431
|
+
attemptsToRun = relevanceResult.toRun.map(a => a.id);
|
|
432
|
+
attemptsSkipped = relevanceResult.toSkip;
|
|
433
|
+
|
|
434
|
+
if (attemptsSkipped.length > 0 && !ciMode) {
|
|
435
|
+
console.log(`\n⊘ Skipping ${attemptsSkipped.length} irrelevant attempt(s):`);
|
|
436
|
+
for (const skip of attemptsSkipped) {
|
|
437
|
+
console.log(` • ${skip.attempt}: ${skip.reason}`);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
282
442
|
// Phase 7.2: Print parallel mode if enabled
|
|
283
443
|
if (!ciMode && validatedParallel > 1) {
|
|
284
444
|
console.log(`\n⚡ PARALLEL: ${validatedParallel} concurrent attempts`);
|
|
@@ -309,12 +469,27 @@ async function executeReality(config) {
|
|
|
309
469
|
|
|
310
470
|
// Phase 7.2: Execute attempts with bounded parallelism
|
|
311
471
|
// Phase 7.3: Pass browser pool to attempts
|
|
472
|
+
// Phase 7.4: Check applicability before executing
|
|
312
473
|
const attemptResults_parallel = await executeParallel(
|
|
313
474
|
attemptsToRun,
|
|
314
475
|
async (attemptId) => {
|
|
315
476
|
const attemptDef = getAttemptDefinition(attemptId);
|
|
316
477
|
if (!attemptDef) {
|
|
317
|
-
|
|
478
|
+
missingAttempts.push(attemptId);
|
|
479
|
+
return {
|
|
480
|
+
attemptId,
|
|
481
|
+
attemptName: attemptId,
|
|
482
|
+
goal: 'Unknown',
|
|
483
|
+
riskCategory: 'UNKNOWN',
|
|
484
|
+
source: 'manual',
|
|
485
|
+
outcome: 'SKIPPED',
|
|
486
|
+
skipReason: 'Attempt not registered',
|
|
487
|
+
skipReasonCode: SKIP_CODES.ENGINE_MISSING,
|
|
488
|
+
exitCode: 0,
|
|
489
|
+
steps: [],
|
|
490
|
+
friction: null,
|
|
491
|
+
error: null
|
|
492
|
+
};
|
|
318
493
|
}
|
|
319
494
|
|
|
320
495
|
if (!ciMode) {
|
|
@@ -330,8 +505,10 @@ async function executeReality(config) {
|
|
|
330
505
|
|
|
331
506
|
let result;
|
|
332
507
|
try {
|
|
333
|
-
//
|
|
508
|
+
// Navigate to site to check prerequisites and applicability
|
|
334
509
|
await page.goto(baseUrl, { waitUntil: 'domcontentloaded', timeout: resolvedTimeout });
|
|
510
|
+
|
|
511
|
+
// Phase 7.4: Check prerequisites before executing attempt
|
|
335
512
|
const prereqCheck = await checkPrerequisites(page, attemptId, 2000);
|
|
336
513
|
|
|
337
514
|
if (!prereqCheck.canProceed) {
|
|
@@ -342,27 +519,73 @@ async function executeReality(config) {
|
|
|
342
519
|
|
|
343
520
|
result = {
|
|
344
521
|
outcome: 'SKIPPED',
|
|
345
|
-
|
|
346
|
-
|
|
522
|
+
skipReason: prereqCheck.reason,
|
|
523
|
+
skipReasonCode: SKIP_CODES.PREREQ,
|
|
524
|
+
exitCode: 0,
|
|
347
525
|
steps: [],
|
|
348
526
|
friction: null,
|
|
349
527
|
error: null
|
|
350
528
|
};
|
|
351
529
|
} else {
|
|
352
|
-
//
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
530
|
+
// Check if this attempt is applicable to the site
|
|
531
|
+
const { AttemptEngine } = require('./attempt-engine');
|
|
532
|
+
const engine = new AttemptEngine({ attemptId, timeout: resolvedTimeout });
|
|
533
|
+
const applicabilityCheck = await engine.checkAttemptApplicability(page, attemptId);
|
|
534
|
+
|
|
535
|
+
if (!applicabilityCheck.applicable && applicabilityCheck.confidence < 0.3) {
|
|
536
|
+
// Attempt not applicable - features not present
|
|
537
|
+
if (!ciMode) {
|
|
538
|
+
console.log(` ⊘ Not applicable: ${applicabilityCheck.reason}`);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
result = {
|
|
542
|
+
outcome: 'NOT_APPLICABLE',
|
|
543
|
+
skipReason: applicabilityCheck.reason,
|
|
544
|
+
skipReasonCode: SKIP_CODES.NOT_APPLICABLE,
|
|
545
|
+
discoverySignals: applicabilityCheck.discoverySignals,
|
|
546
|
+
exitCode: 0,
|
|
547
|
+
steps: [],
|
|
548
|
+
friction: null,
|
|
549
|
+
error: null
|
|
550
|
+
};
|
|
551
|
+
} else if (!applicabilityCheck.applicable && applicabilityCheck.confidence >= 0.3 && applicabilityCheck.confidence < 0.6) {
|
|
552
|
+
// Features possibly present but discovery uncertain - try to execute anyway
|
|
553
|
+
// Mark as DISCOVERY_FAILED if it fails
|
|
554
|
+
result = await executeAttempt({
|
|
555
|
+
baseUrl,
|
|
556
|
+
attemptId,
|
|
557
|
+
artifactsDir: attemptArtifactsDir,
|
|
558
|
+
headful,
|
|
559
|
+
enableTrace,
|
|
560
|
+
enableScreenshots,
|
|
561
|
+
quiet: ciMode,
|
|
562
|
+
timeout: resolvedTimeout,
|
|
563
|
+
browserContext: context,
|
|
564
|
+
browserPage: page
|
|
565
|
+
});
|
|
566
|
+
|
|
567
|
+
// If execution failed due to element not found, reclassify as DISCOVERY_FAILED
|
|
568
|
+
if (result.outcome === 'FAILURE' && result.error && /element|selector|locator/i.test(result.error)) {
|
|
569
|
+
result.outcome = 'DISCOVERY_FAILED';
|
|
570
|
+
result.skipReason = `Element discovery failed: ${result.error}`;
|
|
571
|
+
result.discoverySignals = applicabilityCheck.discoverySignals;
|
|
572
|
+
}
|
|
573
|
+
} else {
|
|
574
|
+
// Applicable or high confidence - execute normally
|
|
575
|
+
result = await executeAttempt({
|
|
576
|
+
baseUrl,
|
|
577
|
+
attemptId,
|
|
578
|
+
artifactsDir: attemptArtifactsDir,
|
|
579
|
+
headful,
|
|
580
|
+
enableTrace,
|
|
581
|
+
enableScreenshots,
|
|
582
|
+
quiet: ciMode,
|
|
583
|
+
timeout: resolvedTimeout,
|
|
584
|
+
// Phase 7.3: Pass context from pool
|
|
585
|
+
browserContext: context,
|
|
586
|
+
browserPage: page
|
|
587
|
+
});
|
|
588
|
+
}
|
|
366
589
|
}
|
|
367
590
|
} finally {
|
|
368
591
|
// Phase 7.3: Cleanup context after attempt
|
|
@@ -378,8 +601,7 @@ async function executeReality(config) {
|
|
|
378
601
|
...result
|
|
379
602
|
};
|
|
380
603
|
|
|
381
|
-
// Phase 7.1: Fail-fast logic (stop on FAILURE, not FRICTION)
|
|
382
|
-
// Phase 7.4: SKIPPED does NOT trigger fail-fast
|
|
604
|
+
// Phase 7.1: Fail-fast logic (stop on FAILURE, not FRICTION or SKIPPED)
|
|
383
605
|
if (failFast && attemptResult.outcome === 'FAILURE') {
|
|
384
606
|
shouldStopScheduling = true;
|
|
385
607
|
if (!ciMode) {
|
|
@@ -395,11 +617,79 @@ async function executeReality(config) {
|
|
|
395
617
|
|
|
396
618
|
// Collect results in order
|
|
397
619
|
for (const result of attemptResults_parallel) {
|
|
398
|
-
if (result
|
|
620
|
+
if (result) {
|
|
399
621
|
attemptResults.push(result);
|
|
400
622
|
}
|
|
401
623
|
}
|
|
402
624
|
|
|
625
|
+
// Add explicit SKIPPED entries for attempts filtered out before execution
|
|
626
|
+
for (const skip of attemptsSkipped) {
|
|
627
|
+
const def = getAttemptDefinition(skip.attempt) || {};
|
|
628
|
+
attemptResults.push({
|
|
629
|
+
attemptId: skip.attempt,
|
|
630
|
+
attemptName: def.name || skip.attempt,
|
|
631
|
+
goal: def.goal,
|
|
632
|
+
riskCategory: def.riskCategory || 'UNKNOWN',
|
|
633
|
+
source: def.source || 'manual',
|
|
634
|
+
outcome: 'SKIPPED',
|
|
635
|
+
skipReason: skip.reason || 'Skipped before execution',
|
|
636
|
+
skipReasonCode: skip.reasonCode || SKIP_CODES.USER_FILTERED,
|
|
637
|
+
exitCode: 0,
|
|
638
|
+
steps: [],
|
|
639
|
+
friction: null,
|
|
640
|
+
error: null
|
|
641
|
+
});
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
// Add explicit SKIPPED entries for attempts disabled by preset (kept for auditability)
|
|
645
|
+
for (const disabledId of presetDisabledAttempts) {
|
|
646
|
+
const def = getAttemptDefinition(disabledId) || {};
|
|
647
|
+
attemptResults.push({
|
|
648
|
+
attemptId: disabledId,
|
|
649
|
+
attemptName: def.name || disabledId,
|
|
650
|
+
goal: def.goal,
|
|
651
|
+
riskCategory: def.riskCategory || 'UNKNOWN',
|
|
652
|
+
source: def.source || 'manual',
|
|
653
|
+
outcome: 'SKIPPED',
|
|
654
|
+
skipReason: 'Disabled by preset',
|
|
655
|
+
skipReasonCode: SKIP_CODES.DISABLED_BY_PRESET,
|
|
656
|
+
exitCode: 0,
|
|
657
|
+
steps: [],
|
|
658
|
+
friction: null,
|
|
659
|
+
error: null,
|
|
660
|
+
disabledByPreset: true
|
|
661
|
+
});
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
// Add explicit SKIPPED for user-filtered attempts that were removed before scheduling
|
|
665
|
+
for (const uf of userFilteredAttempts) {
|
|
666
|
+
const def = getAttemptDefinition(uf.attemptId) || {};
|
|
667
|
+
attemptResults.push({
|
|
668
|
+
attemptId: uf.attemptId,
|
|
669
|
+
attemptName: def.name || uf.attemptId,
|
|
670
|
+
goal: def.goal,
|
|
671
|
+
riskCategory: def.riskCategory || 'UNKNOWN',
|
|
672
|
+
source: def.source || 'manual',
|
|
673
|
+
outcome: 'SKIPPED',
|
|
674
|
+
skipReason: uf.reason,
|
|
675
|
+
skipReasonCode: SKIP_CODES.USER_FILTERED,
|
|
676
|
+
exitCode: 0,
|
|
677
|
+
steps: [],
|
|
678
|
+
friction: null,
|
|
679
|
+
error: null,
|
|
680
|
+
userFiltered: true
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
// Preserve requested ordering for downstream artifacts
|
|
685
|
+
const attemptOrder = new Map(requestedAttempts.map((id, idx) => [id, idx]));
|
|
686
|
+
attemptResults.sort((a, b) => (attemptOrder.get(a.attemptId) ?? 999) - (attemptOrder.get(b.attemptId) ?? 999));
|
|
687
|
+
|
|
688
|
+
// Normalize execution metadata
|
|
689
|
+
for (const result of attemptResults) {
|
|
690
|
+
result.executed = isExecutedAttempt(result);
|
|
691
|
+
}
|
|
692
|
+
|
|
403
693
|
// Phase 3: Execute intent flows (deterministic, curated)
|
|
404
694
|
if (enableFlows) {
|
|
405
695
|
console.log(`\n🎯 Executing intent flows...`);
|
|
@@ -526,12 +816,35 @@ async function executeReality(config) {
|
|
|
526
816
|
});
|
|
527
817
|
}
|
|
528
818
|
|
|
819
|
+
// Ensure artifacts exist for skipped attempts so totals remain canonical and inspectable
|
|
820
|
+
for (const result of attemptResults) {
|
|
821
|
+
if (!isExecutedAttempt(result)) {
|
|
822
|
+
const attemptDir = path.join(runDir, result.attemptId);
|
|
823
|
+
const attemptRunDir = path.join(attemptDir, 'attempt-skipped');
|
|
824
|
+
fs.mkdirSync(path.join(attemptRunDir, 'attempt-screenshots'), { recursive: true });
|
|
825
|
+
|
|
826
|
+
const skipSummary = {
|
|
827
|
+
attemptId: result.attemptId,
|
|
828
|
+
outcome: 'SKIPPED',
|
|
829
|
+
reason: result.skipReason || 'Not executed'
|
|
830
|
+
};
|
|
831
|
+
|
|
832
|
+
const jsonPath = path.join(attemptRunDir, 'attempt-report.json');
|
|
833
|
+
const htmlPath = path.join(attemptRunDir, 'attempt-report.html');
|
|
834
|
+
fs.writeFileSync(jsonPath, JSON.stringify(skipSummary, null, 2));
|
|
835
|
+
fs.writeFileSync(htmlPath, `<html><body><h1>${result.attemptName || result.attemptId}</h1><p>Skipped: ${skipSummary.reason}</p></body></html>`);
|
|
836
|
+
|
|
837
|
+
result.reportJsonPath = result.reportJsonPath || jsonPath;
|
|
838
|
+
result.reportHtmlPath = result.reportHtmlPath || htmlPath;
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
|
|
529
842
|
// Generate market report (existing flow)
|
|
530
843
|
const reporter = new MarketReporter();
|
|
531
844
|
const report = reporter.createReport({
|
|
532
845
|
runId,
|
|
533
846
|
baseUrl,
|
|
534
|
-
attemptsRun:
|
|
847
|
+
attemptsRun: requestedAttempts,
|
|
535
848
|
results: attemptResults.map(r => ({
|
|
536
849
|
attemptId: r.attemptId,
|
|
537
850
|
attemptName: r.attemptName,
|
|
@@ -650,22 +963,381 @@ async function executeReality(config) {
|
|
|
650
963
|
console.warn(`⚠️ Enhanced HTML report failed (non-critical): ${htmlErr.message}`);
|
|
651
964
|
}
|
|
652
965
|
|
|
653
|
-
// Phase
|
|
966
|
+
// Phase 7.3: Cleanup browser pool
|
|
967
|
+
await browserPool.close();
|
|
968
|
+
|
|
969
|
+
// Wave 4: Honest results & near-success
|
|
970
|
+
// Calculate attempt statistics and coverage
|
|
971
|
+
const disabledAttemptResults = attemptResults.filter(a => a.disabledByPreset);
|
|
972
|
+
const eligibleAttempts = attemptResults.filter(a => !a.disabledByPreset);
|
|
973
|
+
const executedAttempts = eligibleAttempts.filter(isExecutedAttempt);
|
|
974
|
+
const skippedAttempts = eligibleAttempts.filter(a => !isExecutedAttempt(a));
|
|
975
|
+
const skippedNotApplicable = eligibleAttempts.filter(a => a.skipReasonCode === SKIP_CODES.NOT_APPLICABLE);
|
|
976
|
+
const skippedMissing = eligibleAttempts.filter(a => a.skipReasonCode === SKIP_CODES.ENGINE_MISSING);
|
|
977
|
+
const skippedUserFiltered = eligibleAttempts.filter(a => a.skipReasonCode === SKIP_CODES.USER_FILTERED || a.userFiltered);
|
|
978
|
+
const skippedDisabledByPreset = disabledAttemptResults;
|
|
979
|
+
const attemptStats = {
|
|
980
|
+
total: eligibleAttempts.length,
|
|
981
|
+
executed: executedAttempts.length,
|
|
982
|
+
executedCount: executedAttempts.length,
|
|
983
|
+
enabledPlannedCount: enabledRequestedAttempts.length,
|
|
984
|
+
disabledPlannedCount: disabledAttemptResults.length,
|
|
985
|
+
successful: executedAttempts.filter(a => a.outcome === 'SUCCESS').length,
|
|
986
|
+
failed: executedAttempts.filter(a => a.outcome === 'FAILURE').length,
|
|
987
|
+
skipped: skippedAttempts.length,
|
|
988
|
+
skippedDetails: skippedAttempts.map(a => ({ attempt: a.attemptId, reason: a.skipReason || 'Not executed', code: a.skipReasonCode })),
|
|
989
|
+
disabled: disabledAttemptResults.length,
|
|
990
|
+
disabledDetails: disabledAttemptResults.map(a => ({ attempt: a.attemptId, reason: a.skipReason || 'Disabled by preset', code: SKIP_CODES.DISABLED_BY_PRESET })),
|
|
991
|
+
userFiltered: skippedUserFiltered.length,
|
|
992
|
+
skippedDisabledByPreset: skippedDisabledByPreset.length,
|
|
993
|
+
skippedNotApplicable: skippedNotApplicable.length,
|
|
994
|
+
skippedMissing: skippedMissing.length,
|
|
995
|
+
skippedUserFiltered: skippedUserFiltered.length
|
|
996
|
+
};
|
|
997
|
+
|
|
998
|
+
const nearSuccessDetails = [];
|
|
999
|
+
for (const a of attemptResults) {
|
|
1000
|
+
if (a.outcome === 'FAILURE') {
|
|
1001
|
+
const errMsg = typeof a.error === 'string' ? a.error : '';
|
|
1002
|
+
const noFailedSteps = Array.isArray(a.steps) ? a.steps.every(s => s.status !== 'failed') : true;
|
|
1003
|
+
if (noFailedSteps && errMsg.includes('Success conditions not met')) {
|
|
1004
|
+
nearSuccessDetails.push({ attempt: a.attemptId, reason: 'Submit succeeded but no confirmation text detected' });
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
attemptStats.nearSuccess = nearSuccessDetails.length;
|
|
1009
|
+
attemptStats.nearSuccessDetails = nearSuccessDetails;
|
|
1010
|
+
|
|
1011
|
+
// Coverage and evidence signals
|
|
1012
|
+
const coverageDenominator = attemptStats.total;
|
|
1013
|
+
const coverageNumerator = attemptStats.executed + skippedNotApplicable.length;
|
|
1014
|
+
const coverageGaps = Math.max(coverageDenominator - coverageNumerator, 0);
|
|
1015
|
+
const coverageSignal = {
|
|
1016
|
+
gaps: coverageGaps,
|
|
1017
|
+
executed: attemptStats.executed,
|
|
1018
|
+
total: coverageDenominator,
|
|
1019
|
+
details: attemptStats.skippedDetails,
|
|
1020
|
+
disabled: attemptStats.disabledDetails,
|
|
1021
|
+
skippedDisabledByPreset: skippedDisabledByPreset.map(a => ({ attempt: a.attemptId, reason: a.skipReason || 'Disabled by preset', code: SKIP_CODES.DISABLED_BY_PRESET })),
|
|
1022
|
+
skippedNotApplicable: skippedNotApplicable.map(a => ({ attempt: a.attemptId, reason: a.skipReason })),
|
|
1023
|
+
skippedMissing: skippedMissing.map(a => ({ attempt: a.attemptId, reason: a.skipReason })),
|
|
1024
|
+
skippedUserFiltered: skippedUserFiltered.map(a => ({ attempt: a.attemptId, reason: a.skipReason })),
|
|
1025
|
+
counts: {
|
|
1026
|
+
executedCount: attemptStats.executed,
|
|
1027
|
+
enabledPlannedCount: attemptStats.enabledPlannedCount,
|
|
1028
|
+
disabledPlannedCount: attemptStats.disabledPlannedCount,
|
|
1029
|
+
skippedDisabledByPreset: skippedDisabledByPreset.length,
|
|
1030
|
+
skippedUserFiltered: skippedUserFiltered.length,
|
|
1031
|
+
skippedNotApplicable: skippedNotApplicable.length,
|
|
1032
|
+
skippedMissing: skippedMissing.length
|
|
1033
|
+
}
|
|
1034
|
+
};
|
|
1035
|
+
|
|
1036
|
+
const evidenceMetrics = {
|
|
1037
|
+
completeness: coverageDenominator > 0 ? coverageNumerator / coverageDenominator : 0,
|
|
1038
|
+
integrity: 0,
|
|
1039
|
+
hashedFiles: 0,
|
|
1040
|
+
totalFiles: 0,
|
|
1041
|
+
screenshotsEnabled: enableScreenshots,
|
|
1042
|
+
tracesEnabled: enableTrace
|
|
1043
|
+
};
|
|
1044
|
+
|
|
1045
|
+
// Build signals used by policy (single source of verdict truth)
|
|
1046
|
+
const policySignals = {
|
|
1047
|
+
coverage: coverageSignal,
|
|
1048
|
+
marketImpact,
|
|
1049
|
+
flows: flowResults,
|
|
1050
|
+
baseline: { baselineCreated, baselineSnapshot, diffResult },
|
|
1051
|
+
attempts: attemptResults,
|
|
1052
|
+
crawlIssues: runSignals.filter(s => s.type === 'coverage'),
|
|
1053
|
+
discoveryIssues: runSignals.filter(s => s.type === 'discovery'),
|
|
1054
|
+
evidence: {
|
|
1055
|
+
metrics: evidenceMetrics,
|
|
1056
|
+
missingScreenshots: !enableScreenshots,
|
|
1057
|
+
missingTraces: !enableTrace
|
|
1058
|
+
},
|
|
1059
|
+
runtimeSignals: runSignals
|
|
1060
|
+
};
|
|
1061
|
+
|
|
1062
|
+
// Resolve policy (strict failure on invalid)
|
|
654
1063
|
let policyEval = null;
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
1064
|
+
let policyObj = null;
|
|
1065
|
+
let presetId = policyName; // Preserve preset ID for naming (e.g., 'startup')
|
|
1066
|
+
let policyHash = null;
|
|
1067
|
+
let policySource = 'default';
|
|
1068
|
+
try {
|
|
1069
|
+
if (policy && typeof policy === 'string' && fs.existsSync(policy)) {
|
|
1070
|
+
policySource = path.resolve(policy);
|
|
1071
|
+
} else if (policy) {
|
|
1072
|
+
policySource = typeof policy === 'string' ? `inline:${policy}` : `preset:${config.preset || policyName}`;
|
|
1073
|
+
} else {
|
|
1074
|
+
policySource = 'default';
|
|
1075
|
+
}
|
|
1076
|
+
policyObj = policy
|
|
1077
|
+
? (typeof policy === 'object' ? policy : parsePolicyOption(policy))
|
|
1078
|
+
: loadPolicy();
|
|
1079
|
+
policyName = policyObj?.name || policyObj?.id || policyName || (policy ? policy : 'default');
|
|
1080
|
+
policyHash = crypto.createHash('sha256').update(JSON.stringify(policyObj)).digest('hex');
|
|
1081
|
+
} catch (policyLoadErr) {
|
|
1082
|
+
console.error(`Error: ${policyLoadErr.message}`);
|
|
1083
|
+
process.exit(2);
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
// First pass policy evaluation (before manifest integrity)
|
|
1087
|
+
policyEval = evaluatePolicy(snapshotBuilder.getSnapshot(), policyObj, policySignals);
|
|
1088
|
+
console.log(`\n🛡️ Evaluating policy... (${policyName})`);
|
|
1089
|
+
console.log(`Policy evaluation result: exitCode=${policyEval.exitCode}, passed=${policyEval.passed}`);
|
|
1090
|
+
if (policyEval.reasons && policyEval.reasons.length > 0) {
|
|
1091
|
+
policyEval.reasons.slice(0, 3).forEach(r => console.log(` • ${r}`));
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
const resolvedConfig = {
|
|
1095
|
+
presetId: config.preset || presetId,
|
|
1096
|
+
policySource,
|
|
1097
|
+
policyId: policyObj?.id || policyObj?.name || policyName,
|
|
1098
|
+
policyHash,
|
|
1099
|
+
mediaRequirements: {
|
|
1100
|
+
requireScreenshots: !!(policyObj?.evidence?.requireScreenshots),
|
|
1101
|
+
requireTraces: !!(policyObj?.evidence?.requireTraces),
|
|
1102
|
+
minCompleteness: policyObj?.evidence?.minCompleteness,
|
|
1103
|
+
minIntegrity: policyObj?.evidence?.minIntegrity
|
|
1104
|
+
},
|
|
1105
|
+
attemptPlan: {
|
|
1106
|
+
enabled: enabledRequestedAttempts,
|
|
1107
|
+
disabled: Array.from(disabledByPreset),
|
|
1108
|
+
userFiltered: userFilteredAttempts.map(u => u.attemptId),
|
|
1109
|
+
missing: missingAttempts
|
|
1110
|
+
},
|
|
1111
|
+
coverage: {
|
|
1112
|
+
total: coverageSignal.total,
|
|
1113
|
+
executed: coverageSignal.executed,
|
|
1114
|
+
executedCount: attemptStats.executed,
|
|
1115
|
+
enabledPlannedCount: attemptStats.enabledPlannedCount,
|
|
1116
|
+
disabledPlannedCount: attemptStats.disabledPlannedCount,
|
|
1117
|
+
skippedNotApplicable: coverageSignal.skippedNotApplicable?.length || 0,
|
|
1118
|
+
skippedMissing: coverageSignal.skippedMissing?.length || 0,
|
|
1119
|
+
skippedUserFiltered: coverageSignal.skippedUserFiltered?.length || 0,
|
|
1120
|
+
skippedDisabledByPreset: coverageSignal.skippedDisabledByPreset?.length || 0,
|
|
1121
|
+
disabled: coverageSignal.disabled?.length || 0
|
|
1122
|
+
},
|
|
1123
|
+
evidenceMetrics
|
|
1124
|
+
};
|
|
1125
|
+
|
|
1126
|
+
// Calculate actual duration from start time
|
|
1127
|
+
const endTime = new Date();
|
|
1128
|
+
const actualDurationMs = endTime.getTime() - startTime.getTime();
|
|
1129
|
+
|
|
1130
|
+
// Rename run directory to status placeholder for auditability
|
|
1131
|
+
let exitCode = policyEval.exitCode;
|
|
1132
|
+
const runResultPreManifest = 'PENDING';
|
|
1133
|
+
const priorRunDir = runDir;
|
|
1134
|
+
const finalRunDirName = makeRunDirName({ timestamp: startTime, url: baseUrl, policy: presetId, result: runResultPreManifest });
|
|
1135
|
+
const finalRunDir = path.join(artifactsDir, finalRunDirName);
|
|
1136
|
+
if (finalRunDir !== runDir) {
|
|
1137
|
+
fs.renameSync(runDir, finalRunDir);
|
|
1138
|
+
runDir = finalRunDir;
|
|
1139
|
+
runDirName = finalRunDirName;
|
|
1140
|
+
}
|
|
1141
|
+
// Rebase attempt artifact paths after rename
|
|
1142
|
+
for (const attempt of attemptResults) {
|
|
1143
|
+
if (attempt.attemptJsonPath) {
|
|
1144
|
+
attempt.attemptJsonPath = attempt.attemptJsonPath.replace(priorRunDir, runDir);
|
|
1145
|
+
}
|
|
1146
|
+
if (attempt.stepsLogPath) {
|
|
1147
|
+
attempt.stepsLogPath = attempt.stepsLogPath.replace(priorRunDir, runDir);
|
|
1148
|
+
}
|
|
1149
|
+
if (attempt.reportJsonPath) {
|
|
1150
|
+
attempt.reportJsonPath = attempt.reportJsonPath.replace(priorRunDir, runDir);
|
|
1151
|
+
}
|
|
1152
|
+
if (attempt.reportHtmlPath) {
|
|
1153
|
+
attempt.reportHtmlPath = attempt.reportHtmlPath.replace(priorRunDir, runDir);
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
const snapshotPathFinal = path.join(runDir, 'snapshot.json');
|
|
1157
|
+
const marketJsonPathFinal = path.join(runDir, path.basename(jsonPath));
|
|
1158
|
+
const marketHtmlPathFinal = path.join(runDir, path.basename(htmlPath));
|
|
1159
|
+
|
|
1160
|
+
// Build decision artifact and summary (first pass)
|
|
1161
|
+
const initialDecision = computeFinalVerdict({
|
|
1162
|
+
marketImpact,
|
|
1163
|
+
policyEval,
|
|
1164
|
+
baseline: { baselineCreated, baselineSnapshot, diffResult },
|
|
1165
|
+
flows: flowResults,
|
|
1166
|
+
attempts: attemptResults
|
|
1167
|
+
});
|
|
1168
|
+
const initialExplanation = buildRealityExplanation({
|
|
1169
|
+
finalDecision: initialDecision,
|
|
1170
|
+
attemptStats,
|
|
1171
|
+
marketImpact,
|
|
1172
|
+
policyEval,
|
|
1173
|
+
baseline: { baselineCreated, baselineSnapshot, diffResult },
|
|
1174
|
+
flows: flowResults,
|
|
1175
|
+
attempts: attemptResults,
|
|
1176
|
+
coverage: coverageSignal
|
|
1177
|
+
});
|
|
1178
|
+
|
|
1179
|
+
const decisionPath = writeDecisionArtifact({
|
|
1180
|
+
runDir,
|
|
1181
|
+
runId,
|
|
1182
|
+
baseUrl,
|
|
1183
|
+
policyName,
|
|
1184
|
+
preset: config.preset || policyName,
|
|
1185
|
+
finalDecision: initialDecision,
|
|
1186
|
+
attemptStats,
|
|
1187
|
+
marketImpact,
|
|
1188
|
+
policyEval,
|
|
1189
|
+
baseline: { baselineCreated, baselineSnapshot, diffResult },
|
|
1190
|
+
flows: flowResults,
|
|
1191
|
+
resolved: resolvedConfig,
|
|
1192
|
+
attempts: attemptResults,
|
|
1193
|
+
coverage: coverageSignal,
|
|
1194
|
+
explanation: initialExplanation
|
|
1195
|
+
});
|
|
1196
|
+
|
|
1197
|
+
const summaryPath = writeRunSummary(runDir, initialDecision, attemptStats, marketImpact, policyEval, initialExplanation);
|
|
1198
|
+
|
|
1199
|
+
// Build integrity manifest over all artifacts and update evidence metrics
|
|
1200
|
+
try {
|
|
1201
|
+
const manifestInfo = writeIntegrityManifest(runDir);
|
|
1202
|
+
evidenceMetrics.hashedFiles = manifestInfo.hashedFiles;
|
|
1203
|
+
evidenceMetrics.totalFiles = manifestInfo.totalFiles;
|
|
1204
|
+
evidenceMetrics.integrity = manifestInfo.totalFiles > 0 ? manifestInfo.hashedFiles / manifestInfo.totalFiles : 0;
|
|
1205
|
+
} catch (manifestErr) {
|
|
1206
|
+
console.warn(`⚠️ Failed to write integrity manifest: ${manifestErr.message}`);
|
|
1207
|
+
evidenceMetrics.integrity = 0;
|
|
1208
|
+
runSignals.push({ id: 'manifest_failed', severity: 'medium', type: 'evidence', description: `Manifest generation failed: ${manifestErr.message}` });
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
// Re-run policy evaluation with final evidence metrics
|
|
1212
|
+
policyEval = evaluatePolicy(snapshotBuilder.getSnapshot(), policyObj, policySignals);
|
|
1213
|
+
const finalDecision = computeFinalVerdict({
|
|
1214
|
+
marketImpact,
|
|
1215
|
+
policyEval,
|
|
1216
|
+
baseline: { baselineCreated, baselineSnapshot, diffResult },
|
|
1217
|
+
flows: flowResults,
|
|
1218
|
+
attempts: attemptResults
|
|
1219
|
+
});
|
|
1220
|
+
const finalExplanation = buildRealityExplanation({
|
|
1221
|
+
finalDecision,
|
|
1222
|
+
attemptStats,
|
|
1223
|
+
marketImpact,
|
|
1224
|
+
policyEval,
|
|
1225
|
+
baseline: { baselineCreated, baselineSnapshot, diffResult },
|
|
1226
|
+
flows: flowResults,
|
|
1227
|
+
attempts: attemptResults,
|
|
1228
|
+
coverage: coverageSignal
|
|
1229
|
+
});
|
|
1230
|
+
exitCode = finalDecision.exitCode;
|
|
1231
|
+
|
|
1232
|
+
// Rewrite decision + summary with final verdict
|
|
1233
|
+
const decisionPathFinal = writeDecisionArtifact({
|
|
1234
|
+
runDir,
|
|
1235
|
+
runId,
|
|
1236
|
+
baseUrl,
|
|
1237
|
+
policyName,
|
|
1238
|
+
preset: config.preset || policyName,
|
|
1239
|
+
finalDecision,
|
|
1240
|
+
attemptStats,
|
|
1241
|
+
marketImpact,
|
|
1242
|
+
policyEval,
|
|
1243
|
+
baseline: { baselineCreated, baselineSnapshot, diffResult },
|
|
1244
|
+
flows: flowResults,
|
|
1245
|
+
resolved: resolvedConfig,
|
|
1246
|
+
attempts: attemptResults,
|
|
1247
|
+
coverage: coverageSignal,
|
|
1248
|
+
explanation: finalExplanation
|
|
1249
|
+
});
|
|
1250
|
+
const summaryPathFinal = writeRunSummary(runDir, finalDecision, attemptStats, marketImpact, policyEval, finalExplanation);
|
|
1251
|
+
|
|
1252
|
+
const runResult = finalDecision.finalVerdict;
|
|
1253
|
+
|
|
1254
|
+
// Persist policy evaluation and meta into snapshot
|
|
1255
|
+
try {
|
|
1256
|
+
const snap = snapshotBuilder.getSnapshot();
|
|
1257
|
+
snap.policyEvaluation = policyEval;
|
|
1258
|
+
snap.policyName = policyName;
|
|
1259
|
+
snap.meta.policyHash = policyHash;
|
|
1260
|
+
snap.meta.preset = config.preset || presetId;
|
|
1261
|
+
snap.meta.evidenceMetrics = evidenceMetrics;
|
|
1262
|
+
snap.meta.coverage = coverageSignal;
|
|
1263
|
+
snap.meta.resolved = resolvedConfig;
|
|
1264
|
+
snap.resolved = resolvedConfig;
|
|
1265
|
+
snap.meta.result = finalDecision.finalVerdict;
|
|
1266
|
+
snap.meta.attemptsSummary = {
|
|
1267
|
+
executed: attemptStats.executed,
|
|
1268
|
+
successful: attemptStats.successful,
|
|
1269
|
+
failed: attemptStats.failed,
|
|
1270
|
+
skipped: attemptStats.skipped,
|
|
1271
|
+
disabled: attemptStats.disabled,
|
|
1272
|
+
nearSuccess: attemptStats.nearSuccess,
|
|
1273
|
+
nearSuccessDetails: attemptStats.nearSuccessDetails
|
|
1274
|
+
};
|
|
1275
|
+
snap.evidenceMetrics = { ...evidenceMetrics, coverage: coverageSignal };
|
|
1276
|
+
snap.coverage = coverageSignal;
|
|
1277
|
+
await saveSnapshot(snap, snapshotPathFinal);
|
|
1278
|
+
// Minimal attestation: sha256(policyHash + snapshotHash + manifestHash + runId)
|
|
1279
|
+
const snapshotHash = hashFile(snapshotPathFinal);
|
|
1280
|
+
let manifestHash = null;
|
|
1281
|
+
try { manifestHash = hashFile(path.join(runDir, 'manifest.json')); } catch {}
|
|
1282
|
+
const attestationHash = crypto.createHash('sha256').update(`${policyHash}|${snapshotHash}|${manifestHash || 'none'}|${runId}`).digest('hex');
|
|
1283
|
+
snap.meta.attestation = { hash: attestationHash, policyHash, snapshotHash, manifestHash, runId };
|
|
1284
|
+
await saveSnapshot(snap, snapshotPathFinal);
|
|
1285
|
+
|
|
1286
|
+
// Rewrite decision to include attestation and auditor-grade summary
|
|
1287
|
+
writeDecisionArtifact({
|
|
1288
|
+
runDir,
|
|
1289
|
+
runId,
|
|
1290
|
+
baseUrl,
|
|
1291
|
+
policyName,
|
|
1292
|
+
preset: config.preset || policyName,
|
|
1293
|
+
finalDecision,
|
|
1294
|
+
attemptStats,
|
|
1295
|
+
marketImpact,
|
|
1296
|
+
policyEval,
|
|
1297
|
+
baseline: { baselineCreated, baselineSnapshot, diffResult },
|
|
1298
|
+
flows: flowResults,
|
|
1299
|
+
resolved: resolvedConfig,
|
|
1300
|
+
attestation: snap.meta.attestation,
|
|
1301
|
+
audit: {
|
|
1302
|
+
executedAttempts: (snapshotBuilder.getSnapshot()?.attempts || []).filter(a => a.executed).map(a => a.attemptId),
|
|
1303
|
+
notTested: {
|
|
1304
|
+
disabledByPreset: (attemptStats.disabledDetails || []).map(d => d.attempt),
|
|
1305
|
+
userFiltered: (snap.coverage?.skippedUserFiltered || []).map(s => s.attempt),
|
|
1306
|
+
notApplicable: (snap.coverage?.skippedNotApplicable || []).map(s => s.attempt),
|
|
1307
|
+
missing: (snap.coverage?.skippedMissing || []).map(s => s.attempt)
|
|
664
1308
|
}
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
|
|
1309
|
+
},
|
|
1310
|
+
attempts: attemptResults,
|
|
1311
|
+
coverage: coverageSignal,
|
|
1312
|
+
explanation: finalExplanation
|
|
1313
|
+
});
|
|
1314
|
+
} catch (_) {}
|
|
1315
|
+
|
|
1316
|
+
// Persist META.json
|
|
1317
|
+
let metaData;
|
|
1318
|
+
try {
|
|
1319
|
+
metaData = {
|
|
1320
|
+
runDir,
|
|
1321
|
+
url: baseUrl,
|
|
1322
|
+
siteSlug,
|
|
1323
|
+
policy: policyName,
|
|
1324
|
+
policyHash,
|
|
1325
|
+
preset: config.preset || presetId,
|
|
1326
|
+
result: runResult,
|
|
1327
|
+
durationMs: actualDurationMs,
|
|
1328
|
+
profile: siteProfile,
|
|
1329
|
+
attempts: attemptStats,
|
|
1330
|
+
evidence: evidenceMetrics,
|
|
1331
|
+
decisionPath: decisionPathFinal || decisionPath,
|
|
1332
|
+
summaryPath: summaryPathFinal || summaryPath
|
|
1333
|
+
};
|
|
1334
|
+
writeMetaJson(metaData);
|
|
1335
|
+
if (process.env.GUARDIAN_DEBUG) {
|
|
1336
|
+
console.log(`\n💾 META.json written successfully`);
|
|
668
1337
|
}
|
|
1338
|
+
} catch (metaErr) {
|
|
1339
|
+
console.warn(`⚠️ Failed to write META.json: ${metaErr.message}`);
|
|
1340
|
+
exitCode = 1;
|
|
669
1341
|
}
|
|
670
1342
|
|
|
671
1343
|
// Phase 5/6: Send webhook notifications
|
|
@@ -688,81 +1360,66 @@ async function executeReality(config) {
|
|
|
688
1360
|
}
|
|
689
1361
|
}
|
|
690
1362
|
|
|
691
|
-
//
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
if (marketImpact.highestSeverity === 'CRITICAL') {
|
|
699
|
-
console.log(`🚨 First run with CRITICAL market risks`);
|
|
700
|
-
exitCode = 1;
|
|
701
|
-
} else if (marketImpact.highestSeverity === 'WARNING') {
|
|
702
|
-
console.log(`⚠️ First run with WARNING market risks`);
|
|
703
|
-
exitCode = 2;
|
|
704
|
-
}
|
|
705
|
-
console.log(`✅ Baseline created`);
|
|
706
|
-
} else if (baselineSnapshot) {
|
|
707
|
-
// Subsequent runs: check for regressions + severity escalation
|
|
708
|
-
const baselineSeverity = baselineSnapshot.marketImpactSummary?.highestSeverity || 'INFO';
|
|
709
|
-
const currentSeverity = marketImpact.highestSeverity;
|
|
710
|
-
const escalation = determineExitCodeFromEscalation(baselineSeverity, currentSeverity);
|
|
711
|
-
|
|
712
|
-
if (escalation.escalated) {
|
|
713
|
-
// Severity escalation is a FAILURE
|
|
714
|
-
exitCode = 1;
|
|
715
|
-
console.log(`🚨 Severity escalated: ${baselineSeverity} → ${currentSeverity}`);
|
|
716
|
-
} else if (diffResult && Object.keys(diffResult.regressions).length > 0) {
|
|
717
|
-
exitCode = 1;
|
|
718
|
-
console.log(`❌ Regressions detected`);
|
|
719
|
-
} else if (currentSeverity !== 'INFO') {
|
|
720
|
-
// Still have market risks but didn't escalate
|
|
721
|
-
exitCode = 2;
|
|
722
|
-
console.log(`⚠️ ${currentSeverity} market risks present`);
|
|
723
|
-
} else {
|
|
724
|
-
exitCode = 0;
|
|
725
|
-
console.log(`✅ No critical changes`);
|
|
1363
|
+
// Wave 2: Update latest pointers with finalized metadata
|
|
1364
|
+
try {
|
|
1365
|
+
const metaContent = readMetaJson(runDir);
|
|
1366
|
+
updateLatestGlobal(runDir, runDirName, metaContent, artifactsDir);
|
|
1367
|
+
updateLatestBySite(runDir, runDirName, metaContent, artifactsDir);
|
|
1368
|
+
if (process.env.GUARDIAN_DEBUG) {
|
|
1369
|
+
console.log(`✅ Latest pointers updated`);
|
|
726
1370
|
}
|
|
1371
|
+
} catch (latestErr) {
|
|
1372
|
+
console.warn(`⚠️ Failed to update latest pointers: ${latestErr.message}`);
|
|
727
1373
|
}
|
|
728
1374
|
|
|
729
|
-
// Override exit code if policy failed
|
|
730
|
-
if (policyEval && !policyEval.passed) {
|
|
731
|
-
exitCode = policyEval.exitCode || 1;
|
|
732
|
-
console.log(`🛡️ Policy override: exit code ${exitCode}`);
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
// Flow-based exit code aggregation (0/1/2)
|
|
736
|
-
const flowExitCode = computeFlowExitCode(flowResults);
|
|
737
|
-
exitCode = flowExitCode;
|
|
738
|
-
if (!ciMode && flowResults.length > 0) {
|
|
739
|
-
console.log(`Exit code (flows): ${flowExitCode}`);
|
|
740
|
-
}
|
|
741
|
-
|
|
742
|
-
// Phase 7.3: Cleanup browser pool
|
|
743
|
-
await browserPool.close();
|
|
744
|
-
|
|
745
1375
|
return {
|
|
746
1376
|
exitCode,
|
|
747
1377
|
report,
|
|
748
1378
|
runDir,
|
|
749
|
-
snapshotPath,
|
|
750
|
-
marketJsonPath:
|
|
751
|
-
marketHtmlPath:
|
|
1379
|
+
snapshotPath: snapshotPathFinal,
|
|
1380
|
+
marketJsonPath: marketJsonPathFinal,
|
|
1381
|
+
marketHtmlPath: marketHtmlPathFinal,
|
|
752
1382
|
attemptResults,
|
|
753
1383
|
flowResults,
|
|
754
1384
|
baselineCreated,
|
|
755
1385
|
diffResult,
|
|
756
|
-
snapshot:
|
|
757
|
-
policyEval
|
|
1386
|
+
snapshot: snapshotBuilder.getSnapshot(),
|
|
1387
|
+
policyEval,
|
|
1388
|
+
resolved: resolvedConfig,
|
|
1389
|
+
finalDecision,
|
|
1390
|
+
explanation: finalExplanation,
|
|
1391
|
+
coverage: coverageSignal
|
|
758
1392
|
};
|
|
759
1393
|
}
|
|
760
1394
|
|
|
761
1395
|
async function runRealityCLI(config) {
|
|
762
1396
|
try {
|
|
763
|
-
|
|
1397
|
+
// Stage II: Environment guard (before any other work)
|
|
1398
|
+
const { checkEnvironment, failWithEnvironmentError } = require('./env-guard');
|
|
1399
|
+
const envCheck = checkEnvironment();
|
|
1400
|
+
if (!envCheck.allOk) {
|
|
1401
|
+
failWithEnvironmentError(envCheck.issues);
|
|
1402
|
+
return; // Exit path, but just in case
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
const { config: effectiveConfig, report: configReport } = applyLocalConfig(config);
|
|
1406
|
+
|
|
1407
|
+
console.log('\nConfig:');
|
|
1408
|
+
console.log(`- source: ${configReport.source}`);
|
|
1409
|
+
if (configReport.path) {
|
|
1410
|
+
console.log(`- path: ${configReport.path}`);
|
|
1411
|
+
}
|
|
1412
|
+
console.log('- effective:');
|
|
1413
|
+
console.log(` - crawl.maxPages: ${configReport.effective.crawl.maxPages}`);
|
|
1414
|
+
console.log(` - crawl.maxDepth: ${configReport.effective.crawl.maxDepth}`);
|
|
1415
|
+
console.log(` - timeouts.navigationMs: ${configReport.effective.timeouts.navigationMs}`);
|
|
1416
|
+
console.log(` - output.dir: ${configReport.effective.output.dir}`);
|
|
1417
|
+
|
|
1418
|
+
const cfg = effectiveConfig;
|
|
1419
|
+
|
|
1420
|
+
if (cfg.watch) {
|
|
764
1421
|
const { startWatchMode } = require('./watch-runner');
|
|
765
|
-
const watchResult = await startWatchMode(
|
|
1422
|
+
const watchResult = await startWatchMode(cfg);
|
|
766
1423
|
if (watchResult && watchResult.watchStarted === false && typeof watchResult.exitCode === 'number') {
|
|
767
1424
|
process.exit(watchResult.exitCode);
|
|
768
1425
|
}
|
|
@@ -770,10 +1427,30 @@ async function runRealityCLI(config) {
|
|
|
770
1427
|
return;
|
|
771
1428
|
}
|
|
772
1429
|
|
|
773
|
-
const result = await executeReality(
|
|
1430
|
+
const result = await executeReality(cfg);
|
|
1431
|
+
|
|
1432
|
+
// Mark first run as complete
|
|
1433
|
+
if (isFirstRun()) {
|
|
1434
|
+
markFirstRunComplete();
|
|
1435
|
+
}
|
|
774
1436
|
|
|
775
1437
|
// Phase 6: Print enhanced CLI summary
|
|
776
1438
|
const ciMode = isCiMode();
|
|
1439
|
+
if (!ciMode) {
|
|
1440
|
+
const resolved = result.resolved || {};
|
|
1441
|
+
console.log(`\nResolved configuration:`);
|
|
1442
|
+
console.log(` Preset: ${resolved.presetId || 'unknown'}`);
|
|
1443
|
+
console.log(` Policy: ${resolved.policyId || 'unknown'} (source: ${resolved.policySource || 'n/a'}, hash: ${resolved.policyHash || 'n/a'})`);
|
|
1444
|
+
if (resolved.mediaRequirements) {
|
|
1445
|
+
console.log(` Media requirements: screenshots=${resolved.mediaRequirements.requireScreenshots}, traces=${resolved.mediaRequirements.requireTraces}`);
|
|
1446
|
+
}
|
|
1447
|
+
if (resolved.attemptPlan) {
|
|
1448
|
+
console.log(` Attempt plan: enabled=${(resolved.attemptPlan.enabled || []).length}, disabled=${(resolved.attemptPlan.disabled || []).length}, userFiltered=${(resolved.attemptPlan.userFiltered || []).length}, missing=${(resolved.attemptPlan.missing || []).length}`);
|
|
1449
|
+
}
|
|
1450
|
+
if (resolved.coverage) {
|
|
1451
|
+
console.log(` Attempt outcomes: executed=${resolved.coverage.executedCount ?? resolved.coverage.executed}, disabled=${resolved.coverage.disabledPlannedCount ?? 0}, skippedDisabledByPreset=${resolved.coverage.skippedDisabledByPreset ?? 0}, skippedUserFiltered=${resolved.coverage.skippedUserFiltered ?? 0}, skippedNotApplicable=${resolved.coverage.skippedNotApplicable ?? 0}, skippedMissing=${resolved.coverage.skippedMissing ?? 0}`);
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
777
1454
|
if (ciMode) {
|
|
778
1455
|
const ciSummary = formatCiSummary({
|
|
779
1456
|
flowResults: result.flowResults || [],
|
|
@@ -784,13 +1461,75 @@ async function runRealityCLI(config) {
|
|
|
784
1461
|
});
|
|
785
1462
|
console.log(ciSummary);
|
|
786
1463
|
} else {
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
1464
|
+
// Strict CLI summary: traceable, factual only
|
|
1465
|
+
const snap = result.snapshot || {};
|
|
1466
|
+
const meta = snap.meta || {};
|
|
1467
|
+
const coverage = snap.coverage || result.coverage || {};
|
|
1468
|
+
const counts = coverage.counts || {};
|
|
1469
|
+
const evidence = snap.evidenceMetrics || {};
|
|
1470
|
+
const resolved = snap.resolved || {};
|
|
1471
|
+
const finalDecision = result.finalDecision || {};
|
|
1472
|
+
const explanation = result.explanation || buildRealityExplanation({
|
|
1473
|
+
finalDecision,
|
|
1474
|
+
attemptStats: snap.meta?.attemptsSummary || {},
|
|
1475
|
+
marketImpact: snap.marketImpact || {},
|
|
1476
|
+
policyEval: result.policyEval,
|
|
1477
|
+
baseline: { diffResult: result.diffResult },
|
|
1478
|
+
flows: result.flowResults,
|
|
1479
|
+
attempts: result.attemptResults,
|
|
1480
|
+
coverage
|
|
1481
|
+
});
|
|
1482
|
+
const sections = explanation.sections || {};
|
|
1483
|
+
const verdictSection = sections['Final Verdict'] || {};
|
|
1484
|
+
console.log('\n' + '━'.repeat(70));
|
|
1485
|
+
console.log('🛡️ Guardian Reality Summary');
|
|
1486
|
+
console.log('━'.repeat(70) + '\n');
|
|
1487
|
+
console.log(`Target: ${meta.url || 'unknown'}`);
|
|
1488
|
+
console.log(`Run ID: ${meta.runId || 'unknown'}\n`);
|
|
1489
|
+
console.log(`Verdict: ${meta.result || finalDecision.finalVerdict || 'unknown'}`);
|
|
1490
|
+
console.log(`Exit Code: ${result.exitCode}`);
|
|
1491
|
+
if (verdictSection.explanation) console.log(`Reason: ${verdictSection.explanation}`);
|
|
1492
|
+
if (verdictSection.whyNot && verdictSection.whyNot.length > 0) {
|
|
1493
|
+
console.log(`Why not alternatives: ${verdictSection.whyNot.join(' ')}`);
|
|
1494
|
+
}
|
|
1495
|
+
const planned = coverage.total ?? (resolved.coverage?.total);
|
|
1496
|
+
const executed = counts.executedCount ?? (resolved.coverage?.executedCount) ?? coverage.executed;
|
|
1497
|
+
console.log(`Executed / Planned: ${executed} / ${planned}`);
|
|
1498
|
+
const completeness = evidence.completeness ?? resolved.evidenceMetrics?.completeness;
|
|
1499
|
+
const integrity = evidence.integrity ?? resolved.evidenceMetrics?.integrity;
|
|
1500
|
+
console.log(`Coverage Completeness: ${typeof completeness === 'number' ? completeness.toFixed(4) : completeness}`);
|
|
1501
|
+
console.log(`Evidence Integrity: ${typeof integrity === 'number' ? integrity.toFixed(4) : integrity}`);
|
|
1502
|
+
if (meta.attestation?.hash) console.log(`Attestation: ${meta.attestation.hash}`);
|
|
1503
|
+
const executedAttempts = (snap.attempts || []).filter(a => a.executed).map(a => a.attemptId);
|
|
1504
|
+
console.log('\nAudit Summary:');
|
|
1505
|
+
console.log(` Tested (${executedAttempts.length}): ${executedAttempts.join(', ') || 'none'}`);
|
|
1506
|
+
const skippedDisabled = (coverage.skippedDisabledByPreset || []).map(s => s.attempt);
|
|
1507
|
+
const skippedUserFiltered = (coverage.skippedUserFiltered || []).map(s => s.attempt);
|
|
1508
|
+
const skippedNotApplicable = (coverage.skippedNotApplicable || []).map(s => s.attempt);
|
|
1509
|
+
const skippedMissing = (coverage.skippedMissing || []).map(s => s.attempt);
|
|
1510
|
+
console.log(` Not Tested — DisabledByPreset (${skippedDisabled.length}): ${skippedDisabled.join(', ') || 'none'}`);
|
|
1511
|
+
console.log(` Not Tested — UserFiltered (${skippedUserFiltered.length}): ${skippedUserFiltered.join(', ') || 'none'}`);
|
|
1512
|
+
console.log(` Not Tested — NotApplicable (${skippedNotApplicable.length}): ${skippedNotApplicable.join(', ') || 'none'}`);
|
|
1513
|
+
console.log(` Not Tested — Missing (${skippedMissing.length}): ${skippedMissing.join(', ') || 'none'}`);
|
|
1514
|
+
if (sections['What Guardian Observed']) {
|
|
1515
|
+
console.log('\nWhat Guardian Observed:');
|
|
1516
|
+
sections['What Guardian Observed'].details.forEach(d => console.log(` • ${d}`));
|
|
1517
|
+
}
|
|
1518
|
+
if (sections['What Guardian Could Not Confirm']) {
|
|
1519
|
+
console.log('\nWhat Guardian Could Not Confirm:');
|
|
1520
|
+
sections['What Guardian Could Not Confirm'].details.forEach(d => console.log(` • ${d}`));
|
|
1521
|
+
}
|
|
1522
|
+
if (sections['Evidence Summary']) {
|
|
1523
|
+
console.log('\nEvidence Summary:');
|
|
1524
|
+
sections['Evidence Summary'].details.forEach(d => console.log(` • ${d}`));
|
|
1525
|
+
}
|
|
1526
|
+
if (sections['Limits of This Run']) {
|
|
1527
|
+
console.log('\nLimits of This Run:');
|
|
1528
|
+
sections['Limits of This Run'].details.forEach(d => console.log(` • ${d}`));
|
|
1529
|
+
}
|
|
1530
|
+
const reportBase = result.runDir || (configReport?.effective?.output?.dir ? path.join(configReport.effective.output.dir, meta.runId || '') : `artifacts/${meta.runId || ''}`);
|
|
1531
|
+
console.log(`\n📁 Full report: ${reportBase}\n`);
|
|
1532
|
+
console.log('━'.repeat(70));
|
|
794
1533
|
}
|
|
795
1534
|
|
|
796
1535
|
process.exit(result.exitCode);
|
|
@@ -880,4 +1619,426 @@ function computeFlowExitCode(flowResults) {
|
|
|
880
1619
|
return 0;
|
|
881
1620
|
}
|
|
882
1621
|
|
|
883
|
-
|
|
1622
|
+
function computeFinalVerdict({ marketImpact, policyEval, baseline, flows, attempts }) {
|
|
1623
|
+
const reasons = [];
|
|
1624
|
+
|
|
1625
|
+
const executedAttempts = Array.isArray(attempts)
|
|
1626
|
+
? attempts.filter(a => a.executed)
|
|
1627
|
+
: [];
|
|
1628
|
+
const successfulAttempts = executedAttempts.filter(a => a.outcome === 'SUCCESS');
|
|
1629
|
+
const failedAttempts = executedAttempts.filter(a => a.outcome === 'FAILURE');
|
|
1630
|
+
const frictionAttempts = executedAttempts.filter(a => a.outcome === 'FRICTION');
|
|
1631
|
+
|
|
1632
|
+
const flowList = Array.isArray(flows) ? flows : [];
|
|
1633
|
+
const failedFlows = flowList.filter(f => f.outcome === 'FAILURE' || f.success === false);
|
|
1634
|
+
const frictionFlows = flowList.filter(f => f.outcome === 'FRICTION');
|
|
1635
|
+
const observedFlows = successfulAttempts.map(a => a.attemptId || a.id || 'unknown');
|
|
1636
|
+
|
|
1637
|
+
// Observation summary always first
|
|
1638
|
+
if (executedAttempts.length > 0) {
|
|
1639
|
+
reasons.push({ code: 'OBSERVED', message: `Observed ${executedAttempts.length} attempted flow(s); successful=${successfulAttempts.length}, failed=${failedAttempts.length}, friction=${frictionAttempts.length}.` });
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
// Baseline regressions
|
|
1643
|
+
const diff = baseline?.diffResult || baseline?.diff;
|
|
1644
|
+
if (diff && diff.regressions && Object.keys(diff.regressions).length > 0) {
|
|
1645
|
+
const regressionAttempts = Object.keys(diff.regressions);
|
|
1646
|
+
reasons.push({ code: 'BASELINE_REGRESSION', message: `Baseline regressions detected for: ${regressionAttempts.join(', ')}.` });
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
// Policy evaluation evidence
|
|
1650
|
+
if (policyEval) {
|
|
1651
|
+
if (!policyEval.passed && policyEval.summary) {
|
|
1652
|
+
reasons.push({ code: 'POLICY', message: policyEval.summary });
|
|
1653
|
+
} else if (!policyEval.passed) {
|
|
1654
|
+
reasons.push({ code: 'POLICY', message: 'Policy conditions not satisfied; evidence insufficient.' });
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
|
|
1658
|
+
// Market impact evidence
|
|
1659
|
+
if (marketImpact && marketImpact.highestSeverity) {
|
|
1660
|
+
reasons.push({ code: 'MARKET_IMPACT', message: `Market impact severity observed: ${marketImpact.highestSeverity}.` });
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
// Flow/attempt failures
|
|
1664
|
+
if (failedAttempts.length > 0) {
|
|
1665
|
+
reasons.push({ code: 'CRITICAL_FLOW_FAILURE', message: `Critical flows failed: ${failedAttempts.map(a => a.attemptId || a.id || 'unknown').join(', ')}.` });
|
|
1666
|
+
}
|
|
1667
|
+
if (failedFlows.length > 0) {
|
|
1668
|
+
reasons.push({ code: 'FLOW_FAILURE', message: `Flow executions failed: ${failedFlows.map(f => f.flowId || f.flowName || 'unknown').join(', ')}.` });
|
|
1669
|
+
}
|
|
1670
|
+
if (frictionAttempts.length > 0 || frictionFlows.length > 0) {
|
|
1671
|
+
const frictionIds = [
|
|
1672
|
+
...frictionAttempts.map(a => a.attemptId || a.id || 'unknown'),
|
|
1673
|
+
...frictionFlows.map(f => f.flowId || f.flowName || 'unknown')
|
|
1674
|
+
];
|
|
1675
|
+
reasons.push({ code: 'FLOW_FRICTION', message: `Flows with friction: ${frictionIds.join(', ')}.` });
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
// Determine verdict
|
|
1679
|
+
let internalVerdict;
|
|
1680
|
+
let finalVerdict;
|
|
1681
|
+
let exitCode;
|
|
1682
|
+
|
|
1683
|
+
if (executedAttempts.length === 0 && flowList.length === 0) {
|
|
1684
|
+
internalVerdict = 'INSUFFICIENT_DATA';
|
|
1685
|
+
reasons.unshift({ code: 'NO_OBSERVATIONS', message: 'No meaningful flows executed; only static or configuration checks available.' });
|
|
1686
|
+
} else if (failedAttempts.length === 0 && failedFlows.length === 0 && frictionAttempts.length === 0 && frictionFlows.length === 0 && (!policyEval || policyEval.passed)) {
|
|
1687
|
+
internalVerdict = 'OBSERVED';
|
|
1688
|
+
if (observedFlows.length > 0) {
|
|
1689
|
+
reasons.push({ code: 'OBSERVED_FLOWS', message: `Observed end-to-end flows: ${observedFlows.join(', ')}.` });
|
|
1690
|
+
}
|
|
1691
|
+
reasons.push({ code: 'SCOPE', message: 'Verdict based solely on executed flows; no claim about business readiness beyond observed actions.' });
|
|
1692
|
+
} else {
|
|
1693
|
+
internalVerdict = 'PARTIAL';
|
|
1694
|
+
if (observedFlows.length > 0) {
|
|
1695
|
+
reasons.push({ code: 'PARTIAL_SCOPE', message: `Some flows observed (${observedFlows.join(', ')}), but at least one flow failed or could not be confirmed.` });
|
|
1696
|
+
} else {
|
|
1697
|
+
reasons.push({ code: 'PARTIAL_SCOPE', message: 'Flows were attempted but outcomes include failures or friction; observations are incomplete.' });
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
// Ensure deterministic ordering: sort by code then message
|
|
1702
|
+
const orderedReasons = reasons
|
|
1703
|
+
.filter(r => r && r.code && r.message)
|
|
1704
|
+
.sort((a, b) => a.code.localeCompare(b.code) || a.message.localeCompare(b.message));
|
|
1705
|
+
|
|
1706
|
+
finalVerdict = toCanonicalVerdict(internalVerdict);
|
|
1707
|
+
exitCode = mapExitCodeFromCanonical(finalVerdict);
|
|
1708
|
+
return { finalVerdict, exitCode, reasons: orderedReasons };
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1711
|
+
function hashFile(filePath) {
|
|
1712
|
+
const data = fs.readFileSync(filePath);
|
|
1713
|
+
return crypto.createHash('sha256').update(data).digest('hex');
|
|
1714
|
+
}
|
|
1715
|
+
|
|
1716
|
+
function writeIntegrityManifest(runDir) {
|
|
1717
|
+
const manifest = {
|
|
1718
|
+
generatedAt: new Date().toISOString(),
|
|
1719
|
+
files: []
|
|
1720
|
+
};
|
|
1721
|
+
|
|
1722
|
+
const allFiles = [];
|
|
1723
|
+
|
|
1724
|
+
const walk = (dir) => {
|
|
1725
|
+
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
1726
|
+
for (const entry of entries) {
|
|
1727
|
+
const full = path.join(dir, entry.name);
|
|
1728
|
+
if (entry.isDirectory()) {
|
|
1729
|
+
walk(full);
|
|
1730
|
+
} else {
|
|
1731
|
+
allFiles.push(full);
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
};
|
|
1735
|
+
|
|
1736
|
+
walk(runDir);
|
|
1737
|
+
|
|
1738
|
+
for (const target of allFiles) {
|
|
1739
|
+
if (fs.existsSync(target)) {
|
|
1740
|
+
manifest.files.push({
|
|
1741
|
+
path: path.relative(runDir, target),
|
|
1742
|
+
sha256: hashFile(target)
|
|
1743
|
+
});
|
|
1744
|
+
}
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
const manifestPath = path.join(runDir, 'manifest.json');
|
|
1748
|
+
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
|
|
1749
|
+
return {
|
|
1750
|
+
manifestPath,
|
|
1751
|
+
hashedFiles: manifest.files.length,
|
|
1752
|
+
totalFiles: allFiles.length
|
|
1753
|
+
};
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1756
|
+
function buildRealityExplanation({ finalDecision = {}, attemptStats = {}, marketImpact = {}, policyEval = {}, baseline = {}, flows = [], attempts = [], coverage = {} }) {
|
|
1757
|
+
const verdict = finalDecision.finalVerdict || 'UNKNOWN';
|
|
1758
|
+
const exitCode = typeof finalDecision.exitCode === 'number' ? finalDecision.exitCode : 1;
|
|
1759
|
+
|
|
1760
|
+
const executedAttempts = (attempts || []).filter(isExecutedAttempt);
|
|
1761
|
+
const successes = executedAttempts.filter(a => a.outcome === 'SUCCESS');
|
|
1762
|
+
const failures = executedAttempts.filter(a => a.outcome === 'FAILURE');
|
|
1763
|
+
const frictions = executedAttempts.filter(a => a.outcome === 'FRICTION');
|
|
1764
|
+
|
|
1765
|
+
const flowList = Array.isArray(flows) ? flows : [];
|
|
1766
|
+
const flowFailures = flowList.filter(f => f.outcome === 'FAILURE' || f.success === false);
|
|
1767
|
+
const flowFrictions = flowList.filter(f => f.outcome === 'FRICTION');
|
|
1768
|
+
const flowSuccesses = flowList.filter(f => f.outcome === 'SUCCESS' || f.success === true);
|
|
1769
|
+
|
|
1770
|
+
const observedDetails = [];
|
|
1771
|
+
if (executedAttempts.length > 0) {
|
|
1772
|
+
const attemptSummary = executedAttempts
|
|
1773
|
+
.map(a => `${a.attemptId || a.id || 'unknown'} (${a.outcome || 'unknown'})`)
|
|
1774
|
+
.sort();
|
|
1775
|
+
observedDetails.push(`Executed ${executedAttempts.length} attempt(s): ${attemptSummary.join(', ')}.`);
|
|
1776
|
+
} else {
|
|
1777
|
+
observedDetails.push('No user journeys executed; evidence limited to crawl/policy signals.');
|
|
1778
|
+
}
|
|
1779
|
+
if (flowSuccesses.length > 0) {
|
|
1780
|
+
const successFlows = flowSuccesses
|
|
1781
|
+
.map(f => f.flowId || f.flowName || 'unknown')
|
|
1782
|
+
.sort();
|
|
1783
|
+
observedDetails.push(`Successful flows: ${successFlows.join(', ')}.`);
|
|
1784
|
+
}
|
|
1785
|
+
|
|
1786
|
+
const couldNotConfirm = [];
|
|
1787
|
+
if (failures.length > 0) {
|
|
1788
|
+
const failureAttempts = failures
|
|
1789
|
+
.map(a => `${a.attemptId || a.id || 'unknown'} (${a.error ? String(a.error).split('\n')[0] : a.outcome})`)
|
|
1790
|
+
.sort();
|
|
1791
|
+
couldNotConfirm.push(`Failures detected in: ${failureAttempts.join(', ')}.`);
|
|
1792
|
+
}
|
|
1793
|
+
if (frictions.length > 0) {
|
|
1794
|
+
const frictionAttempts = frictions.map(a => a.attemptId || a.id || 'unknown').sort();
|
|
1795
|
+
couldNotConfirm.push(`Friction observed in: ${frictionAttempts.join(', ')}.`);
|
|
1796
|
+
}
|
|
1797
|
+
if (flowFailures.length > 0) {
|
|
1798
|
+
const ff = flowFailures.map(f => f.flowId || f.flowName || 'unknown').sort();
|
|
1799
|
+
couldNotConfirm.push(`Flow failures: ${ff.join(', ')}.`);
|
|
1800
|
+
}
|
|
1801
|
+
if (flowFrictions.length > 0) {
|
|
1802
|
+
const ff = flowFrictions.map(f => f.flowId || f.flowName || 'unknown').sort();
|
|
1803
|
+
couldNotConfirm.push(`Flow friction: ${ff.join(', ')}.`);
|
|
1804
|
+
}
|
|
1805
|
+
if (policyEval && policyEval.passed === false) {
|
|
1806
|
+
const reason = policyEval.summary || 'Policy conditions not satisfied.';
|
|
1807
|
+
couldNotConfirm.push(`Policy check failed: ${reason}`);
|
|
1808
|
+
}
|
|
1809
|
+
const diff = baseline?.diffResult || baseline?.diff;
|
|
1810
|
+
if (diff && diff.regressions && Object.keys(diff.regressions).length > 0) {
|
|
1811
|
+
const regressionAttempts = Object.keys(diff.regressions).sort();
|
|
1812
|
+
couldNotConfirm.push(`Baseline regressions: ${regressionAttempts.join(', ')}.`);
|
|
1813
|
+
}
|
|
1814
|
+
if (coverage && typeof coverage.gaps === 'number' && coverage.gaps > 0) {
|
|
1815
|
+
couldNotConfirm.push(`Coverage gaps: ${coverage.gaps} planned attempt(s) not observed.`);
|
|
1816
|
+
}
|
|
1817
|
+
if (couldNotConfirm.length === 0) {
|
|
1818
|
+
couldNotConfirm.push('No outstanding gaps; all planned evidence confirmed for observed scope.');
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
const evidenceSummary = [];
|
|
1822
|
+
const executedCount = attemptStats.executed ?? executedAttempts.length;
|
|
1823
|
+
const totalPlanned = attemptStats.total ?? attemptStats.enabledPlannedCount ?? executedAttempts.length;
|
|
1824
|
+
evidenceSummary.push(`Attempt outcomes: executed=${executedCount}, success=${attemptStats.successful ?? successes.length}, failed=${attemptStats.failed ?? failures.length}, skipped=${attemptStats.skipped ?? 0}.`);
|
|
1825
|
+
evidenceSummary.push(`Flow outcomes: success=${flowSuccesses.length}, failures=${flowFailures.length}, friction=${flowFrictions.length}.`);
|
|
1826
|
+
if (marketImpact && marketImpact.highestSeverity) {
|
|
1827
|
+
evidenceSummary.push(`Market severity observed: ${marketImpact.highestSeverity}.`);
|
|
1828
|
+
} else {
|
|
1829
|
+
evidenceSummary.push('Market severity not reported.');
|
|
1830
|
+
}
|
|
1831
|
+
if (policyEval && typeof policyEval.passed === 'boolean') {
|
|
1832
|
+
evidenceSummary.push(`Policy evaluation: ${policyEval.passed ? 'passed' : 'failed'} (exit ${policyEval.exitCode ?? 'unknown'}).`);
|
|
1833
|
+
} else {
|
|
1834
|
+
evidenceSummary.push('Policy evaluation: not run.');
|
|
1835
|
+
}
|
|
1836
|
+
if (diff && diff.regressions) {
|
|
1837
|
+
const regressions = Object.keys(diff.regressions);
|
|
1838
|
+
evidenceSummary.push(`Baseline regressions: ${regressions.length > 0 ? regressions.join(', ') : 'none'}.`);
|
|
1839
|
+
} else {
|
|
1840
|
+
evidenceSummary.push('Baseline comparison: none or no regressions detected.');
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1843
|
+
const limits = [];
|
|
1844
|
+
const gaps = coverage && typeof coverage.gaps === 'number' ? coverage.gaps : Math.max((totalPlanned || 0) - (executedCount || 0), 0);
|
|
1845
|
+
limits.push(`Coverage: ${executedCount}/${totalPlanned || 0} attempts executed; gaps=${gaps}.`);
|
|
1846
|
+
if (attemptStats.disabled) {
|
|
1847
|
+
limits.push(`Disabled by preset: ${attemptStats.disabled} attempt(s).`);
|
|
1848
|
+
}
|
|
1849
|
+
if (attemptStats.skippedUserFiltered) {
|
|
1850
|
+
limits.push(`User-filtered skips: ${attemptStats.skippedUserFiltered}.`);
|
|
1851
|
+
}
|
|
1852
|
+
if (attemptStats.skippedNotApplicable) {
|
|
1853
|
+
limits.push(`Not-applicable skips: ${attemptStats.skippedNotApplicable}.`);
|
|
1854
|
+
}
|
|
1855
|
+
if (attemptStats.skippedMissing) {
|
|
1856
|
+
limits.push(`Missing engine skips: ${attemptStats.skippedMissing}.`);
|
|
1857
|
+
}
|
|
1858
|
+
if (limits.length === 0) {
|
|
1859
|
+
limits.push('No additional limits detected beyond observed attempts.');
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
let whyThisVerdict;
|
|
1863
|
+
const whyNotList = [];
|
|
1864
|
+
if (verdict === 'OBSERVED') {
|
|
1865
|
+
whyThisVerdict = 'All executed attempts succeeded; no failures, friction, or policy blockers detected.';
|
|
1866
|
+
whyNotList.push('Not PARTIAL because no failures, friction, or policy shortfalls remain.');
|
|
1867
|
+
whyNotList.push(`Not INSUFFICIENT_DATA because ${executedCount} attempt(s) executed with complete evidence.`);
|
|
1868
|
+
} else if (verdict === 'PARTIAL') {
|
|
1869
|
+
const partialDrivers = [];
|
|
1870
|
+
if (failures.length > 0) partialDrivers.push(`${failures.length} failed attempt(s)`);
|
|
1871
|
+
if (frictions.length > 0) partialDrivers.push(`${frictions.length} friction attempt(s)`);
|
|
1872
|
+
if (flowFailures.length > 0) partialDrivers.push(`${flowFailures.length} failed flow(s)`);
|
|
1873
|
+
if (policyEval && policyEval.passed === false) partialDrivers.push('policy not satisfied');
|
|
1874
|
+
whyThisVerdict = partialDrivers.length > 0
|
|
1875
|
+
? `Evidence is mixed: ${partialDrivers.join('; ')}.`
|
|
1876
|
+
: 'Evidence incomplete or mixed; not all planned signals confirmed.';
|
|
1877
|
+
whyNotList.push('Not OBSERVED because at least one failure, friction, or policy issue remains.');
|
|
1878
|
+
whyNotList.push(`Not INSUFFICIENT_DATA because ${executedCount} attempt(s) executed and produced evidence.`);
|
|
1879
|
+
} else if (verdict === 'INSUFFICIENT_DATA') {
|
|
1880
|
+
whyThisVerdict = 'No meaningful user flows were executed; evidence is insufficient to claim readiness.';
|
|
1881
|
+
whyNotList.push('Not OBSERVED because no successful flows were confirmed.');
|
|
1882
|
+
whyNotList.push('Not PARTIAL because there was no executable evidence to partially support readiness.');
|
|
1883
|
+
} else {
|
|
1884
|
+
whyThisVerdict = 'Verdict unavailable; using conservative interpretation of observed evidence.';
|
|
1885
|
+
whyNotList.push('Alternative verdicts not evaluated due to unknown state.');
|
|
1886
|
+
}
|
|
1887
|
+
|
|
1888
|
+
const finalVerdictSection = {
|
|
1889
|
+
verdict,
|
|
1890
|
+
exitCode,
|
|
1891
|
+
explanation: whyThisVerdict,
|
|
1892
|
+
whyNot: whyNotList,
|
|
1893
|
+
reasons: (finalDecision.reasons || []).map(r => `${r.code}: ${r.message}`)
|
|
1894
|
+
};
|
|
1895
|
+
|
|
1896
|
+
const sections = {
|
|
1897
|
+
'Final Verdict': finalVerdictSection,
|
|
1898
|
+
'What Guardian Observed': {
|
|
1899
|
+
summary: observedDetails[0],
|
|
1900
|
+
details: observedDetails
|
|
1901
|
+
},
|
|
1902
|
+
'What Guardian Could Not Confirm': {
|
|
1903
|
+
summary: couldNotConfirm[0],
|
|
1904
|
+
details: couldNotConfirm
|
|
1905
|
+
},
|
|
1906
|
+
'Evidence Summary': {
|
|
1907
|
+
summary: evidenceSummary[0],
|
|
1908
|
+
details: evidenceSummary
|
|
1909
|
+
},
|
|
1910
|
+
'Limits of This Run': {
|
|
1911
|
+
summary: limits[0],
|
|
1912
|
+
details: limits
|
|
1913
|
+
}
|
|
1914
|
+
};
|
|
1915
|
+
|
|
1916
|
+
return { verdict: finalVerdictSection, sections };
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1919
|
+
function writeDecisionArtifact({ runDir, runId, baseUrl, policyName, preset, finalDecision, attemptStats, marketImpact, policyEval, baseline, flows, resolved, attestation, audit, attempts = [], coverage = {}, explanation }) {
|
|
1920
|
+
const structuredExplanation = explanation || buildRealityExplanation({ finalDecision, attemptStats, marketImpact, policyEval, baseline, flows, attempts, coverage });
|
|
1921
|
+
const safePolicyEval = policyEval || { passed: true, exitCode: 0, summary: 'Policy evaluation not run.' };
|
|
1922
|
+
const diff = baseline?.diffResult || baseline?.diff || {};
|
|
1923
|
+
const auditSummary = audit ? {
|
|
1924
|
+
tested: Array.isArray(audit.executedAttempts) ? audit.executedAttempts : [],
|
|
1925
|
+
notTested: {
|
|
1926
|
+
disabledByPreset: audit.notTested?.disabledByPreset || [],
|
|
1927
|
+
userFiltered: audit.notTested?.userFiltered || [],
|
|
1928
|
+
notApplicable: audit.notTested?.notApplicable || [],
|
|
1929
|
+
missing: audit.notTested?.missing || []
|
|
1930
|
+
}
|
|
1931
|
+
} : {
|
|
1932
|
+
tested: [],
|
|
1933
|
+
notTested: { disabledByPreset: [], userFiltered: [], notApplicable: [], missing: [] }
|
|
1934
|
+
};
|
|
1935
|
+
|
|
1936
|
+
const decision = {
|
|
1937
|
+
runId,
|
|
1938
|
+
url: baseUrl,
|
|
1939
|
+
timestamp: new Date().toISOString(),
|
|
1940
|
+
preset: preset || 'default',
|
|
1941
|
+
policyName: policyName || 'unknown',
|
|
1942
|
+
finalVerdict: finalDecision.finalVerdict,
|
|
1943
|
+
exitCode: finalDecision.exitCode,
|
|
1944
|
+
reasons: finalDecision.reasons,
|
|
1945
|
+
resolved: resolved || {},
|
|
1946
|
+
attestation: attestation || {},
|
|
1947
|
+
counts: {
|
|
1948
|
+
attemptsExecuted: attemptStats.executed || 0,
|
|
1949
|
+
successful: attemptStats.successful || 0,
|
|
1950
|
+
failed: attemptStats.failed || 0,
|
|
1951
|
+
skipped: attemptStats.skipped || 0,
|
|
1952
|
+
nearSuccess: attemptStats.nearSuccess || 0
|
|
1953
|
+
},
|
|
1954
|
+
inputs: {
|
|
1955
|
+
policy: safePolicyEval,
|
|
1956
|
+
baseline: diff,
|
|
1957
|
+
market: marketImpact || {},
|
|
1958
|
+
flows: {
|
|
1959
|
+
total: Array.isArray(flows) ? flows.length : 0,
|
|
1960
|
+
failures: Array.isArray(flows) ? flows.filter(f => f.outcome === 'FAILURE' || f.success === false).length : 0,
|
|
1961
|
+
frictions: Array.isArray(flows) ? flows.filter(f => f.outcome === 'FRICTION').length : 0
|
|
1962
|
+
}
|
|
1963
|
+
},
|
|
1964
|
+
coverage: {
|
|
1965
|
+
total: coverage.total || attemptStats.enabledPlannedCount || attemptStats.total || 0,
|
|
1966
|
+
executed: coverage.executed || attemptStats.executed || 0,
|
|
1967
|
+
gaps: coverage.gaps ?? Math.max((attemptStats.total || 0) - (attemptStats.executed || 0), 0),
|
|
1968
|
+
skipped: coverage.details || attemptStats.skippedDetails || [],
|
|
1969
|
+
disabled: coverage.disabled || attemptStats.disabledDetails || []
|
|
1970
|
+
},
|
|
1971
|
+
auditSummary,
|
|
1972
|
+
sections: structuredExplanation.sections,
|
|
1973
|
+
explanation: structuredExplanation.verdict
|
|
1974
|
+
};
|
|
1975
|
+
|
|
1976
|
+
const decisionPath = path.join(runDir, 'decision.json');
|
|
1977
|
+
fs.writeFileSync(decisionPath, JSON.stringify(decision, null, 2));
|
|
1978
|
+
return decisionPath;
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
function writeRunSummary(runDir, finalDecision, attemptStats, marketImpact, policyEval, explanation) {
|
|
1982
|
+
const structuredExplanation = explanation || buildRealityExplanation({ finalDecision, attemptStats, marketImpact, policyEval });
|
|
1983
|
+
const sections = structuredExplanation.sections;
|
|
1984
|
+
|
|
1985
|
+
const lines = [];
|
|
1986
|
+
lines.push(`Final Verdict: ${finalDecision.finalVerdict} (exit ${finalDecision.exitCode})`);
|
|
1987
|
+
lines.push(`Why this verdict: ${sections['Final Verdict'].explanation}`);
|
|
1988
|
+
if (sections['Final Verdict'].whyNot && sections['Final Verdict'].whyNot.length > 0) {
|
|
1989
|
+
lines.push(`Why not alternatives: ${sections['Final Verdict'].whyNot.join(' ')}`);
|
|
1990
|
+
}
|
|
1991
|
+
lines.push('');
|
|
1992
|
+
lines.push('What Guardian Observed:');
|
|
1993
|
+
sections['What Guardian Observed'].details.forEach(d => lines.push(`- ${d}`));
|
|
1994
|
+
lines.push('');
|
|
1995
|
+
lines.push('What Guardian Could Not Confirm:');
|
|
1996
|
+
sections['What Guardian Could Not Confirm'].details.forEach(d => lines.push(`- ${d}`));
|
|
1997
|
+
lines.push('');
|
|
1998
|
+
lines.push('Evidence Summary:');
|
|
1999
|
+
sections['Evidence Summary'].details.forEach(d => lines.push(`- ${d}`));
|
|
2000
|
+
lines.push('');
|
|
2001
|
+
lines.push('Limits of This Run:');
|
|
2002
|
+
sections['Limits of This Run'].details.forEach(d => lines.push(`- ${d}`));
|
|
2003
|
+
|
|
2004
|
+
const summaryPath = path.join(runDir, 'summary.txt');
|
|
2005
|
+
fs.writeFileSync(summaryPath, lines.join('\n'));
|
|
2006
|
+
// Also emit a markdown version for consistency with report discovery
|
|
2007
|
+
try {
|
|
2008
|
+
const mdLines = [];
|
|
2009
|
+
mdLines.push(`# Guardian Reality Summary`);
|
|
2010
|
+
mdLines.push('');
|
|
2011
|
+
|
|
2012
|
+
mdLines.push(`## Final Verdict`);
|
|
2013
|
+
mdLines.push(`- Verdict: ${finalDecision.finalVerdict} (exit ${finalDecision.exitCode})`);
|
|
2014
|
+
mdLines.push(`- Why this verdict: ${sections['Final Verdict'].explanation}`);
|
|
2015
|
+
if (sections['Final Verdict'].whyNot && sections['Final Verdict'].whyNot.length > 0) {
|
|
2016
|
+
sections['Final Verdict'].whyNot.forEach(reason => mdLines.push(`- ${reason}`));
|
|
2017
|
+
}
|
|
2018
|
+
if (sections['Final Verdict'].reasons && sections['Final Verdict'].reasons.length > 0) {
|
|
2019
|
+
mdLines.push(`- Evidence reasons: ${sections['Final Verdict'].reasons.join(' ')}`);
|
|
2020
|
+
}
|
|
2021
|
+
|
|
2022
|
+
mdLines.push('');
|
|
2023
|
+
mdLines.push(`## What Guardian Observed`);
|
|
2024
|
+
sections['What Guardian Observed'].details.forEach(d => mdLines.push(`- ${d}`));
|
|
2025
|
+
|
|
2026
|
+
mdLines.push('');
|
|
2027
|
+
mdLines.push(`## What Guardian Could Not Confirm`);
|
|
2028
|
+
sections['What Guardian Could Not Confirm'].details.forEach(d => mdLines.push(`- ${d}`));
|
|
2029
|
+
|
|
2030
|
+
mdLines.push('');
|
|
2031
|
+
mdLines.push(`## Evidence Summary`);
|
|
2032
|
+
sections['Evidence Summary'].details.forEach(d => mdLines.push(`- ${d}`));
|
|
2033
|
+
|
|
2034
|
+
mdLines.push('');
|
|
2035
|
+
mdLines.push(`## Limits of This Run`);
|
|
2036
|
+
sections['Limits of This Run'].details.forEach(d => mdLines.push(`- ${d}`));
|
|
2037
|
+
|
|
2038
|
+
const summaryMdPath = path.join(runDir, 'summary.md');
|
|
2039
|
+
fs.writeFileSync(summaryMdPath, mdLines.join('\n'));
|
|
2040
|
+
} catch (_) {}
|
|
2041
|
+
return summaryPath;
|
|
2042
|
+
}
|
|
2043
|
+
|
|
2044
|
+
module.exports = { executeReality, runRealityCLI, computeFlowExitCode, applySafeDefaults, writeDecisionArtifact, computeFinalVerdict };
|