@odavl/guardian 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/CHANGELOG.md +210 -210
  2. package/LICENSE +21 -21
  3. package/README.md +297 -184
  4. package/bin/guardian.js +2242 -2221
  5. package/config/README.md +59 -59
  6. package/config/guardian.config.json +54 -54
  7. package/config/guardian.policy.json +12 -12
  8. package/config/profiles/docs.yaml +18 -18
  9. package/config/profiles/ecommerce.yaml +17 -17
  10. package/config/profiles/landing-demo.yaml +16 -16
  11. package/config/profiles/marketing.yaml +18 -18
  12. package/config/profiles/saas.yaml +21 -21
  13. package/flows/example-login-flow.json +36 -36
  14. package/flows/example-signup-flow.json +44 -44
  15. package/package.json +124 -116
  16. package/policies/enterprise.json +12 -12
  17. package/policies/landing-demo.json +22 -22
  18. package/policies/saas.json +12 -12
  19. package/policies/startup.json +12 -12
  20. package/src/enterprise/audit-logger.js +166 -166
  21. package/src/enterprise/pdf-exporter.js +267 -267
  22. package/src/enterprise/rbac-gate.js +142 -142
  23. package/src/enterprise/rbac.js +239 -239
  24. package/src/enterprise/site-manager.js +180 -180
  25. package/src/founder/feedback-system.js +156 -156
  26. package/src/founder/founder-tracker.js +213 -213
  27. package/src/founder/usage-signals.js +141 -141
  28. package/src/guardian/action-hints.js +439 -439
  29. package/src/guardian/alert-ledger.js +121 -121
  30. package/src/guardian/artifact-sanitizer.js +56 -56
  31. package/src/guardian/attempt-engine.js +1069 -1029
  32. package/src/guardian/attempt-registry.js +267 -267
  33. package/src/guardian/attempt-relevance.js +106 -106
  34. package/src/guardian/attempt-reporter.js +513 -507
  35. package/src/guardian/attempt.js +274 -273
  36. package/src/guardian/attempts-filter.js +63 -63
  37. package/src/guardian/auto-attempt-builder.js +283 -283
  38. package/src/guardian/baseline-registry.js +177 -177
  39. package/src/guardian/baseline-reporter.js +143 -143
  40. package/src/guardian/baseline-storage.js +285 -285
  41. package/src/guardian/baseline.js +535 -534
  42. package/src/guardian/behavioral-signals.js +261 -261
  43. package/src/guardian/breakage-intelligence.js +224 -224
  44. package/src/guardian/browser-pool.js +131 -131
  45. package/src/guardian/browser.js +119 -119
  46. package/src/guardian/canonical-truth.js +308 -308
  47. package/src/guardian/ci-cli.js +121 -121
  48. package/src/guardian/ci-gate.js +96 -96
  49. package/src/guardian/ci-mode.js +15 -15
  50. package/src/guardian/ci-output.js +55 -38
  51. package/src/guardian/cli-summary.js +102 -102
  52. package/src/guardian/confidence-signals.js +251 -251
  53. package/src/guardian/config-loader.js +161 -161
  54. package/src/guardian/config-validator.js +285 -283
  55. package/src/guardian/coverage-model.js +239 -239
  56. package/src/guardian/coverage-packs.js +58 -58
  57. package/src/guardian/crawler.js +142 -142
  58. package/src/guardian/data-guardian-detector.js +189 -189
  59. package/src/guardian/decision-authority.js +746 -725
  60. package/src/guardian/detection-layers.js +271 -271
  61. package/src/guardian/determinism.js +146 -146
  62. package/src/guardian/discovery-engine.js +661 -661
  63. package/src/guardian/drift-detector.js +100 -100
  64. package/src/guardian/enhanced-html-reporter.js +522 -522
  65. package/src/guardian/env-guard.js +128 -127
  66. package/src/guardian/error-clarity.js +399 -399
  67. package/src/guardian/export-contract.js +196 -196
  68. package/src/guardian/fail-safe.js +212 -212
  69. package/src/guardian/failure-intelligence.js +173 -173
  70. package/src/guardian/failure-taxonomy.js +169 -169
  71. package/src/guardian/final-outcome.js +206 -206
  72. package/src/guardian/first-run-profile.js +89 -89
  73. package/src/guardian/first-run.js +65 -67
  74. package/src/guardian/flag-validator.js +111 -111
  75. package/src/guardian/flow-executor.js +641 -639
  76. package/src/guardian/flow-registry.js +67 -67
  77. package/src/guardian/honesty.js +394 -394
  78. package/src/guardian/html-reporter.js +416 -416
  79. package/src/guardian/human-intent-resolver.js +296 -296
  80. package/src/guardian/human-interaction-model.js +351 -351
  81. package/src/guardian/human-journey-context.js +184 -184
  82. package/src/guardian/human-navigator.js +544 -544
  83. package/src/guardian/human-reporter.js +435 -431
  84. package/src/guardian/index.js +226 -221
  85. package/src/guardian/init-command.js +143 -143
  86. package/src/guardian/intent-detector.js +148 -146
  87. package/src/guardian/journey-definitions.js +132 -132
  88. package/src/guardian/journey-scan-cli.js +142 -145
  89. package/src/guardian/journey-scanner.js +583 -583
  90. package/src/guardian/junit-reporter.js +281 -281
  91. package/src/guardian/language-detection.js +99 -99
  92. package/src/guardian/live-alert.js +56 -56
  93. package/src/guardian/live-baseline-compare.js +146 -146
  94. package/src/guardian/live-cli.js +95 -95
  95. package/src/guardian/live-guardian.js +210 -210
  96. package/src/guardian/live-scheduler-runner.js +137 -137
  97. package/src/guardian/live-scheduler-state.js +167 -168
  98. package/src/guardian/live-scheduler.js +146 -146
  99. package/src/guardian/live-state.js +110 -110
  100. package/src/guardian/market-criticality.js +335 -335
  101. package/src/guardian/market-reporter.js +577 -577
  102. package/src/guardian/network-trace.js +178 -178
  103. package/src/guardian/obs-logger.js +110 -110
  104. package/src/guardian/observed-capabilities.js +427 -427
  105. package/src/guardian/output-contract.js +154 -0
  106. package/src/guardian/output-readability.js +264 -264
  107. package/src/guardian/parallel-executor.js +116 -116
  108. package/src/guardian/path-safety.js +56 -56
  109. package/src/guardian/pattern-analyzer.js +348 -348
  110. package/src/guardian/policy.js +432 -434
  111. package/src/guardian/prelaunch-gate.js +193 -193
  112. package/src/guardian/prerequisite-checker.js +101 -101
  113. package/src/guardian/preset-loader.js +152 -157
  114. package/src/guardian/profile-loader.js +96 -96
  115. package/src/guardian/reality.js +3025 -2826
  116. package/src/guardian/realworld-scenarios.js +94 -94
  117. package/src/guardian/reporter.js +167 -167
  118. package/src/guardian/retry-policy.js +123 -123
  119. package/src/guardian/root-cause-analysis.js +171 -171
  120. package/src/guardian/rules-engine.js +558 -558
  121. package/src/guardian/run-artifacts.js +212 -212
  122. package/src/guardian/run-cleanup.js +207 -207
  123. package/src/guardian/run-export.js +522 -522
  124. package/src/guardian/run-latest.js +90 -90
  125. package/src/guardian/run-list.js +211 -211
  126. package/src/guardian/run-summary.js +20 -20
  127. package/src/guardian/runtime-root.js +246 -246
  128. package/src/guardian/safety.js +248 -248
  129. package/src/guardian/scan-presets.js +133 -149
  130. package/src/guardian/screenshot.js +152 -152
  131. package/src/guardian/secret-hygiene.js +44 -44
  132. package/src/guardian/selector-fallbacks.js +394 -394
  133. package/src/guardian/semantic-contact-detection.js +255 -255
  134. package/src/guardian/semantic-contact-finder.js +201 -201
  135. package/src/guardian/semantic-targets.js +234 -234
  136. package/src/guardian/site-intelligence.js +588 -588
  137. package/src/guardian/site-introspection.js +257 -257
  138. package/src/guardian/sitemap.js +225 -225
  139. package/src/guardian/smoke.js +283 -258
  140. package/src/guardian/snapshot-schema.js +177 -290
  141. package/src/guardian/snapshot.js +430 -397
  142. package/src/guardian/stability-scorer.js +169 -169
  143. package/src/guardian/success-evaluator.js +214 -214
  144. package/src/guardian/template-command.js +184 -184
  145. package/src/guardian/text-formatters.js +426 -426
  146. package/src/guardian/timeout-profiles.js +57 -57
  147. package/src/guardian/truth/attempt.contract.js +158 -0
  148. package/src/guardian/truth/decision.contract.js +275 -0
  149. package/src/guardian/truth/snapshot.contract.js +363 -0
  150. package/src/guardian/validators.js +323 -323
  151. package/src/guardian/verdict-card.js +474 -474
  152. package/src/guardian/verdict-clarity.js +298 -298
  153. package/src/guardian/verdict-policy.js +363 -363
  154. package/src/guardian/verdict.js +333 -333
  155. package/src/guardian/verdicts.js +79 -74
  156. package/src/guardian/visual-diff.js +247 -247
  157. package/src/guardian/wait-for-outcome.js +119 -119
  158. package/src/guardian/watch-runner.js +181 -181
  159. package/src/guardian/watchdog-diff.js +167 -167
  160. package/src/guardian/webhook.js +206 -206
  161. package/src/payments/stripe-checkout.js +169 -169
  162. package/src/plans/plan-definitions.js +148 -148
  163. package/src/plans/plan-manager.js +211 -211
  164. package/src/plans/usage-tracker.js +210 -210
  165. package/src/recipes/recipe-engine.js +188 -188
  166. package/src/recipes/recipe-failure-analysis.js +159 -159
  167. package/src/recipes/recipe-registry.js +134 -134
  168. package/src/recipes/recipe-runtime.js +507 -507
  169. package/src/recipes/recipe-store.js +410 -410
  170. package/SECURITY.md +0 -77
  171. package/VERSIONING.md +0 -100
  172. package/guardian-contract-v1.md +0 -502
@@ -1,348 +1,348 @@
1
- /**
2
- * Pattern Analyzer
3
- * Detects recurring risk patterns across multiple runs.
4
- *
5
- * Patterns detected:
6
- * - repeated_skipped_attempts: Same attempt consistently skipped
7
- * - recurring_friction_path: Specific attempt/page shows friction across runs
8
- * - confidence_degradation: Confidence score declining over time
9
- * - single_point_failure: One attempt always fails while others succeed
10
- */
11
-
12
- const fs = require('fs');
13
- const path = require('path');
14
-
15
- /**
16
- * Load all recent run metadata for a site
17
- * @param {string} artifactsDir - e.g., ./artifacts
18
- * @param {string} siteSlug - e.g., example-com
19
- * @param {number} maxRuns - Maximum runs to analyze (default 10)
20
- * @returns {array} - [{runDirName, runId, timestamp, meta, snapshotPath}, ...]
21
- */
22
- function loadRecentRunsForSite(artifactsDir, siteSlug, maxRuns = 10) {
23
- const runs = [];
24
-
25
- try {
26
- const entries = fs.readdirSync(artifactsDir, { withFileTypes: true });
27
-
28
- for (const entry of entries) {
29
- if (!entry.isDirectory()) continue;
30
-
31
- const dirName = entry.name;
32
- // Format: YYYY-MM-DD_HH-MM-SS_<siteSlug>_<policy>_<result>
33
- if (!dirName.includes(siteSlug)) continue;
34
-
35
- const metaPath = path.join(artifactsDir, dirName, 'META.json');
36
- const snapshotPath = path.join(artifactsDir, dirName, 'snapshot.json');
37
-
38
- // Check if run has artifacts
39
- if (!fs.existsSync(metaPath)) continue;
40
-
41
- try {
42
- const metaRaw = fs.readFileSync(metaPath, 'utf8');
43
- const meta = JSON.parse(metaRaw);
44
-
45
- runs.push({
46
- runDirName: dirName,
47
- runId: dirName,
48
- timestamp: new Date(meta.timestamp || dirName.split('_')[0]),
49
- meta,
50
- snapshotPath: fs.existsSync(snapshotPath) ? snapshotPath : null,
51
- snapshot: null // lazy-loaded
52
- });
53
- } catch (parseErr) {
54
- // Skip unparseable runs
55
- continue;
56
- }
57
- }
58
- } catch (err) {
59
- return [];
60
- }
61
-
62
- // Sort by timestamp descending, take recent N
63
- runs.sort((a, b) => b.timestamp - a.timestamp);
64
- return runs.slice(0, maxRuns);
65
- }
66
-
67
- /**
68
- * Load snapshot data for a run if available
69
- */
70
- function loadSnapshot(run) {
71
- if (run.snapshot) return run.snapshot;
72
- if (!run.snapshotPath || !fs.existsSync(run.snapshotPath)) return null;
73
-
74
- try {
75
- const raw = fs.readFileSync(run.snapshotPath, 'utf8');
76
- run.snapshot = JSON.parse(raw);
77
- return run.snapshot;
78
- } catch (err) {
79
- return null;
80
- }
81
- }
82
-
83
- /**
84
- * Detect repeated skipped attempts
85
- * If an attempt is SKIPPED in multiple runs, it's a pattern
86
- */
87
- function detectRepeatedSkippedAttempts(runs) {
88
- const patterns = [];
89
- const attemptSkipCounts = {}; // { attemptId: { count, runIds } }
90
-
91
- for (const run of runs) {
92
- const snapshot = loadSnapshot(run);
93
- if (!snapshot || !snapshot.attempts) continue;
94
-
95
- for (const attempt of snapshot.attempts) {
96
- if (attempt.outcome === 'SKIPPED') {
97
- if (!attemptSkipCounts[attempt.attemptId]) {
98
- attemptSkipCounts[attempt.attemptId] = { count: 0, runIds: [] };
99
- }
100
- attemptSkipCounts[attempt.attemptId].count++;
101
- attemptSkipCounts[attempt.attemptId].runIds.push(run.runId);
102
- }
103
- }
104
- }
105
-
106
- for (const [attemptId, data] of Object.entries(attemptSkipCounts)) {
107
- if (data.count >= 2) {
108
- patterns.push({
109
- patternId: `repeated_skipped_${attemptId}`,
110
- type: 'repeated_skipped_attempts',
111
- summary: `Attempt "${attemptId}" was not executed in ${data.count} of the last ${runs.length} runs.`,
112
- whyItMatters: `Skipped attempts leave critical user paths untested. Consider ensuring this attempt runs in every evaluation.`,
113
- recommendedFocus: 'Coverage gap detected; this path has not been exercised.',
114
- evidence: {
115
- attemptId,
116
- occurrences: data.count,
117
- runIds: data.runIds,
118
- basedOnRuns: runs.length
119
- },
120
- confidence: data.count >= 3 ? 'high' : 'medium',
121
- limits: `Based on last ${Math.min(runs.length, 10)} runs. If you intentionally skip this attempt, ignore this pattern.`
122
- });
123
- }
124
- }
125
-
126
- return patterns;
127
- }
128
-
129
- /**
130
- * Detect recurring friction on specific paths/attempts
131
- * If same attempt shows FRICTION in 2+ runs, it's a pattern
132
- */
133
- function detectRecurringFriction(runs) {
134
- const patterns = [];
135
- const frictionCounts = {}; // { attemptId: { count, runIds, totalDuration } }
136
-
137
- for (const run of runs) {
138
- const snapshot = loadSnapshot(run);
139
- if (!snapshot || !snapshot.attempts) continue;
140
-
141
- for (const attempt of snapshot.attempts) {
142
- if (attempt.outcome === 'FRICTION' || (attempt.friction && attempt.friction.isFriction)) {
143
- if (!frictionCounts[attempt.attemptId]) {
144
- frictionCounts[attempt.attemptId] = { count: 0, runIds: [], durations: [] };
145
- }
146
- frictionCounts[attempt.attemptId].count++;
147
- frictionCounts[attempt.attemptId].runIds.push(run.runId);
148
- frictionCounts[attempt.attemptId].durations.push(attempt.totalDurationMs || 0);
149
- }
150
- }
151
- }
152
-
153
- for (const [attemptId, data] of Object.entries(frictionCounts)) {
154
- if (data.count >= 2) {
155
- const avgDuration = data.durations.length > 0
156
- ? Math.round(data.durations.reduce((a, b) => a + b, 0) / data.durations.length)
157
- : 0;
158
-
159
- patterns.push({
160
- patternId: `recurring_friction_${attemptId}`,
161
- type: 'recurring_friction',
162
- summary: `Attempt "${attemptId}" showed friction in ${data.count} of the last ${runs.length} runs (avg ${avgDuration}ms).`,
163
- whyItMatters: `Recurring friction signals friction is not random—there's a systematic issue (slow endpoint, unreliable element, poor UX). This harms user satisfaction and should be investigated.`,
164
- recommendedFocus: 'User experience may be degrading on this path.',
165
- evidence: {
166
- attemptId,
167
- occurrences: data.count,
168
- runIds: data.runIds,
169
- avgDurationMs: avgDuration,
170
- basedOnRuns: runs.length
171
- },
172
- confidence: data.count >= 3 ? 'high' : 'medium',
173
- limits: `Based on last ${Math.min(runs.length, 10)} runs. High variability in network or load may cause friction; consider examining environment factors.`
174
- });
175
- }
176
- }
177
-
178
- return patterns;
179
- }
180
-
181
- /**
182
- * Detect confidence score degradation over time
183
- * If confidence declining across last 3+ runs, user should investigate
184
- */
185
- function detectConfidenceDegradation(runs) {
186
- const patterns = [];
187
- const runsWithVerdicts = runs
188
- .map(run => {
189
- const snapshot = loadSnapshot(run);
190
- const verdict = snapshot && snapshot.verdict;
191
- return {
192
- runId: run.runId,
193
- timestamp: run.timestamp,
194
- score: verdict && verdict.confidence ? verdict.confidence.score : null
195
- };
196
- })
197
- .filter(r => r.score !== null)
198
- .reverse(); // oldest first
199
-
200
- if (runsWithVerdicts.length < 3) return patterns;
201
-
202
- // Check if trend is declining (slope analysis)
203
- const scores = runsWithVerdicts.map(r => r.score);
204
- let isDecreasing = true;
205
- for (let i = 1; i < scores.length; i++) {
206
- if (scores[i] >= scores[i - 1]) {
207
- isDecreasing = false;
208
- break;
209
- }
210
- }
211
-
212
- if (isDecreasing && runsWithVerdicts.length >= 3) {
213
- const firstScore = scores[0];
214
- const lastScore = scores[scores.length - 1];
215
- const drop = firstScore - lastScore;
216
-
217
- if (drop >= 0.2) { // significant drop (20+ percentage points)
218
- patterns.push({
219
- patternId: 'confidence_degradation',
220
- type: 'confidence_degradation',
221
- summary: `Confidence declined from ${(firstScore * 100).toFixed(0)}% to ${(lastScore * 100).toFixed(0)}% over ${runsWithVerdicts.length} runs.`,
222
- whyItMatters: `Declining confidence indicates growing test failures or friction. Site quality may be degrading, or test coverage may be revealing previously hidden issues.`,
223
- recommendedFocus: 'Overall quality signals are trending down across runs.',
224
- evidence: {
225
- runCount: runsWithVerdicts.length,
226
- runIds: runsWithVerdicts.map(r => r.runId),
227
- scores: runsWithVerdicts.map(r => r.score),
228
- trend: 'declining'
229
- },
230
- confidence: drop >= 0.3 ? 'high' : 'medium',
231
- limits: `Based on last ${runsWithVerdicts.length} runs with verdicts. Short-term fluctuations are normal; patterns become clearer with 5+ runs.`
232
- });
233
- }
234
- }
235
-
236
- return patterns;
237
- }
238
-
239
- /**
240
- * Detect single-point-of-failure: one attempt fails consistently while others succeed
241
- */
242
- function detectSinglePointFailure(runs) {
243
- const patterns = [];
244
- const attemptOutcomes = {}; // { attemptId: { success, failure, friction } }
245
-
246
- for (const run of runs) {
247
- const snapshot = loadSnapshot(run);
248
- if (!snapshot || !snapshot.attempts) continue;
249
-
250
- for (const attempt of snapshot.attempts) {
251
- if (!attemptOutcomes[attempt.attemptId]) {
252
- attemptOutcomes[attempt.attemptId] = { success: 0, failure: 0, friction: 0, skipped: 0, runIds: [] };
253
- }
254
- const counts = attemptOutcomes[attempt.attemptId];
255
- if (attempt.outcome === 'SKIPPED') counts.skipped++;
256
- else if (attempt.outcome === 'FAILURE') counts.failure++;
257
- else if (attempt.outcome === 'FRICTION') counts.friction++;
258
- else counts.success++;
259
- counts.runIds.push(run.runId);
260
- }
261
- }
262
-
263
- // Find attempts that fail in most runs while others succeed
264
- const attemptResults = [];
265
- for (const [attemptId, counts] of Object.entries(attemptOutcomes)) {
266
- const executed = counts.success + counts.failure + counts.friction;
267
- if (executed >= 2) {
268
- const failureRate = executed > 0 ? counts.failure / executed : 0;
269
- attemptResults.push({
270
- attemptId,
271
- failureRate,
272
- failureCount: counts.failure,
273
- executedCount: executed,
274
- ...counts
275
- });
276
- }
277
- }
278
-
279
- // Filter to attempts with high failure rate while others succeed
280
- const avgFailureRate = attemptResults.length > 0
281
- ? attemptResults.reduce((sum, a) => sum + a.failureRate, 0) / attemptResults.length
282
- : 0;
283
-
284
- // Detect outliers: 2+ failures AND (rate >= 0.6 OR significantly higher than average)
285
- const outliers = attemptResults.filter(a =>
286
- a.failureCount >= 2 && (a.failureRate >= 0.6 || a.failureRate > avgFailureRate + 0.3)
287
- );
288
-
289
- for (const outlier of outliers) {
290
- patterns.push({
291
- patternId: `single_point_failure_${outlier.attemptId}`,
292
- type: 'single_point_failure',
293
- summary: `Attempt "${outlier.attemptId}" did not complete in ${outlier.failure} of ${outlier.executedCount} runs—much higher than other attempts.`,
294
- whyItMatters: `This attempt is a bottleneck. It's preventing users from reaching critical functionality. Prioritize fixing whatever blocks this path.`,
295
- recommendedFocus: 'This path is a bottleneck and blocks user progress.',
296
- evidence: {
297
- attemptId: outlier.attemptId,
298
- failureCount: outlier.failure,
299
- executedCount: outlier.executedCount,
300
- failureRate: (outlier.failureRate * 100).toFixed(0) + '%',
301
- runIds: outlier.runIds.slice(0, 5) // show first 5
302
- },
303
- confidence: outlier.executedCount >= 4 ? 'high' : 'medium',
304
- limits: `Based on ${outlier.executedCount} executions across last ${runs.length} runs. If this is intentionally experimental, consider removing or documenting it.`
305
- });
306
- }
307
-
308
- return patterns;
309
- }
310
-
311
- /**
312
- * Main: Analyze all patterns for a given site
313
- * @param {string} artifactsDir - Path to artifacts directory
314
- * @param {string} siteSlug - Site slug (e.g., example-com)
315
- * @param {number} maxRuns - Max runs to consider (default 10)
316
- * @returns {array} - Array of pattern objects
317
- */
318
- function analyzePatterns(artifactsDir, siteSlug, maxRuns = 10) {
319
- const runs = loadRecentRunsForSite(artifactsDir, siteSlug, maxRuns);
320
-
321
- if (runs.length < 2) {
322
- // Need at least 2 runs to detect patterns
323
- return [];
324
- }
325
-
326
- const allPatterns = [
327
- ...detectRepeatedSkippedAttempts(runs),
328
- ...detectRecurringFriction(runs),
329
- ...detectConfidenceDegradation(runs),
330
- ...detectSinglePointFailure(runs)
331
- ];
332
-
333
- // Sort by confidence level and type
334
- const confidenceRank = { high: 3, medium: 2, low: 1 };
335
- allPatterns.sort((a, b) => confidenceRank[b.confidence] - confidenceRank[a.confidence]);
336
-
337
- return allPatterns;
338
- }
339
-
340
- module.exports = {
341
- analyzePatterns,
342
- loadRecentRunsForSite,
343
- loadSnapshot,
344
- detectRepeatedSkippedAttempts,
345
- detectRecurringFriction,
346
- detectConfidenceDegradation,
347
- detectSinglePointFailure
348
- };
1
+ /**
2
+ * Pattern Analyzer
3
+ * Detects recurring risk patterns across multiple runs.
4
+ *
5
+ * Patterns detected:
6
+ * - repeated_skipped_attempts: Same attempt consistently skipped
7
+ * - recurring_friction_path: Specific attempt/page shows friction across runs
8
+ * - confidence_degradation: Confidence score declining over time
9
+ * - single_point_failure: One attempt always fails while others succeed
10
+ */
11
+
12
+ const fs = require('fs');
13
+ const path = require('path');
14
+
15
+ /**
16
+ * Load all recent run metadata for a site
17
+ * @param {string} artifactsDir - e.g., ./artifacts
18
+ * @param {string} siteSlug - e.g., example-com
19
+ * @param {number} maxRuns - Maximum runs to analyze (default 10)
20
+ * @returns {array} - [{runDirName, runId, timestamp, meta, snapshotPath}, ...]
21
+ */
22
+ function loadRecentRunsForSite(artifactsDir, siteSlug, maxRuns = 10) {
23
+ const runs = [];
24
+
25
+ try {
26
+ const entries = fs.readdirSync(artifactsDir, { withFileTypes: true });
27
+
28
+ for (const entry of entries) {
29
+ if (!entry.isDirectory()) continue;
30
+
31
+ const dirName = entry.name;
32
+ // Format: YYYY-MM-DD_HH-MM-SS_<siteSlug>_<policy>_<result>
33
+ if (!dirName.includes(siteSlug)) continue;
34
+
35
+ const metaPath = path.join(artifactsDir, dirName, 'META.json');
36
+ const snapshotPath = path.join(artifactsDir, dirName, 'snapshot.json');
37
+
38
+ // Check if run has artifacts
39
+ if (!fs.existsSync(metaPath)) continue;
40
+
41
+ try {
42
+ const metaRaw = fs.readFileSync(metaPath, 'utf8');
43
+ const meta = JSON.parse(metaRaw);
44
+
45
+ runs.push({
46
+ runDirName: dirName,
47
+ runId: dirName,
48
+ timestamp: new Date(meta.timestamp || dirName.split('_')[0]),
49
+ meta,
50
+ snapshotPath: fs.existsSync(snapshotPath) ? snapshotPath : null,
51
+ snapshot: null // lazy-loaded
52
+ });
53
+ } catch (_parseErr) {
54
+ // Skip unparseable runs
55
+ continue;
56
+ }
57
+ }
58
+ } catch (_err) {
59
+ return [];
60
+ }
61
+
62
+ // Sort by timestamp descending, take recent N
63
+ runs.sort((a, b) => b.timestamp - a.timestamp);
64
+ return runs.slice(0, maxRuns);
65
+ }
66
+
67
+ /**
68
+ * Load snapshot data for a run if available
69
+ */
70
+ function loadSnapshot(run) {
71
+ if (run.snapshot) return run.snapshot;
72
+ if (!run.snapshotPath || !fs.existsSync(run.snapshotPath)) return null;
73
+
74
+ try {
75
+ const raw = fs.readFileSync(run.snapshotPath, 'utf8');
76
+ run.snapshot = JSON.parse(raw);
77
+ return run.snapshot;
78
+ } catch (_err) {
79
+ return null;
80
+ }
81
+ }
82
+
83
+ /**
84
+ * Detect repeated skipped attempts
85
+ * If an attempt is SKIPPED in multiple runs, it's a pattern
86
+ */
87
+ function detectRepeatedSkippedAttempts(runs) {
88
+ const patterns = [];
89
+ const attemptSkipCounts = {}; // { attemptId: { count, runIds } }
90
+
91
+ for (const run of runs) {
92
+ const snapshot = loadSnapshot(run);
93
+ if (!snapshot || !snapshot.attempts) continue;
94
+
95
+ for (const attempt of snapshot.attempts) {
96
+ if (attempt.outcome === 'SKIPPED') {
97
+ if (!attemptSkipCounts[attempt.attemptId]) {
98
+ attemptSkipCounts[attempt.attemptId] = { count: 0, runIds: [] };
99
+ }
100
+ attemptSkipCounts[attempt.attemptId].count++;
101
+ attemptSkipCounts[attempt.attemptId].runIds.push(run.runId);
102
+ }
103
+ }
104
+ }
105
+
106
+ for (const [attemptId, data] of Object.entries(attemptSkipCounts)) {
107
+ if (data.count >= 2) {
108
+ patterns.push({
109
+ patternId: `repeated_skipped_${attemptId}`,
110
+ type: 'repeated_skipped_attempts',
111
+ summary: `Attempt "${attemptId}" was not executed in ${data.count} of the last ${runs.length} runs.`,
112
+ whyItMatters: `Skipped attempts leave critical user paths untested. Consider ensuring this attempt runs in every evaluation.`,
113
+ recommendedFocus: 'Coverage gap detected; this path has not been exercised.',
114
+ evidence: {
115
+ attemptId,
116
+ occurrences: data.count,
117
+ runIds: data.runIds,
118
+ basedOnRuns: runs.length
119
+ },
120
+ confidence: data.count >= 3 ? 'high' : 'medium',
121
+ limits: `Based on last ${Math.min(runs.length, 10)} runs. If you intentionally skip this attempt, ignore this pattern.`
122
+ });
123
+ }
124
+ }
125
+
126
+ return patterns;
127
+ }
128
+
129
+ /**
130
+ * Detect recurring friction on specific paths/attempts
131
+ * If same attempt shows FRICTION in 2+ runs, it's a pattern
132
+ */
133
+ function detectRecurringFriction(runs) {
134
+ const patterns = [];
135
+ const frictionCounts = {}; // { attemptId: { count, runIds, totalDuration } }
136
+
137
+ for (const run of runs) {
138
+ const snapshot = loadSnapshot(run);
139
+ if (!snapshot || !snapshot.attempts) continue;
140
+
141
+ for (const attempt of snapshot.attempts) {
142
+ if (attempt.outcome === 'FRICTION' || (attempt.friction && attempt.friction.isFriction)) {
143
+ if (!frictionCounts[attempt.attemptId]) {
144
+ frictionCounts[attempt.attemptId] = { count: 0, runIds: [], durations: [] };
145
+ }
146
+ frictionCounts[attempt.attemptId].count++;
147
+ frictionCounts[attempt.attemptId].runIds.push(run.runId);
148
+ frictionCounts[attempt.attemptId].durations.push(attempt.totalDurationMs || 0);
149
+ }
150
+ }
151
+ }
152
+
153
+ for (const [attemptId, data] of Object.entries(frictionCounts)) {
154
+ if (data.count >= 2) {
155
+ const avgDuration = data.durations.length > 0
156
+ ? Math.round(data.durations.reduce((a, b) => a + b, 0) / data.durations.length)
157
+ : 0;
158
+
159
+ patterns.push({
160
+ patternId: `recurring_friction_${attemptId}`,
161
+ type: 'recurring_friction',
162
+ summary: `Attempt "${attemptId}" showed friction in ${data.count} of the last ${runs.length} runs (avg ${avgDuration}ms).`,
163
+ whyItMatters: `Recurring friction signals friction is not random—there's a systematic issue (slow endpoint, unreliable element, poor UX). This harms user satisfaction and should be investigated.`,
164
+ recommendedFocus: 'User experience may be degrading on this path.',
165
+ evidence: {
166
+ attemptId,
167
+ occurrences: data.count,
168
+ runIds: data.runIds,
169
+ avgDurationMs: avgDuration,
170
+ basedOnRuns: runs.length
171
+ },
172
+ confidence: data.count >= 3 ? 'high' : 'medium',
173
+ limits: `Based on last ${Math.min(runs.length, 10)} runs. High variability in network or load may cause friction; consider examining environment factors.`
174
+ });
175
+ }
176
+ }
177
+
178
+ return patterns;
179
+ }
180
+
181
+ /**
182
+ * Detect confidence score degradation over time
183
+ * If confidence declining across last 3+ runs, user should investigate
184
+ */
185
+ function detectConfidenceDegradation(runs) {
186
+ const patterns = [];
187
+ const runsWithVerdicts = runs
188
+ .map(run => {
189
+ const snapshot = loadSnapshot(run);
190
+ const verdict = snapshot && snapshot.verdict;
191
+ return {
192
+ runId: run.runId,
193
+ timestamp: run.timestamp,
194
+ score: verdict && verdict.confidence ? verdict.confidence.score : null
195
+ };
196
+ })
197
+ .filter(r => r.score !== null)
198
+ .reverse(); // oldest first
199
+
200
+ if (runsWithVerdicts.length < 3) return patterns;
201
+
202
+ // Check if trend is declining (slope analysis)
203
+ const scores = runsWithVerdicts.map(r => r.score);
204
+ let isDecreasing = true;
205
+ for (let i = 1; i < scores.length; i++) {
206
+ if (scores[i] >= scores[i - 1]) {
207
+ isDecreasing = false;
208
+ break;
209
+ }
210
+ }
211
+
212
+ if (isDecreasing && runsWithVerdicts.length >= 3) {
213
+ const firstScore = scores[0];
214
+ const lastScore = scores[scores.length - 1];
215
+ const drop = firstScore - lastScore;
216
+
217
+ if (drop >= 0.2) { // significant drop (20+ percentage points)
218
+ patterns.push({
219
+ patternId: 'confidence_degradation',
220
+ type: 'confidence_degradation',
221
+ summary: `Confidence declined from ${(firstScore * 100).toFixed(0)}% to ${(lastScore * 100).toFixed(0)}% over ${runsWithVerdicts.length} runs.`,
222
+ whyItMatters: `Declining confidence indicates growing test failures or friction. Site quality may be degrading, or test coverage may be revealing previously hidden issues.`,
223
+ recommendedFocus: 'Overall quality signals are trending down across runs.',
224
+ evidence: {
225
+ runCount: runsWithVerdicts.length,
226
+ runIds: runsWithVerdicts.map(r => r.runId),
227
+ scores: runsWithVerdicts.map(r => r.score),
228
+ trend: 'declining'
229
+ },
230
+ confidence: drop >= 0.3 ? 'high' : 'medium',
231
+ limits: `Based on last ${runsWithVerdicts.length} runs with verdicts. Short-term fluctuations are normal; patterns become clearer with 5+ runs.`
232
+ });
233
+ }
234
+ }
235
+
236
+ return patterns;
237
+ }
238
+
239
+ /**
240
+ * Detect single-point-of-failure: one attempt fails consistently while others succeed
241
+ */
242
+ function detectSinglePointFailure(runs) {
243
+ const patterns = [];
244
+ const attemptOutcomes = {}; // { attemptId: { success, failure, friction } }
245
+
246
+ for (const run of runs) {
247
+ const snapshot = loadSnapshot(run);
248
+ if (!snapshot || !snapshot.attempts) continue;
249
+
250
+ for (const attempt of snapshot.attempts) {
251
+ if (!attemptOutcomes[attempt.attemptId]) {
252
+ attemptOutcomes[attempt.attemptId] = { success: 0, failure: 0, friction: 0, skipped: 0, runIds: [] };
253
+ }
254
+ const counts = attemptOutcomes[attempt.attemptId];
255
+ if (attempt.outcome === 'SKIPPED') counts.skipped++;
256
+ else if (attempt.outcome === 'FAILURE') counts.failure++;
257
+ else if (attempt.outcome === 'FRICTION') counts.friction++;
258
+ else counts.success++;
259
+ counts.runIds.push(run.runId);
260
+ }
261
+ }
262
+
263
+ // Find attempts that fail in most runs while others succeed
264
+ const attemptResults = [];
265
+ for (const [attemptId, counts] of Object.entries(attemptOutcomes)) {
266
+ const executed = counts.success + counts.failure + counts.friction;
267
+ if (executed >= 2) {
268
+ const failureRate = executed > 0 ? counts.failure / executed : 0;
269
+ attemptResults.push({
270
+ attemptId,
271
+ failureRate,
272
+ failureCount: counts.failure,
273
+ executedCount: executed,
274
+ ...counts
275
+ });
276
+ }
277
+ }
278
+
279
+ // Filter to attempts with high failure rate while others succeed
280
+ const avgFailureRate = attemptResults.length > 0
281
+ ? attemptResults.reduce((sum, a) => sum + a.failureRate, 0) / attemptResults.length
282
+ : 0;
283
+
284
+ // Detect outliers: 2+ failures AND (rate >= 0.6 OR significantly higher than average)
285
+ const outliers = attemptResults.filter(a =>
286
+ a.failureCount >= 2 && (a.failureRate >= 0.6 || a.failureRate > avgFailureRate + 0.3)
287
+ );
288
+
289
+ for (const outlier of outliers) {
290
+ patterns.push({
291
+ patternId: `single_point_failure_${outlier.attemptId}`,
292
+ type: 'single_point_failure',
293
+ summary: `Attempt "${outlier.attemptId}" did not complete in ${outlier.failure} of ${outlier.executedCount} runs—much higher than other attempts.`,
294
+ whyItMatters: `This attempt is a bottleneck. It's preventing users from reaching critical functionality. Prioritize fixing whatever blocks this path.`,
295
+ recommendedFocus: 'This path is a bottleneck and blocks user progress.',
296
+ evidence: {
297
+ attemptId: outlier.attemptId,
298
+ failureCount: outlier.failure,
299
+ executedCount: outlier.executedCount,
300
+ failureRate: (outlier.failureRate * 100).toFixed(0) + '%',
301
+ runIds: outlier.runIds.slice(0, 5) // show first 5
302
+ },
303
+ confidence: outlier.executedCount >= 4 ? 'high' : 'medium',
304
+ limits: `Based on ${outlier.executedCount} executions across last ${runs.length} runs. If this is intentionally experimental, consider removing or documenting it.`
305
+ });
306
+ }
307
+
308
+ return patterns;
309
+ }
310
+
311
+ /**
312
+ * Main: Analyze all patterns for a given site
313
+ * @param {string} artifactsDir - Path to artifacts directory
314
+ * @param {string} siteSlug - Site slug (e.g., example-com)
315
+ * @param {number} maxRuns - Max runs to consider (default 10)
316
+ * @returns {array} - Array of pattern objects
317
+ */
318
+ function analyzePatterns(artifactsDir, siteSlug, maxRuns = 10) {
319
+ const runs = loadRecentRunsForSite(artifactsDir, siteSlug, maxRuns);
320
+
321
+ if (runs.length < 2) {
322
+ // Need at least 2 runs to detect patterns
323
+ return [];
324
+ }
325
+
326
+ const allPatterns = [
327
+ ...detectRepeatedSkippedAttempts(runs),
328
+ ...detectRecurringFriction(runs),
329
+ ...detectConfidenceDegradation(runs),
330
+ ...detectSinglePointFailure(runs)
331
+ ];
332
+
333
+ // Sort by confidence level and type
334
+ const confidenceRank = { high: 3, medium: 2, low: 1 };
335
+ allPatterns.sort((a, b) => confidenceRank[b.confidence] - confidenceRank[a.confidence]);
336
+
337
+ return allPatterns;
338
+ }
339
+
340
+ module.exports = {
341
+ analyzePatterns,
342
+ loadRecentRunsForSite,
343
+ loadSnapshot,
344
+ detectRepeatedSkippedAttempts,
345
+ detectRecurringFriction,
346
+ detectConfidenceDegradation,
347
+ detectSinglePointFailure
348
+ };