@veraxhq/verax 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +14 -4
- package/src/cli/commands/default.js +244 -86
- package/src/cli/commands/doctor.js +36 -4
- package/src/cli/commands/run.js +253 -69
- package/src/cli/entry.js +5 -5
- package/src/cli/util/detection-engine.js +4 -3
- package/src/cli/util/events.js +76 -0
- package/src/cli/util/expectation-extractor.js +11 -1
- package/src/cli/util/findings-writer.js +1 -0
- package/src/cli/util/observation-engine.js +69 -23
- package/src/cli/util/paths.js +3 -2
- package/src/cli/util/project-discovery.js +20 -0
- package/src/cli/util/redact.js +2 -2
- package/src/cli/util/runtime-budget.js +147 -0
- package/src/cli/util/summary-writer.js +12 -1
- package/src/types/global.d.ts +28 -0
- package/src/types/ts-ast.d.ts +24 -0
- package/src/verax/cli/doctor.js +2 -2
- package/src/verax/cli/init.js +1 -1
- package/src/verax/cli/url-safety.js +12 -2
- package/src/verax/cli/wizard.js +13 -2
- package/src/verax/core/budget-engine.js +1 -1
- package/src/verax/core/decision-snapshot.js +2 -2
- package/src/verax/core/determinism-model.js +35 -6
- package/src/verax/core/incremental-store.js +15 -7
- package/src/verax/core/replay-validator.js +4 -4
- package/src/verax/core/replay.js +1 -1
- package/src/verax/core/silence-impact.js +1 -1
- package/src/verax/core/silence-model.js +9 -7
- package/src/verax/detect/comparison.js +8 -3
- package/src/verax/detect/confidence-engine.js +17 -17
- package/src/verax/detect/detection-engine.js +1 -1
- package/src/verax/detect/evidence-index.js +15 -65
- package/src/verax/detect/expectation-model.js +54 -3
- package/src/verax/detect/explanation-helpers.js +1 -1
- package/src/verax/detect/finding-detector.js +2 -2
- package/src/verax/detect/findings-writer.js +9 -16
- package/src/verax/detect/flow-detector.js +4 -4
- package/src/verax/detect/index.js +37 -11
- package/src/verax/detect/interactive-findings.js +3 -4
- package/src/verax/detect/signal-mapper.js +2 -2
- package/src/verax/detect/skip-classifier.js +4 -4
- package/src/verax/detect/verdict-engine.js +4 -6
- package/src/verax/flow/flow-engine.js +3 -2
- package/src/verax/flow/flow-spec.js +1 -2
- package/src/verax/index.js +15 -3
- package/src/verax/intel/effect-detector.js +1 -1
- package/src/verax/intel/index.js +2 -2
- package/src/verax/intel/route-extractor.js +3 -3
- package/src/verax/intel/vue-navigation-extractor.js +81 -18
- package/src/verax/intel/vue-router-extractor.js +4 -2
- package/src/verax/learn/action-contract-extractor.js +3 -3
- package/src/verax/learn/ast-contract-extractor.js +53 -1
- package/src/verax/learn/index.js +36 -2
- package/src/verax/learn/manifest-writer.js +28 -14
- package/src/verax/learn/route-extractor.js +1 -1
- package/src/verax/learn/route-validator.js +8 -7
- package/src/verax/learn/state-extractor.js +1 -1
- package/src/verax/learn/static-extractor-navigation.js +1 -1
- package/src/verax/learn/static-extractor-validation.js +2 -2
- package/src/verax/learn/static-extractor.js +8 -7
- package/src/verax/learn/ts-contract-resolver.js +14 -12
- package/src/verax/observe/browser.js +22 -3
- package/src/verax/observe/console-sensor.js +2 -2
- package/src/verax/observe/expectation-executor.js +2 -1
- package/src/verax/observe/focus-sensor.js +1 -1
- package/src/verax/observe/human-driver.js +29 -10
- package/src/verax/observe/index.js +10 -7
- package/src/verax/observe/interaction-discovery.js +27 -15
- package/src/verax/observe/interaction-runner.js +6 -6
- package/src/verax/observe/loading-sensor.js +6 -0
- package/src/verax/observe/navigation-sensor.js +1 -1
- package/src/verax/observe/settle.js +1 -0
- package/src/verax/observe/state-sensor.js +8 -4
- package/src/verax/observe/state-ui-sensor.js +7 -1
- package/src/verax/observe/traces-writer.js +27 -16
- package/src/verax/observe/ui-signal-sensor.js +7 -0
- package/src/verax/scan-summary-writer.js +5 -2
- package/src/verax/shared/artifact-manager.js +1 -1
- package/src/verax/shared/budget-profiles.js +2 -2
- package/src/verax/shared/caching.js +1 -1
- package/src/verax/shared/config-loader.js +1 -2
- package/src/verax/shared/dynamic-route-utils.js +12 -6
- package/src/verax/shared/retry-policy.js +1 -6
- package/src/verax/shared/root-artifacts.js +1 -1
- package/src/verax/shared/zip-artifacts.js +1 -0
- package/src/verax/validate/context-validator.js +1 -1
- package/src/verax/observe/index.js.backup +0 -1
- package/src/verax/validate/context-validator.js.bak +0 -0
|
@@ -222,10 +222,12 @@ export class DecisionRecorder {
|
|
|
222
222
|
* @param {Object} budget
|
|
223
223
|
*/
|
|
224
224
|
export function recordBudgetProfile(recorder, profileName, budget) {
|
|
225
|
+
const now = Date.now();
|
|
225
226
|
recorder.recordBatch([
|
|
226
227
|
{
|
|
227
228
|
decision_id: DECISION_IDS.BUDGET_PROFILE_SELECTED,
|
|
228
229
|
category: 'BUDGET',
|
|
230
|
+
timestamp: now,
|
|
229
231
|
inputs: { env_var: process.env.VERAX_BUDGET_PROFILE || 'STANDARD' },
|
|
230
232
|
chosen_value: profileName,
|
|
231
233
|
reason: `Budget profile selected: ${profileName}`
|
|
@@ -233,6 +235,7 @@ export function recordBudgetProfile(recorder, profileName, budget) {
|
|
|
233
235
|
{
|
|
234
236
|
decision_id: DECISION_IDS.BUDGET_MAX_INTERACTIONS,
|
|
235
237
|
category: 'BUDGET',
|
|
238
|
+
timestamp: now,
|
|
236
239
|
inputs: { profile: profileName },
|
|
237
240
|
chosen_value: budget.maxInteractionsPerPage,
|
|
238
241
|
reason: `Max interactions per page from ${profileName} profile`
|
|
@@ -240,6 +243,7 @@ export function recordBudgetProfile(recorder, profileName, budget) {
|
|
|
240
243
|
{
|
|
241
244
|
decision_id: DECISION_IDS.BUDGET_MAX_PAGES,
|
|
242
245
|
category: 'BUDGET',
|
|
246
|
+
timestamp: now,
|
|
243
247
|
inputs: { profile: profileName },
|
|
244
248
|
chosen_value: budget.maxPages,
|
|
245
249
|
reason: `Max pages from ${profileName} profile`
|
|
@@ -247,6 +251,7 @@ export function recordBudgetProfile(recorder, profileName, budget) {
|
|
|
247
251
|
{
|
|
248
252
|
decision_id: DECISION_IDS.BUDGET_SCAN_DURATION,
|
|
249
253
|
category: 'BUDGET',
|
|
254
|
+
timestamp: now,
|
|
250
255
|
inputs: { profile: profileName },
|
|
251
256
|
chosen_value: budget.maxScanDurationMs,
|
|
252
257
|
reason: `Scan duration limit from ${profileName} profile`
|
|
@@ -260,10 +265,12 @@ export function recordBudgetProfile(recorder, profileName, budget) {
|
|
|
260
265
|
* @param {Object} budget
|
|
261
266
|
*/
|
|
262
267
|
export function recordTimeoutConfig(recorder, budget) {
|
|
268
|
+
const now = Date.now();
|
|
263
269
|
recorder.recordBatch([
|
|
264
270
|
{
|
|
265
271
|
decision_id: DECISION_IDS.TIMEOUT_NAVIGATION,
|
|
266
272
|
category: 'TIMEOUT',
|
|
273
|
+
timestamp: now,
|
|
267
274
|
inputs: { budget_config: true },
|
|
268
275
|
chosen_value: budget.navigationTimeoutMs,
|
|
269
276
|
reason: 'Navigation timeout from budget configuration'
|
|
@@ -271,6 +278,7 @@ export function recordTimeoutConfig(recorder, budget) {
|
|
|
271
278
|
{
|
|
272
279
|
decision_id: DECISION_IDS.TIMEOUT_INTERACTION,
|
|
273
280
|
category: 'TIMEOUT',
|
|
281
|
+
timestamp: now,
|
|
274
282
|
inputs: { budget_config: true },
|
|
275
283
|
chosen_value: budget.interactionTimeoutMs,
|
|
276
284
|
reason: 'Interaction timeout from budget configuration'
|
|
@@ -278,6 +286,7 @@ export function recordTimeoutConfig(recorder, budget) {
|
|
|
278
286
|
{
|
|
279
287
|
decision_id: DECISION_IDS.TIMEOUT_SETTLE,
|
|
280
288
|
category: 'TIMEOUT',
|
|
289
|
+
timestamp: now,
|
|
281
290
|
inputs: { budget_config: true },
|
|
282
291
|
chosen_value: budget.settleTimeoutMs,
|
|
283
292
|
reason: 'Settle timeout from budget configuration'
|
|
@@ -285,6 +294,7 @@ export function recordTimeoutConfig(recorder, budget) {
|
|
|
285
294
|
{
|
|
286
295
|
decision_id: DECISION_IDS.TIMEOUT_STABILIZATION,
|
|
287
296
|
category: 'TIMEOUT',
|
|
297
|
+
timestamp: now,
|
|
288
298
|
inputs: { budget_config: true },
|
|
289
299
|
chosen_value: budget.stabilizationWindowMs,
|
|
290
300
|
reason: 'Stabilization window from budget configuration'
|
|
@@ -301,9 +311,11 @@ export function recordTimeoutConfig(recorder, budget) {
|
|
|
301
311
|
* @param {string} reason
|
|
302
312
|
*/
|
|
303
313
|
export function recordAdaptiveStabilization(recorder, enabled, wasExtended = false, extensionMs = 0, reason = '') {
|
|
314
|
+
const now = Date.now();
|
|
304
315
|
recorder.record({
|
|
305
316
|
decision_id: DECISION_IDS.ADAPTIVE_STABILIZATION_ENABLED,
|
|
306
317
|
category: 'ADAPTIVE_STABILIZATION',
|
|
318
|
+
timestamp: now,
|
|
307
319
|
inputs: { budget_config: true },
|
|
308
320
|
chosen_value: enabled,
|
|
309
321
|
reason: enabled ? 'Adaptive stabilization enabled by budget profile' : 'Adaptive stabilization disabled'
|
|
@@ -313,6 +325,7 @@ export function recordAdaptiveStabilization(recorder, enabled, wasExtended = fal
|
|
|
313
325
|
recorder.record({
|
|
314
326
|
decision_id: DECISION_IDS.ADAPTIVE_STABILIZATION_EXTENDED,
|
|
315
327
|
category: 'ADAPTIVE_STABILIZATION',
|
|
328
|
+
timestamp: now,
|
|
316
329
|
inputs: { dom_changing: true, network_active: true },
|
|
317
330
|
chosen_value: extensionMs,
|
|
318
331
|
reason: reason || `Extended stabilization by ${extensionMs}ms due to ongoing changes`
|
|
@@ -333,10 +346,12 @@ export function recordRetryAttempt(recorder, operationType, attemptNumber, delay
|
|
|
333
346
|
DECISION_IDS.RETRY_NAVIGATION_ATTEMPTED :
|
|
334
347
|
DECISION_IDS.RETRY_INTERACTION_ATTEMPTED;
|
|
335
348
|
|
|
349
|
+
const now = Date.now();
|
|
336
350
|
recorder.recordBatch([
|
|
337
351
|
{
|
|
338
352
|
decision_id: decisionId,
|
|
339
353
|
category: 'RETRY',
|
|
354
|
+
timestamp: now,
|
|
340
355
|
inputs: { attempt: attemptNumber, error_type: errorType },
|
|
341
356
|
chosen_value: true,
|
|
342
357
|
reason: `Retry attempt ${attemptNumber} for ${operationType} due to ${errorType}`
|
|
@@ -344,6 +359,7 @@ export function recordRetryAttempt(recorder, operationType, attemptNumber, delay
|
|
|
344
359
|
{
|
|
345
360
|
decision_id: DECISION_IDS.RETRY_BACKOFF_DELAY,
|
|
346
361
|
category: 'RETRY',
|
|
362
|
+
timestamp: now,
|
|
347
363
|
inputs: { attempt: attemptNumber },
|
|
348
364
|
chosen_value: delayMs,
|
|
349
365
|
reason: `Exponential backoff delay: ${delayMs}ms`
|
|
@@ -355,22 +371,32 @@ export function recordRetryAttempt(recorder, operationType, attemptNumber, delay
|
|
|
355
371
|
* Record budget truncation
|
|
356
372
|
* @param {DecisionRecorder} recorder
|
|
357
373
|
* @param {string} truncationType - 'interactions' | 'pages' | 'scan_time'
|
|
358
|
-
* @param {number} limit
|
|
359
|
-
* @param {number} actual
|
|
374
|
+
* @param {Object|number} limitOrOptions - Either a number (limit) or object with {limit, reached/elapsed, scope?}
|
|
375
|
+
* @param {number} [actual] - Actual value (only used if limitOrOptions is a number)
|
|
360
376
|
*/
|
|
361
|
-
export function recordTruncation(recorder, truncationType,
|
|
377
|
+
export function recordTruncation(recorder, truncationType, limitOrOptions, actual = null) {
|
|
362
378
|
const decisionIdMap = {
|
|
363
379
|
interactions: DECISION_IDS.TRUNCATION_INTERACTIONS_CAPPED,
|
|
364
380
|
pages: DECISION_IDS.TRUNCATION_PAGES_CAPPED,
|
|
365
381
|
scan_time: DECISION_IDS.TRUNCATION_SCAN_TIME_EXCEEDED
|
|
366
382
|
};
|
|
367
383
|
|
|
384
|
+
let limit, actualValue;
|
|
385
|
+
if (typeof limitOrOptions === 'object' && limitOrOptions !== null) {
|
|
386
|
+
limit = limitOrOptions.limit;
|
|
387
|
+
actualValue = limitOrOptions.reached || limitOrOptions.elapsed || limitOrOptions.actual || 0;
|
|
388
|
+
} else {
|
|
389
|
+
limit = limitOrOptions;
|
|
390
|
+
actualValue = actual || 0;
|
|
391
|
+
}
|
|
392
|
+
|
|
368
393
|
recorder.record({
|
|
369
394
|
decision_id: decisionIdMap[truncationType] || DECISION_IDS.TRUNCATION_BUDGET_EXCEEDED,
|
|
370
395
|
category: 'TRUNCATION',
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
396
|
+
timestamp: Date.now(),
|
|
397
|
+
inputs: { limit, actual: actualValue },
|
|
398
|
+
chosen_value: actualValue,
|
|
399
|
+
reason: `Budget exceeded: ${truncationType} capped at ${limit} (attempted ${actualValue})`
|
|
374
400
|
});
|
|
375
401
|
}
|
|
376
402
|
|
|
@@ -381,11 +407,13 @@ export function recordTruncation(recorder, truncationType, limit, actual) {
|
|
|
381
407
|
*/
|
|
382
408
|
export function recordEnvironment(recorder, environment) {
|
|
383
409
|
const { browserType = 'unknown', viewport = { width: 1280, height: 720 } } = environment;
|
|
410
|
+
const now = Date.now();
|
|
384
411
|
|
|
385
412
|
recorder.recordBatch([
|
|
386
413
|
{
|
|
387
414
|
decision_id: DECISION_IDS.ENV_BROWSER_DETECTED,
|
|
388
415
|
category: 'ENVIRONMENT',
|
|
416
|
+
timestamp: now,
|
|
389
417
|
inputs: { detected: true },
|
|
390
418
|
chosen_value: browserType,
|
|
391
419
|
reason: `Browser type: ${browserType}`
|
|
@@ -393,6 +421,7 @@ export function recordEnvironment(recorder, environment) {
|
|
|
393
421
|
{
|
|
394
422
|
decision_id: DECISION_IDS.ENV_VIEWPORT_SIZE,
|
|
395
423
|
category: 'ENVIRONMENT',
|
|
424
|
+
timestamp: now,
|
|
396
425
|
inputs: { default_viewport: true },
|
|
397
426
|
chosen_value: viewport,
|
|
398
427
|
reason: `Viewport size: ${viewport.width}x${viewport.height}`
|
|
@@ -9,8 +9,9 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { createHash } from 'crypto';
|
|
12
|
-
import { resolve
|
|
12
|
+
import { resolve } from 'path';
|
|
13
13
|
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
|
|
14
|
+
import { getRunArtifactDir } from './run-id.js';
|
|
14
15
|
|
|
15
16
|
/**
|
|
16
17
|
* Compute deterministic hash/signature for a route
|
|
@@ -39,8 +40,12 @@ export function computeInteractionSignature(interaction, url) {
|
|
|
39
40
|
/**
|
|
40
41
|
* Load previous snapshot if it exists
|
|
41
42
|
*/
|
|
42
|
-
export function loadPreviousSnapshot(projectDir) {
|
|
43
|
-
|
|
43
|
+
export function loadPreviousSnapshot(projectDir, runId) {
|
|
44
|
+
if (!runId) {
|
|
45
|
+
return null; // No runId, no snapshot
|
|
46
|
+
}
|
|
47
|
+
const runDir = getRunArtifactDir(projectDir, runId);
|
|
48
|
+
const snapshotPath = resolve(runDir, 'incremental-snapshot.json');
|
|
44
49
|
if (!existsSync(snapshotPath)) {
|
|
45
50
|
return null;
|
|
46
51
|
}
|
|
@@ -56,10 +61,13 @@ export function loadPreviousSnapshot(projectDir) {
|
|
|
56
61
|
/**
|
|
57
62
|
* Save current snapshot
|
|
58
63
|
*/
|
|
59
|
-
export function saveSnapshot(projectDir, snapshot) {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
64
|
+
export function saveSnapshot(projectDir, snapshot, runId) {
|
|
65
|
+
if (!runId) {
|
|
66
|
+
throw new Error('runId is required for saveSnapshot');
|
|
67
|
+
}
|
|
68
|
+
const runDir = getRunArtifactDir(projectDir, runId);
|
|
69
|
+
mkdirSync(runDir, { recursive: true });
|
|
70
|
+
const snapshotPath = resolve(runDir, 'incremental-snapshot.json');
|
|
63
71
|
|
|
64
72
|
writeFileSync(snapshotPath, JSON.stringify(snapshot, null, 2));
|
|
65
73
|
}
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* - Categorize deviations: truncation_difference, timeout_difference, environment_difference
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import { DecisionRecorder
|
|
16
|
+
import { DecisionRecorder } from './determinism-model.js';
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
19
|
* Compare two decision values and determine if they're equivalent
|
|
@@ -47,10 +47,10 @@ function areValuesEquivalent(value1, value2) {
|
|
|
47
47
|
/**
|
|
48
48
|
* Categorize a decision deviation
|
|
49
49
|
* @param {Object} baseline - Baseline decision
|
|
50
|
-
* @param {Object}
|
|
50
|
+
* @param {Object} _current - Current decision (unused parameter, kept for API compatibility)
|
|
51
51
|
* @returns {string} - Deviation category
|
|
52
52
|
*/
|
|
53
|
-
function categorizeDeviation(baseline,
|
|
53
|
+
function categorizeDeviation(baseline, _current) {
|
|
54
54
|
const category = baseline.category;
|
|
55
55
|
|
|
56
56
|
switch (category) {
|
|
@@ -78,7 +78,7 @@ function categorizeDeviation(baseline, current) {
|
|
|
78
78
|
* @returns {string} - Human-readable explanation
|
|
79
79
|
*/
|
|
80
80
|
function explainDeviation(baseline, current) {
|
|
81
|
-
const
|
|
81
|
+
const _category = baseline.category;
|
|
82
82
|
const devCategory = categorizeDeviation(baseline, current);
|
|
83
83
|
|
|
84
84
|
// Build explanation based on decision type
|
package/src/verax/core/replay.js
CHANGED
|
@@ -141,7 +141,7 @@ export function loadRunArtifacts(runDir) {
|
|
|
141
141
|
* Replay a previous run from artifacts
|
|
142
142
|
*
|
|
143
143
|
* @param {string} runDir - Path to run directory
|
|
144
|
-
* @returns {
|
|
144
|
+
* @returns {Promise<any>} Replay result with observation summary
|
|
145
145
|
*/
|
|
146
146
|
export async function replayRun(runDir) {
|
|
147
147
|
// Load artifacts
|
|
@@ -335,7 +335,7 @@ export function createImpactSummary(silences) {
|
|
|
335
335
|
* @returns {string} Human-readable interpretation
|
|
336
336
|
*/
|
|
337
337
|
function generateConfidenceInterpretation(aggregated) {
|
|
338
|
-
const { coverage, promise_verification, overall } = aggregated;
|
|
338
|
+
const { coverage: _coverage, promise_verification: _promise_verification, overall } = aggregated;
|
|
339
339
|
|
|
340
340
|
if (overall === 0) {
|
|
341
341
|
return 'No silence events - observation confidence is complete within evaluated scope';
|
|
@@ -18,8 +18,10 @@ import { mapSilenceReasonToOutcome } from './canonical-outcomes.js';
|
|
|
18
18
|
*
|
|
19
19
|
* PHASE 4: Enhanced with lifecycle model
|
|
20
20
|
*
|
|
21
|
+
* Note: Fields marked as optional are auto-inferred by record() if not provided
|
|
22
|
+
*
|
|
21
23
|
* @typedef {Object} SilenceEntry
|
|
22
|
-
* @property {string} outcome - Canonical outcome: SILENT_FAILURE | COVERAGE_GAP | UNPROVEN_INTERACTION | SAFETY_BLOCK | INFORMATIONAL
|
|
24
|
+
* @property {string} [outcome] - Canonical outcome: SILENT_FAILURE | COVERAGE_GAP | UNPROVEN_INTERACTION | SAFETY_BLOCK | INFORMATIONAL (auto-inferred if missing)
|
|
23
25
|
* @property {string} scope - What was not evaluated: page | interaction | expectation | sensor | navigation | settle
|
|
24
26
|
* @property {string} reason - Why it wasn't evaluated: timeout | cap | budget_exceeded | incremental_reuse | safety_skip | no_expectation | discovery_failed | sensor_failed
|
|
25
27
|
* @property {string} description - Human-readable description of what wasn't observed
|
|
@@ -28,12 +30,12 @@ import { mapSilenceReasonToOutcome } from './canonical-outcomes.js';
|
|
|
28
30
|
* @property {number} [count] - Number of items affected by this silence (optional)
|
|
29
31
|
* @property {string} [evidenceUrl] - URL where this silence occurred (optional)
|
|
30
32
|
*
|
|
31
|
-
* PHASE 4 Lifecycle Fields:
|
|
32
|
-
* @property {string} silence_type - Technical classification: interaction_not_executed | promise_not_evaluated | sensor_failure | timeout | budget_limit | safety_block | discovery_failure
|
|
33
|
+
* PHASE 4 Lifecycle Fields (all auto-inferred if missing):
|
|
34
|
+
* @property {string} [silence_type] - Technical classification: interaction_not_executed | promise_not_evaluated | sensor_failure | timeout | budget_limit | safety_block | discovery_failure (auto-inferred)
|
|
33
35
|
* @property {Object} [related_promise] - Promise that could not be evaluated (if applicable) or null with reason
|
|
34
|
-
* @property {string} trigger - What triggered this silence: user_action_blocked | navigation_timeout | budget_exhausted | safety_policy | selector_not_found | etc
|
|
35
|
-
* @property {string} evaluation_status - blocked | ambiguous | skipped | timed_out | incomplete
|
|
36
|
-
* @property {Object} confidence_impact - Which confidence metrics are affected: { coverage: -X%, promise_verification: -Y%, overall: -Z% }
|
|
36
|
+
* @property {string} [trigger] - What triggered this silence: user_action_blocked | navigation_timeout | budget_exhausted | safety_policy | selector_not_found | etc (auto-inferred)
|
|
37
|
+
* @property {string} [evaluation_status] - blocked | ambiguous | skipped | timed_out | incomplete (auto-inferred)
|
|
38
|
+
* @property {Object} [confidence_impact] - Which confidence metrics are affected: { coverage: -X%, promise_verification: -Y%, overall: -Z% } (auto-inferred)
|
|
37
39
|
*/
|
|
38
40
|
|
|
39
41
|
/**
|
|
@@ -272,7 +274,7 @@ export class SilenceTracker {
|
|
|
272
274
|
/**
|
|
273
275
|
* PHASE 4: Infer confidence impact from reason and scope
|
|
274
276
|
*/
|
|
275
|
-
_inferConfidenceImpact(reason,
|
|
277
|
+
_inferConfidenceImpact(reason, _scope) {
|
|
276
278
|
// Map reason to which confidence metric(s) are affected
|
|
277
279
|
const impact = {
|
|
278
280
|
coverage: 0,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { resolve } from 'path';
|
|
2
2
|
import { getUrlPath, getScreenshotHash } from './evidence-validator.js';
|
|
3
|
+
import { getScreenshotDir } from '../core/run-id.js';
|
|
3
4
|
|
|
4
5
|
export function hasMeaningfulUrlChange(beforeUrl, afterUrl) {
|
|
5
6
|
const beforePath = getUrlPath(beforeUrl);
|
|
@@ -15,9 +16,13 @@ export function hasMeaningfulUrlChange(beforeUrl, afterUrl) {
|
|
|
15
16
|
return beforeNormalized !== afterNormalized;
|
|
16
17
|
}
|
|
17
18
|
|
|
18
|
-
export function hasVisibleChange(beforeScreenshot, afterScreenshot, projectDir) {
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
export function hasVisibleChange(beforeScreenshot, afterScreenshot, projectDir, runId) {
|
|
20
|
+
if (!runId) {
|
|
21
|
+
throw new Error('runId is required for hasVisibleChange');
|
|
22
|
+
}
|
|
23
|
+
const screenshotsDir = getScreenshotDir(projectDir, runId);
|
|
24
|
+
const beforePath = resolve(screenshotsDir, beforeScreenshot);
|
|
25
|
+
const afterPath = resolve(screenshotsDir, afterScreenshot);
|
|
21
26
|
|
|
22
27
|
const beforeHash = getScreenshotHash(beforePath);
|
|
23
28
|
const afterHash = getScreenshotHash(afterPath);
|
|
@@ -348,10 +348,10 @@ function scoreByFindingType({
|
|
|
348
348
|
expectationStrength,
|
|
349
349
|
networkSummary,
|
|
350
350
|
consoleSummary,
|
|
351
|
-
uiSignals,
|
|
351
|
+
uiSignals: _uiSignals,
|
|
352
352
|
evidenceSignals,
|
|
353
|
-
comparisons,
|
|
354
|
-
attemptMeta,
|
|
353
|
+
comparisons: _comparisons,
|
|
354
|
+
attemptMeta: _attemptMeta,
|
|
355
355
|
boosts,
|
|
356
356
|
penalties
|
|
357
357
|
}) {
|
|
@@ -477,7 +477,7 @@ function scoreByFindingType({
|
|
|
477
477
|
// TYPE-SPECIFIC SCORING FUNCTIONS
|
|
478
478
|
// ============================================================
|
|
479
479
|
|
|
480
|
-
function scoreNetworkSilentFailure({ networkSummary, consoleSummary, evidenceSignals, boosts, penalties }) {
|
|
480
|
+
function scoreNetworkSilentFailure({ networkSummary: _networkSummary, consoleSummary: _consoleSummary, evidenceSignals, boosts, penalties: _penalties }) {
|
|
481
481
|
let total = 0;
|
|
482
482
|
|
|
483
483
|
// +10 if network failed
|
|
@@ -513,7 +513,7 @@ function penalizeNetworkSilentFailure({ evidenceSignals, penalties }) {
|
|
|
513
513
|
return total;
|
|
514
514
|
}
|
|
515
515
|
|
|
516
|
-
function scoreValidationSilentFailure({ networkSummary, consoleSummary, evidenceSignals, boosts, penalties }) {
|
|
516
|
+
function scoreValidationSilentFailure({ networkSummary: _networkSummary, consoleSummary: _consoleSummary, evidenceSignals, boosts, penalties: _penalties }) {
|
|
517
517
|
let total = 0;
|
|
518
518
|
|
|
519
519
|
// +10 if console errors (validation errors logged)
|
|
@@ -543,7 +543,7 @@ function penalizeValidationSilentFailure({ evidenceSignals, penalties }) {
|
|
|
543
543
|
return total;
|
|
544
544
|
}
|
|
545
545
|
|
|
546
|
-
function scoreMissingFeedbackFailure({ networkSummary, evidenceSignals, boosts, penalties }) {
|
|
546
|
+
function scoreMissingFeedbackFailure({ networkSummary: _networkSummary, evidenceSignals, boosts, penalties: _penalties }) {
|
|
547
547
|
let total = 0;
|
|
548
548
|
|
|
549
549
|
// +10 if slow/pending requests
|
|
@@ -573,7 +573,7 @@ function penalizeMissingFeedbackFailure({ evidenceSignals, penalties }) {
|
|
|
573
573
|
return total;
|
|
574
574
|
}
|
|
575
575
|
|
|
576
|
-
function scoreNoEffectSilentFailure({ expectation, evidenceSignals, boosts, penalties }) {
|
|
576
|
+
function scoreNoEffectSilentFailure({ expectation: _expectation, evidenceSignals, boosts, penalties: _penalties }) {
|
|
577
577
|
let total = 0;
|
|
578
578
|
|
|
579
579
|
// +10 if URL should have changed but didn't
|
|
@@ -615,7 +615,7 @@ function penalizeNoEffectSilentFailure({ evidenceSignals, penalties }) {
|
|
|
615
615
|
return total;
|
|
616
616
|
}
|
|
617
617
|
|
|
618
|
-
function scoreMissingNetworkAction({ expectation, expectationStrength, evidenceSignals, boosts, penalties }) {
|
|
618
|
+
function scoreMissingNetworkAction({ expectation, expectationStrength, evidenceSignals, boosts, penalties: _penalties }) {
|
|
619
619
|
let total = 0;
|
|
620
620
|
|
|
621
621
|
// +10 if PROVEN expectation
|
|
@@ -651,7 +651,7 @@ function penalizeMissingNetworkAction({ evidenceSignals, penalties }) {
|
|
|
651
651
|
return total;
|
|
652
652
|
}
|
|
653
653
|
|
|
654
|
-
function scoreMissingStateAction({ expectation, expectationStrength, evidenceSignals, boosts, penalties }) {
|
|
654
|
+
function scoreMissingStateAction({ expectation: _expectation, expectationStrength, evidenceSignals, boosts, penalties: _penalties }) {
|
|
655
655
|
let total = 0;
|
|
656
656
|
|
|
657
657
|
// +10 if PROVEN expectation
|
|
@@ -688,7 +688,7 @@ function penalizeMissingStateAction({ evidenceSignals, penalties }) {
|
|
|
688
688
|
}
|
|
689
689
|
|
|
690
690
|
// NAVIGATION INTELLIGENCE v2: Navigation failure scoring
|
|
691
|
-
function scoreNavigationSilentFailure({ expectation, expectationStrength, evidenceSignals, boosts, penalties }) {
|
|
691
|
+
function scoreNavigationSilentFailure({ expectation: _expectation, expectationStrength: _expectationStrength, evidenceSignals, boosts, penalties: _penalties }) {
|
|
692
692
|
let total = 0;
|
|
693
693
|
|
|
694
694
|
// +10 if URL should have changed but didn't
|
|
@@ -730,7 +730,7 @@ function penalizeNavigationSilentFailure({ evidenceSignals, penalties }) {
|
|
|
730
730
|
return total;
|
|
731
731
|
}
|
|
732
732
|
|
|
733
|
-
function scorePartialNavigationFailure({ expectation, expectationStrength, evidenceSignals, boosts, penalties }) {
|
|
733
|
+
function scorePartialNavigationFailure({ expectation: _expectation, expectationStrength: _expectationStrength, evidenceSignals, boosts, penalties: _penalties }) {
|
|
734
734
|
let total = 0;
|
|
735
735
|
|
|
736
736
|
// +10 if history changed but target not reached
|
|
@@ -764,7 +764,7 @@ function penalizePartialNavigationFailure({ evidenceSignals, penalties }) {
|
|
|
764
764
|
// EXPLANATION GENERATION (ORDERED BY IMPORTANCE)
|
|
765
765
|
// ============================================================
|
|
766
766
|
|
|
767
|
-
function generateExplanations(boosts, penalties, expectationStrength,
|
|
767
|
+
function generateExplanations(boosts, penalties, expectationStrength, _evidenceSignals) {
|
|
768
768
|
const explain = [];
|
|
769
769
|
|
|
770
770
|
// Add penalties first (most important negatives)
|
|
@@ -802,11 +802,11 @@ function generateExplanations(boosts, penalties, expectationStrength, evidenceSi
|
|
|
802
802
|
*/
|
|
803
803
|
function generateConfidenceExplanation({
|
|
804
804
|
level,
|
|
805
|
-
score,
|
|
805
|
+
score: _score,
|
|
806
806
|
expectationStrength,
|
|
807
807
|
sensorsPresent,
|
|
808
808
|
allSensorsPresent,
|
|
809
|
-
evidenceSignals,
|
|
809
|
+
evidenceSignals: _evidenceSignals,
|
|
810
810
|
boosts,
|
|
811
811
|
penalties,
|
|
812
812
|
attemptMeta,
|
|
@@ -914,20 +914,20 @@ function generateConfidenceExplanation({
|
|
|
914
914
|
export { hasNetworkData, hasConsoleData, hasUiData };
|
|
915
915
|
|
|
916
916
|
// Detect error feedback (legacy helper)
|
|
917
|
-
function
|
|
917
|
+
function _detectErrorFeedback(uiSignals) {
|
|
918
918
|
const before = uiSignals?.before || {};
|
|
919
919
|
const after = uiSignals?.after || {};
|
|
920
920
|
return after.hasErrorSignal && !before.hasErrorSignal;
|
|
921
921
|
}
|
|
922
922
|
|
|
923
923
|
// Detect loading feedback (legacy helper)
|
|
924
|
-
function
|
|
924
|
+
function _detectLoadingFeedback(uiSignals) {
|
|
925
925
|
const after = uiSignals?.after || {};
|
|
926
926
|
return after.hasLoadingIndicator;
|
|
927
927
|
}
|
|
928
928
|
|
|
929
929
|
// Detect status feedback (legacy helper)
|
|
930
|
-
function
|
|
930
|
+
function _detectStatusFeedback(uiSignals) {
|
|
931
931
|
const after = uiSignals?.after || {};
|
|
932
932
|
return after.hasStatusSignal || after.hasLiveRegion || after.hasDialog;
|
|
933
933
|
}
|
|
@@ -77,7 +77,7 @@ class DetectionEngine {
|
|
|
77
77
|
* Classify a single expectation against observations
|
|
78
78
|
* @private
|
|
79
79
|
*/
|
|
80
|
-
_classifyExpectation(expectation, observationMap,
|
|
80
|
+
_classifyExpectation(expectation, observationMap, _allObservations) {
|
|
81
81
|
const expectationId = expectation.id;
|
|
82
82
|
const promise = expectation.promise || {};
|
|
83
83
|
|
|
@@ -8,18 +8,15 @@
|
|
|
8
8
|
* with scope='evidence', reason='evidence_missing', preserving full context.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
import path from 'path';
|
|
11
|
+
// fs and path imports removed - currently unused
|
|
13
12
|
|
|
14
13
|
/**
|
|
15
14
|
* Build evidence index from observation traces with validation.
|
|
16
15
|
*
|
|
17
16
|
* @param {Array} traces - Observation traces from observe phase
|
|
18
|
-
* @param {string|null} projectDir - Project directory for file validation (null to skip validation)
|
|
19
|
-
* @param {Object|null} silenceTracker - Silence tracker instance (null to skip silence tracking)
|
|
20
17
|
* @returns {Object} { evidenceIndex, expectationEvidenceMap, findingEvidenceMap }
|
|
21
18
|
*/
|
|
22
|
-
export function buildEvidenceIndex(traces,
|
|
19
|
+
export function buildEvidenceIndex(traces, _projectDir = null, _silenceTracker = null) {
|
|
23
20
|
const evidenceIndex = [];
|
|
24
21
|
const expectationEvidenceMap = new Map();
|
|
25
22
|
const findingEvidenceMap = new Map();
|
|
@@ -29,13 +26,13 @@ export function buildEvidenceIndex(traces, projectDir = null, silenceTracker = n
|
|
|
29
26
|
return { evidenceIndex, expectationEvidenceMap, findingEvidenceMap };
|
|
30
27
|
}
|
|
31
28
|
|
|
32
|
-
// Use fs/path for evidence validation if projectDir provided
|
|
33
|
-
let existsSync = null;
|
|
34
|
-
let resolvePath = null;
|
|
35
|
-
if (projectDir && fs && path) {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
}
|
|
29
|
+
// Use fs/path for evidence validation if projectDir provided (currently unused)
|
|
30
|
+
// let existsSync = null;
|
|
31
|
+
// let resolvePath = null;
|
|
32
|
+
// if (projectDir && fs && path) {
|
|
33
|
+
// existsSync = fs.existsSync;
|
|
34
|
+
// resolvePath = path.resolve;
|
|
35
|
+
// }
|
|
39
36
|
|
|
40
37
|
for (const trace of traces) {
|
|
41
38
|
// Prefer modern trace schema: trace.before/trace.after
|
|
@@ -44,56 +41,7 @@ export function buildEvidenceIndex(traces, projectDir = null, silenceTracker = n
|
|
|
44
41
|
let beforeScreenshot = trace.before?.screenshot ?? trace.evidence?.beforeScreenshot ?? null;
|
|
45
42
|
let afterScreenshot = trace.after?.screenshot ?? trace.evidence?.afterScreenshot ?? null;
|
|
46
43
|
|
|
47
|
-
// PHASE 3:
|
|
48
|
-
if (existsSync && resolvePath && projectDir) {
|
|
49
|
-
const veraxDir = resolvePath(projectDir, '.veraxverax');
|
|
50
|
-
|
|
51
|
-
// Check beforeScreenshot exists
|
|
52
|
-
if (beforeScreenshot) {
|
|
53
|
-
const beforePath = resolvePath(veraxDir, 'observe', beforeScreenshot);
|
|
54
|
-
const fileExists = existsSync(beforePath);
|
|
55
|
-
|
|
56
|
-
if (!fileExists) {
|
|
57
|
-
// Track missing evidence as silence
|
|
58
|
-
if (silenceTracker) {
|
|
59
|
-
silenceTracker.record({
|
|
60
|
-
scope: 'evidence',
|
|
61
|
-
reason: 'evidence_missing',
|
|
62
|
-
description: `Screenshot evidence file not found: ${beforeScreenshot}`,
|
|
63
|
-
context: {
|
|
64
|
-
expectationId: trace.expectationId,
|
|
65
|
-
interaction: trace.interaction?.label,
|
|
66
|
-
expectedPath: beforePath
|
|
67
|
-
},
|
|
68
|
-
impact: 'incomplete_check'
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
beforeScreenshot = null; // Remove invalid evidence reference
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
// Check afterScreenshot exists
|
|
76
|
-
if (afterScreenshot) {
|
|
77
|
-
const afterPath = resolvePath(veraxDir, 'observe', afterScreenshot);
|
|
78
|
-
if (!existsSync(afterPath)) {
|
|
79
|
-
// Track missing evidence as silence
|
|
80
|
-
if (silenceTracker) {
|
|
81
|
-
silenceTracker.record({
|
|
82
|
-
scope: 'evidence',
|
|
83
|
-
reason: 'evidence_missing',
|
|
84
|
-
description: `Screenshot evidence file not found: ${afterScreenshot}`,
|
|
85
|
-
context: {
|
|
86
|
-
expectationId: trace.expectationId,
|
|
87
|
-
interaction: trace.interaction?.label,
|
|
88
|
-
expectedPath: afterPath
|
|
89
|
-
},
|
|
90
|
-
impact: 'incomplete_check'
|
|
91
|
-
});
|
|
92
|
-
}
|
|
93
|
-
afterScreenshot = null; // Remove invalid evidence reference
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
}
|
|
44
|
+
// PHASE 3: Evidence file validation removed - screenshots stored in .verax/runs/<runId>/evidence/
|
|
97
45
|
|
|
98
46
|
const entry = {
|
|
99
47
|
id: `ev-${id}`,
|
|
@@ -135,12 +83,14 @@ export function buildEvidenceIndex(traces, projectDir = null, silenceTracker = n
|
|
|
135
83
|
* @param {string} findingsPath - Path to findings.json
|
|
136
84
|
* @returns {Promise<string>} Path to written evidence-index.json
|
|
137
85
|
*/
|
|
138
|
-
export async function writeEvidenceIndex(projectDir, evidenceIndex, tracesPath, findingsPath, runDirOpt
|
|
86
|
+
export async function writeEvidenceIndex(projectDir, evidenceIndex, tracesPath, findingsPath, runDirOpt) {
|
|
139
87
|
const { resolve } = await import('path');
|
|
140
88
|
const { mkdirSync, writeFileSync } = await import('fs');
|
|
141
89
|
|
|
142
|
-
|
|
143
|
-
|
|
90
|
+
if (!runDirOpt) {
|
|
91
|
+
throw new Error('runDirOpt is required');
|
|
92
|
+
}
|
|
93
|
+
const artifactsDir = resolve(runDirOpt, 'evidence');
|
|
144
94
|
mkdirSync(artifactsDir, { recursive: true });
|
|
145
95
|
|
|
146
96
|
const evidenceIndexPath = resolve(artifactsDir, 'evidence-index.json');
|
|
@@ -38,7 +38,7 @@ function normalizeExpectationType(expectationType) {
|
|
|
38
38
|
*/
|
|
39
39
|
function normalizeSelector(selector) {
|
|
40
40
|
if (!selector) return '';
|
|
41
|
-
return selector.replace(/[
|
|
41
|
+
return selector.replace(/[[\]()]/g, '').trim();
|
|
42
42
|
}
|
|
43
43
|
|
|
44
44
|
/**
|
|
@@ -114,7 +114,7 @@ export function matchExpectation(expectation, interaction, beforeUrl) {
|
|
|
114
114
|
return null;
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
-
function
|
|
117
|
+
function _matchesStaticExpectation(expectation, interaction, afterUrl) {
|
|
118
118
|
if (interaction.type !== 'link') return false;
|
|
119
119
|
|
|
120
120
|
const afterPath = getUrlPath(afterUrl);
|
|
@@ -129,7 +129,7 @@ function matchesStaticExpectation(expectation, interaction, afterUrl) {
|
|
|
129
129
|
|
|
130
130
|
const selectorHint = expectation.evidence.selectorHint || '';
|
|
131
131
|
const interactionSelector = interaction.selector || '';
|
|
132
|
-
const
|
|
132
|
+
const _interactionLabel = (interaction.label || '').toLowerCase().trim();
|
|
133
133
|
|
|
134
134
|
if (selectorHint && interactionSelector) {
|
|
135
135
|
if (selectorHint.includes(interactionSelector) || interactionSelector.includes(selectorHint.replace(/[\]()]/g, ''))) {
|
|
@@ -224,3 +224,54 @@ export function expectsNavigation(manifest, interaction, beforeUrl) {
|
|
|
224
224
|
return false;
|
|
225
225
|
}
|
|
226
226
|
|
|
227
|
+
/**
|
|
228
|
+
* Get expectation for interaction from manifest (unified API for static and action contracts)
|
|
229
|
+
* @param {Object} manifest - Project manifest
|
|
230
|
+
* @param {Object} interaction - Interaction object
|
|
231
|
+
* @param {string} beforeUrl - URL before interaction
|
|
232
|
+
* @param {Object} [attemptMeta] - Optional metadata about the interaction attempt
|
|
233
|
+
* @returns {Object} { hasExpectation: boolean, proof: string, ...expectationData }
|
|
234
|
+
*/
|
|
235
|
+
export function getExpectation(manifest, interaction, beforeUrl, attemptMeta = {}) {
|
|
236
|
+
// Check static expectations first (for static sites)
|
|
237
|
+
if (manifest.staticExpectations && manifest.staticExpectations.length > 0) {
|
|
238
|
+
for (const expectation of manifest.staticExpectations) {
|
|
239
|
+
const matched = matchExpectation(expectation, interaction, beforeUrl);
|
|
240
|
+
if (matched) {
|
|
241
|
+
return {
|
|
242
|
+
hasExpectation: true,
|
|
243
|
+
proof: expectation.proof || 'PROVEN_EXPECTATION',
|
|
244
|
+
expectationType: expectation.type,
|
|
245
|
+
expectedTargetPath: expectation.targetPath,
|
|
246
|
+
...expectation
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Check action contracts (for apps with source-level contracts)
|
|
253
|
+
if (manifest.actionContracts && manifest.actionContracts.length > 0) {
|
|
254
|
+
const sourceRef = attemptMeta?.sourceRef;
|
|
255
|
+
if (sourceRef) {
|
|
256
|
+
for (const contract of manifest.actionContracts) {
|
|
257
|
+
if (contract.source === sourceRef) {
|
|
258
|
+
const expectationType = contract.kind === 'NETWORK_ACTION' ? 'network_action' : 'action';
|
|
259
|
+
return {
|
|
260
|
+
hasExpectation: true,
|
|
261
|
+
proof: 'PROVEN_EXPECTATION',
|
|
262
|
+
expectationType,
|
|
263
|
+
method: contract.method,
|
|
264
|
+
urlPath: contract.urlPath,
|
|
265
|
+
...contract
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
return {
|
|
273
|
+
hasExpectation: false,
|
|
274
|
+
proof: 'UNKNOWN_EXPECTATION'
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
|