@xera-ai/core 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artifact/status.d.ts +12 -0
- package/dist/artifact/status.d.ts.map +1 -1
- package/dist/bin/internal.js +1059 -567
- package/dist/bin-internal/auth-setup.d.ts +2 -0
- package/dist/bin-internal/auth-setup.d.ts.map +1 -0
- package/dist/bin-internal/disputes.d.ts +2 -0
- package/dist/bin-internal/disputes.d.ts.map +1 -0
- package/dist/bin-internal/doctor.d.ts +1 -1
- package/dist/bin-internal/doctor.d.ts.map +1 -1
- package/dist/bin-internal/exec.d.ts.map +1 -1
- package/dist/bin-internal/graph-record-script.d.ts.map +1 -1
- package/dist/bin-internal/graph-record.d.ts.map +1 -1
- package/dist/bin-internal/index.d.ts.map +1 -1
- package/dist/bin-internal/normalize.d.ts.map +1 -1
- package/dist/bin-internal/report.d.ts.map +1 -1
- package/dist/bin-internal/verify-prompts.d.ts.map +1 -1
- package/dist/classifier/aggregate.d.ts.map +1 -1
- package/dist/classifier/auth-expired.d.ts +12 -0
- package/dist/classifier/auth-expired.d.ts.map +1 -0
- package/dist/classifier/contract-drift.d.ts +35 -0
- package/dist/classifier/contract-drift.d.ts.map +1 -0
- package/dist/classifier/rate-limited.d.ts +15 -0
- package/dist/classifier/rate-limited.d.ts.map +1 -0
- package/dist/config/schema.d.ts +32 -3
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/graph/schema.d.ts +9 -0
- package/dist/graph/schema.d.ts.map +1 -1
- package/dist/graph/store.d.ts.map +1 -1
- package/dist/graph/types.d.ts +2 -1
- package/dist/graph/types.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/scrub/index.d.ts +2 -0
- package/dist/scrub/index.d.ts.map +1 -0
- package/dist/scrub/rules.d.ts +12 -0
- package/dist/scrub/rules.d.ts.map +1 -0
- package/dist/src/index.js +110 -5
- package/package.json +4 -3
- package/src/artifact/status.ts +3 -0
- package/src/bin-internal/auth-setup.ts +116 -0
- package/src/bin-internal/disputes.ts +88 -0
- package/src/bin-internal/doctor.ts +13 -1
- package/src/bin-internal/exec.ts +45 -9
- package/src/bin-internal/graph-record-script.ts +37 -8
- package/src/bin-internal/graph-record.ts +3 -0
- package/src/bin-internal/index.ts +4 -0
- package/src/bin-internal/normalize.ts +13 -1
- package/src/bin-internal/report.ts +94 -2
- package/src/bin-internal/verify-prompts.ts +2 -1
- package/src/classifier/aggregate.ts +3 -0
- package/src/classifier/auth-expired.ts +44 -0
- package/src/classifier/contract-drift.ts +111 -0
- package/src/classifier/rate-limited.ts +25 -0
- package/src/config/schema.ts +52 -9
- package/src/graph/schema.ts +3 -0
- package/src/graph/store.ts +8 -1
- package/src/graph/types.ts +5 -1
- package/src/index.ts +2 -0
- package/src/scrub/index.ts +1 -0
- package/src/scrub/rules.ts +69 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { pathToFileURL } from 'node:url';
|
|
4
|
+
import { loadConfig } from '../config/load';
|
|
5
|
+
|
|
6
|
+
interface AuthSetupOpts {
|
|
7
|
+
role?: string;
|
|
8
|
+
shape: 'web' | 'http' | 'all';
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function parseOpts(argv: string[]): AuthSetupOpts {
|
|
12
|
+
const opts: AuthSetupOpts = { shape: 'all' };
|
|
13
|
+
for (let i = 0; i < argv.length; i++) {
|
|
14
|
+
const a = argv[i];
|
|
15
|
+
const next = argv[i + 1];
|
|
16
|
+
if (a === '--role' && next) {
|
|
17
|
+
opts.role = next;
|
|
18
|
+
i++;
|
|
19
|
+
} else if (a === '--shape' && next) {
|
|
20
|
+
if (next === 'web' || next === 'http' || next === 'all') opts.shape = next;
|
|
21
|
+
i++;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return opts;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export async function authSetupCmd(argv: string[]): Promise<number> {
|
|
28
|
+
const opts = parseOpts(argv);
|
|
29
|
+
const cwd = process.cwd();
|
|
30
|
+
const config = await loadConfig(cwd);
|
|
31
|
+
|
|
32
|
+
const authSetupScript = join(cwd, 'shared', 'auth-setup.ts');
|
|
33
|
+
if (!existsSync(authSetupScript)) {
|
|
34
|
+
console.error(
|
|
35
|
+
`[xera:auth-setup] auth-setup.ts not found at ${authSetupScript}. Run 'bunx @xera-ai/cli init' first.`,
|
|
36
|
+
);
|
|
37
|
+
return 1;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const mod = (await import(pathToFileURL(authSetupScript).href)) as {
|
|
41
|
+
web?: unknown;
|
|
42
|
+
http?: unknown;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
let exitCode = 0;
|
|
46
|
+
|
|
47
|
+
// Web roles
|
|
48
|
+
if (
|
|
49
|
+
(opts.shape === 'all' || opts.shape === 'web') &&
|
|
50
|
+
config.web &&
|
|
51
|
+
typeof mod.web === 'function'
|
|
52
|
+
) {
|
|
53
|
+
const { runAuthSetup } = await import('@xera-ai/web');
|
|
54
|
+
const { chromium } = await import('@playwright/test');
|
|
55
|
+
const browser = await chromium.launch();
|
|
56
|
+
try {
|
|
57
|
+
for (const [roleName, roleCreds] of Object.entries(config.web.auth.roles)) {
|
|
58
|
+
if (opts.role && roleName !== opts.role) continue;
|
|
59
|
+
const email = process.env[roleCreds.envEmail];
|
|
60
|
+
const password = process.env[roleCreds.envPassword];
|
|
61
|
+
if (!email || !password) {
|
|
62
|
+
console.error(
|
|
63
|
+
`[xera:auth-setup] missing env vars ${roleCreds.envEmail} / ${roleCreds.envPassword} for role '${roleName}'`,
|
|
64
|
+
);
|
|
65
|
+
exitCode = 1;
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
await runAuthSetup({
|
|
70
|
+
role: roleName,
|
|
71
|
+
creds: { email, password },
|
|
72
|
+
setupScriptPath: authSetupScript,
|
|
73
|
+
authDir: join(cwd, '.xera', '.auth'),
|
|
74
|
+
browser,
|
|
75
|
+
});
|
|
76
|
+
console.log(`[xera:auth-setup] ✓ ${roleName}.json (web)`);
|
|
77
|
+
} catch (e) {
|
|
78
|
+
console.error(`[xera:auth-setup] ✗ web/${roleName}: ${(e as Error).message}`);
|
|
79
|
+
exitCode = 1;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
} finally {
|
|
83
|
+
await browser.close();
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Http roles
|
|
88
|
+
if (
|
|
89
|
+
(opts.shape === 'all' || opts.shape === 'http') &&
|
|
90
|
+
config.http &&
|
|
91
|
+
typeof mod.http === 'function'
|
|
92
|
+
) {
|
|
93
|
+
// The auth-setup.ts template reads config via globalThis; set it for the user's function.
|
|
94
|
+
(globalThis as Record<string, unknown>).__XERA_HTTP_CONFIG__ = config.http;
|
|
95
|
+
|
|
96
|
+
const { runHttpAuthSetup } = await import('@xera-ai/http');
|
|
97
|
+
for (const roleName of Object.keys(config.http.auth.roles)) {
|
|
98
|
+
if (opts.role && roleName !== opts.role) continue;
|
|
99
|
+
try {
|
|
100
|
+
await runHttpAuthSetup({
|
|
101
|
+
authDir: join(cwd, '.xera', '.auth'),
|
|
102
|
+
role: roleName,
|
|
103
|
+
config: config.http,
|
|
104
|
+
setupFn: mod.http as Parameters<typeof runHttpAuthSetup>[0]['setupFn'],
|
|
105
|
+
creds: { email: '', password: '' },
|
|
106
|
+
});
|
|
107
|
+
console.log(`[xera:auth-setup] ✓ http/${roleName}.json`);
|
|
108
|
+
} catch (e) {
|
|
109
|
+
console.error(`[xera:auth-setup] ✗ http/${roleName}: ${(e as Error).message}`);
|
|
110
|
+
exitCode = 1;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return exitCode;
|
|
116
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { loadAllEvents } from '../graph/store';
|
|
2
|
+
import type { ClassificationDisputedPayload, Event } from '../graph/types';
|
|
3
|
+
|
|
4
|
+
function parseDuration(s: string): number {
|
|
5
|
+
// accepts "7d", "30d", "1h", "5m" — returns ms; returns 0 for invalid input
|
|
6
|
+
const match = s.match(/^(\d+)([dhm])$/);
|
|
7
|
+
if (!match) return 0;
|
|
8
|
+
const n = Number.parseInt(match[1]!, 10);
|
|
9
|
+
const unit = match[2]!;
|
|
10
|
+
if (unit === 'd') return n * 86400 * 1000;
|
|
11
|
+
if (unit === 'h') return n * 3600 * 1000;
|
|
12
|
+
if (unit === 'm') return n * 60 * 1000;
|
|
13
|
+
return 0;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface DisputeRow {
|
|
17
|
+
ts: string;
|
|
18
|
+
runId: string;
|
|
19
|
+
scenarioId: string;
|
|
20
|
+
originalClassification: string;
|
|
21
|
+
disputedTo: string;
|
|
22
|
+
qaActor: string;
|
|
23
|
+
qaReason?: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function eventToRow(e: Event & { type: 'classification.disputed' }): DisputeRow {
|
|
27
|
+
const p = e.payload as ClassificationDisputedPayload;
|
|
28
|
+
const row: DisputeRow = {
|
|
29
|
+
ts: e.ts,
|
|
30
|
+
runId: p.runId,
|
|
31
|
+
scenarioId: p.scenarioId,
|
|
32
|
+
originalClassification: p.originalClassification,
|
|
33
|
+
disputedTo: p.disputedTo,
|
|
34
|
+
qaActor: p.qaActor,
|
|
35
|
+
};
|
|
36
|
+
if (p.qaReason) row.qaReason = p.qaReason;
|
|
37
|
+
return row;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function renderText(rows: DisputeRow[]): string {
|
|
41
|
+
if (rows.length === 0) return 'No disputes recorded.\n';
|
|
42
|
+
const lines: string[] = [];
|
|
43
|
+
lines.push(`${rows.length} dispute(s):`);
|
|
44
|
+
for (const r of rows) {
|
|
45
|
+
lines.push(
|
|
46
|
+
` ${r.ts} | ${r.scenarioId} | ${r.originalClassification} → ${r.disputedTo} | ${r.qaActor}`,
|
|
47
|
+
);
|
|
48
|
+
if (r.qaReason) lines.push(` reason: ${r.qaReason}`);
|
|
49
|
+
}
|
|
50
|
+
return `${lines.join('\n')}\n`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export async function disputesCmd(argv: string[]): Promise<number> {
|
|
54
|
+
let since: string | undefined;
|
|
55
|
+
let format: 'text' | 'json' = 'text';
|
|
56
|
+
for (let i = 0; i < argv.length; i++) {
|
|
57
|
+
if (argv[i] === '--since') {
|
|
58
|
+
since = argv[++i];
|
|
59
|
+
} else if (argv[i] === '--format') {
|
|
60
|
+
const v = argv[++i];
|
|
61
|
+
if (v === 'json' || v === 'text') format = v;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const repoRoot = process.cwd();
|
|
66
|
+
const events = loadAllEvents(repoRoot);
|
|
67
|
+
const disputes = events.filter(
|
|
68
|
+
(e): e is Event & { type: 'classification.disputed' } => e.type === 'classification.disputed',
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
let cutoffMs: number | undefined;
|
|
72
|
+
if (since) {
|
|
73
|
+
const sinceMs = parseDuration(since);
|
|
74
|
+
if (sinceMs > 0) cutoffMs = Date.now() - sinceMs;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const rows = disputes
|
|
78
|
+
.filter((e) => cutoffMs === undefined || Date.parse(e.ts) >= cutoffMs)
|
|
79
|
+
.map(eventToRow)
|
|
80
|
+
.sort((a, b) => (a.ts < b.ts ? 1 : -1));
|
|
81
|
+
|
|
82
|
+
if (format === 'json') {
|
|
83
|
+
process.stdout.write(JSON.stringify(rows, null, 2));
|
|
84
|
+
} else {
|
|
85
|
+
process.stdout.write(renderText(rows));
|
|
86
|
+
}
|
|
87
|
+
return 0;
|
|
88
|
+
}
|
|
@@ -117,8 +117,9 @@ function checkRootScripts(repoRoot: string): CheckResult[] {
|
|
|
117
117
|
return missing.map((s) => ({ ok: false, message: `root package.json missing script: ${s}` }));
|
|
118
118
|
}
|
|
119
119
|
|
|
120
|
-
export async function doctorCmd(
|
|
120
|
+
export async function doctorCmd(argv: string[], opts: DoctorOpts = {}): Promise<number> {
|
|
121
121
|
const repoRoot = opts.cwd ?? process.cwd();
|
|
122
|
+
const autoEnrich = argv.includes('--auto-enrich');
|
|
122
123
|
const results: CheckResult[] = [
|
|
123
124
|
...checkGoldenEvalDir(repoRoot),
|
|
124
125
|
...checkRubricPrompt(repoRoot),
|
|
@@ -156,6 +157,17 @@ export async function doctorCmd(_argv: string[], opts: DoctorOpts = {}): Promise
|
|
|
156
157
|
console.log(` These won't participate in v0.6.1+ features (TEST_OUTDATED, /xera-impact).`);
|
|
157
158
|
console.log(` Run: bun run xera:graph-backfill`);
|
|
158
159
|
console.log(` (Use --dry-run to preview.)`);
|
|
160
|
+
if (autoEnrich) {
|
|
161
|
+
console.log('[doctor] --auto-enrich: running backfill for unbackfilled tickets...');
|
|
162
|
+
// Lazy import to avoid circular deps
|
|
163
|
+
const { graphBackfillCmd } = await import('./graph-backfill');
|
|
164
|
+
const exitCode = await graphBackfillCmd([]);
|
|
165
|
+
if (exitCode === 0) {
|
|
166
|
+
console.log(`[doctor] auto-enrich: backfilled ${unbackfilled.length} tickets`);
|
|
167
|
+
} else {
|
|
168
|
+
console.error('[doctor] auto-enrich: backfill failed');
|
|
169
|
+
}
|
|
170
|
+
}
|
|
159
171
|
}
|
|
160
172
|
}
|
|
161
173
|
}
|
package/src/bin-internal/exec.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { existsSync, mkdirSync } from 'node:fs';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { chromium } from '@playwright/test';
|
|
4
4
|
import { runAuthSetup, runPlaywright, stagePlaywrightState } from '@xera-ai/web';
|
|
5
|
+
import { readMeta } from '../artifact/meta';
|
|
5
6
|
import { generateRunId, resolveArtifactPaths } from '../artifact/paths';
|
|
6
7
|
import { needsRefresh } from '../auth/refresh';
|
|
7
8
|
import { readAuthState } from '../auth/state';
|
|
@@ -15,6 +16,8 @@ export async function execCmd(argv: string[]): Promise<number> {
|
|
|
15
16
|
console.error('[xera:exec] usage: exec <TICKET>');
|
|
16
17
|
return 1;
|
|
17
18
|
}
|
|
19
|
+
const grepIdx = argv.indexOf('--grep');
|
|
20
|
+
const grep = grepIdx > -1 ? argv[grepIdx + 1] : undefined;
|
|
18
21
|
const cwd = process.cwd();
|
|
19
22
|
const config = await loadConfig(cwd);
|
|
20
23
|
const paths = resolveArtifactPaths(cwd, ticket);
|
|
@@ -40,16 +43,48 @@ export async function execCmd(argv: string[]): Promise<number> {
|
|
|
40
43
|
|
|
41
44
|
const t0 = Date.now();
|
|
42
45
|
try {
|
|
46
|
+
const meta = readMeta(paths.metaPath);
|
|
47
|
+
const adapter = meta?.adapter ?? 'web';
|
|
48
|
+
|
|
49
|
+
if (adapter === 'http') {
|
|
50
|
+
if (!config.http) {
|
|
51
|
+
throw new Error('http adapter requires http config block');
|
|
52
|
+
}
|
|
53
|
+
const env = process.env['XERA_ENV'] ?? config.http.defaultEnv;
|
|
54
|
+
const { HttpAdapter } = await import('@xera-ai/http');
|
|
55
|
+
const result = await HttpAdapter.execute({
|
|
56
|
+
ticketDir: paths.ticketDir,
|
|
57
|
+
config,
|
|
58
|
+
runId,
|
|
59
|
+
env,
|
|
60
|
+
});
|
|
61
|
+
log.log({
|
|
62
|
+
step: 'exec.complete',
|
|
63
|
+
runId,
|
|
64
|
+
outcome: result.outcome,
|
|
65
|
+
elapsedMs: Date.now() - t0,
|
|
66
|
+
});
|
|
67
|
+
console.log(`[xera:exec] runId=${runId} outcome=${result.outcome}`);
|
|
68
|
+
// Exit 3 means "test failed" (expected vs infra error); lock released in finally
|
|
69
|
+
return result.outcome === 'PASS' ? 0 : 3;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// adapter === 'web' — existing path below unchanged
|
|
73
|
+
if (!config.web) {
|
|
74
|
+
throw new Error('web adapter requires web config block');
|
|
75
|
+
}
|
|
76
|
+
const webConfig = config.web;
|
|
77
|
+
|
|
43
78
|
// Auth refresh per role declared in xera.config.ts
|
|
44
|
-
if (
|
|
79
|
+
if (webConfig.auth.strategy === 'storageState' && webConfig.auth.setupScript) {
|
|
45
80
|
const browser = await chromium.launch();
|
|
46
81
|
try {
|
|
47
|
-
for (const [roleName, roleCreds] of Object.entries(
|
|
82
|
+
for (const [roleName, roleCreds] of Object.entries(webConfig.auth.roles)) {
|
|
48
83
|
const entry = readAuthState(paths.authDir, roleName);
|
|
49
84
|
if (
|
|
50
85
|
needsRefresh(entry, {
|
|
51
|
-
ttl:
|
|
52
|
-
refreshBuffer:
|
|
86
|
+
ttl: webConfig.auth.ttl,
|
|
87
|
+
refreshBuffer: webConfig.auth.refreshBuffer,
|
|
53
88
|
})
|
|
54
89
|
) {
|
|
55
90
|
const email = process.env[roleCreds.envEmail];
|
|
@@ -63,7 +98,7 @@ export async function execCmd(argv: string[]): Promise<number> {
|
|
|
63
98
|
await runAuthSetup({
|
|
64
99
|
role: roleName,
|
|
65
100
|
creds: { email, password },
|
|
66
|
-
setupScriptPath: join(cwd,
|
|
101
|
+
setupScriptPath: join(cwd, webConfig.auth.setupScript),
|
|
67
102
|
authDir: paths.authDir,
|
|
68
103
|
browser,
|
|
69
104
|
});
|
|
@@ -78,8 +113,8 @@ export async function execCmd(argv: string[]): Promise<number> {
|
|
|
78
113
|
// Stage Playwright storageState files at predictable paths
|
|
79
114
|
// (.xera/.auth/.cache/<role>.json) — generated spec.ts references these
|
|
80
115
|
// via test.use({ storageState }) when an authenticated session is needed.
|
|
81
|
-
if (
|
|
82
|
-
for (const roleName of Object.keys(
|
|
116
|
+
if (webConfig.auth.strategy === 'storageState') {
|
|
117
|
+
for (const roleName of Object.keys(webConfig.auth.roles)) {
|
|
83
118
|
if (readAuthState(paths.authDir, roleName)) {
|
|
84
119
|
stagePlaywrightState(paths.authDir, roleName);
|
|
85
120
|
}
|
|
@@ -99,8 +134,8 @@ export async function execCmd(argv: string[]): Promise<number> {
|
|
|
99
134
|
const runDir = paths.runPath(runId).runDir;
|
|
100
135
|
mkdirSync(runDir, { recursive: true });
|
|
101
136
|
|
|
102
|
-
const envName = process.env.XERA_ENV ??
|
|
103
|
-
const baseURL =
|
|
137
|
+
const envName = process.env.XERA_ENV ?? webConfig.defaultEnv;
|
|
138
|
+
const baseURL = webConfig.baseUrl[envName] ?? webConfig.baseUrl[webConfig.defaultEnv]!;
|
|
104
139
|
|
|
105
140
|
const reportJsonPath = join(runDir, 'report.json');
|
|
106
141
|
|
|
@@ -117,6 +152,7 @@ export async function execCmd(argv: string[]): Promise<number> {
|
|
|
117
152
|
// path to read.
|
|
118
153
|
PLAYWRIGHT_JSON_OUTPUT_NAME: reportJsonPath,
|
|
119
154
|
},
|
|
155
|
+
...(grep && { grep }),
|
|
120
156
|
});
|
|
121
157
|
log.log({ step: 'exec.done', runId, exit: r.exitCode, ms: Date.now() - t0 });
|
|
122
158
|
|
|
@@ -30,18 +30,49 @@ const mk = <T extends Event['type']>(
|
|
|
30
30
|
payload,
|
|
31
31
|
}) as Event;
|
|
32
32
|
|
|
33
|
+
const P0_KEYWORDS = [
|
|
34
|
+
'log in',
|
|
35
|
+
'login',
|
|
36
|
+
'sign in',
|
|
37
|
+
'signin',
|
|
38
|
+
'sign up',
|
|
39
|
+
'signup',
|
|
40
|
+
'auth',
|
|
41
|
+
'authentic',
|
|
42
|
+
'payment',
|
|
43
|
+
'pay ',
|
|
44
|
+
'checkout',
|
|
45
|
+
'purchase',
|
|
46
|
+
'charge',
|
|
47
|
+
'password',
|
|
48
|
+
'credential',
|
|
49
|
+
'admin',
|
|
50
|
+
'permission',
|
|
51
|
+
'role',
|
|
52
|
+
'must ',
|
|
53
|
+
'critical',
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
function inferPriority(name: string, gherkin: string): 'p0' | 'p1' {
|
|
57
|
+
const haystack = `${name} ${gherkin}`.toLowerCase();
|
|
58
|
+
for (const kw of P0_KEYWORDS) {
|
|
59
|
+
if (haystack.includes(kw)) return 'p0';
|
|
60
|
+
}
|
|
61
|
+
return 'p1';
|
|
62
|
+
}
|
|
63
|
+
|
|
33
64
|
function parseFeature(
|
|
34
65
|
text: string,
|
|
35
66
|
): Array<{ name: string; priority: 'p0' | 'p1' | 'p2'; gherkin: string }> {
|
|
36
67
|
const scenarios: Array<{ name: string; priority: 'p0' | 'p1' | 'p2'; gherkin: string }> = [];
|
|
37
68
|
const lines = text.split('\n');
|
|
38
|
-
let
|
|
69
|
+
let explicitTag: 'p0' | 'p1' | 'p2' | null = null;
|
|
39
70
|
let i = 0;
|
|
40
71
|
while (i < lines.length) {
|
|
41
72
|
const line = lines[i]!.trim();
|
|
42
73
|
if (line.startsWith('@')) {
|
|
43
74
|
const tag = line.slice(1).split(/\s+/)[0]!.toLowerCase();
|
|
44
|
-
if (tag === 'p0' || tag === 'p1' || tag === 'p2')
|
|
75
|
+
if (tag === 'p0' || tag === 'p1' || tag === 'p2') explicitTag = tag;
|
|
45
76
|
i++;
|
|
46
77
|
continue;
|
|
47
78
|
}
|
|
@@ -55,12 +86,10 @@ function parseFeature(
|
|
|
55
86
|
!lines[i]!.trim().startsWith('@')
|
|
56
87
|
)
|
|
57
88
|
i++;
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
});
|
|
63
|
-
currentTagPriority = 'p1';
|
|
89
|
+
const gherkin = lines.slice(start, i).join('\n');
|
|
90
|
+
const priority = explicitTag !== null ? explicitTag : inferPriority(name, gherkin);
|
|
91
|
+
scenarios.push({ name, priority, gherkin });
|
|
92
|
+
explicitTag = null;
|
|
64
93
|
continue;
|
|
65
94
|
}
|
|
66
95
|
i++;
|
|
@@ -261,6 +261,9 @@ export async function graphRecordCmd(argv: string[]): Promise<number> {
|
|
|
261
261
|
'FLAKY',
|
|
262
262
|
'PASS',
|
|
263
263
|
'TEST_OUTDATED',
|
|
264
|
+
'CONTRACT_DRIFT',
|
|
265
|
+
'RATE_LIMITED',
|
|
266
|
+
'AUTH_EXPIRED',
|
|
264
267
|
];
|
|
265
268
|
if (!validClass.includes(from) || !validClass.includes(to)) {
|
|
266
269
|
console.error(
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { authSetupCmd } from './auth-setup';
|
|
2
|
+
import { disputesCmd } from './disputes';
|
|
1
3
|
import { doctorCmd } from './doctor';
|
|
2
4
|
import { evalDeterministicCmd } from './eval-deterministic';
|
|
3
5
|
import { evalPrepareCmd } from './eval-prepare';
|
|
@@ -24,6 +26,8 @@ import { validateFeatureCmd } from './validate-feature';
|
|
|
24
26
|
import { verifyPromptsCmd } from './verify-prompts';
|
|
25
27
|
|
|
26
28
|
const COMMANDS: Record<string, (argv: string[]) => Promise<number>> = {
|
|
29
|
+
'auth-setup': authSetupCmd,
|
|
30
|
+
disputes: disputesCmd,
|
|
27
31
|
doctor: doctorCmd,
|
|
28
32
|
'eval-deterministic': evalDeterministicCmd,
|
|
29
33
|
'eval-prepare': evalPrepareCmd,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { existsSync, readdirSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
-
import {
|
|
3
|
+
import { readMeta } from '../artifact/meta';
|
|
4
4
|
import { resolveArtifactPaths } from '../artifact/paths';
|
|
5
5
|
|
|
6
6
|
export async function normalizeCmd(argv: string[]): Promise<number> {
|
|
@@ -26,6 +26,18 @@ export async function normalizeCmd(argv: string[]): Promise<number> {
|
|
|
26
26
|
console.error(`[xera:normalize] runs/${runId} missing`);
|
|
27
27
|
return 1;
|
|
28
28
|
}
|
|
29
|
+
|
|
30
|
+
const meta = readMeta(paths.metaPath);
|
|
31
|
+
const adapter = meta?.adapter ?? 'web';
|
|
32
|
+
|
|
33
|
+
if (adapter === 'http') {
|
|
34
|
+
const { normalizeHttpRun } = await import('@xera-ai/http');
|
|
35
|
+
await normalizeHttpRun({ runId, runDir });
|
|
36
|
+
console.log(`[xera:normalize] wrote normalized.json (http)`);
|
|
37
|
+
return 0;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const { normalizeRun } = await import('@xera-ai/web');
|
|
29
41
|
const r = await normalizeRun({ runId, runDir });
|
|
30
42
|
console.log(
|
|
31
43
|
`[xera:normalize] wrote normalized.json (scrubbed_fields_count=${r.scrubbed_fields_count})`,
|
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import { readMeta } from '../artifact/meta';
|
|
3
4
|
import { resolveArtifactPaths } from '../artifact/paths';
|
|
5
|
+
import { readAuthState } from '../auth/state';
|
|
4
6
|
import { aggregateScenarios } from '../classifier/aggregate';
|
|
7
|
+
import { type AuthFileSummary, classifyAuthExpired } from '../classifier/auth-expired';
|
|
8
|
+
import { classifyContractDrift } from '../classifier/contract-drift';
|
|
9
|
+
import { classifyRateLimited } from '../classifier/rate-limited';
|
|
5
10
|
import type { ScenarioClassification } from '../classifier/types';
|
|
11
|
+
import { loadConfig } from '../config/load';
|
|
6
12
|
import type { OutdatedDecision } from '../graph/classify';
|
|
7
13
|
import { enhanceClassification } from '../graph/classify';
|
|
8
14
|
import { deriveSnapshot, loadAllEvents } from '../graph/store';
|
|
@@ -22,10 +28,96 @@ export async function reportCmd(argv: string[]): Promise<number> {
|
|
|
22
28
|
console.error('[xera:report] usage: report <TICKET> --input=<classifier-output.json>');
|
|
23
29
|
return 1;
|
|
24
30
|
}
|
|
25
|
-
const
|
|
31
|
+
const cwd = process.cwd();
|
|
32
|
+
const paths = resolveArtifactPaths(cwd, ticket);
|
|
26
33
|
const input = JSON.parse(readFileSync(inputArg.slice('--input='.length), 'utf8')) as ReportInput;
|
|
27
34
|
|
|
28
|
-
|
|
35
|
+
// v0.7: apply deterministic HTTP classifier rules before aggregation.
|
|
36
|
+
// When adapter === 'http', scan normalized.json http.calls for rate-limit,
|
|
37
|
+
// auth-expired, and contract-drift signals and override failing scenario classes.
|
|
38
|
+
interface HttpRuleOverride {
|
|
39
|
+
class: ScenarioClassification['class'];
|
|
40
|
+
rationale: string;
|
|
41
|
+
}
|
|
42
|
+
let httpRuleOverride: HttpRuleOverride | null = null;
|
|
43
|
+
|
|
44
|
+
const meta = readMeta(paths.metaPath);
|
|
45
|
+
if (meta?.adapter === 'http') {
|
|
46
|
+
const config = await loadConfig(cwd);
|
|
47
|
+
if (config.http) {
|
|
48
|
+
const normalizedPath = join(paths.ticketDir, 'runs', input.runId, 'normalized.json');
|
|
49
|
+
if (existsSync(normalizedPath)) {
|
|
50
|
+
const norm = JSON.parse(readFileSync(normalizedPath, 'utf8')) as {
|
|
51
|
+
http?: {
|
|
52
|
+
calls?: Array<{ method: string; url: string; status: number; respBody?: unknown }>;
|
|
53
|
+
};
|
|
54
|
+
};
|
|
55
|
+
const calls = norm.http?.calls ?? [];
|
|
56
|
+
|
|
57
|
+
// RATE_LIMITED
|
|
58
|
+
const rate = classifyRateLimited({ calls });
|
|
59
|
+
if (rate) httpRuleOverride = rate;
|
|
60
|
+
|
|
61
|
+
// AUTH_EXPIRED — needs auth files
|
|
62
|
+
if (!httpRuleOverride) {
|
|
63
|
+
const authFiles: Record<string, AuthFileSummary> = {};
|
|
64
|
+
const httpAuthDir = join(cwd, '.xera', '.auth', 'http');
|
|
65
|
+
for (const role of Object.keys(config.http.auth.roles)) {
|
|
66
|
+
const entry = readAuthState(httpAuthDir, role);
|
|
67
|
+
if (entry) {
|
|
68
|
+
const p = entry.payload as {
|
|
69
|
+
token: string;
|
|
70
|
+
type: 'bearer' | 'apiKey' | 'basic' | 'cookie';
|
|
71
|
+
};
|
|
72
|
+
if (typeof p.token === 'string' && typeof p.type === 'string') {
|
|
73
|
+
authFiles[role] = {
|
|
74
|
+
token: p.token,
|
|
75
|
+
type: p.type as AuthFileSummary['type'],
|
|
76
|
+
expires_at: entry.expires_at,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
const authExp = classifyAuthExpired({ calls, authFiles });
|
|
82
|
+
if (authExp) httpRuleOverride = authExp;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// CONTRACT_DRIFT — needs openapi
|
|
86
|
+
if (!httpRuleOverride && config.http.spec) {
|
|
87
|
+
const { loadOpenApi } = await import('@xera-ai/http');
|
|
88
|
+
const openapi = await loadOpenApi(config.http.spec);
|
|
89
|
+
if (openapi) {
|
|
90
|
+
const drift = classifyContractDrift({
|
|
91
|
+
calls: calls.map((c) => ({
|
|
92
|
+
method: c.method,
|
|
93
|
+
url: c.url,
|
|
94
|
+
status: c.status,
|
|
95
|
+
respBody: c.respBody,
|
|
96
|
+
})),
|
|
97
|
+
openapi,
|
|
98
|
+
});
|
|
99
|
+
if (drift) httpRuleOverride = drift;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Apply override: if a deterministic rule fired, stamp every FAIL scenario with it.
|
|
107
|
+
const scenariosForAggregation: ScenarioClassification[] = httpRuleOverride
|
|
108
|
+
? input.scenarios.map((s) =>
|
|
109
|
+
s.outcome === 'FAIL'
|
|
110
|
+
? {
|
|
111
|
+
...s,
|
|
112
|
+
class: httpRuleOverride.class,
|
|
113
|
+
rationale: httpRuleOverride.rationale,
|
|
114
|
+
confidence: 'high' as const,
|
|
115
|
+
}
|
|
116
|
+
: s,
|
|
117
|
+
)
|
|
118
|
+
: input.scenarios;
|
|
119
|
+
|
|
120
|
+
const aggregated = aggregateScenarios(scenariosForAggregation);
|
|
29
121
|
|
|
30
122
|
// v0.6.1: TEST_OUTDATED enhancement.
|
|
31
123
|
// The /xera-report skill writes outdated-decisions.json BEFORE invoking this subcommand,
|
|
@@ -2,6 +2,9 @@ import type { ClassifyOutput, Confidence, ScenarioClassification } from './types
|
|
|
2
2
|
|
|
3
3
|
const CLASS_PRIORITY: Array<ClassifyOutput['overall']> = [
|
|
4
4
|
'REAL_BUG',
|
|
5
|
+
'CONTRACT_DRIFT',
|
|
6
|
+
'AUTH_EXPIRED',
|
|
7
|
+
'RATE_LIMITED',
|
|
5
8
|
'TEST_OUTDATED',
|
|
6
9
|
'TEST_BUG',
|
|
7
10
|
'SELECTOR_DRIFT',
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { ClassifyResult, HttpCallSummary } from './rate-limited';
|
|
2
|
+
|
|
3
|
+
export interface AuthFileSummary {
|
|
4
|
+
token: string;
|
|
5
|
+
type: 'bearer' | 'apiKey' | 'basic' | 'cookie';
|
|
6
|
+
expires_at: string;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface ClassifyAuthExpiredInput {
|
|
10
|
+
calls: readonly HttpCallSummary[];
|
|
11
|
+
authFiles: Record<string, AuthFileSummary>;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function jwtExpPast(jwt: string, now: number): boolean {
|
|
15
|
+
const parts = jwt.split('.');
|
|
16
|
+
if (parts.length !== 3) return false;
|
|
17
|
+
try {
|
|
18
|
+
const payloadB64 = parts[1];
|
|
19
|
+
if (!payloadB64) return false;
|
|
20
|
+
const payload = JSON.parse(Buffer.from(payloadB64, 'base64url').toString('utf8')) as {
|
|
21
|
+
exp?: number;
|
|
22
|
+
};
|
|
23
|
+
return typeof payload.exp === 'number' && payload.exp * 1000 < now;
|
|
24
|
+
} catch {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function classifyAuthExpired(input: ClassifyAuthExpiredInput): ClassifyResult | null {
|
|
30
|
+
const has401 = input.calls.some((c) => c.status === 401);
|
|
31
|
+
if (!has401) return null;
|
|
32
|
+
const now = Date.now();
|
|
33
|
+
for (const [role, entry] of Object.entries(input.authFiles)) {
|
|
34
|
+
const fileExpired = new Date(entry.expires_at).getTime() < now;
|
|
35
|
+
const jwtExpired = entry.type === 'bearer' && jwtExpPast(entry.token, now);
|
|
36
|
+
if (fileExpired || jwtExpired) {
|
|
37
|
+
return {
|
|
38
|
+
class: 'AUTH_EXPIRED',
|
|
39
|
+
rationale: `HTTP 401 captured; auth file for role '${role}' is past expiry. Run: bun run xera:auth-setup --role ${role}`,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return null;
|
|
44
|
+
}
|