@hasna/testers 0.0.28 → 0.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -154
- package/README.md +92 -0
- package/dist/cli/index.js +1354 -79
- package/dist/db/database.d.ts.map +1 -1
- package/dist/db/personas.d.ts.map +1 -1
- package/dist/db/runs.d.ts +29 -0
- package/dist/db/runs.d.ts.map +1 -1
- package/dist/db/scenarios.d.ts +12 -0
- package/dist/db/scenarios.d.ts.map +1 -1
- package/dist/db/sessions.d.ts +36 -0
- package/dist/db/sessions.d.ts.map +1 -0
- package/dist/db/step-results.d.ts +30 -0
- package/dist/db/step-results.d.ts.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +818 -25
- package/dist/lib/a11y-audit.d.ts +54 -0
- package/dist/lib/a11y-audit.d.ts.map +1 -0
- package/dist/lib/api-discovery.d.ts +46 -0
- package/dist/lib/api-discovery.d.ts.map +1 -0
- package/dist/lib/assertions.d.ts.map +1 -1
- package/dist/lib/auth-profiles.d.ts +16 -0
- package/dist/lib/auth-profiles.d.ts.map +1 -0
- package/dist/lib/auth-session-pool.d.ts +57 -0
- package/dist/lib/auth-session-pool.d.ts.map +1 -0
- package/dist/lib/batch-actions.d.ts +44 -0
- package/dist/lib/batch-actions.d.ts.map +1 -0
- package/dist/lib/browser-compat.d.ts +14 -0
- package/dist/lib/browser-compat.d.ts.map +1 -0
- package/dist/lib/browser.d.ts +6 -7
- package/dist/lib/browser.d.ts.map +1 -1
- package/dist/lib/ci.d.ts +12 -0
- package/dist/lib/ci.d.ts.map +1 -1
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/discovery.d.ts +33 -0
- package/dist/lib/discovery.d.ts.map +1 -0
- package/dist/lib/dom-mutation.d.ts +53 -0
- package/dist/lib/dom-mutation.d.ts.map +1 -0
- package/dist/lib/environment.d.ts +26 -0
- package/dist/lib/environment.d.ts.map +1 -0
- package/dist/lib/health-scan.d.ts +2 -1
- package/dist/lib/health-scan.d.ts.map +1 -1
- package/dist/lib/hybrid-runner.d.ts.map +1 -1
- package/dist/lib/junit-export.d.ts +24 -0
- package/dist/lib/junit-export.d.ts.map +1 -0
- package/dist/lib/network-mock.d.ts +38 -0
- package/dist/lib/network-mock.d.ts.map +1 -0
- package/dist/lib/offline-mode.d.ts +31 -0
- package/dist/lib/offline-mode.d.ts.map +1 -0
- package/dist/lib/pdf-export.d.ts +27 -0
- package/dist/lib/pdf-export.d.ts.map +1 -0
- package/dist/lib/performance.d.ts +65 -0
- package/dist/lib/performance.d.ts.map +1 -0
- package/dist/lib/pr-comment.d.ts +27 -0
- package/dist/lib/pr-comment.d.ts.map +1 -0
- package/dist/lib/preview-detect.d.ts +27 -0
- package/dist/lib/preview-detect.d.ts.map +1 -0
- package/dist/lib/prod-debug.d.ts +77 -0
- package/dist/lib/prod-debug.d.ts.map +1 -0
- package/dist/lib/recorder.d.ts +42 -0
- package/dist/lib/recorder.d.ts.map +1 -1
- package/dist/lib/repo-discovery.d.ts +102 -0
- package/dist/lib/repo-discovery.d.ts.map +1 -0
- package/dist/lib/repo-executor.d.ts +56 -0
- package/dist/lib/repo-executor.d.ts.map +1 -0
- package/dist/lib/responsive.d.ts +43 -0
- package/dist/lib/responsive.d.ts.map +1 -0
- package/dist/lib/runner.d.ts +1 -0
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/lib/scenario-chain.d.ts +52 -0
- package/dist/lib/scenario-chain.d.ts.map +1 -0
- package/dist/lib/templates.d.ts.map +1 -1
- package/dist/lib/webhooks.d.ts +3 -0
- package/dist/lib/webhooks.d.ts.map +1 -1
- package/dist/mcp/index.js +852 -74
- package/dist/sdk/index.d.ts +48 -0
- package/dist/sdk/index.d.ts.map +1 -0
- package/dist/server/index.js +276 -29
- package/dist/types/index.d.ts +66 -2
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SDK/Library API for programmatic use of open-testers.
|
|
3
|
+
* Use this as the entry point when integrating open-testers into
|
|
4
|
+
* Node.js/TypeScript projects without going through the MCP server.
|
|
5
|
+
*
|
|
6
|
+
* ```typescript
|
|
7
|
+
* import { createScenario, listScenarioTemplates, RunOptions } from "open-testers/sdk";
|
|
8
|
+
* ```
|
|
9
|
+
*/
|
|
10
|
+
export type { Scenario, Run, Result, CreateScenarioInput, UpdateScenarioInput, Persona, CreatePersonaInput, UpdatePersonaInput, PersonaAuth, AuthProfile, AuthStrategy, Assertion, AssertionType, ScenarioPriority, ResultStatus, ModelPreset, BrowserEngine, } from "../types/index.js";
|
|
11
|
+
export type { MockRule } from "../lib/network-mock.js";
|
|
12
|
+
export type { BatchAction, BatchActionResult } from "../lib/batch-actions.js";
|
|
13
|
+
export type { MutationEvent, MutationOptions } from "../lib/dom-mutation.js";
|
|
14
|
+
export type { WebVitals, PerformanceBudget, BudgetViolation, PerformanceResult } from "../lib/performance.js";
|
|
15
|
+
export type { A11yAuditResult, A11yAuditOptions, A11yViolation } from "../lib/a11y-audit.js";
|
|
16
|
+
export type { Environment, EnvironmentInfo } from "../lib/environment.js";
|
|
17
|
+
export type { ThrottleProfile } from "../lib/offline-mode.js";
|
|
18
|
+
export type { ChainOutput, ChainLink } from "../lib/scenario-chain.js";
|
|
19
|
+
export type { Webhook, WebhookPayload, ApiCheckWebhookPayload } from "../lib/webhooks.js";
|
|
20
|
+
export { createScenario, getScenario, getScenarioByShortId, listScenarios, updateScenario, deleteScenario, findStaleScenarios, } from "../db/scenarios.js";
|
|
21
|
+
export { getRun, listRuns, updateRun, countRuns, } from "../db/runs.js";
|
|
22
|
+
export { createResult, getResult, listResults, getResultsByRun, updateResult, } from "../db/results.js";
|
|
23
|
+
export { createStepResult, getStepResult, listStepResults, updateStepResult, } from "../db/step-results.js";
|
|
24
|
+
export { createPersona, getPersona, listPersonas, updatePersona, deletePersona, listAuthenticatedPersonas, savePersonaAuthCookies, } from "../db/personas.js";
|
|
25
|
+
export { getTemplate, listTemplateNames, SCENARIO_TEMPLATES, } from "../lib/templates.js";
|
|
26
|
+
export { generateHtmlReport, generateLatestReport, imageToBase64, } from "../lib/report.js";
|
|
27
|
+
export { saveHtmlReport, generatePdfReport, } from "../lib/pdf-export.js";
|
|
28
|
+
export { toJUnitXml, } from "../lib/junit-export.js";
|
|
29
|
+
export { DEVICE_PRESETS, setDevicePreset, setViewport, captureResponsiveScreenshots, isMobileViewport, listDevicePresets, } from "../lib/responsive.js";
|
|
30
|
+
export type { DevicePreset } from "../lib/responsive.js";
|
|
31
|
+
export { batchActions, hasBatchFailures, formatBatchResults, } from "../lib/batch-actions.js";
|
|
32
|
+
export { watchMutations, waitForElement, waitForElementRemoved, waitForText, snapshotDOM, compareSnapshots, extractElements, } from "../lib/dom-mutation.js";
|
|
33
|
+
export { collectPerformanceMetrics, collectWebVitals, checkBudget, formatPerformanceResult, DEFAULT_BUDGET, } from "../lib/performance.js";
|
|
34
|
+
export { runA11yAudit, hasA11yIssues, formatA11yResults, } from "../lib/a11y-audit.js";
|
|
35
|
+
export { evaluateAssertions, parseAssertionString, allAssertionsPassed, formatAssertionResults, } from "../lib/assertions.js";
|
|
36
|
+
export type { AssertionResult } from "../lib/assertions.js";
|
|
37
|
+
export { getEnvInfo, detectEnvironment, getEnvironmentOverride, } from "../lib/environment.js";
|
|
38
|
+
export { setupNetworkMocks, MockPresets, } from "../lib/network-mock.js";
|
|
39
|
+
export { goOffline, goOnline, testOfflineHandling, enableThrottling, disableThrottling, THROTTLE_PROFILES, } from "../lib/offline-mode.js";
|
|
40
|
+
export { applyChainOutput, resolveChain, extractChainOutput, hasChainDependency, } from "../lib/scenario-chain.js";
|
|
41
|
+
export { authenticateWithProfile, serializeProfile, deserializeProfile, } from "../lib/auth-profiles.js";
|
|
42
|
+
export { discoverApiEndpoints, generateApiScenarios, groupEndpoints, summarizeEndpoints, } from "../lib/api-discovery.js";
|
|
43
|
+
export { setBaseline, getBaseline, compareImages, compareRunScreenshots, formatVisualDiffTerminal, } from "../lib/visual-diff.js";
|
|
44
|
+
export type { VisualDiffResult } from "../lib/visual-diff.js";
|
|
45
|
+
export { startRunAsync, runSingleScenario, runBatch, runByFilter, onRunEvent, } from "../lib/runner.js";
|
|
46
|
+
export type { RunOptions, RunEvent, RunEventHandler } from "../lib/runner.js";
|
|
47
|
+
export { launchBrowser, getPage, closeBrowser, BrowserPool, launchBrowserEngine, installBrowser, } from "../lib/browser.js";
|
|
48
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sdk/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,YAAY,EACV,QAAQ,EACR,GAAG,EACH,MAAM,EACN,mBAAmB,EACnB,mBAAmB,EACnB,OAAO,EACP,kBAAkB,EAClB,kBAAkB,EAClB,WAAW,EACX,WAAW,EACX,YAAY,EACZ,SAAS,EACT,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,aAAa,GACd,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC9E,YAAY,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC7E,YAAY,EAAE,SAAS,EAAE,iBAAiB,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC9G,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC7F,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC1E,YAAY,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACvE,YAAY,EAAE,OAAO,EAAE,cAAc,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAI1F,OAAO,EACL,cAAc,EACd,WAAW,EACX,oBAAoB,EACpB,aAAa,EACb,cAAc,EACd,cAAc,EACd,kBAAkB,GACnB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EACL,MAAM,EACN,QAAQ,EACR,SAAS,EACT,SAAS,GACV,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,EACf,YAAY,GACb,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EACL,aAAa,EACb,UAAU,EACV,YAAY,EACZ,aAAa,EACb,aAAa,EACb,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,aAAa,GACd,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EACL,cAAc,EACd,iBAAiB,GAClB,MAAM,sBAAsB,CAAC;AAI9B,OAAO,EACL,UAAU,GACX,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EACL,cAAc,EACd,eAAe,EACf,WAAW,EACX,4BAA4B,EAC5B,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAIzD,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EACL,cAAc,EACd,cAAc,EACd,qBAAqB,EACrB,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,eAAe,GAChB,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,WAAW,EACX,uBAAuB,EACvB,cAAc,GACf,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EACL,YAAY,EACZ,aAAa,EACb,iBAAiB,GAClB,MAAM,sBAAsB,CAAC;AAI9B,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,sBAAsB,GACvB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAI5D,OAAO,EACL,UAAU,EACV,iBAAiB,EACjB,sBAAsB,GACvB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EACL,iBAAiB,EACjB,WAAW,GACZ,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EACL,SAAS,EACT,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EACL,gBAAgB,EAChB,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAIlC,OAAO,EACL,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,cAAc,EACd,kBAAkB,GACnB,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EACL,WAAW,EACX,WAAW,EACX,aAAa,EACb,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAI9D,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,WAAW,EACX,UAAU,GACX,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAI9E,OAAO,EACL,aAAa,EACb,OAAO,EACP,YAAY,EACZ,WAAW,EACX,mBAAmB,EACnB,cAAc,GACf,MAAM,mBAAmB,CAAC"}
|
package/dist/server/index.js
CHANGED
|
@@ -77,7 +77,8 @@ function scenarioFromRow(row) {
|
|
|
77
77
|
createdAt: row.created_at,
|
|
78
78
|
updatedAt: row.updated_at,
|
|
79
79
|
lastPassedAt: row.last_passed_at ?? null,
|
|
80
|
-
lastPassedUrl: row.last_passed_url ?? null
|
|
80
|
+
lastPassedUrl: row.last_passed_url ?? null,
|
|
81
|
+
parameters: row.parameters ? JSON.parse(row.parameters) : null
|
|
81
82
|
};
|
|
82
83
|
}
|
|
83
84
|
function runFromRow(row) {
|
|
@@ -97,7 +98,14 @@ function runFromRow(row) {
|
|
|
97
98
|
metadata: row.metadata ? JSON.parse(row.metadata) : null,
|
|
98
99
|
isBaseline: row.is_baseline === 1,
|
|
99
100
|
samples: row.samples ?? 1,
|
|
100
|
-
flakinessThreshold: row.flakiness_threshold ?? 0.95
|
|
101
|
+
flakinessThreshold: row.flakiness_threshold ?? 0.95,
|
|
102
|
+
prNumber: row.pr_number ?? null,
|
|
103
|
+
prTitle: row.pr_title ?? null,
|
|
104
|
+
prBranch: row.pr_branch ?? null,
|
|
105
|
+
prBaseBranch: row.pr_base_branch ?? null,
|
|
106
|
+
prCommitSha: row.pr_commit_sha ?? null,
|
|
107
|
+
prUrl: row.pr_url ?? null,
|
|
108
|
+
ghAppInstallationId: row.gh_app_installation_id ?? null
|
|
101
109
|
};
|
|
102
110
|
}
|
|
103
111
|
function resultFromRow(row) {
|
|
@@ -118,7 +126,8 @@ function resultFromRow(row) {
|
|
|
118
126
|
createdAt: row.created_at,
|
|
119
127
|
personaId: row.persona_id ?? null,
|
|
120
128
|
personaName: row.persona_name ?? null,
|
|
121
|
-
failureAnalysis: row.failure_analysis ? JSON.parse(row.failure_analysis) : null
|
|
129
|
+
failureAnalysis: row.failure_analysis ? JSON.parse(row.failure_analysis) : null,
|
|
130
|
+
harPath: row.har_path ?? null
|
|
122
131
|
};
|
|
123
132
|
}
|
|
124
133
|
function screenshotFromRow(row) {
|
|
@@ -210,7 +219,10 @@ function personaFromRow(row) {
|
|
|
210
219
|
email: row.auth_email,
|
|
211
220
|
password: row.auth_password,
|
|
212
221
|
loginPath: row.auth_login_path ?? "/login",
|
|
213
|
-
cookies: row.auth_cookies ? JSON.parse(row.auth_cookies) : null
|
|
222
|
+
cookies: row.auth_cookies ? JSON.parse(row.auth_cookies) : null,
|
|
223
|
+
strategy: row.auth_strategy ?? "form-login",
|
|
224
|
+
headers: row.auth_headers ? JSON.parse(row.auth_headers) : undefined,
|
|
225
|
+
customScript: row.auth_script ?? undefined
|
|
214
226
|
} : null
|
|
215
227
|
};
|
|
216
228
|
}
|
|
@@ -10141,6 +10153,43 @@ ALTER TABLE scenarios ADD COLUMN required_role TEXT;
|
|
|
10141
10153
|
machine_id TEXT,
|
|
10142
10154
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
10143
10155
|
);
|
|
10156
|
+
`,
|
|
10157
|
+
`
|
|
10158
|
+
ALTER TABLE results ADD COLUMN har_path TEXT;
|
|
10159
|
+
`,
|
|
10160
|
+
`
|
|
10161
|
+
ALTER TABLE scenarios ADD COLUMN parameters TEXT;
|
|
10162
|
+
`,
|
|
10163
|
+
`
|
|
10164
|
+
ALTER TABLE personas ADD COLUMN auth_strategy TEXT DEFAULT 'form-login';
|
|
10165
|
+
ALTER TABLE personas ADD COLUMN auth_headers TEXT;
|
|
10166
|
+
ALTER TABLE personas ADD COLUMN auth_script TEXT;
|
|
10167
|
+
`,
|
|
10168
|
+
`
|
|
10169
|
+
CREATE TABLE IF NOT EXISTS step_results (
|
|
10170
|
+
id TEXT PRIMARY KEY,
|
|
10171
|
+
result_id TEXT NOT NULL REFERENCES results(id) ON DELETE CASCADE,
|
|
10172
|
+
step_number INTEGER NOT NULL,
|
|
10173
|
+
action TEXT NOT NULL,
|
|
10174
|
+
status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('passed','failed','error','running','skipped')),
|
|
10175
|
+
tool_name TEXT,
|
|
10176
|
+
tool_input TEXT,
|
|
10177
|
+
tool_result TEXT,
|
|
10178
|
+
thinking TEXT,
|
|
10179
|
+
error TEXT,
|
|
10180
|
+
duration_ms INTEGER,
|
|
10181
|
+
screenshot_id TEXT REFERENCES screenshots(id),
|
|
10182
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
10183
|
+
);
|
|
10184
|
+
`,
|
|
10185
|
+
`
|
|
10186
|
+
ALTER TABLE runs ADD COLUMN pr_number INTEGER;
|
|
10187
|
+
ALTER TABLE runs ADD COLUMN pr_title TEXT;
|
|
10188
|
+
ALTER TABLE runs ADD COLUMN pr_branch TEXT;
|
|
10189
|
+
ALTER TABLE runs ADD COLUMN pr_base_branch TEXT;
|
|
10190
|
+
ALTER TABLE runs ADD COLUMN pr_commit_sha TEXT;
|
|
10191
|
+
ALTER TABLE runs ADD COLUMN pr_url TEXT;
|
|
10192
|
+
ALTER TABLE runs ADD COLUMN gh_app_installation_id TEXT;
|
|
10144
10193
|
`
|
|
10145
10194
|
];
|
|
10146
10195
|
});
|
|
@@ -10862,6 +10911,16 @@ async function launchBrowser(options) {
|
|
|
10862
10911
|
const headless = options?.headless ?? true;
|
|
10863
10912
|
const viewport = options?.viewport ?? DEFAULT_VIEWPORT;
|
|
10864
10913
|
try {
|
|
10914
|
+
if (engine === "playwright-firefox") {
|
|
10915
|
+
const { firefox } = await import("playwright");
|
|
10916
|
+
const browser = await firefox.launch({ headless });
|
|
10917
|
+
return browser;
|
|
10918
|
+
}
|
|
10919
|
+
if (engine === "playwright-webkit") {
|
|
10920
|
+
const { webkit } = await import("playwright");
|
|
10921
|
+
const browser = await webkit.launch({ headless });
|
|
10922
|
+
return browser;
|
|
10923
|
+
}
|
|
10865
10924
|
return await launchPlaywright({ headless, viewport });
|
|
10866
10925
|
} catch (error) {
|
|
10867
10926
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -10978,8 +11037,9 @@ async function installBrowser(engine) {
|
|
|
10978
11037
|
const { installLightpanda: installLightpanda2 } = await Promise.resolve().then(() => (init_browser_lightpanda(), exports_browser_lightpanda));
|
|
10979
11038
|
return installLightpanda2();
|
|
10980
11039
|
}
|
|
11040
|
+
const browserName = engine === "playwright-firefox" ? "firefox" : engine === "playwright-webkit" ? "webkit" : "chromium";
|
|
10981
11041
|
try {
|
|
10982
|
-
execSync(
|
|
11042
|
+
execSync(`bunx playwright install ${browserName}`, {
|
|
10983
11043
|
stdio: "inherit"
|
|
10984
11044
|
});
|
|
10985
11045
|
} catch (error) {
|
|
@@ -11111,7 +11171,7 @@ function getDefaultConfig() {
|
|
|
11111
11171
|
browser: {
|
|
11112
11172
|
headless: true,
|
|
11113
11173
|
viewport: { width: 1280, height: 720 },
|
|
11114
|
-
timeout:
|
|
11174
|
+
timeout: 120000
|
|
11115
11175
|
},
|
|
11116
11176
|
screenshots: {
|
|
11117
11177
|
dir: join8(getTestersDir(), "screenshots"),
|
|
@@ -11141,7 +11201,8 @@ function loadConfig() {
|
|
|
11141
11201
|
judgeModel: fileConfig.judgeModel,
|
|
11142
11202
|
judgeProvider: fileConfig.judgeProvider,
|
|
11143
11203
|
selfHeal: fileConfig.selfHeal ?? false,
|
|
11144
|
-
conversationsSpace: fileConfig.conversationsSpace
|
|
11204
|
+
conversationsSpace: fileConfig.conversationsSpace,
|
|
11205
|
+
prodDebug: fileConfig.prodDebug
|
|
11145
11206
|
};
|
|
11146
11207
|
const envModel = process.env["TESTERS_MODEL"];
|
|
11147
11208
|
if (envModel) {
|
|
@@ -12510,7 +12571,7 @@ var init_scan_issues = __esm(() => {
|
|
|
12510
12571
|
// src/server/index.ts
|
|
12511
12572
|
init_paths();
|
|
12512
12573
|
import { existsSync as existsSync10 } from "fs";
|
|
12513
|
-
import { join as
|
|
12574
|
+
import { join as join13 } from "path";
|
|
12514
12575
|
|
|
12515
12576
|
// node_modules/zod/v3/external.js
|
|
12516
12577
|
var exports_external = {};
|
|
@@ -16506,9 +16567,9 @@ function createScenario(input) {
|
|
|
16506
16567
|
const short_id = nextShortId(input.projectId);
|
|
16507
16568
|
const timestamp = now();
|
|
16508
16569
|
db2.query(`
|
|
16509
|
-
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, assertions, version, created_at, updated_at)
|
|
16510
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
16511
|
-
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, JSON.stringify(input.assertions ?? []), timestamp, timestamp);
|
|
16570
|
+
INSERT INTO scenarios (id, short_id, project_id, name, description, steps, tags, priority, model, timeout_ms, target_path, requires_auth, auth_config, metadata, assertions, parameters, version, created_at, updated_at)
|
|
16571
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
16572
|
+
`).run(id, short_id, input.projectId ?? null, input.name, input.description, JSON.stringify(input.steps ?? []), JSON.stringify(input.tags ?? []), input.priority ?? "medium", input.model ?? null, input.timeoutMs ?? null, input.targetPath ?? null, input.requiresAuth ? 1 : 0, input.authConfig ? JSON.stringify(input.authConfig) : null, input.metadata ? JSON.stringify(input.metadata) : null, JSON.stringify(input.assertions ?? []), input.parameters ? JSON.stringify(input.parameters) : null, timestamp, timestamp);
|
|
16512
16573
|
return getScenario(id);
|
|
16513
16574
|
}
|
|
16514
16575
|
function getScenario(id) {
|
|
@@ -16658,6 +16719,10 @@ function updateScenario(id, input, version) {
|
|
|
16658
16719
|
sets.push("assertions = ?");
|
|
16659
16720
|
params.push(JSON.stringify(input.assertions));
|
|
16660
16721
|
}
|
|
16722
|
+
if (input.parameters !== undefined) {
|
|
16723
|
+
sets.push("parameters = ?");
|
|
16724
|
+
params.push(JSON.stringify(input.parameters));
|
|
16725
|
+
}
|
|
16661
16726
|
if (sets.length === 0) {
|
|
16662
16727
|
return existing;
|
|
16663
16728
|
}
|
|
@@ -16723,9 +16788,9 @@ function createRun(input) {
|
|
|
16723
16788
|
const id = uuid();
|
|
16724
16789
|
const timestamp = now();
|
|
16725
16790
|
db2.query(`
|
|
16726
|
-
INSERT INTO runs (id, project_id, status, url, model, headed, parallel, total, passed, failed, started_at, finished_at, metadata, samples, flakiness_threshold)
|
|
16727
|
-
VALUES (?, ?, 'pending', ?, ?, ?, ?, 0, 0, 0, ?, NULL, ?, ?, ?)
|
|
16728
|
-
`).run(id, input.projectId ?? null, input.url, input.model, input.headed ? 1 : 0, input.parallel ?? 1, timestamp,
|
|
16791
|
+
INSERT INTO runs (id, project_id, status, url, model, headed, parallel, total, passed, failed, started_at, finished_at, metadata, samples, flakiness_threshold, pr_number, pr_title, pr_branch, pr_base_branch, pr_commit_sha, pr_url, gh_app_installation_id)
|
|
16792
|
+
VALUES (?, ?, 'pending', ?, ?, ?, ?, 0, 0, 0, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
16793
|
+
`).run(id, input.projectId ?? null, input.url, input.model, input.headed ? 1 : 0, input.parallel ?? 1, timestamp, JSON.stringify({}), input.samples ?? 1, input.flakinessThreshold ?? 0.95, input.prNumber ?? null, input.prTitle ?? null, input.prBranch ?? null, input.prBaseBranch ?? null, input.prCommitSha ?? null, input.prUrl ?? null, input.ghAppInstallationId ?? null);
|
|
16729
16794
|
return getRun(id);
|
|
16730
16795
|
}
|
|
16731
16796
|
function getRun(id) {
|
|
@@ -16753,6 +16818,14 @@ function listRuns(filter) {
|
|
|
16753
16818
|
conditions.push("status = ?");
|
|
16754
16819
|
params.push(filter.status);
|
|
16755
16820
|
}
|
|
16821
|
+
if (filter?.since) {
|
|
16822
|
+
conditions.push("started_at >= ?");
|
|
16823
|
+
params.push(filter.since);
|
|
16824
|
+
}
|
|
16825
|
+
if (filter?.until) {
|
|
16826
|
+
conditions.push("started_at <= ?");
|
|
16827
|
+
params.push(filter.until);
|
|
16828
|
+
}
|
|
16756
16829
|
let sql = "SELECT * FROM runs";
|
|
16757
16830
|
if (conditions.length > 0) {
|
|
16758
16831
|
sql += " WHERE " + conditions.join(" AND ");
|
|
@@ -16784,6 +16857,14 @@ function countRuns(filter) {
|
|
|
16784
16857
|
conditions.push("status = ?");
|
|
16785
16858
|
params.push(filter.status);
|
|
16786
16859
|
}
|
|
16860
|
+
if (filter?.since) {
|
|
16861
|
+
conditions.push("started_at >= ?");
|
|
16862
|
+
params.push(filter.since);
|
|
16863
|
+
}
|
|
16864
|
+
if (filter?.until) {
|
|
16865
|
+
conditions.push("started_at <= ?");
|
|
16866
|
+
params.push(filter.until);
|
|
16867
|
+
}
|
|
16787
16868
|
let sql = "SELECT COUNT(*) as count FROM runs";
|
|
16788
16869
|
if (conditions.length > 0)
|
|
16789
16870
|
sql += " WHERE " + conditions.join(" AND ");
|
|
@@ -17378,6 +17459,8 @@ async function runPipelineScenario(scenario, options) {
|
|
|
17378
17459
|
|
|
17379
17460
|
// src/lib/runner.ts
|
|
17380
17461
|
init_results();
|
|
17462
|
+
import { mkdirSync as mkdirSync6 } from "fs";
|
|
17463
|
+
import { join as join12 } from "path";
|
|
17381
17464
|
|
|
17382
17465
|
// src/lib/failure-analyzer.ts
|
|
17383
17466
|
function analyzeFailure(error, reasoning) {
|
|
@@ -17500,6 +17583,74 @@ function estimateRunCostCents(scenarioCount, model, samples = 1) {
|
|
|
17500
17583
|
return scenarioCount * costPerScenario * Math.max(1, samples);
|
|
17501
17584
|
}
|
|
17502
17585
|
|
|
17586
|
+
// src/db/step-results.ts
|
|
17587
|
+
init_database();
|
|
17588
|
+
function createStepResult(input) {
|
|
17589
|
+
const db2 = getDatabase();
|
|
17590
|
+
const id = uuid();
|
|
17591
|
+
const timestamp = now();
|
|
17592
|
+
db2.query(`
|
|
17593
|
+
INSERT INTO step_results (id, result_id, step_number, action, status, tool_name, tool_input, thinking, created_at)
|
|
17594
|
+
VALUES (?, ?, ?, ?, 'running', ?, ?, ?, ?)
|
|
17595
|
+
`).run(id, input.resultId, input.stepNumber, input.action, input.toolName ?? null, input.toolInput ? JSON.stringify(input.toolInput) : null, input.thinking ?? null, timestamp);
|
|
17596
|
+
return getStepResult(id);
|
|
17597
|
+
}
|
|
17598
|
+
function getStepResult(id) {
|
|
17599
|
+
const db2 = getDatabase();
|
|
17600
|
+
const row = db2.query("SELECT * FROM step_results WHERE id = ?").get(id);
|
|
17601
|
+
return row ? stepResultFromRow(row) : null;
|
|
17602
|
+
}
|
|
17603
|
+
function updateStepResult(id, updates) {
|
|
17604
|
+
const db2 = getDatabase();
|
|
17605
|
+
const existing = getStepResult(id);
|
|
17606
|
+
if (!existing)
|
|
17607
|
+
return null;
|
|
17608
|
+
const sets = [];
|
|
17609
|
+
const params = [];
|
|
17610
|
+
if (updates.status !== undefined) {
|
|
17611
|
+
sets.push("status = ?");
|
|
17612
|
+
params.push(updates.status);
|
|
17613
|
+
}
|
|
17614
|
+
if (updates.toolResult !== undefined) {
|
|
17615
|
+
sets.push("tool_result = ?");
|
|
17616
|
+
params.push(updates.toolResult);
|
|
17617
|
+
}
|
|
17618
|
+
if (updates.error !== undefined) {
|
|
17619
|
+
sets.push("error = ?");
|
|
17620
|
+
params.push(updates.error);
|
|
17621
|
+
}
|
|
17622
|
+
if (updates.durationMs !== undefined) {
|
|
17623
|
+
sets.push("duration_ms = ?");
|
|
17624
|
+
params.push(updates.durationMs);
|
|
17625
|
+
}
|
|
17626
|
+
if (updates.screenshotId !== undefined) {
|
|
17627
|
+
sets.push("screenshot_id = ?");
|
|
17628
|
+
params.push(updates.screenshotId);
|
|
17629
|
+
}
|
|
17630
|
+
if (sets.length === 0)
|
|
17631
|
+
return existing;
|
|
17632
|
+
params.push(id);
|
|
17633
|
+
db2.query(`UPDATE step_results SET ${sets.join(", ")} WHERE id = ?`).run(...params);
|
|
17634
|
+
return getStepResult(id);
|
|
17635
|
+
}
|
|
17636
|
+
function stepResultFromRow(row) {
|
|
17637
|
+
return {
|
|
17638
|
+
id: row.id,
|
|
17639
|
+
resultId: row.result_id,
|
|
17640
|
+
stepNumber: row.step_number,
|
|
17641
|
+
action: row.action,
|
|
17642
|
+
status: row.status,
|
|
17643
|
+
toolName: row.tool_name,
|
|
17644
|
+
toolInput: row.tool_input ? JSON.parse(row.tool_input) : null,
|
|
17645
|
+
toolResult: row.tool_result,
|
|
17646
|
+
thinking: row.thinking,
|
|
17647
|
+
error: row.error,
|
|
17648
|
+
durationMs: row.duration_ms,
|
|
17649
|
+
screenshotId: row.screenshot_id,
|
|
17650
|
+
createdAt: row.created_at
|
|
17651
|
+
};
|
|
17652
|
+
}
|
|
17653
|
+
|
|
17503
17654
|
// src/db/personas.ts
|
|
17504
17655
|
init_types();
|
|
17505
17656
|
init_database();
|
|
@@ -17509,9 +17660,9 @@ function createPersona(input) {
|
|
|
17509
17660
|
const short_id = shortUuid();
|
|
17510
17661
|
const timestamp = now();
|
|
17511
17662
|
db2.query(`
|
|
17512
|
-
INSERT INTO personas (id, short_id, project_id, name, description, role, instructions, traits, goals, behaviors, expertise_level, demographics, pain_points, metadata, enabled, auth_email, auth_password, auth_login_path, version, created_at, updated_at)
|
|
17513
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
17514
|
-
`).run(id, short_id, input.projectId ?? null, input.name, input.description ?? "", input.role, input.instructions ?? "", JSON.stringify(input.traits ?? []), JSON.stringify(input.goals ?? []), JSON.stringify(input.behaviors ?? []), input.expertiseLevel ?? "intermediate", JSON.stringify(input.demographics ?? {}), JSON.stringify(input.painPoints ?? []), input.metadata ? JSON.stringify(input.metadata) : "{}", input.enabled === false ? 0 : 1, input.authEmail ?? null, input.authPassword ?? null, input.authLoginPath ?? null, timestamp, timestamp);
|
|
17663
|
+
INSERT INTO personas (id, short_id, project_id, name, description, role, instructions, traits, goals, behaviors, expertise_level, demographics, pain_points, metadata, enabled, auth_email, auth_password, auth_login_path, auth_cookies, auth_strategy, auth_headers, auth_script, version, created_at, updated_at)
|
|
17664
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
17665
|
+
`).run(id, short_id, input.projectId ?? null, input.name, input.description ?? "", input.role, input.instructions ?? "", JSON.stringify(input.traits ?? []), JSON.stringify(input.goals ?? []), JSON.stringify(input.behaviors ?? []), input.expertiseLevel ?? "intermediate", JSON.stringify(input.demographics ?? {}), JSON.stringify(input.painPoints ?? []), input.metadata ? JSON.stringify(input.metadata) : "{}", input.enabled === false ? 0 : 1, input.authEmail ?? null, input.authPassword ?? null, input.authLoginPath ?? null, null, input.authStrategy ?? "form-login", input.authHeaders ? JSON.stringify(input.authHeaders) : null, input.authCustomScript ?? null, timestamp, timestamp);
|
|
17515
17666
|
return getPersona(id);
|
|
17516
17667
|
}
|
|
17517
17668
|
function getPersona(id) {
|
|
@@ -17629,6 +17780,18 @@ function updatePersona(id, updates, version) {
|
|
|
17629
17780
|
sets.push("auth_cookies = ?");
|
|
17630
17781
|
params.push(updates.authCookies ? JSON.stringify(updates.authCookies) : null);
|
|
17631
17782
|
}
|
|
17783
|
+
if (updates.authStrategy !== undefined) {
|
|
17784
|
+
sets.push("auth_strategy = ?");
|
|
17785
|
+
params.push(updates.authStrategy);
|
|
17786
|
+
}
|
|
17787
|
+
if (updates.authHeaders !== undefined) {
|
|
17788
|
+
sets.push("auth_headers = ?");
|
|
17789
|
+
params.push(JSON.stringify(updates.authHeaders));
|
|
17790
|
+
}
|
|
17791
|
+
if (updates.authCustomScript !== undefined) {
|
|
17792
|
+
sets.push("auth_script = ?");
|
|
17793
|
+
params.push(updates.authCustomScript);
|
|
17794
|
+
}
|
|
17632
17795
|
if (sets.length === 0) {
|
|
17633
17796
|
return existing;
|
|
17634
17797
|
}
|
|
@@ -18166,6 +18329,24 @@ function signPayload(body, secret) {
|
|
|
18166
18329
|
}
|
|
18167
18330
|
return `sha256=${Math.abs(hash).toString(16).padStart(16, "0")}`;
|
|
18168
18331
|
}
|
|
18332
|
+
function formatDiscordPayload(payload) {
|
|
18333
|
+
const isPassed = payload.run.status === "passed";
|
|
18334
|
+
const color = isPassed ? 2278750 : 15680580;
|
|
18335
|
+
return {
|
|
18336
|
+
username: "open-testers",
|
|
18337
|
+
embeds: [
|
|
18338
|
+
{
|
|
18339
|
+
title: `Test Run ${payload.run.status.toUpperCase()}`,
|
|
18340
|
+
color,
|
|
18341
|
+
description: `URL: ${payload.run.url}
|
|
18342
|
+
` + `Results: ${payload.run.passed}/${payload.run.total} passed` + (payload.run.failed > 0 ? ` (${payload.run.failed} failed)` : "") + (payload.schedule ? `
|
|
18343
|
+
Schedule: ${payload.schedule.name}` : ""),
|
|
18344
|
+
timestamp: payload.timestamp,
|
|
18345
|
+
footer: { text: "open-testers" }
|
|
18346
|
+
}
|
|
18347
|
+
]
|
|
18348
|
+
};
|
|
18349
|
+
}
|
|
18169
18350
|
function formatSlackPayload(payload) {
|
|
18170
18351
|
const status = payload.run.status === "passed" ? ":white_check_mark:" : ":x:";
|
|
18171
18352
|
const color = payload.run.status === "passed" ? "#22c55e" : "#ef4444";
|
|
@@ -18208,7 +18389,8 @@ async function dispatchWebhooks(event, run, schedule) {
|
|
|
18208
18389
|
if (!webhook.events.includes(event) && !webhook.events.includes("*"))
|
|
18209
18390
|
continue;
|
|
18210
18391
|
const isSlack = webhook.url.includes("hooks.slack.com");
|
|
18211
|
-
const
|
|
18392
|
+
const isDiscord = webhook.url.includes("discord.com/api/webhooks") || webhook.url.includes("discordapp.com/api/webhooks");
|
|
18393
|
+
const body = isSlack ? JSON.stringify(formatSlackPayload(payload)) : isDiscord ? JSON.stringify(formatDiscordPayload(payload)) : JSON.stringify(payload);
|
|
18212
18394
|
const headers = {
|
|
18213
18395
|
"Content-Type": "application/json"
|
|
18214
18396
|
};
|
|
@@ -18529,13 +18711,35 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
18529
18711
|
emit({ type: "scenario:start", scenarioId: scenario.id, scenarioName: scenario.name, resultId: result.id, runId });
|
|
18530
18712
|
let browser = null;
|
|
18531
18713
|
let page = null;
|
|
18714
|
+
let context = null;
|
|
18715
|
+
let harPath = null;
|
|
18532
18716
|
let stopNetworkLogging = null;
|
|
18533
18717
|
const networkErrors = [];
|
|
18534
18718
|
try {
|
|
18535
18719
|
browser = await launchBrowser({ headless: !(effectiveOptions.headed ?? false), engine: effectiveOptions.engine });
|
|
18536
|
-
|
|
18537
|
-
|
|
18538
|
-
|
|
18720
|
+
const useHar = effectiveOptions.engine !== "lightpanda" && effectiveOptions.engine !== "bun";
|
|
18721
|
+
if (useHar) {
|
|
18722
|
+
const testersDir = process.env["HASNA_TESTERS_DIR"] || join12(process.env["HOME"] || "", ".hasna", "testers");
|
|
18723
|
+
const harDir = join12(testersDir, "hars");
|
|
18724
|
+
mkdirSync6(harDir, { recursive: true });
|
|
18725
|
+
harPath = join12(harDir, `${result.id}.har`);
|
|
18726
|
+
const contextOptions = {
|
|
18727
|
+
viewport: config.browser.viewport,
|
|
18728
|
+
recordHar: { path: harPath, mode: "full" }
|
|
18729
|
+
};
|
|
18730
|
+
if (effectiveOptions.recordVideo) {
|
|
18731
|
+
const videoDir = join12(testersDir, "videos");
|
|
18732
|
+
mkdirSync6(videoDir, { recursive: true });
|
|
18733
|
+
contextOptions.recordVideo = { dir: videoDir, size: config.browser.viewport };
|
|
18734
|
+
}
|
|
18735
|
+
context = await browser.newContext(contextOptions);
|
|
18736
|
+
page = await context.newPage();
|
|
18737
|
+
} else {
|
|
18738
|
+
page = await getPage(browser, {
|
|
18739
|
+
viewport: config.browser.viewport,
|
|
18740
|
+
engine: effectiveOptions.engine
|
|
18741
|
+
});
|
|
18742
|
+
}
|
|
18539
18743
|
const targetUrl = scenario.targetPath ? `${options.url.replace(/\/$/, "")}${scenario.targetPath}` : options.url;
|
|
18540
18744
|
const scenarioTimeout = scenario.timeoutMs ?? options.timeout ?? config.browser.timeout ?? 60000;
|
|
18541
18745
|
registerSession({
|
|
@@ -18555,7 +18759,11 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
18555
18759
|
}
|
|
18556
18760
|
});
|
|
18557
18761
|
const consoleErrors = [];
|
|
18762
|
+
const consoleLogs = [];
|
|
18763
|
+
let currentStep = 0;
|
|
18558
18764
|
page.on("console", (msg) => {
|
|
18765
|
+
const logEntry = { step: currentStep > 0 ? currentStep : null, type: msg.type(), text: msg.text(), timestamp: Date.now() };
|
|
18766
|
+
consoleLogs.push(logEntry);
|
|
18559
18767
|
if (msg.type() === "error")
|
|
18560
18768
|
consoleErrors.push(msg.text());
|
|
18561
18769
|
});
|
|
@@ -18587,6 +18795,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
18587
18795
|
}
|
|
18588
18796
|
await page.goto(targetUrl, { timeout: Math.min(scenarioTimeout, 30000) });
|
|
18589
18797
|
const stepStartTimes = new Map;
|
|
18798
|
+
const stepResultIds = new Map;
|
|
18590
18799
|
const agentResult = await withTimeout(runAgentLoop({
|
|
18591
18800
|
client,
|
|
18592
18801
|
page,
|
|
@@ -18609,13 +18818,32 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
18609
18818
|
onStep: (stepEvent) => {
|
|
18610
18819
|
let stepDurationMs;
|
|
18611
18820
|
if (stepEvent.type === "tool_call") {
|
|
18821
|
+
currentStep = stepEvent.stepNumber;
|
|
18612
18822
|
stepStartTimes.set(stepEvent.stepNumber, Date.now());
|
|
18823
|
+
const stepResult = createStepResult({
|
|
18824
|
+
resultId: result.id,
|
|
18825
|
+
stepNumber: stepEvent.stepNumber,
|
|
18826
|
+
action: stepEvent.toolName ?? `step-${stepEvent.stepNumber}`,
|
|
18827
|
+
toolName: stepEvent.toolName,
|
|
18828
|
+
toolInput: stepEvent.toolInput,
|
|
18829
|
+
thinking: stepEvent.thinking
|
|
18830
|
+
});
|
|
18831
|
+
stepResultIds.set(stepEvent.stepNumber, stepResult.id);
|
|
18613
18832
|
} else if (stepEvent.type === "tool_result") {
|
|
18614
18833
|
const startTime = stepStartTimes.get(stepEvent.stepNumber);
|
|
18615
18834
|
if (startTime !== undefined) {
|
|
18616
18835
|
stepDurationMs = Date.now() - startTime;
|
|
18617
18836
|
stepStartTimes.delete(stepEvent.stepNumber);
|
|
18618
18837
|
}
|
|
18838
|
+
const stepResultId = stepResultIds.get(stepEvent.stepNumber);
|
|
18839
|
+
if (stepResultId) {
|
|
18840
|
+
const isSuccess = !stepEvent.toolResult?.toLowerCase().includes("error") && !stepEvent.toolResult?.toLowerCase().includes("failed");
|
|
18841
|
+
updateStepResult(stepResultId, {
|
|
18842
|
+
status: isSuccess ? "passed" : "failed",
|
|
18843
|
+
toolResult: stepEvent.toolResult,
|
|
18844
|
+
durationMs: stepDurationMs
|
|
18845
|
+
});
|
|
18846
|
+
}
|
|
18619
18847
|
}
|
|
18620
18848
|
emit({
|
|
18621
18849
|
type: `step:${stepEvent.type}`,
|
|
@@ -18664,7 +18892,7 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
18664
18892
|
durationMs: Date.now() - new Date(result.createdAt).getTime(),
|
|
18665
18893
|
tokensUsed: agentResult.tokensUsed,
|
|
18666
18894
|
costCents: estimateCost(model, agentResult.tokensUsed),
|
|
18667
|
-
metadata: networkErrors.length > 0 ? networkMeta :
|
|
18895
|
+
metadata: { consoleLogs, ...networkErrors.length > 0 ? networkMeta : {} }
|
|
18668
18896
|
});
|
|
18669
18897
|
if (agentResult.status === "failed" || agentResult.status === "error") {
|
|
18670
18898
|
const failureAnalysis = analyzeFailure(null, agentResult.reasoning ?? null);
|
|
@@ -18700,8 +18928,16 @@ async function runSingleScenario(scenario, runId, options) {
|
|
|
18700
18928
|
emit({ type: "scenario:error", scenarioId: scenario.id, scenarioName: scenario.name, error: errorMsg, runId });
|
|
18701
18929
|
return updatedResult;
|
|
18702
18930
|
} finally {
|
|
18703
|
-
if (
|
|
18704
|
-
|
|
18931
|
+
if (harPath) {
|
|
18932
|
+
try {
|
|
18933
|
+
updateResult(result.id, { metadata: { harPath } });
|
|
18934
|
+
} catch {}
|
|
18935
|
+
}
|
|
18936
|
+
if (browser) {
|
|
18937
|
+
try {
|
|
18938
|
+
await closeBrowser(browser, effectiveOptions.engine);
|
|
18939
|
+
} catch {}
|
|
18940
|
+
}
|
|
18705
18941
|
}
|
|
18706
18942
|
}
|
|
18707
18943
|
async function runBatch(scenarios, options) {
|
|
@@ -20229,7 +20465,7 @@ async function handleRequest(req) {
|
|
|
20229
20465
|
if (pathname === "/api/status" && method === "GET") {
|
|
20230
20466
|
const config = loadConfig();
|
|
20231
20467
|
getDatabase();
|
|
20232
|
-
const dbPath = process.env["HASNA_TESTERS_DB_PATH"] ?? process.env["TESTERS_DB_PATH"] ??
|
|
20468
|
+
const dbPath = process.env["HASNA_TESTERS_DB_PATH"] ?? process.env["TESTERS_DB_PATH"] ?? join13(getTestersDir(), "testers.db");
|
|
20233
20469
|
const scenarios = listScenarios();
|
|
20234
20470
|
const runs = listRuns();
|
|
20235
20471
|
return jsonResponse({
|
|
@@ -20888,7 +21124,7 @@ async function handleRequest(req) {
|
|
|
20888
21124
|
return jsonResponse({ routes, apiRoutes, totalCovered: coverageMap.size });
|
|
20889
21125
|
}
|
|
20890
21126
|
if (!pathname.startsWith("/api")) {
|
|
20891
|
-
const dashboardDir =
|
|
21127
|
+
const dashboardDir = join13(import.meta.dir, "..", "..", "dashboard", "dist");
|
|
20892
21128
|
if (!existsSync10(dashboardDir)) {
|
|
20893
21129
|
return new Response(`<!DOCTYPE html>
|
|
20894
21130
|
<html>
|
|
@@ -20907,7 +21143,7 @@ async function handleRequest(req) {
|
|
|
20907
21143
|
}
|
|
20908
21144
|
});
|
|
20909
21145
|
}
|
|
20910
|
-
const filePath =
|
|
21146
|
+
const filePath = join13(dashboardDir, pathname === "/" ? "index.html" : pathname);
|
|
20911
21147
|
if (existsSync10(filePath)) {
|
|
20912
21148
|
const file = Bun.file(filePath);
|
|
20913
21149
|
return new Response(file, {
|
|
@@ -20917,7 +21153,7 @@ async function handleRequest(req) {
|
|
|
20917
21153
|
}
|
|
20918
21154
|
});
|
|
20919
21155
|
}
|
|
20920
|
-
const indexPath =
|
|
21156
|
+
const indexPath = join13(dashboardDir, "index.html");
|
|
20921
21157
|
if (existsSync10(indexPath)) {
|
|
20922
21158
|
const file = Bun.file(indexPath);
|
|
20923
21159
|
return new Response(file, {
|
|
@@ -20931,8 +21167,19 @@ async function handleRequest(req) {
|
|
|
20931
21167
|
return errorResponse("Not found", 404);
|
|
20932
21168
|
}
|
|
20933
21169
|
var port = parseInt(process.env["TESTERS_PORT"] ?? "19450", 10);
|
|
21170
|
+
process.on("unhandledRejection", (reason) => {
|
|
21171
|
+
const msg = reason instanceof Error ? reason.stack ?? reason.message : String(reason);
|
|
21172
|
+
console.error(`[testers-serve] Unhandled promise rejection: ${msg}`);
|
|
21173
|
+
});
|
|
21174
|
+
process.on("uncaughtException", (err) => {
|
|
21175
|
+
console.error(`[testers-serve] Uncaught exception: ${err.stack ?? err.message}`);
|
|
21176
|
+
});
|
|
20934
21177
|
var server = Bun.serve({
|
|
20935
21178
|
port,
|
|
20936
|
-
fetch: handleRequest
|
|
21179
|
+
fetch: handleRequest,
|
|
21180
|
+
error(err) {
|
|
21181
|
+
console.error(`[testers-serve] Request error: ${err.stack ?? err.message}`);
|
|
21182
|
+
return errorResponse("Internal server error", 500);
|
|
21183
|
+
}
|
|
20937
21184
|
});
|
|
20938
21185
|
console.log(`Open Testers server running at http://localhost:${server.port}`);
|