@hasna/testers 0.0.8 → 0.0.11

package/dist/lib/runner.d.ts

@@ -5,13 +5,14 @@ export interface RunOptions {
     headed?: boolean;
     parallel?: number;
     timeout?: number;
+    retry?: number;
     projectId?: string;
     apiKey?: string;
     screenshotDir?: string;
     engine?: "playwright" | "lightpanda";
 }
 export interface RunEvent {
-    type: "scenario:start" | "scenario:pass" | "scenario:fail" | "scenario:error" | "screenshot:captured" | "run:complete" | "step:tool_call" | "step:tool_result" | "step:thinking";
+    type: "scenario:start" | "scenario:pass" | "scenario:fail" | "scenario:error" | "screenshot:captured" | "run:complete" | "step:tool_call" | "step:tool_result" | "step:thinking" | "scenario:timeout_warning";
     scenarioId?: string;
     scenarioName?: string;
     resultId?: string;
@@ -23,6 +24,11 @@ export interface RunEvent {
     toolResult?: string;
     thinking?: string;
     stepNumber?: number;
+    retryAttempt?: number;
+    maxRetries?: number;
+    stepDurationMs?: number;
+    timeoutMs?: number;
+    elapsedMs?: number;
 }
 export type RunEventHandler = (event: RunEvent) => void;
 export declare function onRunEvent(handler: RunEventHandler): void;

package/dist/lib/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAY/D,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,YAAY,GAAG,YAAY,CAAC;CACtC;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EACA,gBAAgB,GAChB,eAAe,GACf,eAAe,GACf,gBAAgB,GAChB,qBAAqB,GACrB,cAAc,GACd,gBAAgB,GAChB,kBAAkB,GAClB,eAAe,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;AAIxD,wBAAgB,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,CAEzD;AAkBD,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,QAAQ,EAClB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,MAAM,CAAC,CAmGjB;AAED,wBAAsB,QAAQ,CAC5B,SAAS,EAAE,QAAQ,EAAE,EACrB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAwH1C;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,UAAU,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GACnF,OAAO,CAAC;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAuB1C;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,UAAU,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GACnF;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,CAmF1C"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAa/D,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,YAAY,GAAG,YAAY,CAAC;CACtC;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EACA,gBAAgB,GAChB,eAAe,GACf,eAAe,GACf,gBAAgB,GAChB,qBAAqB,GACrB,cAAc,GACd,gBAAgB,GAChB,kBAAkB,GAClB,eAAe,GACf,0BAA0B,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;AAIxD,wBAAgB,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,CAEzD;AA+BD,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,QAAQ,EAClB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,MAAM,CAAC,CAiHjB;AAED,wBAAsB,QAAQ,CAC5B,SAAS,EAAE,QAAQ,EAAE,EACrB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAsI1C;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,UAAU,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GACnF,OAAO,CAAC;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAuB1C;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,UAAU,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GACnF;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,CAmF1C"}

package/dist/mcp/index.js

@@ -4813,7 +4813,10 @@ function listScenarios(filter) {
   if (conditions.length > 0) {
     sql += " WHERE " + conditions.join(" AND ");
   }
-  sql += " ORDER BY created_at DESC";
+  const sortField = filter?.sort ?? "date";
+  const sortDir = filter?.desc === false ? "ASC" : "DESC";
+  const orderByCol = sortField === "name" ? "name" : sortField === "priority" ? "CASE priority WHEN 'critical' THEN 0 WHEN 'high' THEN 1 WHEN 'medium' THEN 2 WHEN 'low' THEN 3 ELSE 4 END" : "created_at";
+  sql += ` ORDER BY ${orderByCol} ${sortDir}`;
   if (filter?.limit) {
     sql += " LIMIT ?";
     params.push(filter.limit);
@@ -4950,7 +4953,10 @@ function listRuns(filter) {
   if (conditions.length > 0) {
     sql += " WHERE " + conditions.join(" AND ");
   }
-  sql += " ORDER BY started_at DESC";
+  const sortField = filter?.sort ?? "date";
+  const sortDir = filter?.desc === false ? "ASC" : "DESC";
+  const orderByCol = sortField === "duration" ? "(CASE WHEN finished_at IS NULL THEN NULL ELSE (julianday(finished_at) - julianday(started_at)) * 86400000 END)" : sortField === "cost" ? "(SELECT COALESCE(SUM(cost_cents), 0) FROM results WHERE run_id = runs.id)" : "started_at";
+  sql += ` ORDER BY ${orderByCol} ${sortDir}`;
   if (filter?.limit) {
     sql += " LIMIT ?";
     params.push(filter.limit);
@@ -6272,6 +6278,38 @@ async function dispatchWebhooks(event, run, schedule) {
   }
 }
 
+// src/lib/logs-integration.ts
+async function pushFailedRunToLogs(run, failedResults, scenarios) {
+  const logsUrl = process.env.LOGS_URL;
+  if (!logsUrl)
+    return;
+  const scenarioMap = new Map(scenarios.map((s) => [s.id, s]));
+  const entries = failedResults.map((result) => {
+    const scenario = scenarioMap.get(result.scenarioId);
+    return {
+      level: "error",
+      source: "sdk",
+      service: "testers",
+      message: `[testers] Scenario failed: ${scenario?.name ?? result.scenarioId}${result.error ? ` \u2014 ${result.error}` : ""}`,
+      metadata: {
+        run_id: run.id,
+        scenario_id: result.scenarioId,
+        scenario_name: scenario?.name,
+        url: run.url,
+        status: result.status,
+        duration_ms: result.durationMs
+      }
+    };
+  });
+  try {
+    await fetch(`${logsUrl.replace(/\/$/, "")}/api/logs`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(entries)
+    });
+  } catch {}
+}
+
 // src/lib/runner.ts
 var eventHandler = null;
 function emit(event) {
@@ -6280,14 +6318,26 @@ function emit(event) {
 }
 function withTimeout(promise, ms, label) {
   return new Promise((resolve, reject) => {
+    const warningAt = Math.floor(ms * 0.8);
+    const warningTimer = setTimeout(() => {
+      emit({
+        type: "scenario:timeout_warning",
+        scenarioName: label,
+        timeoutMs: ms,
+        elapsedMs: warningAt
+      });
+    }, warningAt);
     const timer = setTimeout(() => {
-      reject(new Error(`Scenario timeout after ${ms}ms: ${label}`));
+      clearTimeout(warningTimer);
+      reject(new Error(`Scenario '${label}' timed out after ${ms}ms. Try: testers run --timeout ${ms * 2} or simplify the scenario steps.`));
     }, ms);
     promise.then((val) => {
       clearTimeout(timer);
+      clearTimeout(warningTimer);
       resolve(val);
     }, (err) => {
       clearTimeout(timer);
+      clearTimeout(warningTimer);
       reject(err);
     });
   });
@@ -6316,6 +6366,7 @@ async function runSingleScenario(scenario, runId, options) {
     const targetUrl = scenario.targetPath ? `${options.url.replace(/\/$/, "")}${scenario.targetPath}` : options.url;
     const scenarioTimeout = scenario.timeoutMs ?? options.timeout ?? config.browser.timeout ?? 60000;
     await page.goto(targetUrl, { timeout: Math.min(scenarioTimeout, 30000) });
+    const stepStartTimes = new Map;
     const agentResult = await withTimeout(runAgentLoop({
       client,
       page,
@@ -6325,6 +6376,16 @@ async function runSingleScenario(scenario, runId, options) {
       runId,
       maxTurns: 30,
       onStep: (stepEvent) => {
+        let stepDurationMs;
+        if (stepEvent.type === "tool_call") {
+          stepStartTimes.set(stepEvent.stepNumber, Date.now());
+        } else if (stepEvent.type === "tool_result") {
+          const startTime = stepStartTimes.get(stepEvent.stepNumber);
+          if (startTime !== undefined) {
+            stepDurationMs = Date.now() - startTime;
+            stepStartTimes.delete(stepEvent.stepNumber);
+          }
+        }
         emit({
           type: `step:${stepEvent.type}`,
           scenarioId: scenario.id,
@@ -6334,7 +6395,8 @@ async function runSingleScenario(scenario, runId, options) {
           toolInput: stepEvent.toolInput,
           toolResult: stepEvent.toolResult,
           thinking: stepEvent.thinking,
-          stepNumber: stepEvent.stepNumber
+          stepNumber: stepEvent.stepNumber,
+          stepDurationMs
         });
       }
     }), scenarioTimeout, scenario.name);
@@ -6414,6 +6476,7 @@ async function runBatch(scenarios, options) {
     } catch {}
     return true;
   };
+  const maxRetries = options.retry ?? 0;
   if (parallel <= 1) {
     for (const scenario of sortedScenarios) {
       if (!await canRun(scenario)) {
@@ -6424,7 +6487,13 @@ async function runBatch(scenarios, options) {
         emit({ type: "scenario:error", scenarioId: scenario.id, scenarioName: scenario.name, error: "Dependency failed \u2014 skipped", runId: run.id });
         continue;
       }
-      const result = await runSingleScenario(scenario, run.id, options);
+      let result = await runSingleScenario(scenario, run.id, options);
+      let attempt = 1;
+      while ((result.status === "failed" || result.status === "error") && attempt <= maxRetries) {
+        emit({ type: "scenario:start", scenarioId: scenario.id, scenarioName: scenario.name, runId: run.id, retryAttempt: attempt + 1, maxRetries: maxRetries + 1 });
+        result = await runSingleScenario(scenario, run.id, options);
+        attempt++;
+      }
       results.push(result);
       if (result.status === "failed" || result.status === "error") {
         failedScenarioIds.add(scenario.id);
@@ -6471,6 +6540,10 @@ async function runBatch(scenarios, options) {
   emit({ type: "run:complete", runId: run.id });
   const eventType = finalRun.status === "failed" ? "failed" : "completed";
   dispatchWebhooks(eventType, finalRun).catch(() => {});
+  if (finalRun.status === "failed") {
+    const failedResults = results.filter((r) => r.status === "failed" || r.status === "error");
+    pushFailedRunToLogs(finalRun, failedResults, scenarios).catch(() => {});
+  }
   return { run: finalRun, results };
 }
 async function runByFilter(options) {