@forkpoint/agent-lighthouse-core 4.0.0 → 4.1.0

package/dist/index.mjs

@@ -5,13 +5,14 @@ var ORIGIN_EVIDENCE_VERSION = "v1";
 var DEFAULT_ORIGIN_CACHE_TTL_MS = 60 * 60 * 1e3;
 var DEFAULT_ORIGIN_CACHE_MAX_ENTRIES = 256;
 var REQUEST_TIMEOUT_MS = 1e4;
-var SCAN_TIMEOUT_MS = 6e4;
+var SCAN_TIMEOUT_MS = 18e4;
 var MAX_RESPONSE_BODY_BYTES = 5 * 1024 * 1024;
 var MAX_CONCURRENT_REQUESTS = 10;
 var SCANNER_USER_AGENT = "AgentLighthouse/1.0 (+https://github.com/ForkPoint/agent-lighthouse)";
 var TAG_SKIPPED_PAGE_TYPE = "skipped:page-type";
 var TAG_SCAN_ERROR = "scan-error";
 var TAG_SKIPPED_NO_EVIDENCE = "skipped:no-evidence";
+var TAG_SKIPPED_SCAN_BUDGET = "skipped:scan-budget";
 var CATEGORY_NAMES = {
   "access-crawl-control": "Access & Crawl Control",
   "content-extraction": "Content Extraction",
@@ -1100,7 +1101,14 @@ var ScanConditionsSchema = z.object({
     informativeCount: z.number().int().nonnegative(),
     gatedCount: z.number().int().nonnegative(),
     reasons: z.record(z.string(), z.number().int().nonnegative())
-  })
+  }),
+  // Optional: reports written before the budget existed carry no block.
+  budget: z.object({
+    limitMs: z.number().nonnegative(),
+    elapsedMs: z.number().nonnegative(),
+    exhausted: z.boolean(),
+    skippedCount: z.number().int().nonnegative()
+  }).optional()
 });
 
 // src/audit.ts
@@ -1311,16 +1319,19 @@ function calculateOverallScore(categories) {
 }
 var GATED_MASS_UNSCORED_THRESHOLD = 0.35;
 function gatedMassShare(checks2) {
-  let gated = 0;
+  return skippedMassShare(checks2, TAG_SKIPPED_NO_EVIDENCE);
+}
+function skippedMassShare(checks2, tag2) {
+  let skipped = 0;
   let total = 0;
   for (const check of checks2) {
     if (isInformative(check)) continue;
     const mass = check.weight ?? 0;
     if (mass <= 0) continue;
     total += mass;
-    if (check.tags?.includes(TAG_SKIPPED_NO_EVIDENCE)) gated += mass;
+    if (check.tags?.includes(tag2)) skipped += mass;
   }
-  return total === 0 ? 0 : gated / total;
+  return total === 0 ? 0 : skipped / total;
 }
 
 // src/audits/access-crawl-control/no-nofollow.ts
@@ -40630,6 +40641,7 @@ function outcomeOf(check) {
   if (tags.includes(TAG_SCAN_ERROR)) return "error";
   if (tags.includes(TAG_SKIPPED_PAGE_TYPE)) return "skipped";
   if (tags.includes(TAG_SKIPPED_NO_EVIDENCE)) return "gated";
+  if (tags.includes(TAG_SKIPPED_SCAN_BUDGET)) return "budget";
   return "ran";
 }
 function traceFromCheck(check, durationMs) {
@@ -40782,9 +40794,28 @@ function planAudits(ctx, config, options = {}) {
   }
   return { runnable, skipped };
 }
-async function runAudits(ctx, config, onEvent, plan, onTrace) {
+function formatBudget(ms) {
+  return ms >= 1e3 ? `${Math.round(ms / 1e3)} s` : `${ms} ms`;
+}
+function budgetReason(signal) {
+  const reason = signal.reason;
+  if (reason instanceof Error && !(reason instanceof DOMException) && reason.message)
+    return reason.message;
+  return "The scan budget ran out.";
+}
+async function runAudits(ctx, config, onEvent, plan, onTrace, budget) {
   const { runnable, skipped } = plan ?? planAudits(ctx, config);
   const allChecks = [...skipped];
+  const budgetStub = (reg2) => {
+    const label3 = `${reg2.meta.id} ${reg2.meta.title}`;
+    const stub = stubCheck(
+      reg2.meta,
+      TAG_SKIPPED_SCAN_BUDGET,
+      `Not assessed: ${budgetReason(budget)} This audit had not started.`
+    );
+    if (typeof onEvent === "function") onEvent({ type: "unit:done", label: label3 });
+    return stub;
+  };
   const tracing = Boolean(onTrace) || logger.level === "debug";
   const trace = (check, durationMs) => {
     if (!tracing) return;
@@ -40795,9 +40826,22 @@ async function runAudits(ctx, config, onEvent, plan, onTrace) {
   for (const stub of skipped) trace(stub, 0);
   const batchSize = 20;
   for (let i = 0; i < runnable.length; i += batchSize) {
+    if (budget?.aborted) {
+      for (const { reg: reg2 } of runnable.slice(i)) {
+        const stub = budgetStub(reg2);
+        trace(stub, 0);
+        allChecks.push(stub);
+      }
+      break;
+    }
     const batch = runnable.slice(i, i + batchSize);
     const batchResults = await Promise.all(
       batch.map(async ({ reg: reg2, scopedPages, scoreDisplayMode }) => {
+        if (budget?.aborted) {
+          const stub = budgetStub(reg2);
+          trace(stub, 0);
+          return stub;
+        }
         const label3 = `${reg2.meta.id} ${reg2.meta.title}`;
         const startedAt = tracing ? performance.now() : 0;
         const elapsed = () => tracing ? Math.round(performance.now() - startedAt) : 0;
@@ -40805,6 +40849,17 @@ async function runAudits(ctx, config, onEvent, plan, onTrace) {
           const instance = reg2.create();
           const scopedCtx = scopedPages && scopedPages !== ctx.pages ? { ...ctx, pages: scopedPages, cacheOwner: cacheOwner(ctx) } : ctx;
           const result = await instance.audit(scopedCtx);
+          if (budget?.aborted) {
+            const stub = stubCheck(
+              reg2.meta,
+              TAG_SKIPPED_SCAN_BUDGET,
+              `Not assessed: ${budgetReason(budget)} This audit was still running.`
+            );
+            if (typeof onEvent === "function")
+              onEvent({ type: "unit:done", label: label3 });
+            trace(stub, elapsed());
+            return stub;
+          }
           const check = instance.toCheckResult(result, scoreDisplayMode);
           if (typeof onEvent === "function")
             onEvent({ type: "unit:done", label: label3 });
@@ -41134,8 +41189,22 @@ var A11Y_MAX_PAGES = Math.max(
 );
 var RATE_LIMIT_BACKOFF_MS = 5e3;
 var MAX_RETRY_AFTER_MS = 3e4;
-async function fetchPageWithRetry(fetcher, url, signal) {
-  const first5 = await fetcher.fetch({ url, signal });
+function unsentResult(url, error) {
+  return {
+    url,
+    finalUrl: url,
+    status: 0,
+    headers: {},
+    body: "",
+    ttfbMs: 0,
+    totalMs: 0,
+    contentType: "",
+    contentLength: 0,
+    error
+  };
+}
+async function fetchPageWithRetry(fetcher, url, fetchSignal, cancel) {
+  const first5 = await fetcher.fetch({ url, signal: fetchSignal });
   if (first5.status !== 429) return first5;
   const header = Number(first5.headers["retry-after"]);
   const waitMs = Number.isFinite(header) && header > 0 ? Math.min(header * 1e3, MAX_RETRY_AFTER_MS) : RATE_LIMIT_BACKOFF_MS;
@@ -41143,14 +41212,46 @@ async function fetchPageWithRetry(fetcher, url, signal) {
     { url, waitMs },
     `[orchestrator] Page answered 429; retrying once in ${waitMs}ms`
   );
-  await new Promise((resolve4) => setTimeout(resolve4, waitMs));
-  signal?.throwIfAborted();
-  return fetcher.fetch({ url, signal });
+  if (!fetchSignal?.aborted)
+    await new Promise((resolve4) => {
+      const timer = setTimeout(done, waitMs);
+      function done() {
+        clearTimeout(timer);
+        fetchSignal?.removeEventListener("abort", done);
+        resolve4();
+      }
+      fetchSignal?.addEventListener("abort", done, { once: true });
+    });
+  cancel?.throwIfAborted();
+  if (fetchSignal?.aborted) return first5;
+  return fetcher.fetch({ url, signal: fetchSignal });
 }
 async function runScan(url, options) {
+  const limitMs = options?.timeoutMs ?? SCAN_TIMEOUT_MS;
+  if (!Number.isFinite(limitMs) || limitMs < 0)
+    throw new RangeError(
+      `timeoutMs must be a non-negative number of milliseconds, got ${String(options?.timeoutMs)}`
+    );
+  const budget = new AbortController();
+  const timer = limitMs > 0 ? setTimeout(() => {
+    const message = `The scan budget of ${formatBudget(limitMs)} ran out.`;
+    logger.warn(`[orchestrator] ${message} ${splitCredentials(url).url}`);
+    budget.abort(new Error(message));
+  }, limitMs) : void 0;
+  try {
+    return await scanWithinBudget(url, options, {
+      signal: budget.signal,
+      limitMs
+    });
+  } finally {
+    if (timer !== void 0) clearTimeout(timer);
+  }
+}
+async function scanWithinBudget(url, options, budget) {
   const onEvent = options?.onEvent;
   const pageOverrides = options?.pages;
   const signal = options?.signal;
+  const fetchSignal = signal ? AbortSignal.any([signal, budget.signal]) : budget.signal;
   const tracker = new ProgressTracker((event) => onEvent?.(event));
   const start = performance.now();
   const fetcher = createFetcher({
@@ -41248,7 +41349,7 @@ async function runScan(url, options) {
           tracker.unitDone(path);
           return Promise.resolve(prefetchedRobots);
         }
-        return fetcher.fetch({ url: `${baseUrl}${path}`, signal }).then((result) => {
+        return fetcher.fetch({ url: `${baseUrl}${path}`, signal: fetchSignal }).then((result) => {
           tracker.unitDone(path);
           return result;
         });
@@ -41259,7 +41360,7 @@ async function runScan(url, options) {
       rootFiles[path] = rootResults[i];
     });
     const isHomepageScan = url === `${baseUrl}/` || url === baseUrl;
-    originHomepageResult = isHomepageScan ? void 0 : await fetchPageWithRetry(fetcher, `${baseUrl}/`, signal);
+    originHomepageResult = isHomepageScan ? void 0 : await fetchPageWithRetry(fetcher, `${baseUrl}/`, fetchSignal, signal);
     originReadAt = (/* @__PURE__ */ new Date()).toISOString();
     originFresh = true;
     tracker.phaseDone();
@@ -41268,7 +41369,12 @@ async function runScan(url, options) {
   signal?.throwIfAborted();
   logger.debug("[orchestrator] Phase 2: Fetching page");
   tracker.phaseStart("fetch-pages", 1 + overrideUrls.length);
-  const pageResult = await fetchPageWithRetry(fetcher, url, signal);
+  const pageResult = await fetchPageWithRetry(
+    fetcher,
+    url,
+    fetchSignal,
+    signal
+  );
   tracker.unitDone(displayUrl);
   if (!originHomepageResult && (url === `${baseUrl}/` || url === baseUrl)) {
     originHomepageResult = pageResult;
@@ -41284,7 +41390,7 @@ async function runScan(url, options) {
   }
   const extraResults = await Promise.all(
     overrideUrls.map(
-      (pageUrl) => fetcher.fetch({ url: pageUrl, signal }).then((result) => {
+      (pageUrl) => fetcher.fetch({ url: pageUrl, signal: fetchSignal }).then((result) => {
         tracker.unitDone(pageUrl);
         return result;
       })
@@ -41362,7 +41468,11 @@ async function runScan(url, options) {
     pages,
     domain,
     baseUrl,
-    fetch: (options2) => fetcher.fetch({ ...options2, signal }),
+    // Once the budget is gone no request leaves: the audit reads an error
+    // result at once, the same shape a refused connection gives it.
+    fetch: (options2) => budget.signal.aborted ? Promise.resolve(
+      unsentResult(options2.url, budgetReason(budget.signal))
+    ) : fetcher.fetch({ ...options2, signal: fetchSignal }),
     wafProtection: wafProtection ?? void 0,
     evidence,
     originEvidence: {
@@ -41393,9 +41503,12 @@ async function runScan(url, options) {
       else tracker.unitFail(event.label, event.error);
     },
     auditPlan,
-    options?.onAuditTrace
+    options?.onAuditTrace,
+    budget.signal
   );
   tracker.phaseDone();
+  signal?.throwIfAborted();
+  const budgetExhausted = budget.signal.aborted;
   logger.debug("[orchestrator] Phase 3 complete: Audits finished");
   tracker.phaseStart("report", 1);
   logger.debug("[orchestrator] Phase 4: Building final report");
@@ -41421,7 +41534,11 @@ async function runScan(url, options) {
   );
   const gatedShare = gatedMassShare(allChecks);
   const escalated = gatedShare > GATED_MASS_UNSCORED_THRESHOLD;
-  const unscoredReason = !evidence.judgeable ? unjudgeableReason(evidence) : escalated ? `The scan could not feed ${Math.round(gatedShare * 100)}% of the registry's evidence mass, so what remains is not a reading of this site.` : void 0;
+  const budgetShare = skippedMassShare(allChecks, TAG_SKIPPED_SCAN_BUDGET);
+  const unassessedShare = gatedShare + budgetShare;
+  const cut = !escalated && budgetShare > 0 && unassessedShare > GATED_MASS_UNSCORED_THRESHOLD;
+  const budgetSentence = budgetExhausted ? budgetReason(budget.signal) : "";
+  const unscoredReason = !evidence.judgeable ? budgetExhausted ? `${budgetSentence} ${unjudgeableReason(evidence)}` : unjudgeableReason(evidence) : escalated ? `The scan could not feed ${Math.round(gatedShare * 100)}% of the registry's evidence mass, so what remains is not a reading of this site.` : cut ? `${budgetSentence} ${Math.round(unassessedShare * 100)}% of the registry's evidence mass was never assessed, so what remains is not a reading of this site.` : void 0;
   const scored = unscoredReason === void 0;
   const allScoredAudits = Object.values(config.audits).flat().filter((a) => a.meta.tier === "scored");
   let registryMass = 0;
@@ -41456,6 +41573,8 @@ async function runScan(url, options) {
         unscoredReasons["skipped-no-evidence"] = (unscoredReasons["skipped-no-evidence"] ?? 0) + 1;
       } else if (check.tags?.includes(TAG_SKIPPED_PAGE_TYPE)) {
         unscoredReasons["skipped-page-type"] = (unscoredReasons["skipped-page-type"] ?? 0) + 1;
+      } else if (check.tags?.includes(TAG_SKIPPED_SCAN_BUDGET)) {
+        unscoredReasons["skipped-scan-budget"] = (unscoredReasons["skipped-scan-budget"] ?? 0) + 1;
       } else {
         unscoredReasons["not-applicable"] = (unscoredReasons["not-applicable"] ?? 0) + 1;
       }
@@ -41493,6 +41612,17 @@ async function runScan(url, options) {
       informativeCount,
       gatedCount,
       reasons: unscoredReasons
+    },
+    // Counted over every check, advisory ones included: the reasons map
+    // files an informative stub under `informative`, but the budget cut it
+    // all the same.
+    budget: {
+      limitMs: budget.limitMs,
+      elapsedMs: durationMs,
+      exhausted: budgetExhausted,
+      skippedCount: allChecks.filter(
+        (c) => c.tags?.includes(TAG_SKIPPED_SCAN_BUDGET)
+      ).length
     }
   };
   const report = {
@@ -41728,9 +41858,11 @@ export {
   TAG_SCAN_ERROR,
   TAG_SKIPPED_NO_EVIDENCE,
   TAG_SKIPPED_PAGE_TYPE,
+  TAG_SKIPPED_SCAN_BUDGET,
   allEvidenceMet,
   allJsonLdNodes,
   boundedDispatcher,
+  budgetReason,
   buildCategoryResult,
   buildScanEvidence,
   calculateCategoryScore,
@@ -41762,6 +41894,7 @@ export {
   extractStylesheetUrls,
   filterConfig,
   flattenJsonLd,
+  formatBudget,
   formatTrace,
   getMainContentText,
   getPreset,
@@ -41799,6 +41932,7 @@ export {
   runScan,
   sampleEntries,
   shouldBypassOriginCache,
+  skippedMassShare,
   stripBom,
   topLevelJsonLd,
   traceFromCheck,