@sagentix-inc/evidence-collector 0.4.0 → 0.4.1

package/collector/src/sources/directory.js

@@ -128,8 +128,45 @@ export function jumpcloudDirectory(api) {
         },
     };
 }
+// One directory per identity provider for the whole run (2026-11-07): BambooHR and Rippling joined with the same provider
+// share it, and a leaver both name is looked up once. Keyed by the run's clients object, which every source of one run
+// shares (cli.ts) and each test builds afresh.
+const directories = new WeakMap();
+/** A directory whose answers are kept per email for the rest of the run; an error is not kept, so the next asker tries again. */
+export function memoDirectory(d) {
+    const seen = new Map();
+    return {
+        idp: d.idp,
+        lookup(email) {
+            const k = email.toLowerCase();
+            let p = seen.get(k);
+            if (!p) {
+                p = d.lookup(email).then((a) => { if (a.state === "error")
+                    seen.delete(k); return a; });
+                seen.set(k, p);
+            }
+            return p;
+        },
+    };
+}
 /** The directory for an identity provider, built from the client a test injected or that provider's own variables; else why not. */
 export function directoryFor(idp, ctx) {
+    const key = `${idp}\n${String(ctx.opts["jumpcloud-region"] ?? "")}\n${String(ctx.opts["jumpcloud-org-id"] ?? "")}`;
+    let byIdp = directories.get(ctx.clients);
+    if (!byIdp) {
+        byIdp = new Map();
+        directories.set(ctx.clients, byIdp);
+    }
+    let d = byIdp.get(key);
+    if (d === undefined) {
+        d = buildDirectory(idp, ctx);
+        if (typeof d !== "string")
+            d = memoDirectory(d);
+        byIdp.set(key, d);
+    }
+    return d;
+}
+function buildDirectory(idp, ctx) {
     const sleep = ctx.clients.sleep ?? realSleep;
     const pick = (key, build) => {
         const given = ctx.clients[key];

package/collector/src/sources/entra.js

@@ -80,7 +80,7 @@
 //
 // Personal data: results carry counts and Entra object ids, never user principal names or names.
 import { readFileSync } from "node:fs";
-import { cachedToken, entraClientAssertion, tokenRequest } from "./oauth.js";
+import { cachedToken, entraClientAssertion, sharedToken, tokenRequest } from "./oauth.js";
 import { DAY_MS, Emitter, arr, enc, fetchRest, firstIds, getAll, intOpt, isObj, ok, realSleep, redact, str, why, withRetry, listRead } from "./readonly_http.js";
 export const GRAPH_BASE = "https://graph.microsoft.com/v1.0";
 export const ENTRA_ENV = { tenant: "ENTRA_TENANT_ID", client: "ENTRA_CLIENT_ID", secret: "ENTRA_CLIENT_SECRET", cert: "ENTRA_CLIENT_CERT_FILE" };
@@ -109,7 +109,8 @@ export function entraFromEnv(env, fetchFn = fetch, target = { base: GRAPH_BASE,
     if (!tenant || !client || (!secret && !certFile))
         return `${ENTRA_ENV.tenant}, ${ENTRA_ENV.client} and ${ENTRA_ENV.secret} or ${ENTRA_ENV.cert} are not all set`;
     const url = `https://login.microsoftonline.com/${enc(tenant)}/oauth2/v2.0/token`;
-    const token = cachedToken(() => {
+    // Shared with every other client of this app and scope in the run (Intune, the HR leaver join): one token request.
+    const token = sharedToken(fetchFn, ["entra", tenant, client, target.scope, secret ?? "", certFile ?? ""], () => cachedToken(() => {
         const form = { grant_type: "client_credentials", client_id: client, scope: target.scope };
         if (certFile) {
             form.client_assertion_type = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer";
@@ -118,7 +119,7 @@ export function entraFromEnv(env, fetchFn = fetch, target = { base: GRAPH_BASE,
         else
             form.client_secret = secret;
         return tokenRequest(url, form, fetchFn);
-    });
+    }));
     return fetchRest({ base: target.base, fetchFn, headers: async () => ({ authorization: `Bearer ${await token()}` }) });
 }
 const enabled = (p) => p.state === "enabled";

package/collector/src/sources/google_oauth.js

@@ -19,7 +19,7 @@
 import { randomBytes, createHash } from "node:crypto";
 import { closeSync, fstatSync, openSync, readFileSync, renameSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { cachedToken, tokenRequest } from "./oauth.js";
+import { cachedToken, sharedToken, tokenRequest } from "./oauth.js";
 export const GOOGLE_AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth";
 export const GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token";
 export const OAUTH_ENV = { clientFile: "GOOGLE_OAUTH_CLIENT_FILE", tokenFile: "GOOGLE_OAUTH_TOKEN_FILE" };
@@ -113,7 +113,7 @@ export function googleOAuthFromEnv(env, scopes, fetchFn = fetch) {
     const refresh = readTokenFile(tokenPath(env), client, scopes);
     if (typeof refresh !== "string")
         return refresh.error;
-    return cachedToken(async () => {
+    return sharedToken(fetchFn, ["google-user", clientFile, client.clientId, refresh, ...scopes], () => cachedToken(async () => {
         try {
             return await tokenRequest(GOOGLE_TOKEN_URL, {
                 grant_type: "refresh_token", refresh_token: refresh, client_id: client.clientId,
@@ -125,5 +125,5 @@ export function googleOAuthFromEnv(env, scopes, fetchFn = fetch) {
             // invalid_grant: revoked, unused for six months, or pushed out by newer tokens; only a new sign-in mends it.
             throw new Error(/invalid_grant/.test(m) ? `${m}: the sign-in was revoked or has expired; run \`collector google-login\` again` : m);
         }
-    });
+    }));
 }

package/collector/src/sources/google_workspace.js

@@ -22,7 +22,7 @@
 // GOOGLE_ADMIN_EMAIL): an install whose domain-wide delegation lacks the scope, or whose admin is not a super
 // admin, gets drive_external_sharing_restricted skipped and keeps every other check.
 import { readFileSync } from "node:fs";
-import { cachedToken, googleAssertion, tokenRequest } from "./oauth.js";
+import { cachedToken, googleAssertion, sharedToken, tokenRequest } from "./oauth.js";
 import { OAUTH_ENV, googleOAuthFromEnv } from "./google_oauth.js";
 import { DAY_MS, Emitter, enc, fetchRest, firstIds, getAll, intOpt, isObj, ok, realSleep, redact, str, why, withRetry, listRead, stampRead } from "./readonly_http.js";
 export const ADMIN_BASE = "https://admin.googleapis.com";
@@ -77,10 +77,11 @@ export function googleFromEnv(env, fetchFn = fetch) {
     // The key file names its own token endpoint; a tampered file could send the signed assertion elsewhere.
     if (!isGoogleTokenUri(tokenUri))
         return `${GOOGLE_ENV.keyFile} names a token_uri outside googleapis.com over HTTPS`;
-    const token = cachedToken(() => tokenRequest(tokenUri, {
+    // Shared with the HR leaver join, which builds this same client (directory.ts): one token request per run.
+    const token = sharedToken(fetchFn, ["google-sa", tokenUri, key.client_email, admin, ...GOOGLE_SCOPES], () => cachedToken(() => tokenRequest(tokenUri, {
         grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
         assertion: googleAssertion(key, admin, GOOGLE_SCOPES, Math.floor(Date.now() / 1000)),
-    }, fetchFn));
+    }, fetchFn)));
     return fetchRest({ base: ADMIN_BASE, fetchFn, headers: async () => ({ authorization: `Bearer ${await token()}` }) });
 }
 /** A client for the Cloud Identity Policy API with a token that holds only POLICY_SCOPE. */

package/collector/src/sources/oauth.js

@@ -71,3 +71,25 @@ export function cachedToken(fetchToken) {
         return cur.token;
     };
 }
+// One token per credential and scope for the whole run, whichever source asks first (2026-11-07): Entra ID, Intune and
+// an HR source joined with Entra ID used to make three identical client-credentials requests. Keyed by the fetch
+// function the clients share (one per run, src/report.ts hold()) and a SHA-256 of the credential's parts, so no secret
+// is kept as a key and a test's fake fetch never sees another test's token.
+const shared = new WeakMap();
+/** The token getter for these credential parts, made once per fetch function by `make`. */
+export function sharedToken(fetchFn, parts, make) {
+    let byKey = shared.get(fetchFn);
+    if (!byKey) {
+        byKey = new Map();
+        shared.set(fetchFn, byKey);
+    }
+    const key = createHash("sha256").update(parts.join("\u0000")).digest("hex");
+    let t = byKey.get(key);
+    if (!t) {
+        const getter = make();
+        // A failed request is not kept: the next source asks again (and says why on its own).
+        t = () => getter().catch((e) => { byKey.delete(key); throw e; });
+        byKey.set(key, t);
+    }
+    return t;
+}

package/collector/src/sources/prowler.js

@@ -15,11 +15,12 @@
 // The collector never passes a Prowler option that writes, fixes or sends anywhere (Security Hub, S3,
 // Slack, Shodan, Prowler Cloud, secret validation), and refuses one given in --prowler-args.
 import { spawn } from "node:child_process";
-import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
+import { accessSync, constants, createReadStream, mkdtempSync, readdirSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { DEFAULT_CATALOG_DIR, loadCatalog } from "../catalog.js";
 import { scrub } from "./shared-cloud.js";
+import { pool } from "../pool.js";
 export const PROWLER_PROVIDERS = ["aws", "azure", "gcp", "kubernetes", "m365", "github", "googleworkspace", "okta", "cloudflare"];
 /** The version the catalogues were built from (apps/sos-app/scripts/build_prowler_catalog.py). */
 export const PROWLER_VERSION = "5.47.0";
@@ -115,37 +116,33 @@ export function outcomeOf(f) {
     return code === "PASS" ? "pass" : code === "FAIL" ? "fail" : "skipped";
 }
 /**
- * Maps a Prowler OCSF array (Detection Findings) to results. Findings whose check the catalogue does not list
- * are dropped and counted. Several findings on one (check, resource) merge into one result: fail if any
- * failed, else pass if any passed, else skipped; their details are joined.
+ * Maps Prowler OCSF findings (Detection Findings), one at a time, to results. Findings whose check the catalogue does
+ * not list are dropped and counted. Several findings on one (check, resource) merge into one result: fail if any
+ * failed, else pass if any passed, else skipped; their details are joined (only as far as the 1,000 characters sent).
  */
-export function parseOcsf(text, provider, known) {
-    let doc;
-    try {
-        doc = JSON.parse(text);
-    }
-    catch (e) {
-        throw new Error(`the Prowler output is not JSON (${e.message}); pass a json-ocsf file.`);
-    }
-    if (!Array.isArray(doc))
-        throw new Error("the Prowler output is not an OCSF array of findings; pass a json-ocsf file.");
-    const summary = { findings: doc.length, kept: 0, dropped: 0, droppedChecks: [], muted: 0, manual: 0, merged: 0 };
-    const dropped = new Set();
-    const byKey = new Map();
-    for (const f of doc) {
+class OcsfMerger {
+    summary = { findings: 0, kept: 0, dropped: 0, droppedChecks: [], muted: 0, manual: 0, merged: 0 };
+    dropped = new Set();
+    byKey = new Map();
+    provider;
+    known;
+    constructor(provider, known) { this.provider = provider; this.known = known; }
+    add(f) {
+        this.summary.findings++;
         if (!isObj(f))
-            continue;
+            return;
+        const summary = this.summary;
         const meta = isObj(f.metadata) ? f.metadata : {};
         const info = isObj(f.finding_info) ? f.finding_info : {};
         const analytic = isObj(info.analytic) ? info.analytic : {};
         const checkId = s(meta.event_code) || s(analytic.uid);
         const unmapped = isObj(f.unmapped) ? f.unmapped : {};
-        const prov = s(unmapped.provider) || provider;
+        const prov = s(unmapped.provider) || this.provider;
         const check = `prowler.${prov}.${checkId}`;
-        if (!checkId || !known.has(check)) {
+        if (!checkId || !this.known.has(check)) {
             summary.dropped++;
-            dropped.add(check);
-            continue;
+            this.dropped.add(check);
+            return;
         }
         if (s(f.status) === "Suppressed")
             summary.muted++;
@@ -165,22 +162,142 @@ export function parseOcsf(text, provider, known) {
         const detail = (s(f.status_detail) || s(f.message)).replace(EMAIL, "[email]");
         const result = outcomeOf(f);
         const key = `${check}|${resource}`;
-        const prev = byKey.get(key);
+        const prev = this.byKey.get(key);
         if (!prev) {
-            byKey.set(key, { r: { check, resource, result, detail: "" }, details: detail ? [detail] : [] });
-            continue;
+            this.byKey.set(key, { r: { check, resource, result, detail: "" }, details: detail ? [detail] : [], len: detail.length });
+            return;
         }
         summary.merged++;
         const rank = (o) => (o === "fail" ? 2 : o === "pass" ? 1 : 0);
         if (rank(result) > rank(prev.r.result))
             prev.r.result = result;
-        if (detail && !prev.details.includes(detail))
+        // Past 1,000 characters a further detail cannot change what is sent, so it is not kept (a large account merges
+        // thousands of findings into one result).
+        if (detail && prev.len < 1000 && !prev.details.includes(detail)) {
+            prev.len += (prev.details.length ? 3 : 0) + detail.length;
             prev.details.push(detail);
+        }
+    }
+    finish() {
+        const results = [...this.byKey.values()].map(({ r, details }) => ({ ...r, detail: details.join(" | ").slice(0, 1000) }));
+        this.summary.kept = results.length;
+        this.summary.droppedChecks = [...this.dropped].sort();
+        return { results, summary: this.summary };
     }
-    const results = [...byKey.values()].map(({ r, details }) => ({ ...r, detail: details.join(" | ").slice(0, 1000) }));
-    summary.kept = results.length;
-    summary.droppedChecks = [...dropped].sort();
-    return { results, summary };
+}
+/** The output is not OCSF: the file is wrong, whichever way it was made. */
+export class OcsfError extends Error {
+}
+const NOT_JSON = (why) => new OcsfError(`the Prowler output is not JSON (${why}); pass a json-ocsf file.`);
+const NOT_ARRAY = () => new OcsfError("the Prowler output is not an OCSF array of findings; pass a json-ocsf file.");
+/** Maps a Prowler OCSF array held in memory (tests, small files). */
+export function parseOcsf(text, provider, known) {
+    let doc;
+    try {
+        doc = JSON.parse(text);
+    }
+    catch (e) {
+        throw NOT_JSON(e.message);
+    }
+    if (!Array.isArray(doc))
+        throw NOT_ARRAY();
+    const m = new OcsfMerger(provider, known);
+    for (const f of doc)
+        m.add(f);
+    return m.finish();
+}
+const WS = (c) => c === 0x20 || c === 0x0a || c === 0x0d || c === 0x09;
+/**
+ * The elements of a top-level JSON array, as text, read chunk by chunk: only one element is held at a time, so a
+ * Prowler output of any size never becomes one string (V8 caps a string near 512 MiB; a large AWS organization's
+ * OCSF file passes that). Strings and escapes are tracked so brackets inside them do not count. No regular expression.
+ */
+export async function* jsonArrayElements(chunks) {
+    let buf = "", i = 0, start = -1, depth = 0, inStr = false, esc = false, state = "pre", afterComma = false;
+    for await (const chunk of chunks) {
+        buf += chunk;
+        for (; i < buf.length; i++) {
+            const c = buf.charCodeAt(i);
+            if (state === "pre") {
+                if (WS(c) || c === 0xfeff)
+                    continue;
+                if (c === 0x5b /* [ */) {
+                    state = "in";
+                    continue;
+                }
+                throw NOT_ARRAY();
+            }
+            if (state === "post") {
+                if (WS(c))
+                    continue;
+                throw NOT_JSON(`text after the closing bracket at offset ${i}`);
+            }
+            if (inStr) {
+                if (esc)
+                    esc = false;
+                else if (c === 0x5c /* \ */)
+                    esc = true;
+                else if (c === 0x22 /* " */)
+                    inStr = false;
+                continue;
+            }
+            if (c === 0x22) {
+                inStr = true;
+                if (start < 0)
+                    start = i;
+                continue;
+            }
+            if (depth === 0) {
+                if (c === 0x2c /* , */ || c === 0x5d /* ] */) {
+                    if (start >= 0) {
+                        yield buf.slice(start, i);
+                        start = -1;
+                        afterComma = c === 0x2c;
+                    }
+                    else if (c === 0x2c || afterComma)
+                        throw NOT_JSON("an empty array element");
+                    if (c === 0x5d)
+                        state = "post";
+                    continue;
+                }
+                if (WS(c))
+                    continue;
+                if (start < 0)
+                    start = i;
+            }
+            if (c === 0x7b /* { */ || c === 0x5b)
+                depth++;
+            else if (c === 0x7d /* } */ || c === 0x5d) {
+                if (--depth < 0)
+                    throw NOT_JSON("unbalanced brackets");
+            }
+        }
+        // Drop what has been read; keep the element in progress.
+        const keep = start >= 0 ? start : i;
+        buf = buf.slice(keep);
+        i -= keep;
+        if (start >= 0)
+            start = 0;
+    }
+    if (state === "pre")
+        throw NOT_ARRAY();
+    if (state !== "post")
+        throw NOT_JSON("the file ends inside the array");
+}
+/** Maps a Prowler OCSF file, streamed (see jsonArrayElements). */
+export async function parseOcsfFile(path, provider, known) {
+    const m = new OcsfMerger(provider, known);
+    for await (const el of jsonArrayElements(createReadStream(path, { encoding: "utf8", highWaterMark: 1 << 20 }))) {
+        let f;
+        try {
+            f = JSON.parse(el);
+        }
+        catch (e) {
+            throw NOT_JSON(e.message);
+        }
+        m.add(f);
+    }
+    return m.finish();
 }
 export const spawnProwler = (bin, args, env) => new Promise((resolve, reject) => {
     const p = spawn(bin, args, { env, stdio: ["ignore", "ignore", "pipe"] });
@@ -189,7 +306,7 @@ export const spawnProwler = (bin, args, env) => new Promise((resolve, reject) =>
     p.on("error", reject);
     p.on("close", (code) => resolve({ code, stderr: err }));
 });
-/** Runs Prowler into a fresh directory and returns the OCSF text. A non-zero exit is an error. */
+/** Runs Prowler into a fresh directory and maps its OCSF file, streamed, before the directory goes. A non-zero exit is an error. */
 export async function runProwler(provider, o) {
     const bad = o.extra.filter((a) => REFUSED_ARGS.includes(a.split("=")[0]));
     if (bad.length)
@@ -210,12 +327,14 @@ export async function runProwler(provider, o) {
         const file = readdirSync(dir).find((f) => f.endsWith(".ocsf.json"));
         if (!file)
             throw new Error(`${o.bin} ${provider} wrote no .ocsf.json in ${dir}.`);
-        return readFileSync(join(dir, file), "utf8");
+        return await parseOcsfFile(join(dir, file), provider, o.known);
     }
     finally {
         rmSync(dir, { recursive: true, force: true });
     }
 }
+/** Prowler scans run side by side, at most this many at once (each is a Python process of its own). */
+export const PROWLER_PARALLEL = 2;
 // ── Source ──────────────────────────────────────────────────────────────────────────────────────────
 export const prowler = {
     name: "prowler",
@@ -240,36 +359,43 @@ export const prowler = {
         if (file && providers.length > 1)
             throw new Error("--prowler-output reads one provider's file; name one --prowler-provider with it.");
         const known = new Set(catalogIds(one(ctx.opts.catalog) ?? DEFAULT_CATALOG_DIR));
-        const results = [];
-        const failures = [];
         for (const provider of providers) {
             if (![...known].some((id) => id.startsWith(`prowler.${provider}.`))) {
                 ctx.warn(`prowler: sos-app's catalogue lists no ${provider} check; every finding will be dropped.`);
             }
-            let text;
-            try {
-                if (file) {
-                    try {
-                        text = readFileSync(file, "utf8");
-                    }
-                    catch (e) {
-                        throw new Error(`cannot read --prowler-output ${file}: ${e.code ?? String(e)}`);
-                    }
+        }
+        // Each provider read or scanned, PROWLER_PARALLEL at a time; the outcomes are then taken in the order given, so the
+        // results and their order are those of one provider after another.
+        const one_ = async (provider) => {
+            if (file) {
+                try {
+                    accessSync(file, constants.R_OK);
                 }
-                else {
-                    const spawner = ctx.clients.prowler ?? spawnProwler;
-                    const extra = one(ctx.opts["prowler-args"]) ? splitArgs(one(ctx.opts["prowler-args"])) : [];
-                    text = await runProwler(provider, { bin: one(ctx.opts["prowler-bin"]) ?? "prowler", extra, env: prowlerEnv(provider, ctx.env), spawner });
+                catch (e) {
+                    throw new Error(`cannot read --prowler-output ${file}: ${e.code ?? String(e)}`);
                 }
+                return parseOcsfFile(file, provider, known);
             }
-            catch (e) {
+            const spawner = ctx.clients.prowler ?? spawnProwler;
+            const extra = one(ctx.opts["prowler-args"]) ? splitArgs(one(ctx.opts["prowler-args"])) : [];
+            return runProwler(provider, { bin: one(ctx.opts["prowler-bin"]) ?? "prowler", extra, env: prowlerEnv(provider, ctx.env), spawner, known });
+        };
+        const outcomes = await pool(providers, PROWLER_PARALLEL, one_);
+        const results = [];
+        const failures = [];
+        for (const [i, o] of outcomes.entries()) {
+            const provider = providers[i];
+            if (o.status === "rejected") {
+                // An output that is not OCSF is the file's fault, not the scan's: it stops the source, as it always has.
+                if (o.reason instanceof OcsfError)
+                    throw o.reason;
                 // A second way that fails leaves the control to its other ways: say so and go on with the next provider.
-                failures.push(e);
+                failures.push(o.reason);
                 if (providers.length > 1)
-                    ctx.warn(`prowler: ${provider} did not run (${e.message}); its controls keep their other ways.`);
+                    ctx.warn(`prowler: ${provider} did not run (${o.reason.message}); its controls keep their other ways.`);
                 continue;
             }
-            const { results: r, summary } = parseOcsf(text, provider, known);
+            const { results: r, summary } = o.value;
             results.push(...r);
             ctx.warn(`prowler: ${summary.findings} findings; ${summary.kept} results (${summary.merged} merged into the same check and resource, ` +
                 `${summary.muted} muted and ${summary.manual} manual sent as skipped); ${summary.dropped} findings dropped from ${summary.droppedChecks.length} checks not in the catalogue` +

package/collector/src/sources/readonly_http.js

@@ -2,6 +2,18 @@
 // GitHub): a GET-only JSON client, retry on rate limits, pagination, "skipped" details and token
 // redaction. The only non-GET request any of these sources makes is the OAuth token request in
 // oauth.ts, to the identity provider's token endpoint.
+import { channel } from "node:diagnostics_channel";
+/** The recorder (src/report.ts RETRY_CHANNEL) counts retries per source from here; a plain string, so this file needs no import of it. */
+const retries = channel("sagentix:collector:retry");
+/** Tells the run's recorder that a call is about to be retried (a no-op when nothing listens). */
+export function announceRetry(status, waitMs, attempt) {
+    if (retries.hasSubscribers)
+        retries.publish({ status, waitMs, attempt });
+}
+/** Gateway and availability errors worth one or two more tries; a 500 is usually the request itself and is not retried. */
+export const TRANSIENT_STATUS = new Set([502, 503, 504]);
+/** Retries after a transient error (a 502/503/504 or a network failure), on top of the rate-limit retries. */
+export const TRANSIENT_RETRIES = 2;
 export const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
 export const str = (v) => (typeof v === "string" ? v : undefined);
 export const arr = (v) => (Array.isArray(v) ? v : []);
@@ -71,21 +83,30 @@ export function retryDelayMs(h, attempt, nowMs) {
         return Math.min(cap, Math.max(1000, reset * 1000 - nowMs + 1000));
     return Math.min(cap, 1000 * 2 ** attempt);
 }
-/** Retries a rate-limited GET up to `max` times; a thrown network error becomes status 0. */
+/**
+ * Retries a rate-limited GET up to `max` times, and a transient failure (502, 503, 504, or a thrown network error or
+ * timeout) up to TRANSIENT_RETRIES times, waiting Retry-After when the answer gives one. A thrown error becomes status 0.
+ */
 export function withRetry(api, sleep = realSleep, max = 4, now = Date.now) {
     return {
         async get(url) {
+            let transient = 0;
             for (let attempt = 0;; attempt++) {
-                let r;
+                let r, thrown = false;
                 try {
                     r = await api.get(url);
                 }
                 catch (e) {
                     r = { status: 0, headers: {}, body: { message: `request failed: ${String(e)}` } };
+                    thrown = true;
                 }
-                if (!isRateLimited(r) || attempt >= max)
+                const rate = isRateLimited(r);
+                const again = rate ? attempt < max : (thrown || TRANSIENT_STATUS.has(r.status)) && transient++ < TRANSIENT_RETRIES;
+                if (!again)
                     return r;
-                await sleep(retryDelayMs(r.headers, attempt, now()));
+                const wait = retryDelayMs(r.headers, rate ? attempt : transient - 1, now());
+                announceRetry(r.status, wait, attempt + 1);
+                await sleep(wait);
             }
         },
     };
@@ -120,7 +141,8 @@ export function why(r) {
     msg = msg.replace(/\s+/g, " ").trim().slice(0, 300);
     const hint = r.status === 401 || r.status === 403
         ? " (the credential may lack the read permission for this call, or the feature is not licensed)"
-        : r.status === 429 ? " (rate limited; retries exhausted)" : "";
+        : r.status === 429 ? " (rate limited; retries exhausted)"
+            : TRANSIENT_STATUS.has(r.status) ? " (the provider is unavailable; retried, try the run again later)" : "";
     return `HTTP ${r.status}${msg ? `: ${msg}` : ""}${hint}`;
 }
 /**

package/collector/src/sources/shared-cloud.js

@@ -1,6 +1,6 @@
 // Helpers shared by the AWS, Azure, GCP, Jira and LMS sources: option lists, retry with backoff on
 // throttling, a result builder, and a scrubber that keeps secret values out of every detail.
-import { redact } from "./readonly_http.js";
+import { TRANSIENT_RETRIES, TRANSIENT_STATUS, announceRetry, redact } from "./readonly_http.js";
 export const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
 export const arr = (v) => (Array.isArray(v) ? v : []);
 export const str = (v) => (typeof v === "string" ? v : undefined);
@@ -20,25 +20,33 @@ export function retryAfterMs(headers, now = Date.now()) {
 }
 /**
  * Calls `send` until it gives a reply that is not throttled, or the attempts run out (then the last
- * throttled reply is returned, and the caller turns it into "skipped"). A thrown error is a reply
- * with status 0.
+ * throttled reply is returned, and the caller turns it into "skipped"). A transient failure (502 or 504,
+ * or a thrown network error or timeout) is tried again up to TRANSIENT_RETRIES times within the same
+ * attempts. A thrown error is a reply with status 0.
  */
 export async function withRetry(send, o = {}) {
     const attempts = o.attempts ?? 5, base = o.baseMs ?? 500, max = o.maxMs ?? 20_000;
     const sleep = o.sleep ?? realSleep;
     const throttled = o.throttled ?? ((r) => r.status === 429 || r.status === 503);
     let last = { status: 0, headers: {}, body: "" };
+    let transient = 0;
     for (let i = 0; i < attempts; i++) {
+        let thrown = false;
         try {
             last = await send();
         }
         catch (e) {
             last = { status: 0, headers: {}, body: `request failed: ${String(e)}` };
+            thrown = true;
         }
-        if (!throttled(last))
+        const again = throttled(last) || ((thrown || TRANSIENT_STATUS.has(last.status)) && transient++ < TRANSIENT_RETRIES);
+        if (!again)
             return last;
-        if (i < attempts - 1)
-            await sleep(Math.min(max, retryAfterMs(last.headers) ?? base * 2 ** i));
+        if (i < attempts - 1) {
+            const wait = Math.min(max, retryAfterMs(last.headers) ?? base * 2 ** i);
+            announceRetry(last.status, wait, i + 1);
+            await sleep(wait);
+        }
     }
     return last;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sagentix-inc/evidence-collector",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "Customer-side evidence collector: reads systems read-only, runs checks, signs the results and pushes them outbound to sos-app.",
   "type": "module",
   "engines": {