@sagentix-inc/evidence-collector 0.3.5 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +137 -9
  2. package/apps/sos-app/src/data/checks/aws.json +12 -0
  3. package/apps/sos-app/src/data/checks/azure.json +18 -0
  4. package/apps/sos-app/src/data/checks/bamboohr.json +40 -0
  5. package/apps/sos-app/src/data/checks/entra.json +1 -1
  6. package/apps/sos-app/src/data/checks/gcp.json +12 -0
  7. package/apps/sos-app/src/data/checks/intune.json +7 -1
  8. package/apps/sos-app/src/data/checks/jamf.json +8 -2
  9. package/apps/sos-app/src/data/checks/kandji.json +7 -1
  10. package/apps/sos-app/src/data/checks/rippling.json +40 -0
  11. package/apps/sos-app/src/data/checks/vulnscan.json +27 -0
  12. package/apps/sos-app/src/data/checks/web.json +31 -1
  13. package/collector/package.json +1 -1
  14. package/collector/src/cli.js +344 -37
  15. package/collector/src/config.js +12 -0
  16. package/collector/src/doctor.js +382 -0
  17. package/collector/src/policies.js +160 -0
  18. package/collector/src/pool.js +19 -0
  19. package/collector/src/report.js +88 -16
  20. package/collector/src/sources/aws/checks.js +64 -0
  21. package/collector/src/sources/aws.js +6 -1
  22. package/collector/src/sources/azure/api.js +4 -0
  23. package/collector/src/sources/azure/checks.js +91 -1
  24. package/collector/src/sources/azure.js +8 -2
  25. package/collector/src/sources/bamboohr.js +133 -0
  26. package/collector/src/sources/cloudflare/api.js +24 -4
  27. package/collector/src/sources/device_common.js +42 -0
  28. package/collector/src/sources/directory.js +198 -0
  29. package/collector/src/sources/engines.js +2 -1
  30. package/collector/src/sources/entra.js +4 -3
  31. package/collector/src/sources/gcp/project.js +47 -3
  32. package/collector/src/sources/gcp.js +5 -2
  33. package/collector/src/sources/github.js +46 -8
  34. package/collector/src/sources/google_oauth.js +3 -3
  35. package/collector/src/sources/google_workspace.js +4 -3
  36. package/collector/src/sources/hris_common.js +189 -0
  37. package/collector/src/sources/index.js +3 -0
  38. package/collector/src/sources/intune.js +40 -3
  39. package/collector/src/sources/jamf.js +65 -8
  40. package/collector/src/sources/kandji.js +46 -7
  41. package/collector/src/sources/oauth.js +22 -0
  42. package/collector/src/sources/prowler.js +178 -52
  43. package/collector/src/sources/readonly_http.js +27 -5
  44. package/collector/src/sources/rippling.js +114 -0
  45. package/collector/src/sources/shared-cloud.js +14 -6
  46. package/collector/src/sources/vulnscan.js +386 -0
  47. package/collector/src/sources/web.js +106 -11
  48. package/collector/src/sources/web_privacy.js +266 -0
  49. package/package.json +1 -1
@@ -15,11 +15,12 @@
15
15
  // The collector never passes a Prowler option that writes, fixes or sends anywhere (Security Hub, S3,
16
16
  // Slack, Shodan, Prowler Cloud, secret validation), and refuses one given in --prowler-args.
17
17
  import { spawn } from "node:child_process";
18
- import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
18
+ import { accessSync, constants, createReadStream, mkdtempSync, readdirSync, rmSync } from "node:fs";
19
19
  import { tmpdir } from "node:os";
20
20
  import { join } from "node:path";
21
21
  import { DEFAULT_CATALOG_DIR, loadCatalog } from "../catalog.js";
22
22
  import { scrub } from "./shared-cloud.js";
23
+ import { pool } from "../pool.js";
23
24
  export const PROWLER_PROVIDERS = ["aws", "azure", "gcp", "kubernetes", "m365", "github", "googleworkspace", "okta", "cloudflare"];
24
25
  /** The version the catalogues were built from (apps/sos-app/scripts/build_prowler_catalog.py). */
25
26
  export const PROWLER_VERSION = "5.47.0";
@@ -115,37 +116,33 @@ export function outcomeOf(f) {
115
116
  return code === "PASS" ? "pass" : code === "FAIL" ? "fail" : "skipped";
116
117
  }
117
118
  /**
118
- * Maps a Prowler OCSF array (Detection Findings) to results. Findings whose check the catalogue does not list
119
- * are dropped and counted. Several findings on one (check, resource) merge into one result: fail if any
120
- * failed, else pass if any passed, else skipped; their details are joined.
119
+ * Maps Prowler OCSF findings (Detection Findings), one at a time, to results. Findings whose check the catalogue does
120
+ * not list are dropped and counted. Several findings on one (check, resource) merge into one result: fail if any
121
+ * failed, else pass if any passed, else skipped; their details are joined (only as far as the 1,000 characters sent).
121
122
  */
122
- export function parseOcsf(text, provider, known) {
123
- let doc;
124
- try {
125
- doc = JSON.parse(text);
126
- }
127
- catch (e) {
128
- throw new Error(`the Prowler output is not JSON (${e.message}); pass a json-ocsf file.`);
129
- }
130
- if (!Array.isArray(doc))
131
- throw new Error("the Prowler output is not an OCSF array of findings; pass a json-ocsf file.");
132
- const summary = { findings: doc.length, kept: 0, dropped: 0, droppedChecks: [], muted: 0, manual: 0, merged: 0 };
133
- const dropped = new Set();
134
- const byKey = new Map();
135
- for (const f of doc) {
123
+ class OcsfMerger {
124
+ summary = { findings: 0, kept: 0, dropped: 0, droppedChecks: [], muted: 0, manual: 0, merged: 0 };
125
+ dropped = new Set();
126
+ byKey = new Map();
127
+ provider;
128
+ known;
129
+ constructor(provider, known) { this.provider = provider; this.known = known; }
130
+ add(f) {
131
+ this.summary.findings++;
136
132
  if (!isObj(f))
137
- continue;
133
+ return;
134
+ const summary = this.summary;
138
135
  const meta = isObj(f.metadata) ? f.metadata : {};
139
136
  const info = isObj(f.finding_info) ? f.finding_info : {};
140
137
  const analytic = isObj(info.analytic) ? info.analytic : {};
141
138
  const checkId = s(meta.event_code) || s(analytic.uid);
142
139
  const unmapped = isObj(f.unmapped) ? f.unmapped : {};
143
- const prov = s(unmapped.provider) || provider;
140
+ const prov = s(unmapped.provider) || this.provider;
144
141
  const check = `prowler.${prov}.${checkId}`;
145
- if (!checkId || !known.has(check)) {
142
+ if (!checkId || !this.known.has(check)) {
146
143
  summary.dropped++;
147
- dropped.add(check);
148
- continue;
144
+ this.dropped.add(check);
145
+ return;
149
146
  }
150
147
  if (s(f.status) === "Suppressed")
151
148
  summary.muted++;
@@ -165,22 +162,142 @@ export function parseOcsf(text, provider, known) {
165
162
  const detail = (s(f.status_detail) || s(f.message)).replace(EMAIL, "[email]");
166
163
  const result = outcomeOf(f);
167
164
  const key = `${check}|${resource}`;
168
- const prev = byKey.get(key);
165
+ const prev = this.byKey.get(key);
169
166
  if (!prev) {
170
- byKey.set(key, { r: { check, resource, result, detail: "" }, details: detail ? [detail] : [] });
171
- continue;
167
+ this.byKey.set(key, { r: { check, resource, result, detail: "" }, details: detail ? [detail] : [], len: detail.length });
168
+ return;
172
169
  }
173
170
  summary.merged++;
174
171
  const rank = (o) => (o === "fail" ? 2 : o === "pass" ? 1 : 0);
175
172
  if (rank(result) > rank(prev.r.result))
176
173
  prev.r.result = result;
177
- if (detail && !prev.details.includes(detail))
174
+ // Past 1,000 characters a further detail cannot change what is sent, so it is not kept (a large account merges
175
+ // thousands of findings into one result).
176
+ if (detail && prev.len < 1000 && !prev.details.includes(detail)) {
177
+ prev.len += (prev.details.length ? 3 : 0) + detail.length;
178
178
  prev.details.push(detail);
179
+ }
180
+ }
181
+ finish() {
182
+ const results = [...this.byKey.values()].map(({ r, details }) => ({ ...r, detail: details.join(" | ").slice(0, 1000) }));
183
+ this.summary.kept = results.length;
184
+ this.summary.droppedChecks = [...this.dropped].sort();
185
+ return { results, summary: this.summary };
179
186
  }
180
- const results = [...byKey.values()].map(({ r, details }) => ({ ...r, detail: details.join(" | ").slice(0, 1000) }));
181
- summary.kept = results.length;
182
- summary.droppedChecks = [...dropped].sort();
183
- return { results, summary };
187
+ }
188
+ /** The output is not OCSF: the file is wrong, whichever way it was made. */
189
+ export class OcsfError extends Error {
190
+ }
191
+ const NOT_JSON = (why) => new OcsfError(`the Prowler output is not JSON (${why}); pass a json-ocsf file.`);
192
+ const NOT_ARRAY = () => new OcsfError("the Prowler output is not an OCSF array of findings; pass a json-ocsf file.");
193
+ /** Maps a Prowler OCSF array held in memory (tests, small files). */
194
+ export function parseOcsf(text, provider, known) {
195
+ let doc;
196
+ try {
197
+ doc = JSON.parse(text);
198
+ }
199
+ catch (e) {
200
+ throw NOT_JSON(e.message);
201
+ }
202
+ if (!Array.isArray(doc))
203
+ throw NOT_ARRAY();
204
+ const m = new OcsfMerger(provider, known);
205
+ for (const f of doc)
206
+ m.add(f);
207
+ return m.finish();
208
+ }
209
+ const WS = (c) => c === 0x20 || c === 0x0a || c === 0x0d || c === 0x09;
210
+ /**
211
+ * The elements of a top-level JSON array, as text, read chunk by chunk: only one element is held at a time, so a
212
+ * Prowler output of any size never becomes one string (V8 caps a string near 512 MiB; a large AWS organization's
213
+ * OCSF file passes that). Strings and escapes are tracked so brackets inside them do not count. No regular expression.
214
+ */
215
+ export async function* jsonArrayElements(chunks) {
216
+ let buf = "", i = 0, start = -1, depth = 0, inStr = false, esc = false, state = "pre", afterComma = false;
217
+ for await (const chunk of chunks) {
218
+ buf += chunk;
219
+ for (; i < buf.length; i++) {
220
+ const c = buf.charCodeAt(i);
221
+ if (state === "pre") {
222
+ if (WS(c) || c === 0xfeff)
223
+ continue;
224
+ if (c === 0x5b /* [ */) {
225
+ state = "in";
226
+ continue;
227
+ }
228
+ throw NOT_ARRAY();
229
+ }
230
+ if (state === "post") {
231
+ if (WS(c))
232
+ continue;
233
+ throw NOT_JSON(`text after the closing bracket at offset ${i}`);
234
+ }
235
+ if (inStr) {
236
+ if (esc)
237
+ esc = false;
238
+ else if (c === 0x5c /* \ */)
239
+ esc = true;
240
+ else if (c === 0x22 /* " */)
241
+ inStr = false;
242
+ continue;
243
+ }
244
+ if (c === 0x22) {
245
+ inStr = true;
246
+ if (start < 0)
247
+ start = i;
248
+ continue;
249
+ }
250
+ if (depth === 0) {
251
+ if (c === 0x2c /* , */ || c === 0x5d /* ] */) {
252
+ if (start >= 0) {
253
+ yield buf.slice(start, i);
254
+ start = -1;
255
+ afterComma = c === 0x2c;
256
+ }
257
+ else if (c === 0x2c || afterComma)
258
+ throw NOT_JSON("an empty array element");
259
+ if (c === 0x5d)
260
+ state = "post";
261
+ continue;
262
+ }
263
+ if (WS(c))
264
+ continue;
265
+ if (start < 0)
266
+ start = i;
267
+ }
268
+ if (c === 0x7b /* { */ || c === 0x5b)
269
+ depth++;
270
+ else if (c === 0x7d /* } */ || c === 0x5d) {
271
+ if (--depth < 0)
272
+ throw NOT_JSON("unbalanced brackets");
273
+ }
274
+ }
275
+ // Drop what has been read; keep the element in progress.
276
+ const keep = start >= 0 ? start : i;
277
+ buf = buf.slice(keep);
278
+ i -= keep;
279
+ if (start >= 0)
280
+ start = 0;
281
+ }
282
+ if (state === "pre")
283
+ throw NOT_ARRAY();
284
+ if (state !== "post")
285
+ throw NOT_JSON("the file ends inside the array");
286
+ }
287
+ /** Maps a Prowler OCSF file, streamed (see jsonArrayElements). */
288
+ export async function parseOcsfFile(path, provider, known) {
289
+ const m = new OcsfMerger(provider, known);
290
+ for await (const el of jsonArrayElements(createReadStream(path, { encoding: "utf8", highWaterMark: 1 << 20 }))) {
291
+ let f;
292
+ try {
293
+ f = JSON.parse(el);
294
+ }
295
+ catch (e) {
296
+ throw NOT_JSON(e.message);
297
+ }
298
+ m.add(f);
299
+ }
300
+ return m.finish();
184
301
  }
185
302
  export const spawnProwler = (bin, args, env) => new Promise((resolve, reject) => {
186
303
  const p = spawn(bin, args, { env, stdio: ["ignore", "ignore", "pipe"] });
@@ -189,7 +306,7 @@ export const spawnProwler = (bin, args, env) => new Promise((resolve, reject) =>
189
306
  p.on("error", reject);
190
307
  p.on("close", (code) => resolve({ code, stderr: err }));
191
308
  });
192
- /** Runs Prowler into a fresh directory and returns the OCSF text. A non-zero exit is an error. */
309
+ /** Runs Prowler into a fresh directory and maps its OCSF file, streamed, before the directory goes. A non-zero exit is an error. */
193
310
  export async function runProwler(provider, o) {
194
311
  const bad = o.extra.filter((a) => REFUSED_ARGS.includes(a.split("=")[0]));
195
312
  if (bad.length)
@@ -210,12 +327,14 @@ export async function runProwler(provider, o) {
210
327
  const file = readdirSync(dir).find((f) => f.endsWith(".ocsf.json"));
211
328
  if (!file)
212
329
  throw new Error(`${o.bin} ${provider} wrote no .ocsf.json in ${dir}.`);
213
- return readFileSync(join(dir, file), "utf8");
330
+ return await parseOcsfFile(join(dir, file), provider, o.known);
214
331
  }
215
332
  finally {
216
333
  rmSync(dir, { recursive: true, force: true });
217
334
  }
218
335
  }
336
+ /** Prowler scans run side by side, at most this many at once (each is a Python process of its own). */
337
+ export const PROWLER_PARALLEL = 2;
219
338
  // ── Source ──────────────────────────────────────────────────────────────────────────────────────────
220
339
  export const prowler = {
221
340
  name: "prowler",
@@ -240,36 +359,43 @@ export const prowler = {
240
359
  if (file && providers.length > 1)
241
360
  throw new Error("--prowler-output reads one provider's file; name one --prowler-provider with it.");
242
361
  const known = new Set(catalogIds(one(ctx.opts.catalog) ?? DEFAULT_CATALOG_DIR));
243
- const results = [];
244
- const failures = [];
245
362
  for (const provider of providers) {
246
363
  if (![...known].some((id) => id.startsWith(`prowler.${provider}.`))) {
247
364
  ctx.warn(`prowler: sos-app's catalogue lists no ${provider} check; every finding will be dropped.`);
248
365
  }
249
- let text;
250
- try {
251
- if (file) {
252
- try {
253
- text = readFileSync(file, "utf8");
254
- }
255
- catch (e) {
256
- throw new Error(`cannot read --prowler-output ${file}: ${e.code ?? String(e)}`);
257
- }
366
+ }
367
+ // Each provider read or scanned, PROWLER_PARALLEL at a time; the outcomes are then taken in the order given, so the
368
+ // results and their order are those of one provider after another.
369
+ const one_ = async (provider) => {
370
+ if (file) {
371
+ try {
372
+ accessSync(file, constants.R_OK);
258
373
  }
259
- else {
260
- const spawner = ctx.clients.prowler ?? spawnProwler;
261
- const extra = one(ctx.opts["prowler-args"]) ? splitArgs(one(ctx.opts["prowler-args"])) : [];
262
- text = await runProwler(provider, { bin: one(ctx.opts["prowler-bin"]) ?? "prowler", extra, env: prowlerEnv(provider, ctx.env), spawner });
374
+ catch (e) {
375
+ throw new Error(`cannot read --prowler-output ${file}: ${e.code ?? String(e)}`);
263
376
  }
377
+ return parseOcsfFile(file, provider, known);
264
378
  }
265
- catch (e) {
379
+ const spawner = ctx.clients.prowler ?? spawnProwler;
380
+ const extra = one(ctx.opts["prowler-args"]) ? splitArgs(one(ctx.opts["prowler-args"])) : [];
381
+ return runProwler(provider, { bin: one(ctx.opts["prowler-bin"]) ?? "prowler", extra, env: prowlerEnv(provider, ctx.env), spawner, known });
382
+ };
383
+ const outcomes = await pool(providers, PROWLER_PARALLEL, one_);
384
+ const results = [];
385
+ const failures = [];
386
+ for (const [i, o] of outcomes.entries()) {
387
+ const provider = providers[i];
388
+ if (o.status === "rejected") {
389
+ // An output that is not OCSF is the file's fault, not the scan's: it stops the source, as it always has.
390
+ if (o.reason instanceof OcsfError)
391
+ throw o.reason;
266
392
  // A second way that fails leaves the control to its other ways: say so and go on with the next provider.
267
- failures.push(e);
393
+ failures.push(o.reason);
268
394
  if (providers.length > 1)
269
- ctx.warn(`prowler: ${provider} did not run (${e.message}); its controls keep their other ways.`);
395
+ ctx.warn(`prowler: ${provider} did not run (${o.reason.message}); its controls keep their other ways.`);
270
396
  continue;
271
397
  }
272
- const { results: r, summary } = parseOcsf(text, provider, known);
398
+ const { results: r, summary } = o.value;
273
399
  results.push(...r);
274
400
  ctx.warn(`prowler: ${summary.findings} findings; ${summary.kept} results (${summary.merged} merged into the same check and resource, ` +
275
401
  `${summary.muted} muted and ${summary.manual} manual sent as skipped); ${summary.dropped} findings dropped from ${summary.droppedChecks.length} checks not in the catalogue` +
@@ -2,6 +2,18 @@
2
2
  // GitHub): a GET-only JSON client, retry on rate limits, pagination, "skipped" details and token
3
3
  // redaction. The only non-GET request any of these sources makes is the OAuth token request in
4
4
  // oauth.ts, to the identity provider's token endpoint.
5
+ import { channel } from "node:diagnostics_channel";
6
+ /** The recorder (src/report.ts RETRY_CHANNEL) counts retries per source from here; a plain string, so this file needs no import of it. */
7
+ const retries = channel("sagentix:collector:retry");
8
+ /** Tells the run's recorder that a call is about to be retried (a no-op when nothing listens). */
9
+ export function announceRetry(status, waitMs, attempt) {
10
+ if (retries.hasSubscribers)
11
+ retries.publish({ status, waitMs, attempt });
12
+ }
13
+ /** Gateway and availability errors worth one or two more tries; a 500 is usually the request itself and is not retried. */
14
+ export const TRANSIENT_STATUS = new Set([502, 503, 504]);
15
+ /** Retries after a transient error (a 502/503/504 or a network failure), on top of the rate-limit retries. */
16
+ export const TRANSIENT_RETRIES = 2;
5
17
  export const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
6
18
  export const str = (v) => (typeof v === "string" ? v : undefined);
7
19
  export const arr = (v) => (Array.isArray(v) ? v : []);
@@ -71,21 +83,30 @@ export function retryDelayMs(h, attempt, nowMs) {
71
83
  return Math.min(cap, Math.max(1000, reset * 1000 - nowMs + 1000));
72
84
  return Math.min(cap, 1000 * 2 ** attempt);
73
85
  }
74
- /** Retries a rate-limited GET up to `max` times; a thrown network error becomes status 0. */
86
+ /**
87
+ * Retries a rate-limited GET up to `max` times, and a transient failure (502, 503, 504, or a thrown network error or
88
+ * timeout) up to TRANSIENT_RETRIES times, waiting Retry-After when the answer gives one. A thrown error becomes status 0.
89
+ */
75
90
  export function withRetry(api, sleep = realSleep, max = 4, now = Date.now) {
76
91
  return {
77
92
  async get(url) {
93
+ let transient = 0;
78
94
  for (let attempt = 0;; attempt++) {
79
- let r;
95
+ let r, thrown = false;
80
96
  try {
81
97
  r = await api.get(url);
82
98
  }
83
99
  catch (e) {
84
100
  r = { status: 0, headers: {}, body: { message: `request failed: ${String(e)}` } };
101
+ thrown = true;
85
102
  }
86
- if (!isRateLimited(r) || attempt >= max)
103
+ const rate = isRateLimited(r);
104
+ const again = rate ? attempt < max : (thrown || TRANSIENT_STATUS.has(r.status)) && transient++ < TRANSIENT_RETRIES;
105
+ if (!again)
87
106
  return r;
88
- await sleep(retryDelayMs(r.headers, attempt, now()));
107
+ const wait = retryDelayMs(r.headers, rate ? attempt : transient - 1, now());
108
+ announceRetry(r.status, wait, attempt + 1);
109
+ await sleep(wait);
89
110
  }
90
111
  },
91
112
  };
@@ -120,7 +141,8 @@ export function why(r) {
120
141
  msg = msg.replace(/\s+/g, " ").trim().slice(0, 300);
121
142
  const hint = r.status === 401 || r.status === 403
122
143
  ? " (the credential may lack the read permission for this call, or the feature is not licensed)"
123
- : r.status === 429 ? " (rate limited; retries exhausted)" : "";
144
+ : r.status === 429 ? " (rate limited; retries exhausted)"
145
+ : TRANSIENT_STATUS.has(r.status) ? " (the provider is unavailable; retried, try the run again later)" : "";
124
146
  return `HTTP ${r.status}${msg ? `: ${msg}` : ""}${hint}`;
125
147
  }
126
148
  /**
@@ -0,0 +1,114 @@
1
+ // Rippling (GET only), with an API token a Rippling admin makes under Developer > API credentials and grants only the
2
+ // workers.read scope, read as joiners, leavers and the org chart (hris_common.ts).
3
+ //
4
+ // From Rippling's REST API documentation (all read 2026-11-07):
5
+ // Tokens: made at https://app.rippling.com/api-tokens/tokens by a user who may manage API tokens; scopes are chosen at
6
+ // creation and limited by the creator's own permissions; a token is revoked when its owner leaves or after 30 days
7
+ // unused, so a daily or weekly run keeps it alive. Requests go to https://rest.ripplingapis.com with
8
+ // Authorization: Bearer <token>. https://developer.rippling.com/documentation/rest-api/essentials/api-tokens
9
+ // GET /workers/?limit=100 workers.read
10
+ // -> { __meta, results: [ { id, work_email, status (INIT, HIRED, ACCEPTED, ACTIVE, TERMINATED), start_date,
11
+ // end_date, manager_id, title, custom_fields, termination_details } ], next_link }
12
+ // https://developer.rippling.com/documentation/rest-api/reference/list-workers.md
13
+ // Pages: send the first request without a cursor, then follow next_link unchanged until it is null; limit is at most
14
+ // 100. https://developer.rippling.com/documentation/rest-api/essentials/pagination
15
+ // Rate: 600 requests per 10 seconds per token and path; a 429 is retried after Retry-After.
16
+ // https://developer.rippling.com/documentation/rest-api/essentials/api-limits
17
+ // The acknowledgement and the background check have no standard worker field: the customer names the custom field that
18
+ // records each date (--rippling-ack-field, --rippling-background-field). custom_fields' item shape is open in the
19
+ // reference, so both { "<name>": value } and { name|label|api_name, value } items are read; a field never found is
20
+ // reported as such rather than failing every hire.
21
+ //
22
+ // Personal data: names, compensation and every expandable object are never asked for (no expand). A person is named in
23
+ // a result by the Rippling worker id; the work email is used only for the identity-provider lookup and the LMS join.
24
+ import { HRIS_CHECKS, hrisChecks, hrisOptionSpecs, hrisOptions, hrisSkipped, isoDay } from "./hris_common.js";
25
+ import { IDP_SECRETS } from "./directory.js";
26
+ import { arr, fetchRest, isObj, ok, realSleep, redact, str, why, withRetry } from "./readonly_http.js";
27
+ export const RIPPLING_ENV = { token: "RIPPLING_API_TOKEN" };
28
+ export const RIPPLING_BASE = "https://rest.ripplingapis.com";
29
+ export const WORKERS = "/workers/?limit=100";
30
+ const FIELD = /^[A-Za-z0-9][A-Za-z0-9_ .-]{0,63}$/;
31
+ export function ripplingFromEnv(env, fetchFn = fetch) {
32
+ const token = env[RIPPLING_ENV.token];
33
+ if (!token)
34
+ return `${RIPPLING_ENV.token} is not set`;
35
+ return fetchRest({ base: RIPPLING_BASE, fetchFn, headers: async () => ({ authorization: `Bearer ${token}` }) });
36
+ }
37
+ /** A custom field's value, from either item shape the reference allows. */
38
+ export function customField(w, name) {
39
+ const want = name.toLowerCase();
40
+ for (const it of arr(w.custom_fields).filter(isObj)) {
41
+ for (const [k, v] of Object.entries(it))
42
+ if (k.toLowerCase() === want)
43
+ return v;
44
+ const key = [it.name, it.label, it.api_name, it.field].find((x) => typeof x === "string" && x.toLowerCase() === want);
45
+ if (key !== undefined)
46
+ return it.value;
47
+ }
48
+ return undefined;
49
+ }
50
+ export function personOf(w, ackField, backgroundField) {
51
+ const s = String(w.status ?? "").toUpperCase();
52
+ const status = s === "ACTIVE" ? "active" : s === "TERMINATED" ? "terminated" : s.toLowerCase() || "unknown";
53
+ return {
54
+ id: String(w.id ?? "?").slice(0, 40),
55
+ email: str(w.work_email)?.trim() || undefined,
56
+ status,
57
+ hired: isoDay(w.start_date),
58
+ left: status === "terminated" ? isoDay(w.end_date) : undefined,
59
+ managerId: str(w.manager_id)?.trim() || undefined,
60
+ title: str(w.title)?.trim() || undefined,
61
+ dates: { ack: ackField ? isoDay(customField(w, ackField)) : undefined, background: backgroundField ? isoDay(customField(w, backgroundField)) : undefined },
62
+ };
63
+ }
64
+ export const rippling = {
65
+ name: "rippling",
66
+ options: {
67
+ "rippling-company": { type: "string", help: "a name for the Rippling company in results, e.g. its domain (the token decides what is read)" },
68
+ ...hrisOptionSpecs("rippling"),
69
+ },
70
+ emits: HRIS_CHECKS.map((c) => `rippling.${c}`),
71
+ describe: (opts) => `Rippling company ${typeof opts["rippling-company"] === "string" ? opts["rippling-company"] : "(token's company)"} (REST API, workers.read)`,
72
+ validate(opts) {
73
+ for (const k of ["rippling-ack-field", "rippling-background-field"]) {
74
+ const v = opts[k];
75
+ if (typeof v === "string" && v.trim() && !FIELD.test(v.trim()))
76
+ return `--${k} must be a custom field name (letters, digits, space, _ . -)`;
77
+ }
78
+ return undefined;
79
+ },
80
+ async run(ctx) {
81
+ const company = typeof ctx.opts["rippling-company"] === "string" && ctx.opts["rippling-company"].trim() ? ctx.opts["rippling-company"].trim().slice(0, 100) : "company";
82
+ const resource = `rippling ${company}`;
83
+ const o = hrisOptions("rippling", ctx);
84
+ if (typeof o === "string")
85
+ return hrisSkipped("rippling", resource, o);
86
+ const given = ctx.clients.rippling;
87
+ const built = given !== undefined ? given : ripplingFromEnv(ctx.env);
88
+ if (!built || typeof built === "string")
89
+ return hrisSkipped("rippling", resource, typeof built === "string" ? built : "no Rippling client");
90
+ const api = withRetry(built, ctx.clients.sleep ?? realSleep);
91
+ const workers = [];
92
+ let url = WORKERS, pages = 0;
93
+ while (url && pages < 500) {
94
+ const r = await api.get(url);
95
+ if (!ok(r) || !isObj(r.body) || !Array.isArray(r.body.results)) {
96
+ return redact(hrisSkipped("rippling", resource, `workers not readable: ${why(r)}`, r), [ctx.env[RIPPLING_ENV.token]]);
97
+ }
98
+ pages++;
99
+ workers.push(...r.body.results.filter(isObj));
100
+ url = str(r.body.next_link) || undefined;
101
+ }
102
+ const people = workers.map((w) => personOf(w, o.ackField, o.backgroundField));
103
+ const read = { method: "config-read", itemsRead: people.length, pagesRead: pages, complete: !url };
104
+ let rs = await hrisChecks("rippling", resource, people, read, o, ctx);
105
+ // A custom field no worker carries is more likely misnamed or not shared with the token than empty for everyone.
106
+ for (const [check, field] of [["rippling.hire_acknowledgement", o.ackField], ["rippling.hire_background_check", o.backgroundField]]) {
107
+ if (!field || workers.some((w) => customField(w, field) !== undefined))
108
+ continue;
109
+ rs = rs.filter((r) => r.check !== check);
110
+ rs.push({ check, resource, result: "skipped", detail: `no worker carries the custom field "${field}"; check its name and that the token can read custom fields` });
111
+ }
112
+ return redact(rs, [ctx.env[RIPPLING_ENV.token], ...IDP_SECRETS(ctx.env)]);
113
+ },
114
+ };
@@ -1,6 +1,6 @@
1
1
  // Helpers shared by the AWS, Azure, GCP, Jira and LMS sources: option lists, retry with backoff on
2
2
  // throttling, a result builder, and a scrubber that keeps secret values out of every detail.
3
- import { redact } from "./readonly_http.js";
3
+ import { TRANSIENT_RETRIES, TRANSIENT_STATUS, announceRetry, redact } from "./readonly_http.js";
4
4
  export const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
5
5
  export const arr = (v) => (Array.isArray(v) ? v : []);
6
6
  export const str = (v) => (typeof v === "string" ? v : undefined);
@@ -20,25 +20,33 @@ export function retryAfterMs(headers, now = Date.now()) {
20
20
  }
21
21
  /**
22
22
  * Calls `send` until it gives a reply that is not throttled, or the attempts run out (then the last
23
- * throttled reply is returned, and the caller turns it into "skipped"). A thrown error is a reply
24
- * with status 0.
23
+ * throttled reply is returned, and the caller turns it into "skipped"). A transient failure (502 or 504,
24
+ * or a thrown network error or timeout) is tried again up to TRANSIENT_RETRIES times within the same
25
+ * attempts. A thrown error is a reply with status 0.
25
26
  */
26
27
  export async function withRetry(send, o = {}) {
27
28
  const attempts = o.attempts ?? 5, base = o.baseMs ?? 500, max = o.maxMs ?? 20_000;
28
29
  const sleep = o.sleep ?? realSleep;
29
30
  const throttled = o.throttled ?? ((r) => r.status === 429 || r.status === 503);
30
31
  let last = { status: 0, headers: {}, body: "" };
32
+ let transient = 0;
31
33
  for (let i = 0; i < attempts; i++) {
34
+ let thrown = false;
32
35
  try {
33
36
  last = await send();
34
37
  }
35
38
  catch (e) {
36
39
  last = { status: 0, headers: {}, body: `request failed: ${String(e)}` };
40
+ thrown = true;
37
41
  }
38
- if (!throttled(last))
42
+ const again = throttled(last) || ((thrown || TRANSIENT_STATUS.has(last.status)) && transient++ < TRANSIENT_RETRIES);
43
+ if (!again)
39
44
  return last;
40
- if (i < attempts - 1)
41
- await sleep(Math.min(max, retryAfterMs(last.headers) ?? base * 2 ** i));
45
+ if (i < attempts - 1) {
46
+ const wait = Math.min(max, retryAfterMs(last.headers) ?? base * 2 ** i);
47
+ announceRetry(last.status, wait, i + 1);
48
+ await sleep(wait);
49
+ }
42
50
  }
43
51
  return last;
44
52
  }