@sagentix-inc/evidence-collector 0.3.5 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -9
- package/apps/sos-app/src/data/checks/aws.json +12 -0
- package/apps/sos-app/src/data/checks/azure.json +18 -0
- package/apps/sos-app/src/data/checks/bamboohr.json +40 -0
- package/apps/sos-app/src/data/checks/entra.json +1 -1
- package/apps/sos-app/src/data/checks/gcp.json +12 -0
- package/apps/sos-app/src/data/checks/intune.json +7 -1
- package/apps/sos-app/src/data/checks/jamf.json +8 -2
- package/apps/sos-app/src/data/checks/kandji.json +7 -1
- package/apps/sos-app/src/data/checks/rippling.json +40 -0
- package/apps/sos-app/src/data/checks/vulnscan.json +27 -0
- package/apps/sos-app/src/data/checks/web.json +31 -1
- package/collector/package.json +1 -1
- package/collector/src/cli.js +344 -37
- package/collector/src/config.js +12 -0
- package/collector/src/doctor.js +382 -0
- package/collector/src/policies.js +160 -0
- package/collector/src/pool.js +19 -0
- package/collector/src/report.js +88 -16
- package/collector/src/sources/aws/checks.js +64 -0
- package/collector/src/sources/aws.js +6 -1
- package/collector/src/sources/azure/api.js +4 -0
- package/collector/src/sources/azure/checks.js +91 -1
- package/collector/src/sources/azure.js +8 -2
- package/collector/src/sources/bamboohr.js +133 -0
- package/collector/src/sources/cloudflare/api.js +24 -4
- package/collector/src/sources/device_common.js +42 -0
- package/collector/src/sources/directory.js +198 -0
- package/collector/src/sources/engines.js +2 -1
- package/collector/src/sources/entra.js +4 -3
- package/collector/src/sources/gcp/project.js +47 -3
- package/collector/src/sources/gcp.js +5 -2
- package/collector/src/sources/github.js +46 -8
- package/collector/src/sources/google_oauth.js +3 -3
- package/collector/src/sources/google_workspace.js +4 -3
- package/collector/src/sources/hris_common.js +189 -0
- package/collector/src/sources/index.js +3 -0
- package/collector/src/sources/intune.js +40 -3
- package/collector/src/sources/jamf.js +65 -8
- package/collector/src/sources/kandji.js +46 -7
- package/collector/src/sources/oauth.js +22 -0
- package/collector/src/sources/prowler.js +178 -52
- package/collector/src/sources/readonly_http.js +27 -5
- package/collector/src/sources/rippling.js +114 -0
- package/collector/src/sources/shared-cloud.js +14 -6
- package/collector/src/sources/vulnscan.js +386 -0
- package/collector/src/sources/web.js +106 -11
- package/collector/src/sources/web_privacy.js +266 -0
- package/package.json +1 -1
|
@@ -15,11 +15,12 @@
|
|
|
15
15
|
// The collector never passes a Prowler option that writes, fixes or sends anywhere (Security Hub, S3,
|
|
16
16
|
// Slack, Shodan, Prowler Cloud, secret validation), and refuses one given in --prowler-args.
|
|
17
17
|
import { spawn } from "node:child_process";
|
|
18
|
-
import { mkdtempSync, readdirSync,
|
|
18
|
+
import { accessSync, constants, createReadStream, mkdtempSync, readdirSync, rmSync } from "node:fs";
|
|
19
19
|
import { tmpdir } from "node:os";
|
|
20
20
|
import { join } from "node:path";
|
|
21
21
|
import { DEFAULT_CATALOG_DIR, loadCatalog } from "../catalog.js";
|
|
22
22
|
import { scrub } from "./shared-cloud.js";
|
|
23
|
+
import { pool } from "../pool.js";
|
|
23
24
|
export const PROWLER_PROVIDERS = ["aws", "azure", "gcp", "kubernetes", "m365", "github", "googleworkspace", "okta", "cloudflare"];
|
|
24
25
|
/** The version the catalogues were built from (apps/sos-app/scripts/build_prowler_catalog.py). */
|
|
25
26
|
export const PROWLER_VERSION = "5.47.0";
|
|
@@ -115,37 +116,33 @@ export function outcomeOf(f) {
|
|
|
115
116
|
return code === "PASS" ? "pass" : code === "FAIL" ? "fail" : "skipped";
|
|
116
117
|
}
|
|
117
118
|
/**
|
|
118
|
-
* Maps
|
|
119
|
-
* are dropped and counted. Several findings on one (check, resource) merge into one result: fail if any
|
|
120
|
-
* failed, else pass if any passed, else skipped; their details are joined.
|
|
119
|
+
* Maps Prowler OCSF findings (Detection Findings), one at a time, to results. Findings whose check the catalogue does
|
|
120
|
+
* not list are dropped and counted. Several findings on one (check, resource) merge into one result: fail if any
|
|
121
|
+
* failed, else pass if any passed, else skipped; their details are joined (only as far as the 1,000 characters sent).
|
|
121
122
|
*/
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
throw new Error("the Prowler output is not an OCSF array of findings; pass a json-ocsf file.");
|
|
132
|
-
const summary = { findings: doc.length, kept: 0, dropped: 0, droppedChecks: [], muted: 0, manual: 0, merged: 0 };
|
|
133
|
-
const dropped = new Set();
|
|
134
|
-
const byKey = new Map();
|
|
135
|
-
for (const f of doc) {
|
|
123
|
+
class OcsfMerger {
|
|
124
|
+
summary = { findings: 0, kept: 0, dropped: 0, droppedChecks: [], muted: 0, manual: 0, merged: 0 };
|
|
125
|
+
dropped = new Set();
|
|
126
|
+
byKey = new Map();
|
|
127
|
+
provider;
|
|
128
|
+
known;
|
|
129
|
+
constructor(provider, known) { this.provider = provider; this.known = known; }
|
|
130
|
+
add(f) {
|
|
131
|
+
this.summary.findings++;
|
|
136
132
|
if (!isObj(f))
|
|
137
|
-
|
|
133
|
+
return;
|
|
134
|
+
const summary = this.summary;
|
|
138
135
|
const meta = isObj(f.metadata) ? f.metadata : {};
|
|
139
136
|
const info = isObj(f.finding_info) ? f.finding_info : {};
|
|
140
137
|
const analytic = isObj(info.analytic) ? info.analytic : {};
|
|
141
138
|
const checkId = s(meta.event_code) || s(analytic.uid);
|
|
142
139
|
const unmapped = isObj(f.unmapped) ? f.unmapped : {};
|
|
143
|
-
const prov = s(unmapped.provider) || provider;
|
|
140
|
+
const prov = s(unmapped.provider) || this.provider;
|
|
144
141
|
const check = `prowler.${prov}.${checkId}`;
|
|
145
|
-
if (!checkId || !known.has(check)) {
|
|
142
|
+
if (!checkId || !this.known.has(check)) {
|
|
146
143
|
summary.dropped++;
|
|
147
|
-
dropped.add(check);
|
|
148
|
-
|
|
144
|
+
this.dropped.add(check);
|
|
145
|
+
return;
|
|
149
146
|
}
|
|
150
147
|
if (s(f.status) === "Suppressed")
|
|
151
148
|
summary.muted++;
|
|
@@ -165,22 +162,142 @@ export function parseOcsf(text, provider, known) {
|
|
|
165
162
|
const detail = (s(f.status_detail) || s(f.message)).replace(EMAIL, "[email]");
|
|
166
163
|
const result = outcomeOf(f);
|
|
167
164
|
const key = `${check}|${resource}`;
|
|
168
|
-
const prev = byKey.get(key);
|
|
165
|
+
const prev = this.byKey.get(key);
|
|
169
166
|
if (!prev) {
|
|
170
|
-
byKey.set(key, { r: { check, resource, result, detail: "" }, details: detail ? [detail] : [] });
|
|
171
|
-
|
|
167
|
+
this.byKey.set(key, { r: { check, resource, result, detail: "" }, details: detail ? [detail] : [], len: detail.length });
|
|
168
|
+
return;
|
|
172
169
|
}
|
|
173
170
|
summary.merged++;
|
|
174
171
|
const rank = (o) => (o === "fail" ? 2 : o === "pass" ? 1 : 0);
|
|
175
172
|
if (rank(result) > rank(prev.r.result))
|
|
176
173
|
prev.r.result = result;
|
|
177
|
-
|
|
174
|
+
// Past 1,000 characters a further detail cannot change what is sent, so it is not kept (a large account merges
|
|
175
|
+
// thousands of findings into one result).
|
|
176
|
+
if (detail && prev.len < 1000 && !prev.details.includes(detail)) {
|
|
177
|
+
prev.len += (prev.details.length ? 3 : 0) + detail.length;
|
|
178
178
|
prev.details.push(detail);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
finish() {
|
|
182
|
+
const results = [...this.byKey.values()].map(({ r, details }) => ({ ...r, detail: details.join(" | ").slice(0, 1000) }));
|
|
183
|
+
this.summary.kept = results.length;
|
|
184
|
+
this.summary.droppedChecks = [...this.dropped].sort();
|
|
185
|
+
return { results, summary: this.summary };
|
|
179
186
|
}
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
187
|
+
}
|
|
188
|
+
/** The output is not OCSF: the file is wrong, whichever way it was made. */
|
|
189
|
+
export class OcsfError extends Error {
|
|
190
|
+
}
|
|
191
|
+
const NOT_JSON = (why) => new OcsfError(`the Prowler output is not JSON (${why}); pass a json-ocsf file.`);
|
|
192
|
+
const NOT_ARRAY = () => new OcsfError("the Prowler output is not an OCSF array of findings; pass a json-ocsf file.");
|
|
193
|
+
/** Maps a Prowler OCSF array held in memory (tests, small files). */
|
|
194
|
+
export function parseOcsf(text, provider, known) {
|
|
195
|
+
let doc;
|
|
196
|
+
try {
|
|
197
|
+
doc = JSON.parse(text);
|
|
198
|
+
}
|
|
199
|
+
catch (e) {
|
|
200
|
+
throw NOT_JSON(e.message);
|
|
201
|
+
}
|
|
202
|
+
if (!Array.isArray(doc))
|
|
203
|
+
throw NOT_ARRAY();
|
|
204
|
+
const m = new OcsfMerger(provider, known);
|
|
205
|
+
for (const f of doc)
|
|
206
|
+
m.add(f);
|
|
207
|
+
return m.finish();
|
|
208
|
+
}
|
|
209
|
+
const WS = (c) => c === 0x20 || c === 0x0a || c === 0x0d || c === 0x09;
|
|
210
|
+
/**
|
|
211
|
+
* The elements of a top-level JSON array, as text, read chunk by chunk: only one element is held at a time, so a
|
|
212
|
+
* Prowler output of any size never becomes one string (V8 caps a string near 512 MiB; a large AWS organization's
|
|
213
|
+
* OCSF file passes that). Strings and escapes are tracked so brackets inside them do not count. No regular expression.
|
|
214
|
+
*/
|
|
215
|
+
export async function* jsonArrayElements(chunks) {
|
|
216
|
+
let buf = "", i = 0, start = -1, depth = 0, inStr = false, esc = false, state = "pre", afterComma = false;
|
|
217
|
+
for await (const chunk of chunks) {
|
|
218
|
+
buf += chunk;
|
|
219
|
+
for (; i < buf.length; i++) {
|
|
220
|
+
const c = buf.charCodeAt(i);
|
|
221
|
+
if (state === "pre") {
|
|
222
|
+
if (WS(c) || c === 0xfeff)
|
|
223
|
+
continue;
|
|
224
|
+
if (c === 0x5b /* [ */) {
|
|
225
|
+
state = "in";
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
throw NOT_ARRAY();
|
|
229
|
+
}
|
|
230
|
+
if (state === "post") {
|
|
231
|
+
if (WS(c))
|
|
232
|
+
continue;
|
|
233
|
+
throw NOT_JSON(`text after the closing bracket at offset ${i}`);
|
|
234
|
+
}
|
|
235
|
+
if (inStr) {
|
|
236
|
+
if (esc)
|
|
237
|
+
esc = false;
|
|
238
|
+
else if (c === 0x5c /* \ */)
|
|
239
|
+
esc = true;
|
|
240
|
+
else if (c === 0x22 /* " */)
|
|
241
|
+
inStr = false;
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
if (c === 0x22) {
|
|
245
|
+
inStr = true;
|
|
246
|
+
if (start < 0)
|
|
247
|
+
start = i;
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
if (depth === 0) {
|
|
251
|
+
if (c === 0x2c /* , */ || c === 0x5d /* ] */) {
|
|
252
|
+
if (start >= 0) {
|
|
253
|
+
yield buf.slice(start, i);
|
|
254
|
+
start = -1;
|
|
255
|
+
afterComma = c === 0x2c;
|
|
256
|
+
}
|
|
257
|
+
else if (c === 0x2c || afterComma)
|
|
258
|
+
throw NOT_JSON("an empty array element");
|
|
259
|
+
if (c === 0x5d)
|
|
260
|
+
state = "post";
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
if (WS(c))
|
|
264
|
+
continue;
|
|
265
|
+
if (start < 0)
|
|
266
|
+
start = i;
|
|
267
|
+
}
|
|
268
|
+
if (c === 0x7b /* { */ || c === 0x5b)
|
|
269
|
+
depth++;
|
|
270
|
+
else if (c === 0x7d /* } */ || c === 0x5d) {
|
|
271
|
+
if (--depth < 0)
|
|
272
|
+
throw NOT_JSON("unbalanced brackets");
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
// Drop what has been read; keep the element in progress.
|
|
276
|
+
const keep = start >= 0 ? start : i;
|
|
277
|
+
buf = buf.slice(keep);
|
|
278
|
+
i -= keep;
|
|
279
|
+
if (start >= 0)
|
|
280
|
+
start = 0;
|
|
281
|
+
}
|
|
282
|
+
if (state === "pre")
|
|
283
|
+
throw NOT_ARRAY();
|
|
284
|
+
if (state !== "post")
|
|
285
|
+
throw NOT_JSON("the file ends inside the array");
|
|
286
|
+
}
|
|
287
|
+
/** Maps a Prowler OCSF file, streamed (see jsonArrayElements). */
|
|
288
|
+
export async function parseOcsfFile(path, provider, known) {
|
|
289
|
+
const m = new OcsfMerger(provider, known);
|
|
290
|
+
for await (const el of jsonArrayElements(createReadStream(path, { encoding: "utf8", highWaterMark: 1 << 20 }))) {
|
|
291
|
+
let f;
|
|
292
|
+
try {
|
|
293
|
+
f = JSON.parse(el);
|
|
294
|
+
}
|
|
295
|
+
catch (e) {
|
|
296
|
+
throw NOT_JSON(e.message);
|
|
297
|
+
}
|
|
298
|
+
m.add(f);
|
|
299
|
+
}
|
|
300
|
+
return m.finish();
|
|
184
301
|
}
|
|
185
302
|
export const spawnProwler = (bin, args, env) => new Promise((resolve, reject) => {
|
|
186
303
|
const p = spawn(bin, args, { env, stdio: ["ignore", "ignore", "pipe"] });
|
|
@@ -189,7 +306,7 @@ export const spawnProwler = (bin, args, env) => new Promise((resolve, reject) =>
|
|
|
189
306
|
p.on("error", reject);
|
|
190
307
|
p.on("close", (code) => resolve({ code, stderr: err }));
|
|
191
308
|
});
|
|
192
|
-
/** Runs Prowler into a fresh directory and
|
|
309
|
+
/** Runs Prowler into a fresh directory and maps its OCSF file, streamed, before the directory goes. A non-zero exit is an error. */
|
|
193
310
|
export async function runProwler(provider, o) {
|
|
194
311
|
const bad = o.extra.filter((a) => REFUSED_ARGS.includes(a.split("=")[0]));
|
|
195
312
|
if (bad.length)
|
|
@@ -210,12 +327,14 @@ export async function runProwler(provider, o) {
|
|
|
210
327
|
const file = readdirSync(dir).find((f) => f.endsWith(".ocsf.json"));
|
|
211
328
|
if (!file)
|
|
212
329
|
throw new Error(`${o.bin} ${provider} wrote no .ocsf.json in ${dir}.`);
|
|
213
|
-
return
|
|
330
|
+
return await parseOcsfFile(join(dir, file), provider, o.known);
|
|
214
331
|
}
|
|
215
332
|
finally {
|
|
216
333
|
rmSync(dir, { recursive: true, force: true });
|
|
217
334
|
}
|
|
218
335
|
}
|
|
336
|
+
/** Prowler scans run side by side, at most this many at once (each is a Python process of its own). */
|
|
337
|
+
export const PROWLER_PARALLEL = 2;
|
|
219
338
|
// ── Source ──────────────────────────────────────────────────────────────────────────────────────────
|
|
220
339
|
export const prowler = {
|
|
221
340
|
name: "prowler",
|
|
@@ -240,36 +359,43 @@ export const prowler = {
|
|
|
240
359
|
if (file && providers.length > 1)
|
|
241
360
|
throw new Error("--prowler-output reads one provider's file; name one --prowler-provider with it.");
|
|
242
361
|
const known = new Set(catalogIds(one(ctx.opts.catalog) ?? DEFAULT_CATALOG_DIR));
|
|
243
|
-
const results = [];
|
|
244
|
-
const failures = [];
|
|
245
362
|
for (const provider of providers) {
|
|
246
363
|
if (![...known].some((id) => id.startsWith(`prowler.${provider}.`))) {
|
|
247
364
|
ctx.warn(`prowler: sos-app's catalogue lists no ${provider} check; every finding will be dropped.`);
|
|
248
365
|
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
throw new Error(`cannot read --prowler-output ${file}: ${e.code ?? String(e)}`);
|
|
257
|
-
}
|
|
366
|
+
}
|
|
367
|
+
// Each provider read or scanned, PROWLER_PARALLEL at a time; the outcomes are then taken in the order given, so the
|
|
368
|
+
// results and their order are those of one provider after another.
|
|
369
|
+
const one_ = async (provider) => {
|
|
370
|
+
if (file) {
|
|
371
|
+
try {
|
|
372
|
+
accessSync(file, constants.R_OK);
|
|
258
373
|
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
const extra = one(ctx.opts["prowler-args"]) ? splitArgs(one(ctx.opts["prowler-args"])) : [];
|
|
262
|
-
text = await runProwler(provider, { bin: one(ctx.opts["prowler-bin"]) ?? "prowler", extra, env: prowlerEnv(provider, ctx.env), spawner });
|
|
374
|
+
catch (e) {
|
|
375
|
+
throw new Error(`cannot read --prowler-output ${file}: ${e.code ?? String(e)}`);
|
|
263
376
|
}
|
|
377
|
+
return parseOcsfFile(file, provider, known);
|
|
264
378
|
}
|
|
265
|
-
|
|
379
|
+
const spawner = ctx.clients.prowler ?? spawnProwler;
|
|
380
|
+
const extra = one(ctx.opts["prowler-args"]) ? splitArgs(one(ctx.opts["prowler-args"])) : [];
|
|
381
|
+
return runProwler(provider, { bin: one(ctx.opts["prowler-bin"]) ?? "prowler", extra, env: prowlerEnv(provider, ctx.env), spawner, known });
|
|
382
|
+
};
|
|
383
|
+
const outcomes = await pool(providers, PROWLER_PARALLEL, one_);
|
|
384
|
+
const results = [];
|
|
385
|
+
const failures = [];
|
|
386
|
+
for (const [i, o] of outcomes.entries()) {
|
|
387
|
+
const provider = providers[i];
|
|
388
|
+
if (o.status === "rejected") {
|
|
389
|
+
// An output that is not OCSF is the file's fault, not the scan's: it stops the source, as it always has.
|
|
390
|
+
if (o.reason instanceof OcsfError)
|
|
391
|
+
throw o.reason;
|
|
266
392
|
// A second way that fails leaves the control to its other ways: say so and go on with the next provider.
|
|
267
|
-
failures.push(
|
|
393
|
+
failures.push(o.reason);
|
|
268
394
|
if (providers.length > 1)
|
|
269
|
-
ctx.warn(`prowler: ${provider} did not run (${
|
|
395
|
+
ctx.warn(`prowler: ${provider} did not run (${o.reason.message}); its controls keep their other ways.`);
|
|
270
396
|
continue;
|
|
271
397
|
}
|
|
272
|
-
const { results: r, summary } =
|
|
398
|
+
const { results: r, summary } = o.value;
|
|
273
399
|
results.push(...r);
|
|
274
400
|
ctx.warn(`prowler: ${summary.findings} findings; ${summary.kept} results (${summary.merged} merged into the same check and resource, ` +
|
|
275
401
|
`${summary.muted} muted and ${summary.manual} manual sent as skipped); ${summary.dropped} findings dropped from ${summary.droppedChecks.length} checks not in the catalogue` +
|
|
@@ -2,6 +2,18 @@
|
|
|
2
2
|
// GitHub): a GET-only JSON client, retry on rate limits, pagination, "skipped" details and token
|
|
3
3
|
// redaction. The only non-GET request any of these sources makes is the OAuth token request in
|
|
4
4
|
// oauth.ts, to the identity provider's token endpoint.
|
|
5
|
+
import { channel } from "node:diagnostics_channel";
|
|
6
|
+
/** The recorder (src/report.ts RETRY_CHANNEL) counts retries per source from here; a plain string, so this file needs no import of it. */
|
|
7
|
+
const retries = channel("sagentix:collector:retry");
|
|
8
|
+
/** Tells the run's recorder that a call is about to be retried (a no-op when nothing listens). */
|
|
9
|
+
export function announceRetry(status, waitMs, attempt) {
|
|
10
|
+
if (retries.hasSubscribers)
|
|
11
|
+
retries.publish({ status, waitMs, attempt });
|
|
12
|
+
}
|
|
13
|
+
/** Gateway and availability errors worth one or two more tries; a 500 is usually the request itself and is not retried. */
|
|
14
|
+
export const TRANSIENT_STATUS = new Set([502, 503, 504]);
|
|
15
|
+
/** Retries after a transient error (a 502/503/504 or a network failure), on top of the rate-limit retries. */
|
|
16
|
+
export const TRANSIENT_RETRIES = 2;
|
|
5
17
|
export const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
6
18
|
export const str = (v) => (typeof v === "string" ? v : undefined);
|
|
7
19
|
export const arr = (v) => (Array.isArray(v) ? v : []);
|
|
@@ -71,21 +83,30 @@ export function retryDelayMs(h, attempt, nowMs) {
|
|
|
71
83
|
return Math.min(cap, Math.max(1000, reset * 1000 - nowMs + 1000));
|
|
72
84
|
return Math.min(cap, 1000 * 2 ** attempt);
|
|
73
85
|
}
|
|
74
|
-
/**
|
|
86
|
+
/**
|
|
87
|
+
* Retries a rate-limited GET up to `max` times, and a transient failure (502, 503, 504, or a thrown network error or
|
|
88
|
+
* timeout) up to TRANSIENT_RETRIES times, waiting Retry-After when the answer gives one. A thrown error becomes status 0.
|
|
89
|
+
*/
|
|
75
90
|
export function withRetry(api, sleep = realSleep, max = 4, now = Date.now) {
|
|
76
91
|
return {
|
|
77
92
|
async get(url) {
|
|
93
|
+
let transient = 0;
|
|
78
94
|
for (let attempt = 0;; attempt++) {
|
|
79
|
-
let r;
|
|
95
|
+
let r, thrown = false;
|
|
80
96
|
try {
|
|
81
97
|
r = await api.get(url);
|
|
82
98
|
}
|
|
83
99
|
catch (e) {
|
|
84
100
|
r = { status: 0, headers: {}, body: { message: `request failed: ${String(e)}` } };
|
|
101
|
+
thrown = true;
|
|
85
102
|
}
|
|
86
|
-
|
|
103
|
+
const rate = isRateLimited(r);
|
|
104
|
+
const again = rate ? attempt < max : (thrown || TRANSIENT_STATUS.has(r.status)) && transient++ < TRANSIENT_RETRIES;
|
|
105
|
+
if (!again)
|
|
87
106
|
return r;
|
|
88
|
-
|
|
107
|
+
const wait = retryDelayMs(r.headers, rate ? attempt : transient - 1, now());
|
|
108
|
+
announceRetry(r.status, wait, attempt + 1);
|
|
109
|
+
await sleep(wait);
|
|
89
110
|
}
|
|
90
111
|
},
|
|
91
112
|
};
|
|
@@ -120,7 +141,8 @@ export function why(r) {
|
|
|
120
141
|
msg = msg.replace(/\s+/g, " ").trim().slice(0, 300);
|
|
121
142
|
const hint = r.status === 401 || r.status === 403
|
|
122
143
|
? " (the credential may lack the read permission for this call, or the feature is not licensed)"
|
|
123
|
-
: r.status === 429 ? " (rate limited; retries exhausted)"
|
|
144
|
+
: r.status === 429 ? " (rate limited; retries exhausted)"
|
|
145
|
+
: TRANSIENT_STATUS.has(r.status) ? " (the provider is unavailable; retried, try the run again later)" : "";
|
|
124
146
|
return `HTTP ${r.status}${msg ? `: ${msg}` : ""}${hint}`;
|
|
125
147
|
}
|
|
126
148
|
/**
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// Rippling (GET only), with an API token a Rippling admin makes under Developer > API credentials and grants only the
|
|
2
|
+
// workers.read scope, read as joiners, leavers and the org chart (hris_common.ts).
|
|
3
|
+
//
|
|
4
|
+
// From Rippling's REST API documentation (all read 2026-11-07):
|
|
5
|
+
// Tokens: made at https://app.rippling.com/api-tokens/tokens by a user who may manage API tokens; scopes are chosen at
|
|
6
|
+
// creation and limited by the creator's own permissions; a token is revoked when its owner leaves or after 30 days
|
|
7
|
+
// unused, so a daily or weekly run keeps it alive. Requests go to https://rest.ripplingapis.com with
|
|
8
|
+
// Authorization: Bearer <token>. https://developer.rippling.com/documentation/rest-api/essentials/api-tokens
|
|
9
|
+
// GET /workers/?limit=100 workers.read
|
|
10
|
+
// -> { __meta, results: [ { id, work_email, status (INIT, HIRED, ACCEPTED, ACTIVE, TERMINATED), start_date,
|
|
11
|
+
// end_date, manager_id, title, custom_fields, termination_details } ], next_link }
|
|
12
|
+
// https://developer.rippling.com/documentation/rest-api/reference/list-workers.md
|
|
13
|
+
// Pages: send the first request without a cursor, then follow next_link unchanged until it is null; limit is at most
|
|
14
|
+
// 100. https://developer.rippling.com/documentation/rest-api/essentials/pagination
|
|
15
|
+
// Rate: 600 requests per 10 seconds per token and path; a 429 is retried after Retry-After.
|
|
16
|
+
// https://developer.rippling.com/documentation/rest-api/essentials/api-limits
|
|
17
|
+
// The acknowledgement and the background check have no standard worker field: the customer names the custom field that
|
|
18
|
+
// records each date (--rippling-ack-field, --rippling-background-field). custom_fields' item shape is open in the
|
|
19
|
+
// reference, so both { "<name>": value } and { name|label|api_name, value } items are read; a field never found is
|
|
20
|
+
// reported as such rather than failing every hire.
|
|
21
|
+
//
|
|
22
|
+
// Personal data: names, compensation and every expandable object are never asked for (no expand). A person is named in
|
|
23
|
+
// a result by the Rippling worker id; the work email is used only for the identity-provider lookup and the LMS join.
|
|
24
|
+
import { HRIS_CHECKS, hrisChecks, hrisOptionSpecs, hrisOptions, hrisSkipped, isoDay } from "./hris_common.js";
|
|
25
|
+
import { IDP_SECRETS } from "./directory.js";
|
|
26
|
+
import { arr, fetchRest, isObj, ok, realSleep, redact, str, why, withRetry } from "./readonly_http.js";
|
|
27
|
+
export const RIPPLING_ENV = { token: "RIPPLING_API_TOKEN" };
|
|
28
|
+
export const RIPPLING_BASE = "https://rest.ripplingapis.com";
|
|
29
|
+
export const WORKERS = "/workers/?limit=100";
|
|
30
|
+
const FIELD = /^[A-Za-z0-9][A-Za-z0-9_ .-]{0,63}$/;
|
|
31
|
+
export function ripplingFromEnv(env, fetchFn = fetch) {
|
|
32
|
+
const token = env[RIPPLING_ENV.token];
|
|
33
|
+
if (!token)
|
|
34
|
+
return `${RIPPLING_ENV.token} is not set`;
|
|
35
|
+
return fetchRest({ base: RIPPLING_BASE, fetchFn, headers: async () => ({ authorization: `Bearer ${token}` }) });
|
|
36
|
+
}
|
|
37
|
+
/** A custom field's value, from either item shape the reference allows. */
|
|
38
|
+
export function customField(w, name) {
|
|
39
|
+
const want = name.toLowerCase();
|
|
40
|
+
for (const it of arr(w.custom_fields).filter(isObj)) {
|
|
41
|
+
for (const [k, v] of Object.entries(it))
|
|
42
|
+
if (k.toLowerCase() === want)
|
|
43
|
+
return v;
|
|
44
|
+
const key = [it.name, it.label, it.api_name, it.field].find((x) => typeof x === "string" && x.toLowerCase() === want);
|
|
45
|
+
if (key !== undefined)
|
|
46
|
+
return it.value;
|
|
47
|
+
}
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
export function personOf(w, ackField, backgroundField) {
|
|
51
|
+
const s = String(w.status ?? "").toUpperCase();
|
|
52
|
+
const status = s === "ACTIVE" ? "active" : s === "TERMINATED" ? "terminated" : s.toLowerCase() || "unknown";
|
|
53
|
+
return {
|
|
54
|
+
id: String(w.id ?? "?").slice(0, 40),
|
|
55
|
+
email: str(w.work_email)?.trim() || undefined,
|
|
56
|
+
status,
|
|
57
|
+
hired: isoDay(w.start_date),
|
|
58
|
+
left: status === "terminated" ? isoDay(w.end_date) : undefined,
|
|
59
|
+
managerId: str(w.manager_id)?.trim() || undefined,
|
|
60
|
+
title: str(w.title)?.trim() || undefined,
|
|
61
|
+
dates: { ack: ackField ? isoDay(customField(w, ackField)) : undefined, background: backgroundField ? isoDay(customField(w, backgroundField)) : undefined },
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export const rippling = {
|
|
65
|
+
name: "rippling",
|
|
66
|
+
options: {
|
|
67
|
+
"rippling-company": { type: "string", help: "a name for the Rippling company in results, e.g. its domain (the token decides what is read)" },
|
|
68
|
+
...hrisOptionSpecs("rippling"),
|
|
69
|
+
},
|
|
70
|
+
emits: HRIS_CHECKS.map((c) => `rippling.${c}`),
|
|
71
|
+
describe: (opts) => `Rippling company ${typeof opts["rippling-company"] === "string" ? opts["rippling-company"] : "(token's company)"} (REST API, workers.read)`,
|
|
72
|
+
validate(opts) {
|
|
73
|
+
for (const k of ["rippling-ack-field", "rippling-background-field"]) {
|
|
74
|
+
const v = opts[k];
|
|
75
|
+
if (typeof v === "string" && v.trim() && !FIELD.test(v.trim()))
|
|
76
|
+
return `--${k} must be a custom field name (letters, digits, space, _ . -)`;
|
|
77
|
+
}
|
|
78
|
+
return undefined;
|
|
79
|
+
},
|
|
80
|
+
async run(ctx) {
|
|
81
|
+
const company = typeof ctx.opts["rippling-company"] === "string" && ctx.opts["rippling-company"].trim() ? ctx.opts["rippling-company"].trim().slice(0, 100) : "company";
|
|
82
|
+
const resource = `rippling ${company}`;
|
|
83
|
+
const o = hrisOptions("rippling", ctx);
|
|
84
|
+
if (typeof o === "string")
|
|
85
|
+
return hrisSkipped("rippling", resource, o);
|
|
86
|
+
const given = ctx.clients.rippling;
|
|
87
|
+
const built = given !== undefined ? given : ripplingFromEnv(ctx.env);
|
|
88
|
+
if (!built || typeof built === "string")
|
|
89
|
+
return hrisSkipped("rippling", resource, typeof built === "string" ? built : "no Rippling client");
|
|
90
|
+
const api = withRetry(built, ctx.clients.sleep ?? realSleep);
|
|
91
|
+
const workers = [];
|
|
92
|
+
let url = WORKERS, pages = 0;
|
|
93
|
+
while (url && pages < 500) {
|
|
94
|
+
const r = await api.get(url);
|
|
95
|
+
if (!ok(r) || !isObj(r.body) || !Array.isArray(r.body.results)) {
|
|
96
|
+
return redact(hrisSkipped("rippling", resource, `workers not readable: ${why(r)}`, r), [ctx.env[RIPPLING_ENV.token]]);
|
|
97
|
+
}
|
|
98
|
+
pages++;
|
|
99
|
+
workers.push(...r.body.results.filter(isObj));
|
|
100
|
+
url = str(r.body.next_link) || undefined;
|
|
101
|
+
}
|
|
102
|
+
const people = workers.map((w) => personOf(w, o.ackField, o.backgroundField));
|
|
103
|
+
const read = { method: "config-read", itemsRead: people.length, pagesRead: pages, complete: !url };
|
|
104
|
+
let rs = await hrisChecks("rippling", resource, people, read, o, ctx);
|
|
105
|
+
// A custom field no worker carries is more likely misnamed or not shared with the token than empty for everyone.
|
|
106
|
+
for (const [check, field] of [["rippling.hire_acknowledgement", o.ackField], ["rippling.hire_background_check", o.backgroundField]]) {
|
|
107
|
+
if (!field || workers.some((w) => customField(w, field) !== undefined))
|
|
108
|
+
continue;
|
|
109
|
+
rs = rs.filter((r) => r.check !== check);
|
|
110
|
+
rs.push({ check, resource, result: "skipped", detail: `no worker carries the custom field "${field}"; check its name and that the token can read custom fields` });
|
|
111
|
+
}
|
|
112
|
+
return redact(rs, [ctx.env[RIPPLING_ENV.token], ...IDP_SECRETS(ctx.env)]);
|
|
113
|
+
},
|
|
114
|
+
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Helpers shared by the AWS, Azure, GCP, Jira and LMS sources: option lists, retry with backoff on
|
|
2
2
|
// throttling, a result builder, and a scrubber that keeps secret values out of every detail.
|
|
3
|
-
import { redact } from "./readonly_http.js";
|
|
3
|
+
import { TRANSIENT_RETRIES, TRANSIENT_STATUS, announceRetry, redact } from "./readonly_http.js";
|
|
4
4
|
export const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
5
5
|
export const arr = (v) => (Array.isArray(v) ? v : []);
|
|
6
6
|
export const str = (v) => (typeof v === "string" ? v : undefined);
|
|
@@ -20,25 +20,33 @@ export function retryAfterMs(headers, now = Date.now()) {
|
|
|
20
20
|
}
|
|
21
21
|
/**
|
|
22
22
|
* Calls `send` until it gives a reply that is not throttled, or the attempts run out (then the last
|
|
23
|
-
* throttled reply is returned, and the caller turns it into "skipped"). A
|
|
24
|
-
*
|
|
23
|
+
* throttled reply is returned, and the caller turns it into "skipped"). A transient failure (502 or 504,
|
|
24
|
+
* or a thrown network error or timeout) is tried again up to TRANSIENT_RETRIES times within the same
|
|
25
|
+
* attempts. A thrown error is a reply with status 0.
|
|
25
26
|
*/
|
|
26
27
|
export async function withRetry(send, o = {}) {
|
|
27
28
|
const attempts = o.attempts ?? 5, base = o.baseMs ?? 500, max = o.maxMs ?? 20_000;
|
|
28
29
|
const sleep = o.sleep ?? realSleep;
|
|
29
30
|
const throttled = o.throttled ?? ((r) => r.status === 429 || r.status === 503);
|
|
30
31
|
let last = { status: 0, headers: {}, body: "" };
|
|
32
|
+
let transient = 0;
|
|
31
33
|
for (let i = 0; i < attempts; i++) {
|
|
34
|
+
let thrown = false;
|
|
32
35
|
try {
|
|
33
36
|
last = await send();
|
|
34
37
|
}
|
|
35
38
|
catch (e) {
|
|
36
39
|
last = { status: 0, headers: {}, body: `request failed: ${String(e)}` };
|
|
40
|
+
thrown = true;
|
|
37
41
|
}
|
|
38
|
-
|
|
42
|
+
const again = throttled(last) || ((thrown || TRANSIENT_STATUS.has(last.status)) && transient++ < TRANSIENT_RETRIES);
|
|
43
|
+
if (!again)
|
|
39
44
|
return last;
|
|
40
|
-
if (i < attempts - 1)
|
|
41
|
-
|
|
45
|
+
if (i < attempts - 1) {
|
|
46
|
+
const wait = Math.min(max, retryAfterMs(last.headers) ?? base * 2 ** i);
|
|
47
|
+
announceRetry(last.status, wait, i + 1);
|
|
48
|
+
await sleep(wait);
|
|
49
|
+
}
|
|
42
50
|
}
|
|
43
51
|
return last;
|
|
44
52
|
}
|