@cclabsnz/sf-audit 1.13.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -0
- package/README.md +15 -4
- package/lib/checks/impl/AuditTrailCheck.js +18 -1
- package/lib/checks/impl/LegacyApiVersionCheck.js +14 -2
- package/lib/checks/impl/LightningMessageChannelCheck.d.ts +10 -0
- package/lib/checks/impl/LightningMessageChannelCheck.js +107 -0
- package/lib/checks/impl/MfaEnforcementCheck.js +19 -1
- package/lib/checks/impl/NamedCredentialsCheck.js +42 -29
- package/lib/checks/registry.js +2 -0
- package/lib/commands/audit/incident/collect.d.ts +22 -0
- package/lib/commands/audit/incident/collect.js +46 -0
- package/lib/commands/audit/incident/report.d.ts +29 -0
- package/lib/commands/audit/incident/report.js +87 -0
- package/lib/compliance/mapping.js +2 -1
- package/lib/findings/CheckMeta.js +1 -0
- package/lib/incident/analyse/actions.d.ts +22 -0
- package/lib/incident/analyse/actions.js +73 -0
- package/lib/incident/analyse/actors.d.ts +56 -0
- package/lib/incident/analyse/actors.js +231 -0
- package/lib/incident/analyse/baseline.d.ts +24 -0
- package/lib/incident/analyse/baseline.js +66 -0
- package/lib/incident/analyse/configDelta.d.ts +33 -0
- package/lib/incident/analyse/configDelta.js +98 -0
- package/lib/incident/analyse/index.d.ts +41 -0
- package/lib/incident/analyse/index.js +68 -0
- package/lib/incident/analyse/outcomes.d.ts +28 -0
- package/lib/incident/analyse/outcomes.js +32 -0
- package/lib/incident/analyse/responses.d.ts +59 -0
- package/lib/incident/analyse/responses.js +201 -0
- package/lib/incident/analyse/verdict.d.ts +40 -0
- package/lib/incident/analyse/verdict.js +149 -0
- package/lib/incident/bundleIo.d.ts +34 -0
- package/lib/incident/bundleIo.js +74 -0
- package/lib/incident/collect/collectBundle.d.ts +21 -0
- package/lib/incident/collect/collectBundle.js +130 -0
- package/lib/incident/collect/discoverWaves.d.ts +15 -0
- package/lib/incident/collect/discoverWaves.js +90 -0
- package/lib/incident/collect/followUp.d.ts +14 -0
- package/lib/incident/collect/followUp.js +64 -0
- package/lib/incident/collect/orgErrors.d.ts +5 -0
- package/lib/incident/collect/orgErrors.js +11 -0
- package/lib/incident/collect/snapshotOrg.d.ts +18 -0
- package/lib/incident/collect/snapshotOrg.js +117 -0
- package/lib/incident/collect/streamGuestLogs.d.ts +17 -0
- package/lib/incident/collect/streamGuestLogs.js +116 -0
- package/lib/incident/csv.d.ts +32 -0
- package/lib/incident/csv.js +176 -0
- package/lib/incident/ipRanges.d.ts +23 -0
- package/lib/incident/ipRanges.js +64 -0
- package/lib/incident/model.d.ts +132 -0
- package/lib/incident/model.js +26 -0
- package/lib/incident/render/evidence.d.ts +15 -0
- package/lib/incident/render/evidence.js +52 -0
- package/lib/incident/render/html.d.ts +7 -0
- package/lib/incident/render/html.js +144 -0
- package/lib/incident/render/labels.d.ts +9 -0
- package/lib/incident/render/labels.js +22 -0
- package/lib/incident/render/markdown.d.ts +17 -0
- package/lib/incident/render/markdown.js +91 -0
- package/lib/incident/render/redact.d.ts +7 -0
- package/lib/incident/render/redact.js +62 -0
- package/lib/incident/text.d.ts +9 -0
- package/lib/incident/text.js +43 -0
- package/package.json +20 -8
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { degrades } from './orgErrors.js';
|
|
2
|
+
import { DEFAULTS, id15 } from '../model.js';
|
|
3
|
+
const quoteIds = (ids) => ids.map((i) => `'${i.replace(/[^A-Za-z0-9]/g, '')}'`).join(',');
|
|
4
|
+
/** Active guests plus any user an anomaly event named, so a since-deactivated guest's rows survive filtering. */
|
|
5
|
+
export async function snapshotGuests(soql, extraIds, warn = () => { }) {
|
|
6
|
+
const warnings = [];
|
|
7
|
+
const active = await soql.queryAll("SELECT Id, Username, Name, Profile.Name, IsActive FROM User WHERE UserType = 'Guest' AND IsActive = true");
|
|
8
|
+
const known = new Set(active.map((u) => id15(u.Id)));
|
|
9
|
+
const missing = [...new Set(extraIds.map((x) => id15(x)).filter((x) => x && !known.has(x)))];
|
|
10
|
+
const extra = missing.length
|
|
11
|
+
? await soql.queryAll(`SELECT Id, Username, Name, Profile.Name, IsActive FROM User WHERE Id IN (${quoteIds(missing)})`)
|
|
12
|
+
: [];
|
|
13
|
+
const optional = async (object, q) => {
|
|
14
|
+
try {
|
|
15
|
+
return await soql.queryAll(q);
|
|
16
|
+
}
|
|
17
|
+
catch (e) {
|
|
18
|
+
if (!degrades(e))
|
|
19
|
+
throw e;
|
|
20
|
+
const m = `Could not read ${object} (${String(e)}); continuing without it.`;
|
|
21
|
+
warnings.push(m);
|
|
22
|
+
warn(m);
|
|
23
|
+
return [];
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
const sites = await optional('Site', 'SELECT Name, GuestUserId FROM Site');
|
|
27
|
+
const users = [...active, ...extra];
|
|
28
|
+
const perms = users.length
|
|
29
|
+
? await optional('PermissionSetAssignment', `SELECT AssigneeId, PermissionSet.Label FROM PermissionSetAssignment WHERE AssigneeId IN (${quoteIds(users.map((u) => u.Id))}) AND PermissionSet.IsOwnedByProfile = false`)
|
|
30
|
+
: [];
|
|
31
|
+
const guests = users.map((u) => {
|
|
32
|
+
const i = id15(u.Id);
|
|
33
|
+
return {
|
|
34
|
+
id15: i,
|
|
35
|
+
username: u.Username,
|
|
36
|
+
name: u.Name,
|
|
37
|
+
profileName: u.Profile?.Name ?? '',
|
|
38
|
+
permissionSetLabels: perms.filter((p) => id15(p.AssigneeId) === i).map((p) => p.PermissionSet.Label).sort(),
|
|
39
|
+
siteNames: [...new Set(sites.filter((s) => id15(s.GuestUserId) === i).map((s) => s.Name))].sort(),
|
|
40
|
+
active: u.IsActive && known.has(i),
|
|
41
|
+
};
|
|
42
|
+
});
|
|
43
|
+
return { guests, warnings };
|
|
44
|
+
}
|
|
45
|
+
export async function readLimits(soql) {
|
|
46
|
+
try {
|
|
47
|
+
const r = await soql.queryAll('SELECT PermissionsQueryAllFiles, PermissionsViewAllData FROM UserPermissionAccess');
|
|
48
|
+
return { queryAllFiles: Boolean(r[0]?.PermissionsQueryAllFiles), viewAllData: Boolean(r[0]?.PermissionsViewAllData) };
|
|
49
|
+
}
|
|
50
|
+
catch (e) {
|
|
51
|
+
if (!degrades(e))
|
|
52
|
+
throw e;
|
|
53
|
+
return { queryAllFiles: false, viewAllData: false };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function relevant(display, guests) {
|
|
57
|
+
const t = display.toLowerCase();
|
|
58
|
+
if (/^created new customer user|^reset password for user/i.test(display))
|
|
59
|
+
return true;
|
|
60
|
+
return guests.some((g) => [g.profileName, g.name, ...g.permissionSetLabels, ...g.siteNames].some((l) => l && t.includes(l.toLowerCase())))
|
|
61
|
+
|| /\bsite\b|experience|network|sharing rule|guest/i.test(display);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* SetupAuditTrail cannot filter on Section and queryAll stops at 10,000 rows, so the range is
|
|
65
|
+
* walked in 4-day windows and any window that comes back full is split in half until it fits.
|
|
66
|
+
*/
|
|
67
|
+
export async function fetchAuditTrail(soql, from, to, guests) {
|
|
68
|
+
const coverage = { from, to, truncatedWindows: [], inaccessible: false };
|
|
69
|
+
const rows = [];
|
|
70
|
+
const iso = (ms) => new Date(ms).toISOString().replace('.000Z', 'Z');
|
|
71
|
+
const fetchWindow = async (startMs, endMs) => {
|
|
72
|
+
const recs = await soql.queryAll(`SELECT CreatedDate, CreatedBy.Name, Section, Action, Display FROM SetupAuditTrail WHERE CreatedDate >= ${iso(startMs)} AND CreatedDate < ${iso(endMs)}`);
|
|
73
|
+
if (recs.length >= DEFAULTS.auditPageCap) {
|
|
74
|
+
if (endMs - startMs <= 60_000) {
|
|
75
|
+
coverage.truncatedWindows.push({ from: iso(startMs), to: iso(endMs) });
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
const mid = startMs + Math.floor((endMs - startMs) / 2);
|
|
79
|
+
await fetchWindow(startMs, mid);
|
|
80
|
+
await fetchWindow(mid, endMs);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
for (const r of recs)
|
|
85
|
+
if (relevant(r.Display ?? '', guests))
|
|
86
|
+
rows.push({ createdDate: r.CreatedDate, createdBy: r.CreatedBy?.Name ?? '', section: r.Section, action: r.Action, display: r.Display ?? '' });
|
|
87
|
+
};
|
|
88
|
+
try {
|
|
89
|
+
const startMs = Date.parse(`${from}T00:00:00Z`);
|
|
90
|
+
const endMs = Date.parse(`${to}T00:00:00Z`) + 86_400_000;
|
|
91
|
+
for (let s = startMs; s < endMs; s += DEFAULTS.auditWindowDays * 86_400_000)
|
|
92
|
+
await fetchWindow(s, Math.min(endMs, s + DEFAULTS.auditWindowDays * 86_400_000));
|
|
93
|
+
}
|
|
94
|
+
catch (e) {
|
|
95
|
+
if (!degrades(e))
|
|
96
|
+
throw e;
|
|
97
|
+
coverage.inaccessible = true;
|
|
98
|
+
}
|
|
99
|
+
rows.sort((a, c) => a.createdDate.localeCompare(c.createdDate));
|
|
100
|
+
return { rows, coverage };
|
|
101
|
+
}
|
|
102
|
+
export async function loginsByDay(soql, days) {
|
|
103
|
+
if (days.length === 0)
|
|
104
|
+
return [];
|
|
105
|
+
const first = days[0];
|
|
106
|
+
const last = new Date(Date.parse(`${days[days.length - 1]}T00:00:00Z`) + 86_400_000).toISOString().slice(0, 10);
|
|
107
|
+
try {
|
|
108
|
+
const r = await soql.queryAll(`SELECT DAY_ONLY(LoginTime) d, Status, COUNT(Id) n FROM LoginHistory WHERE LoginTime >= ${first}T00:00:00Z AND LoginTime < ${last}T00:00:00Z GROUP BY DAY_ONLY(LoginTime), Status`);
|
|
109
|
+
return r.filter((x) => days.includes(x.d)).map((x) => ({ day: x.d, status: x.Status, count: Number(x.n) }));
|
|
110
|
+
}
|
|
111
|
+
catch (e) {
|
|
112
|
+
if (!degrades(e))
|
|
113
|
+
throw e;
|
|
114
|
+
return [];
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
//# sourceMappingURL=snapshotOrg.js.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { RestClient, SoqlClient } from '@cclabsnz/sf-core';
|
|
2
|
+
import { type LogCoverage, type LogType, type Wave } from '../model.js';
|
|
3
|
+
/** Columns a log of this type needs but its header lacks. Every type needs a user column. */
|
|
4
|
+
export declare function missingColumns(type: LogType, header: string[]): string[];
|
|
5
|
+
export declare function collectionDays(waves: Wave[]): string[];
|
|
6
|
+
/**
|
|
7
|
+
* Downloads each daily log with sf-core's getRawToFile (fetch is banned in src/ by the
|
|
8
|
+
* read-only invariant), filters it to guest rows with a streaming parser, then deletes the raw
|
|
9
|
+
* file. Peak disk use is one raw log, up to ~600 MB on a spike day.
|
|
10
|
+
*/
|
|
11
|
+
export declare function streamGuestLogs(deps: {
|
|
12
|
+
soql: SoqlClient;
|
|
13
|
+
rest: RestClient;
|
|
14
|
+
}, bundleDir: string, days: string[], guestIds: ReadonlySet<string>, warn: (m: string) => void, reuse?: (type: LogType, day: string) => LogCoverage | undefined,
|
|
15
|
+
/** Called as each log is settled, so the caller can checkpoint the manifest and a killed run can resume. */
|
|
16
|
+
onLogDone?: (coverage: LogCoverage) => void | Promise<void>): Promise<LogCoverage[]>;
|
|
17
|
+
//# sourceMappingURL=streamGuestLogs.d.ts.map
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// src/incident/collect/streamGuestLogs.ts
|
|
2
|
+
import { mkdirSync, rmSync, renameSync, statfsSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { degrades } from './orgErrors.js';
|
|
5
|
+
import { filterLogFile, MissingColumnsError } from '../csv.js';
|
|
6
|
+
import { logPath } from '../bundleIo.js';
|
|
7
|
+
import { LOG_TYPES } from '../model.js';
|
|
8
|
+
const nextDay = (d, n) => new Date(Date.parse(`${d}T00:00:00Z`) + n * 86_400_000).toISOString().slice(0, 10);
|
|
9
|
+
const REQUIRED_COLUMNS = {
|
|
10
|
+
AuraRequest: ['TIMESTAMP_DERIVED', 'CLIENT_IP', 'REQUEST_ID', 'ACTION_MESSAGE'],
|
|
11
|
+
Sites: ['REQUEST_ID', 'RESPONSE_SIZE'],
|
|
12
|
+
};
|
|
13
|
+
/** Columns a log of this type needs but its header lacks. Every type needs a user column. */
|
|
14
|
+
export function missingColumns(type, header) {
|
|
15
|
+
const have = new Set(header.map((h) => h.trim()));
|
|
16
|
+
const out = (REQUIRED_COLUMNS[type] ?? []).filter((c) => !have.has(c));
|
|
17
|
+
if (!have.has('USER_ID') && !have.has('USER_ID_DERIVED'))
|
|
18
|
+
out.push('USER_ID or USER_ID_DERIVED');
|
|
19
|
+
return out;
|
|
20
|
+
}
|
|
21
|
+
export function collectionDays(waves) {
|
|
22
|
+
return [...new Set(waves.flatMap((w) => w.days.flatMap((d) => [nextDay(d, -1), d, nextDay(d, 1)])))].sort();
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Downloads each daily log with sf-core's getRawToFile (fetch is banned in src/ by the
|
|
26
|
+
* read-only invariant), filters it to guest rows with a streaming parser, then deletes the raw
|
|
27
|
+
* file. Peak disk use is one raw log, up to ~600 MB on a spike day.
|
|
28
|
+
*/
|
|
29
|
+
export async function streamGuestLogs(deps, bundleDir, days, guestIds, warn, reuse = () => undefined,
|
|
30
|
+
/** Called as each log is settled, so the caller can checkpoint the manifest and a killed run can resume. */
|
|
31
|
+
onLogDone = () => { }) {
|
|
32
|
+
const coverage = [];
|
|
33
|
+
const settle = async (c) => { coverage.push(c); await onLogDone(c); };
|
|
34
|
+
const blank = (type, day, status, detail) => ({ type, day, status, totalRows: 0, guestRows: 0, guestRowsByUser: {}, malformed: 0, detail });
|
|
35
|
+
if (days.length === 0)
|
|
36
|
+
return coverage;
|
|
37
|
+
let files;
|
|
38
|
+
try {
|
|
39
|
+
files = await deps.soql.queryAll(`SELECT Id, EventType, LogDate, LogFileLength FROM EventLogFile WHERE Interval = 'Daily' AND EventType IN (${LOG_TYPES.map((t) => `'${t}'`).join(',')}) ` +
|
|
40
|
+
`AND LogDate >= ${days[0]}T00:00:00Z AND LogDate <= ${days[days.length - 1]}T00:00:00Z`);
|
|
41
|
+
}
|
|
42
|
+
catch (e) {
|
|
43
|
+
if (!degrades(e))
|
|
44
|
+
throw e;
|
|
45
|
+
return days.flatMap((d) => LOG_TYPES.map((t) => blank(t, d, 'no-permission', 'EventLogFile is not readable: the running user needs the View Event Log Files permission.')));
|
|
46
|
+
}
|
|
47
|
+
const tmp = join(bundleDir, '.tmp');
|
|
48
|
+
// A killed earlier run can leave raw, unfiltered logs here, which contain non-guest data.
|
|
49
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
50
|
+
mkdirSync(tmp, { recursive: true });
|
|
51
|
+
for (const day of days) {
|
|
52
|
+
for (const type of LOG_TYPES) {
|
|
53
|
+
const prior = reuse(type, day);
|
|
54
|
+
if (prior) {
|
|
55
|
+
await settle(prior);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const matches = files.filter((x) => x.EventType === type && x.LogDate.slice(0, 10) === day);
|
|
59
|
+
if (matches.length === 0) {
|
|
60
|
+
await settle(blank(type, day, 'missing', 'No EventLogFile for this day (past retention, not generated yet, or no activity).'));
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
// Two Daily files for one type and day cannot be told apart; picking one could hide the other.
|
|
64
|
+
if (matches.length > 1) {
|
|
65
|
+
warn(`Could not collect ${type} for ${day}: multiple daily log files.`);
|
|
66
|
+
await settle(blank(type, day, 'failed', 'multiple daily log files'));
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
const f = matches[0];
|
|
70
|
+
const raw = join(tmp, `${f.Id}.csv`);
|
|
71
|
+
const rel = logPath(type, day);
|
|
72
|
+
const part = join(bundleDir, `${rel}.part`);
|
|
73
|
+
let freeBytes;
|
|
74
|
+
try {
|
|
75
|
+
const s = statfsSync(bundleDir);
|
|
76
|
+
freeBytes = Number(s.bavail) * Number(s.bsize);
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
freeBytes = undefined;
|
|
80
|
+
}
|
|
81
|
+
if (freeBytes !== undefined && f.LogFileLength && freeBytes < 1.2 * f.LogFileLength) {
|
|
82
|
+
const mb = (n) => Math.ceil(n / 1_048_576);
|
|
83
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
84
|
+
throw new Error(`Not enough free disk space for ${type} ${day} (${mb(1.2 * f.LogFileLength)} MB needed, ${Math.floor(freeBytes / 1_048_576)} MB free). Free space and re-run; completed logs are kept.`);
|
|
85
|
+
}
|
|
86
|
+
let settled;
|
|
87
|
+
try {
|
|
88
|
+
await deps.rest.getRawToFile(`/sobjects/EventLogFile/${f.Id}/LogFile`, raw);
|
|
89
|
+
const r = await filterLogFile(raw, part, guestIds, { requiredColumns: (h) => missingColumns(type, h) });
|
|
90
|
+
renameSync(part, join(bundleDir, rel));
|
|
91
|
+
settled = { type, day, status: 'collected', ...r, file: rel };
|
|
92
|
+
}
|
|
93
|
+
catch (e) {
|
|
94
|
+
if (e?.code === 'ENOSPC' || /HTTP 401/.test(String(e))) {
|
|
95
|
+
rmSync(part, { force: true });
|
|
96
|
+
rmSync(raw, { force: true });
|
|
97
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
98
|
+
throw e;
|
|
99
|
+
}
|
|
100
|
+
warn(`Could not collect ${type} for ${day}: ${e instanceof MissingColumnsError ? e.message : String(e)}`);
|
|
101
|
+
settled = e instanceof MissingColumnsError
|
|
102
|
+
? blank(type, day, 'failed', e.message)
|
|
103
|
+
: blank(type, day, /INSUFFICIENT_ACCESS|HTTP 403/i.test(String(e)) ? 'no-permission' : 'failed', String(e));
|
|
104
|
+
rmSync(part, { force: true });
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
rmSync(raw, { force: true });
|
|
108
|
+
}
|
|
109
|
+
// Outside the try: a checkpoint failure must not be recorded as a second, failed entry.
|
|
110
|
+
await settle(settled);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
114
|
+
return coverage;
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=streamGuestLogs.js.map
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streams CSV rows from disk. Spike-day event logs exceed V8's ~512 MB string limit, so the
|
|
3
|
+
* whole-file parser in timeline/loadCaptures.ts cannot be used here. The state machine carries
|
|
4
|
+
* across read chunks, including a quote that ends one chunk.
|
|
5
|
+
*/
|
|
6
|
+
export declare function readCsvCells(path: string, opts?: {
|
|
7
|
+
chunkSize?: number;
|
|
8
|
+
}): AsyncGenerator<string[]>;
|
|
9
|
+
export declare function readCsvRecords(path: string, opts?: {
|
|
10
|
+
chunkSize?: number;
|
|
11
|
+
}): AsyncGenerator<Record<string, string>>;
|
|
12
|
+
export declare function csvLine(cells: readonly string[]): string;
|
|
13
|
+
export interface FilterResult {
|
|
14
|
+
totalRows: number;
|
|
15
|
+
guestRows: number;
|
|
16
|
+
guestRowsByUser: Record<string, number>;
|
|
17
|
+
malformed: number;
|
|
18
|
+
}
|
|
19
|
+
/** A log whose header lacks columns the analysis needs. Thrown before any row is written. */
|
|
20
|
+
export declare class MissingColumnsError extends Error {
|
|
21
|
+
readonly missing: string[];
|
|
22
|
+
constructor(missing: string[]);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Copies the rows whose USER_ID or USER_ID_DERIVED belongs to a guest user; counts every row.
|
|
26
|
+
* A UTF-8 BOM is stripped from the header. `requiredColumns`, when given, returns the columns
|
|
27
|
+
* the header lacks; any at all fails the file closed with MissingColumnsError.
|
|
28
|
+
*/
|
|
29
|
+
export declare function filterLogFile(rawPath: string, outPath: string, guestIds: ReadonlySet<string>, opts?: {
|
|
30
|
+
requiredColumns?: (header: string[]) => string[];
|
|
31
|
+
}): Promise<FilterResult>;
|
|
32
|
+
//# sourceMappingURL=csv.d.ts.map
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { createReadStream, createWriteStream, rmSync, existsSync, statSync } from 'node:fs';
|
|
2
|
+
import { mkdir } from 'node:fs/promises';
|
|
3
|
+
import { dirname } from 'node:path';
|
|
4
|
+
import { finished } from 'node:stream/promises';
|
|
5
|
+
import { id15 } from './model.js';
|
|
6
|
+
/**
|
|
7
|
+
* Streams CSV rows from disk. Spike-day event logs exceed V8's ~512 MB string limit, so the
|
|
8
|
+
* whole-file parser in timeline/loadCaptures.ts cannot be used here. The state machine carries
|
|
9
|
+
* across read chunks, including a quote that ends one chunk.
|
|
10
|
+
*/
|
|
11
|
+
export async function* readCsvCells(path, opts) {
|
|
12
|
+
const chunkSize = opts?.chunkSize ?? (1 << 20);
|
|
13
|
+
const stream = createReadStream(path, { encoding: 'utf8', highWaterMark: chunkSize });
|
|
14
|
+
let field = '';
|
|
15
|
+
let row = [];
|
|
16
|
+
let inQuotes = false;
|
|
17
|
+
let quotePending = false; // saw a quote inside a quoted field; next char decides
|
|
18
|
+
let started = false;
|
|
19
|
+
for await (const chunk of stream) {
|
|
20
|
+
for (let i = 0; i < chunk.length; i++) {
|
|
21
|
+
const c = chunk[i];
|
|
22
|
+
if (inQuotes) {
|
|
23
|
+
if (quotePending) {
|
|
24
|
+
quotePending = false;
|
|
25
|
+
if (c === '"') {
|
|
26
|
+
field += '"';
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
inQuotes = false; // that quote closed the field; handle c below
|
|
30
|
+
}
|
|
31
|
+
else if (c === '"') {
|
|
32
|
+
quotePending = true;
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
field += c;
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (c === '"' && field === '') {
|
|
41
|
+
inQuotes = true;
|
|
42
|
+
started = true;
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (c === ',') {
|
|
46
|
+
row.push(field);
|
|
47
|
+
field = '';
|
|
48
|
+
started = true;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (c === '\r')
|
|
52
|
+
continue;
|
|
53
|
+
if (c === '\n') {
|
|
54
|
+
row.push(field);
|
|
55
|
+
field = '';
|
|
56
|
+
if (!(row.length === 1 && row[0] === '' && !started))
|
|
57
|
+
yield row;
|
|
58
|
+
row = [];
|
|
59
|
+
started = false;
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
field += c;
|
|
63
|
+
started = true;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
if (started || row.length > 0) {
|
|
67
|
+
row.push(field);
|
|
68
|
+
yield row;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export async function* readCsvRecords(path, opts) {
|
|
72
|
+
let header;
|
|
73
|
+
for await (const cells of readCsvCells(path, opts)) {
|
|
74
|
+
if (!header) {
|
|
75
|
+
header = cells;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
if (cells.length !== header.length)
|
|
79
|
+
continue;
|
|
80
|
+
const rec = {};
|
|
81
|
+
for (let i = 0; i < header.length; i++)
|
|
82
|
+
rec[header[i]] = cells[i];
|
|
83
|
+
yield rec;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
export function csvLine(cells) {
|
|
87
|
+
return cells.map((v) => `"${v.replace(/"/g, '""')}"`).join(',') + '\n';
|
|
88
|
+
}
|
|
89
|
+
/** A log whose header lacks columns the analysis needs. Thrown before any row is written. */
|
|
90
|
+
export class MissingColumnsError extends Error {
|
|
91
|
+
missing;
|
|
92
|
+
constructor(missing) {
|
|
93
|
+
super(`missing columns: ${missing.join(', ')}`);
|
|
94
|
+
this.missing = missing;
|
|
95
|
+
this.name = 'MissingColumnsError';
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Copies the rows whose USER_ID or USER_ID_DERIVED belongs to a guest user; counts every row.
|
|
100
|
+
* A UTF-8 BOM is stripped from the header. `requiredColumns`, when given, returns the columns
|
|
101
|
+
* the header lacks; any at all fails the file closed with MissingColumnsError.
|
|
102
|
+
*/
|
|
103
|
+
export async function filterLogFile(rawPath, outPath, guestIds, opts = {}) {
|
|
104
|
+
await mkdir(dirname(outPath), { recursive: true });
|
|
105
|
+
const out = createWriteStream(outPath);
|
|
106
|
+
const result = { totalRows: 0, guestRows: 0, guestRowsByUser: {}, malformed: 0 };
|
|
107
|
+
let header;
|
|
108
|
+
let userCols = [];
|
|
109
|
+
let streamError = null;
|
|
110
|
+
out.on('error', (err) => { if (!streamError)
|
|
111
|
+
streamError = err; });
|
|
112
|
+
const write = async (s) => {
|
|
113
|
+
if (streamError)
|
|
114
|
+
throw streamError;
|
|
115
|
+
if (!out.write(s))
|
|
116
|
+
await new Promise((resolve, reject) => {
|
|
117
|
+
const onDrain = () => {
|
|
118
|
+
out.removeListener('error', onError);
|
|
119
|
+
if (streamError)
|
|
120
|
+
reject(streamError);
|
|
121
|
+
else
|
|
122
|
+
resolve();
|
|
123
|
+
};
|
|
124
|
+
const onError = (err) => {
|
|
125
|
+
out.removeListener('drain', onDrain);
|
|
126
|
+
streamError = err;
|
|
127
|
+
reject(err);
|
|
128
|
+
};
|
|
129
|
+
out.once('drain', onDrain);
|
|
130
|
+
out.once('error', onError);
|
|
131
|
+
});
|
|
132
|
+
};
|
|
133
|
+
try {
|
|
134
|
+
for await (const cells of readCsvCells(rawPath)) {
|
|
135
|
+
if (!header) {
|
|
136
|
+
if (cells.length > 0 && cells[0].startsWith('\uFEFF'))
|
|
137
|
+
cells[0] = cells[0].slice(1);
|
|
138
|
+
const missing = opts.requiredColumns?.(cells) ?? [];
|
|
139
|
+
if (missing.length > 0)
|
|
140
|
+
throw new MissingColumnsError(missing);
|
|
141
|
+
header = cells;
|
|
142
|
+
userCols = ['USER_ID', 'USER_ID_DERIVED'].map((h) => header.indexOf(h)).filter((i) => i >= 0);
|
|
143
|
+
await write(csvLine(cells));
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
result.totalRows++;
|
|
147
|
+
if (cells.length !== header.length) {
|
|
148
|
+
result.malformed++;
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
const guest = userCols.map((i) => id15(cells[i])).find((u) => u !== undefined && guestIds.has(u));
|
|
152
|
+
if (!guest)
|
|
153
|
+
continue;
|
|
154
|
+
result.guestRows++;
|
|
155
|
+
result.guestRowsByUser[guest] = (result.guestRowsByUser[guest] ?? 0) + 1;
|
|
156
|
+
await write(csvLine(cells));
|
|
157
|
+
}
|
|
158
|
+
out.end();
|
|
159
|
+
await finished(out);
|
|
160
|
+
}
|
|
161
|
+
catch (err) {
|
|
162
|
+
out.destroy();
|
|
163
|
+
try {
|
|
164
|
+
// Only delete if it's a file, not a directory
|
|
165
|
+
if (existsSync(outPath) && statSync(outPath).isFile()) {
|
|
166
|
+
rmSync(outPath, { force: true });
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
catch {
|
|
170
|
+
// Ignore cleanup errors; rethrow original error
|
|
171
|
+
}
|
|
172
|
+
throw err;
|
|
173
|
+
}
|
|
174
|
+
return result;
|
|
175
|
+
}
|
|
176
|
+
//# sourceMappingURL=csv.js.map
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hosting-provider classification from range files the user downloads and passes with
|
|
3
|
+
* --ip-ranges. The plugin never fetches them: the authenticated org is its only network
|
|
4
|
+
* destination (docs/TRUST.md), and network-egress.test.ts enforces that. IPv4 only.
|
|
5
|
+
*/
|
|
6
|
+
interface Range {
|
|
7
|
+
base: number;
|
|
8
|
+
bits: number;
|
|
9
|
+
label: string;
|
|
10
|
+
}
|
|
11
|
+
export interface IpRangeSet {
|
|
12
|
+
lookup(ip: string): string | null;
|
|
13
|
+
sources: string[];
|
|
14
|
+
}
|
|
15
|
+
export declare function parseIpRangeFile(name: string, text: string): Range[];
|
|
16
|
+
export declare function loadIpRanges(files: Array<{
|
|
17
|
+
name: string;
|
|
18
|
+
text: string;
|
|
19
|
+
}>): IpRangeSet;
|
|
20
|
+
/** The original file name of a bundled range file: collect stores it as `ip-ranges/{index}-{name}`. */
|
|
21
|
+
export declare function rangeFileName(rel: string): string;
|
|
22
|
+
export {};
|
|
23
|
+
//# sourceMappingURL=ipRanges.d.ts.map
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
function ipv4ToInt(ip) {
|
|
2
|
+
const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(ip.trim());
|
|
3
|
+
if (!m)
|
|
4
|
+
return null;
|
|
5
|
+
const parts = m.slice(1).map(Number);
|
|
6
|
+
if (parts.some((p) => p > 255))
|
|
7
|
+
return null;
|
|
8
|
+
return ((parts[0] << 24) >>> 0) + (parts[1] << 16) + (parts[2] << 8) + parts[3];
|
|
9
|
+
}
|
|
10
|
+
function cidr(prefix, label) {
|
|
11
|
+
const [addr, bitsText] = prefix.split('/');
|
|
12
|
+
const base = ipv4ToInt(addr);
|
|
13
|
+
const bits = Number(bitsText);
|
|
14
|
+
if (base === null || !bitsText?.trim() || !Number.isInteger(bits) || bits < 0 || bits > 32)
|
|
15
|
+
return null;
|
|
16
|
+
return { base, bits, label };
|
|
17
|
+
}
|
|
18
|
+
export function parseIpRangeFile(name, text) {
|
|
19
|
+
const out = [];
|
|
20
|
+
const push = (r) => { if (r)
|
|
21
|
+
out.push(r); };
|
|
22
|
+
const trimmed = text.trim();
|
|
23
|
+
if (trimmed.startsWith('{')) {
|
|
24
|
+
const j = JSON.parse(trimmed);
|
|
25
|
+
for (const p of j.prefixes ?? []) {
|
|
26
|
+
if (p.ip_prefix)
|
|
27
|
+
push(cidr(p.ip_prefix, `AWS ${p.region ?? ''} (${p.service ?? ''})`.replace(' ()', '')));
|
|
28
|
+
else if (p.ipv4Prefix)
|
|
29
|
+
push(cidr(p.ipv4Prefix, `GCP ${p.scope ?? ''}`.trim()));
|
|
30
|
+
}
|
|
31
|
+
for (const v of j.values ?? [])
|
|
32
|
+
for (const a of v.properties?.addressPrefixes ?? [])
|
|
33
|
+
push(cidr(a, `Azure ${v.name}`));
|
|
34
|
+
return out;
|
|
35
|
+
}
|
|
36
|
+
for (const line of trimmed.split(/\r?\n/)) {
|
|
37
|
+
const l = line.trim();
|
|
38
|
+
if (l && !l.startsWith('#'))
|
|
39
|
+
push(cidr(l.includes('/') ? l : `${l}/32`, name));
|
|
40
|
+
}
|
|
41
|
+
return out;
|
|
42
|
+
}
|
|
43
|
+
export function loadIpRanges(files) {
|
|
44
|
+
const ranges = files.flatMap((f) => parseIpRangeFile(f.name, f.text)).sort((a, b) => b.bits - a.bits);
|
|
45
|
+
return {
|
|
46
|
+
sources: files.map((f) => f.name),
|
|
47
|
+
lookup(ip) {
|
|
48
|
+
const n = ipv4ToInt(ip);
|
|
49
|
+
if (n === null)
|
|
50
|
+
return null;
|
|
51
|
+
for (const r of ranges) {
|
|
52
|
+
const mask = r.bits === 0 ? 0 : (~0 << (32 - r.bits)) >>> 0;
|
|
53
|
+
if (((n & mask) >>> 0) === ((r.base & mask) >>> 0))
|
|
54
|
+
return r.label;
|
|
55
|
+
}
|
|
56
|
+
return null;
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/** The original file name of a bundled range file: collect stores it as `ip-ranges/{index}-{name}`. */
|
|
61
|
+
export function rangeFileName(rel) {
|
|
62
|
+
return (rel.split('/').pop() ?? rel).replace(/^\d+-/, '');
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=ipRanges.js.map
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types shared by `sf audit incident collect` and `report`. The bundle on disk is the contract
|
|
3
|
+
* between the two commands: collect writes it, report only reads it.
|
|
4
|
+
*/
|
|
5
|
+
export declare const LOG_TYPES: readonly ['AuraRequest', 'Sites', 'URI', 'ApexExecution', 'VisualforceRequest', 'ContentTransfer', 'GraphQlQueryExecution', 'UniqueQuery', 'LightningError', 'ApexUnexpectedException'];
|
|
6
|
+
export type LogType = (typeof LOG_TYPES)[number];
|
|
7
|
+
/** Without these two, response sizes cannot be assessed, so a wave's result is `not-assessed`. */
|
|
8
|
+
export declare const REQUIRED_LOG_TYPES: readonly LogType[];
|
|
9
|
+
export declare const DEFAULTS: {
|
|
10
|
+
readonly spikeRatio: 5;
|
|
11
|
+
readonly emptyBandBytes: 64;
|
|
12
|
+
readonly auditWindowDays: 4;
|
|
13
|
+
/** jsforce autoFetchQuery stops here; a window returning this many rows may be truncated. */
|
|
14
|
+
readonly auditPageCap: 10000;
|
|
15
|
+
readonly outlierFloor: 100;
|
|
16
|
+
readonly outlierMultiple: 3;
|
|
17
|
+
/** An IP with logins from more distinct users than this is shared egress (proxy, NAT). */
|
|
18
|
+
readonly sharedEgressUsers: 8;
|
|
19
|
+
};
|
|
20
|
+
export declare function id15(id: string | undefined | null): string | undefined;
|
|
21
|
+
export interface AnomalyEvent {
|
|
22
|
+
eventIdentifier: string;
|
|
23
|
+
eventDate: string;
|
|
24
|
+
score: number;
|
|
25
|
+
userId15: string;
|
|
26
|
+
username: string;
|
|
27
|
+
sourceIp?: string;
|
|
28
|
+
userAgent?: string;
|
|
29
|
+
summary?: string;
|
|
30
|
+
totalControllerEvents?: number;
|
|
31
|
+
requestedEntities?: string;
|
|
32
|
+
soqlCommands?: string;
|
|
33
|
+
}
|
|
34
|
+
export interface GuestUser {
|
|
35
|
+
id15: string;
|
|
36
|
+
username: string;
|
|
37
|
+
name: string;
|
|
38
|
+
profileName: string;
|
|
39
|
+
permissionSetLabels: string[];
|
|
40
|
+
siteNames: string[];
|
|
41
|
+
/** false when the user came only from an anomaly event and is no longer an active guest. */
|
|
42
|
+
active: boolean;
|
|
43
|
+
}
|
|
44
|
+
export interface Wave {
|
|
45
|
+
id: string;
|
|
46
|
+
guestId15: string;
|
|
47
|
+
site: string;
|
|
48
|
+
days: string[];
|
|
49
|
+
eventIds: string[];
|
|
50
|
+
}
|
|
51
|
+
export type LogStatus = 'collected' | 'missing' | 'no-permission' | 'failed';
|
|
52
|
+
export interface LogCoverage {
|
|
53
|
+
type: LogType;
|
|
54
|
+
day: string;
|
|
55
|
+
status: LogStatus;
|
|
56
|
+
totalRows: number;
|
|
57
|
+
guestRows: number;
|
|
58
|
+
guestRowsByUser: Record<string, number>;
|
|
59
|
+
malformed: number;
|
|
60
|
+
file?: string;
|
|
61
|
+
detail?: string;
|
|
62
|
+
}
|
|
63
|
+
export interface AuditRow {
|
|
64
|
+
createdDate: string;
|
|
65
|
+
createdBy: string;
|
|
66
|
+
section: string | null;
|
|
67
|
+
action: string;
|
|
68
|
+
display: string;
|
|
69
|
+
}
|
|
70
|
+
export interface AuditCoverage {
|
|
71
|
+
from: string;
|
|
72
|
+
to: string;
|
|
73
|
+
truncatedWindows: Array<{
|
|
74
|
+
from: string;
|
|
75
|
+
to: string;
|
|
76
|
+
}>;
|
|
77
|
+
inaccessible: boolean;
|
|
78
|
+
}
|
|
79
|
+
export interface LoginRow {
|
|
80
|
+
loginTime: string;
|
|
81
|
+
userId15: string;
|
|
82
|
+
sourceIp: string;
|
|
83
|
+
status: string;
|
|
84
|
+
loginUrl?: string;
|
|
85
|
+
browser?: string;
|
|
86
|
+
}
|
|
87
|
+
export interface LoginDayCount {
|
|
88
|
+
day: string;
|
|
89
|
+
status: string;
|
|
90
|
+
count: number;
|
|
91
|
+
}
|
|
92
|
+
export interface LinkedUser {
|
|
93
|
+
id15: string;
|
|
94
|
+
name: string;
|
|
95
|
+
email: string;
|
|
96
|
+
createdDate: string;
|
|
97
|
+
createdById15: string;
|
|
98
|
+
profileName: string;
|
|
99
|
+
isActive: boolean;
|
|
100
|
+
}
|
|
101
|
+
export interface RunningUserLimits {
|
|
102
|
+
queryAllFiles: boolean;
|
|
103
|
+
viewAllData: boolean;
|
|
104
|
+
}
|
|
105
|
+
export interface BundleManifest {
|
|
106
|
+
version: 1;
|
|
107
|
+
complete: boolean;
|
|
108
|
+
orgId: string;
|
|
109
|
+
orgName: string;
|
|
110
|
+
collectedAt: string;
|
|
111
|
+
sinceDays: number;
|
|
112
|
+
detectorAvailable: boolean;
|
|
113
|
+
waves: Wave[];
|
|
114
|
+
guests: GuestUser[];
|
|
115
|
+
logs: LogCoverage[];
|
|
116
|
+
audit: AuditCoverage;
|
|
117
|
+
limits: RunningUserLimits;
|
|
118
|
+
ipRangeFiles: string[];
|
|
119
|
+
/** Degraded reads while snapshotting guest configuration (optional: older bundles lack it). */
|
|
120
|
+
snapshotWarnings?: string[];
|
|
121
|
+
files: Record<string, string>;
|
|
122
|
+
}
|
|
123
|
+
export interface FollowUp {
|
|
124
|
+
logins: LoginRow[];
|
|
125
|
+
users: LinkedUser[];
|
|
126
|
+
/** True when a LoginHistory batch returned the queryAll cap, so some logins may be missing. */
|
|
127
|
+
truncated?: boolean;
|
|
128
|
+
}
|
|
129
|
+
export type ActionClass = 'data-access' | 'auth' | 'plumbing' | 'unknown';
|
|
130
|
+
export type Classification = 'internal-testing' | 'automated-scan' | 'organic' | 'indeterminate';
|
|
131
|
+
export type WaveOutcome = 'access-gained' | 'content-returned' | 'no-evidence' | 'not-assessed';
|
|
132
|
+
//# sourceMappingURL=model.d.ts.map
|