@cclabsnz/sf-audit 1.13.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -0
- package/README.md +15 -4
- package/lib/checks/impl/AuditTrailCheck.js +18 -1
- package/lib/checks/impl/LegacyApiVersionCheck.js +14 -2
- package/lib/checks/impl/LightningMessageChannelCheck.d.ts +10 -0
- package/lib/checks/impl/LightningMessageChannelCheck.js +107 -0
- package/lib/checks/impl/MfaEnforcementCheck.js +19 -1
- package/lib/checks/impl/NamedCredentialsCheck.js +42 -29
- package/lib/checks/registry.js +2 -0
- package/lib/commands/audit/incident/collect.d.ts +22 -0
- package/lib/commands/audit/incident/collect.js +46 -0
- package/lib/commands/audit/incident/report.d.ts +29 -0
- package/lib/commands/audit/incident/report.js +87 -0
- package/lib/compliance/mapping.js +2 -1
- package/lib/findings/CheckMeta.js +1 -0
- package/lib/incident/analyse/actions.d.ts +22 -0
- package/lib/incident/analyse/actions.js +73 -0
- package/lib/incident/analyse/actors.d.ts +56 -0
- package/lib/incident/analyse/actors.js +231 -0
- package/lib/incident/analyse/baseline.d.ts +24 -0
- package/lib/incident/analyse/baseline.js +66 -0
- package/lib/incident/analyse/configDelta.d.ts +33 -0
- package/lib/incident/analyse/configDelta.js +98 -0
- package/lib/incident/analyse/index.d.ts +41 -0
- package/lib/incident/analyse/index.js +68 -0
- package/lib/incident/analyse/outcomes.d.ts +28 -0
- package/lib/incident/analyse/outcomes.js +32 -0
- package/lib/incident/analyse/responses.d.ts +59 -0
- package/lib/incident/analyse/responses.js +201 -0
- package/lib/incident/analyse/verdict.d.ts +40 -0
- package/lib/incident/analyse/verdict.js +149 -0
- package/lib/incident/bundleIo.d.ts +34 -0
- package/lib/incident/bundleIo.js +74 -0
- package/lib/incident/collect/collectBundle.d.ts +21 -0
- package/lib/incident/collect/collectBundle.js +130 -0
- package/lib/incident/collect/discoverWaves.d.ts +15 -0
- package/lib/incident/collect/discoverWaves.js +90 -0
- package/lib/incident/collect/followUp.d.ts +14 -0
- package/lib/incident/collect/followUp.js +64 -0
- package/lib/incident/collect/orgErrors.d.ts +5 -0
- package/lib/incident/collect/orgErrors.js +11 -0
- package/lib/incident/collect/snapshotOrg.d.ts +18 -0
- package/lib/incident/collect/snapshotOrg.js +117 -0
- package/lib/incident/collect/streamGuestLogs.d.ts +17 -0
- package/lib/incident/collect/streamGuestLogs.js +116 -0
- package/lib/incident/csv.d.ts +32 -0
- package/lib/incident/csv.js +176 -0
- package/lib/incident/ipRanges.d.ts +23 -0
- package/lib/incident/ipRanges.js +64 -0
- package/lib/incident/model.d.ts +132 -0
- package/lib/incident/model.js +26 -0
- package/lib/incident/render/evidence.d.ts +15 -0
- package/lib/incident/render/evidence.js +52 -0
- package/lib/incident/render/html.d.ts +7 -0
- package/lib/incident/render/html.js +144 -0
- package/lib/incident/render/labels.d.ts +9 -0
- package/lib/incident/render/labels.js +22 -0
- package/lib/incident/render/markdown.d.ts +17 -0
- package/lib/incident/render/markdown.js +91 -0
- package/lib/incident/render/redact.d.ts +7 -0
- package/lib/incident/render/redact.js +62 -0
- package/lib/incident/text.d.ts +9 -0
- package/lib/incident/text.js +43 -0
- package/package.json +20 -8
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { containsWord } from '../text.js';
|
|
2
|
+
import { isSelfRegistration } from './outcomes.js';
|
|
3
|
+
function siteOf(g) { return g.siteNames[0] ?? g.username; }
|
|
4
|
+
function labelsOf(g) {
|
|
5
|
+
return [g.profileName, g.name, ...g.permissionSetLabels, ...g.siteNames].filter((l) => !!l && l.trim() !== '');
|
|
6
|
+
}
|
|
7
|
+
function mentions(text, label) {
|
|
8
|
+
return containsWord(text, label);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Sites credited with a change. Labels belonging to more than one guest are "shared" and never
|
|
12
|
+
* attribute a change; `onlyShared` is true when a change matched solely through shared labels.
|
|
13
|
+
*/
|
|
14
|
+
function sitesTouched(row, guests) {
|
|
15
|
+
const owners = new Map();
|
|
16
|
+
for (const g of guests)
|
|
17
|
+
for (const l of labelsOf(g)) {
|
|
18
|
+
const k = l.trim().toLowerCase();
|
|
19
|
+
if (!owners.has(k))
|
|
20
|
+
owners.set(k, new Set());
|
|
21
|
+
owners.get(k).add(g.id15);
|
|
22
|
+
}
|
|
23
|
+
const sites = new Set();
|
|
24
|
+
let sharedHit = false;
|
|
25
|
+
for (const g of guests)
|
|
26
|
+
for (const l of labelsOf(g)) {
|
|
27
|
+
if (!mentions(row.display, l))
|
|
28
|
+
continue;
|
|
29
|
+
if ((owners.get(l.trim().toLowerCase())?.size ?? 0) > 1)
|
|
30
|
+
sharedHit = true;
|
|
31
|
+
else
|
|
32
|
+
sites.add(siteOf(g));
|
|
33
|
+
}
|
|
34
|
+
return { sites: [...sites], onlyShared: sites.size === 0 && sharedHit };
|
|
35
|
+
}
|
|
36
|
+
export function computeConfigDelta(audit, guests, waves) {
|
|
37
|
+
const changes = audit.filter((r) => !isSelfRegistration(r))
|
|
38
|
+
.map((r) => ({ at: r.createdDate, by: r.createdBy, section: r.section, display: r.display, ...sitesTouched(r, guests) }))
|
|
39
|
+
.filter((c) => c.sites.length > 0 || c.onlyShared)
|
|
40
|
+
.sort((a, c) => a.at.localeCompare(c.at));
|
|
41
|
+
// Merge waves into distinct windows (several sites can share a day).
|
|
42
|
+
const spanMap = new Map();
|
|
43
|
+
for (const w of waves) {
|
|
44
|
+
const from = w.days[0], to = w.days[w.days.length - 1];
|
|
45
|
+
const k = `${from}|${to}`;
|
|
46
|
+
const cur = spanMap.get(k);
|
|
47
|
+
if (cur)
|
|
48
|
+
cur.ids.push(w.id);
|
|
49
|
+
else
|
|
50
|
+
spanMap.set(k, { from, to, ids: [w.id] });
|
|
51
|
+
}
|
|
52
|
+
const spans = [...spanMap.values()].sort((a, c) => a.from.localeCompare(c.from) || a.to.localeCompare(c.to));
|
|
53
|
+
const bounds = [];
|
|
54
|
+
const during = (s) => bounds.push({ label: `During ${s.ids.join(', ')} (${s.from === s.to ? s.from : `${s.from} to ${s.to}`})`, kind: 'during', from: s.from, to: s.to });
|
|
55
|
+
if (spans.length)
|
|
56
|
+
bounds.push({ label: `Before ${spans[0].from}`, kind: 'before', from: null, to: spans[0].from });
|
|
57
|
+
spans.forEach((sp, i) => {
|
|
58
|
+
if (i > 0)
|
|
59
|
+
bounds.push({ label: `${spans[i - 1].to} to ${sp.from}`, kind: 'between', from: spans[i - 1].to, to: sp.from });
|
|
60
|
+
during(sp);
|
|
61
|
+
});
|
|
62
|
+
if (spans.length)
|
|
63
|
+
bounds.push({ label: `After ${spans[spans.length - 1].to}`, kind: 'after', from: spans[spans.length - 1].to, to: null });
|
|
64
|
+
const periods = bounds.map((p) => {
|
|
65
|
+
const bySite = {};
|
|
66
|
+
const shared = [];
|
|
67
|
+
for (const { onlyShared, ...c } of changes) {
|
|
68
|
+
const day = c.at.slice(0, 10);
|
|
69
|
+
const inside = p.kind === 'during'
|
|
70
|
+
? day >= p.from && day <= p.to
|
|
71
|
+
: (p.from === null || day > p.from) && (p.to === null || day < p.to);
|
|
72
|
+
if (!inside)
|
|
73
|
+
continue;
|
|
74
|
+
if (onlyShared)
|
|
75
|
+
shared.push(c);
|
|
76
|
+
else
|
|
77
|
+
for (const s of c.sites)
|
|
78
|
+
(bySite[s] ??= []).push(c);
|
|
79
|
+
}
|
|
80
|
+
return { ...p, bySite, shared };
|
|
81
|
+
});
|
|
82
|
+
const asymmetries = [];
|
|
83
|
+
for (const w of waves) {
|
|
84
|
+
// Asymmetry compares only the between-wave periods, never a wave's own days.
|
|
85
|
+
const period = periods.find((p) => p.kind === 'between' && p.to === w.days[0]);
|
|
86
|
+
if (!period)
|
|
87
|
+
continue;
|
|
88
|
+
const hitBefore = new Set(waves.filter((x) => x.days[x.days.length - 1] <= period.from).map((x) => x.site));
|
|
89
|
+
const own = period.bySite[w.site]?.length ?? 0;
|
|
90
|
+
for (const other of [...hitBefore].filter((s) => s !== w.site).sort()) {
|
|
91
|
+
const theirs = period.bySite[other]?.length ?? 0;
|
|
92
|
+
if (theirs > own)
|
|
93
|
+
asymmetries.push({ waveId: w.id, site: w.site, changes: own, comparedSite: other, comparedChanges: theirs, period: period.label });
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return { periods, asymmetries };
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=configDelta.js.map
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { type AuditCoverage, type Classification, type GuestUser, type LogCoverage, type RunningUserLimits, type WaveOutcome } from '../model.js';
|
|
2
|
+
import { type ActionSummary } from './actions.js';
|
|
3
|
+
import { type DayVolume } from './baseline.js';
|
|
4
|
+
import { type Asymmetry, type Period } from './configDelta.js';
|
|
5
|
+
import { type VerdictInput } from './verdict.js';
|
|
6
|
+
export { FORBIDDEN_WORDS } from './verdict.js';
|
|
7
|
+
export interface WaveResult extends VerdictInput {
|
|
8
|
+
actions: ActionSummary;
|
|
9
|
+
classification: Classification;
|
|
10
|
+
result: WaveOutcome;
|
|
11
|
+
limits: string[];
|
|
12
|
+
nextSteps: string[];
|
|
13
|
+
asymmetries: Asymmetry[];
|
|
14
|
+
}
|
|
15
|
+
export interface IncidentResult {
|
|
16
|
+
orgId: string;
|
|
17
|
+
orgName: string;
|
|
18
|
+
collectedAt: string;
|
|
19
|
+
manifestSha256: string;
|
|
20
|
+
guests: GuestUser[];
|
|
21
|
+
volumes: DayVolume[];
|
|
22
|
+
waves: WaveResult[];
|
|
23
|
+
/** Org-wide limits, shown with the per-wave limits even when there are zero waves. */
|
|
24
|
+
globalLimits: string[];
|
|
25
|
+
config: {
|
|
26
|
+
periods: Period[];
|
|
27
|
+
asymmetries: Asymmetry[];
|
|
28
|
+
};
|
|
29
|
+
coverage: {
|
|
30
|
+
logs: LogCoverage[];
|
|
31
|
+
audit: AuditCoverage;
|
|
32
|
+
limits: RunningUserLimits;
|
|
33
|
+
detectorAvailable: boolean;
|
|
34
|
+
ipRanges: string[];
|
|
35
|
+
};
|
|
36
|
+
withinBaseline: boolean;
|
|
37
|
+
}
|
|
38
|
+
export declare function analyseBundle(dir: string, opts?: {
|
|
39
|
+
spikeRatio?: number;
|
|
40
|
+
}): Promise<IncidentResult>;
|
|
41
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// src/incident/analyse/index.ts
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { loadBundle } from '../bundleIo.js';
|
|
5
|
+
import { DEFAULTS, REQUIRED_LOG_TYPES } from '../model.js';
|
|
6
|
+
import { loadIpRanges, rangeFileName } from '../ipRanges.js';
|
|
7
|
+
import { findActors } from './actors.js';
|
|
8
|
+
import { summariseActionCounts } from './actions.js';
|
|
9
|
+
import { assessSpikes, computeDayVolumes } from './baseline.js';
|
|
10
|
+
import { computeConfigDelta } from './configDelta.js';
|
|
11
|
+
import { computeOutcomes } from './outcomes.js';
|
|
12
|
+
import { analyseResponses } from './responses.js';
|
|
13
|
+
import { classify, globalLimitsFor, limitsFor, nextStepsFor, outcomeOf } from './verdict.js';
|
|
14
|
+
export { FORBIDDEN_WORDS } from './verdict.js';
|
|
15
|
+
function requiredPresent(m, days) {
|
|
16
|
+
return days.every((d) => REQUIRED_LOG_TYPES.every((t) => m.logs.some((l) => l.type === t && l.day === d && l.status === 'collected')));
|
|
17
|
+
}
|
|
18
|
+
export async function analyseBundle(dir, opts = {}) {
|
|
19
|
+
const b = await loadBundle(dir);
|
|
20
|
+
const m = b.manifest;
|
|
21
|
+
const ranges = m.ipRangeFiles.length
|
|
22
|
+
? loadIpRanges(m.ipRangeFiles.map((rel) => ({ name: rangeFileName(rel), text: readFileSync(join(dir, rel), 'utf-8') })))
|
|
23
|
+
: null;
|
|
24
|
+
const volumes = await computeDayVolumes(b);
|
|
25
|
+
const config = computeConfigDelta(b.audit, m.guests, m.waves);
|
|
26
|
+
const waves = [];
|
|
27
|
+
for (const wave of m.waves) {
|
|
28
|
+
const actors = await findActors(b, wave, ranges);
|
|
29
|
+
const v = {
|
|
30
|
+
wave,
|
|
31
|
+
spikes: assessSpikes(b, volumes, wave, opts.spikeRatio ?? DEFAULTS.spikeRatio),
|
|
32
|
+
actors,
|
|
33
|
+
responses: await analyseResponses(b, wave, actors),
|
|
34
|
+
outcomes: computeOutcomes(b, wave, actors),
|
|
35
|
+
requiredLogsPresent: requiredPresent(m, wave.days),
|
|
36
|
+
};
|
|
37
|
+
const asymmetries = config.asymmetries.filter((a) => a.waveId === wave.id);
|
|
38
|
+
waves.push({
|
|
39
|
+
...v,
|
|
40
|
+
actions: summariseActionCounts(actors.map((a) => a.actionCounts)),
|
|
41
|
+
classification: classify(v),
|
|
42
|
+
result: outcomeOf(v),
|
|
43
|
+
limits: limitsFor(v),
|
|
44
|
+
nextSteps: nextStepsFor(v, asymmetries, m),
|
|
45
|
+
asymmetries,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
return {
|
|
49
|
+
orgId: m.orgId,
|
|
50
|
+
orgName: m.orgName,
|
|
51
|
+
collectedAt: m.collectedAt,
|
|
52
|
+
manifestSha256: b.manifestSha256,
|
|
53
|
+
guests: m.guests,
|
|
54
|
+
volumes,
|
|
55
|
+
waves,
|
|
56
|
+
globalLimits: globalLimitsFor(m, b.followUp),
|
|
57
|
+
config,
|
|
58
|
+
coverage: { logs: m.logs, audit: m.audit, limits: m.limits, detectorAvailable: m.detectorAvailable, ipRanges: m.ipRangeFiles },
|
|
59
|
+
// Zero waves, or a wave with no baseline, is "nothing assessed", never "within baseline".
|
|
60
|
+
withinBaseline: waves.length > 0 && waves.every((w) => w.requiredLogsPresent
|
|
61
|
+
&& w.spikes.every((s) => s.baselineMedian !== null)
|
|
62
|
+
&& !w.spikes.some((s) => s.isSpike)
|
|
63
|
+
&& w.actors.length === 0
|
|
64
|
+
// Never above a wave that returned content, gained access, or could not be assessed.
|
|
65
|
+
&& w.result === 'no-evidence'),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Bundle } from '../bundleIo.js';
|
|
2
|
+
import type { AuditRow, LoginRow, Wave } from '../model.js';
|
|
3
|
+
import { type Actor } from './actors.js';
|
|
4
|
+
export interface IdentityLink {
|
|
5
|
+
ip: string;
|
|
6
|
+
userId15: string;
|
|
7
|
+
userName: string;
|
|
8
|
+
email: string;
|
|
9
|
+
userCreatedDate: string;
|
|
10
|
+
createdByGuest: string;
|
|
11
|
+
loginTime: string;
|
|
12
|
+
}
|
|
13
|
+
export interface Outcomes {
|
|
14
|
+
actorLogins: Array<LoginRow & {
|
|
15
|
+
actorId: string;
|
|
16
|
+
}>;
|
|
17
|
+
/** Successful logins from actor IPs that are NOT shared egress. */
|
|
18
|
+
successfulLogins: number;
|
|
19
|
+
/** Successful logins from shared-egress actor IPs: listed, never counted as access. */
|
|
20
|
+
sharedEgressLogins: number;
|
|
21
|
+
failedLogins: number;
|
|
22
|
+
/** Cannot be tied to an IP (the audit trail has none); listed for review, never counted as access. */
|
|
23
|
+
selfRegistrationsInActorWindow: AuditRow[];
|
|
24
|
+
identityLinks: IdentityLink[];
|
|
25
|
+
}
|
|
26
|
+
export declare function isSelfRegistration(row: AuditRow): boolean;
|
|
27
|
+
export declare function computeOutcomes(b: Bundle, wave: Wave, actors: Actor[]): Outcomes;
|
|
28
|
+
//# sourceMappingURL=outcomes.d.ts.map
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { normaliseIp } from './actors.js';
|
|
2
|
+
export function isSelfRegistration(row) {
|
|
3
|
+
return /^Created new Customer User\b/.test(row.display) || /^Reset password for user\b/.test(row.display);
|
|
4
|
+
}
|
|
5
|
+
export function computeOutcomes(b, wave, actors) {
|
|
6
|
+
const actorOf = new Map(actors.flatMap((a) => a.ips.map((ip) => [normaliseIp(ip), a.id])));
|
|
7
|
+
const actorLogins = b.followUp.logins.filter((l) => actorOf.has(normaliseIp(l.sourceIp))).map((l) => ({ ...l, actorId: actorOf.get(normaliseIp(l.sourceIp)) }));
|
|
8
|
+
const shared = new Set(actors.flatMap((a) => a.sharedEgressIps.map(normaliseIp)));
|
|
9
|
+
const success = actorLogins.filter((l) => l.status === 'Success');
|
|
10
|
+
const ok = success.filter((l) => !shared.has(normaliseIp(l.sourceIp)));
|
|
11
|
+
const guestNames = new Map(b.manifest.guests.map((g) => [g.id15, g.name]));
|
|
12
|
+
const waveGuestName = guestNames.get(wave.guestId15);
|
|
13
|
+
const identityLinks = [];
|
|
14
|
+
for (const l of ok) {
|
|
15
|
+
const u = b.followUp.users.find((x) => x.id15 === l.userId15);
|
|
16
|
+
const creator = u ? guestNames.get(u.createdById15) : undefined;
|
|
17
|
+
if (!u || !creator)
|
|
18
|
+
continue;
|
|
19
|
+
identityLinks.push({ ip: l.sourceIp, userId15: u.id15, userName: u.name, email: u.email, userCreatedDate: u.createdDate, createdByGuest: creator, loginTime: l.loginTime });
|
|
20
|
+
}
|
|
21
|
+
const windows = actors.map((a) => [a.firstSeen, a.lastSeen]);
|
|
22
|
+
const selfRegistrationsInActorWindow = b.audit.filter((r) => r.createdBy === waveGuestName && /^Created new Customer User\b/.test(r.display) &&
|
|
23
|
+
windows.some(([f, t]) => {
|
|
24
|
+
const at = Date.parse(r.createdDate), from = Date.parse(f), to = Date.parse(t);
|
|
25
|
+
return !Number.isNaN(at) && !Number.isNaN(from) && !Number.isNaN(to) && at >= from && at <= to;
|
|
26
|
+
}));
|
|
27
|
+
return {
|
|
28
|
+
actorLogins, successfulLogins: ok.length, sharedEgressLogins: success.length - ok.length,
|
|
29
|
+
failedLogins: actorLogins.length - success.length, selfRegistrationsInActorWindow, identityLinks,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=outcomes.js.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { Bundle } from '../bundleIo.js';
|
|
2
|
+
import { type Wave } from '../model.js';
|
|
3
|
+
import { type Actor } from './actors.js';
|
|
4
|
+
export interface ReturnedContent {
|
|
5
|
+
requestId: string;
|
|
6
|
+
timestamp: string;
|
|
7
|
+
ip: string;
|
|
8
|
+
size: number;
|
|
9
|
+
actions: string[];
|
|
10
|
+
/** The actor block that sent the call, or '' when it came from no isolated block. */
|
|
11
|
+
actorId: string;
|
|
12
|
+
/** The action the reply is attributed to (the batch's first data-access action). */
|
|
13
|
+
action: string;
|
|
14
|
+
}
|
|
15
|
+
export interface ResponseSummary {
|
|
16
|
+
/**
|
|
17
|
+
* Most common reply size in the reference set: the guest's auth replies on the wave days plus
|
|
18
|
+
* its data-access replies on baseline days, excluding blocks also active on the wave days.
|
|
19
|
+
* Learned from replies that are NOT under test.
|
|
20
|
+
*/
|
|
21
|
+
emptySize: number | null;
|
|
22
|
+
/** True when the reference set was empty and the size was taken from the replies under test. */
|
|
23
|
+
emptySizeInferred: boolean;
|
|
24
|
+
band: number;
|
|
25
|
+
/** All of the wave guest's data-access calls on the wave days, from any IP. */
|
|
26
|
+
dataAccessCalls: number;
|
|
27
|
+
/** Data-access and auth calls on the wave days whose REQUEST_ID joined to a Sites size. */
|
|
28
|
+
joined: number;
|
|
29
|
+
/** Data-access calls on the wave days whose REQUEST_ID joined to a Sites size. */
|
|
30
|
+
dataAccessJoined: number;
|
|
31
|
+
/** Replies in the reference set the empty size was learned from. */
|
|
32
|
+
referenceReplies: number;
|
|
33
|
+
returnedContent: ReturnedContent[];
|
|
34
|
+
blankRequestIdsDropped: number;
|
|
35
|
+
/** Data-access and auth calls on the wave days whose REQUEST_ID had no Sites size that day. */
|
|
36
|
+
unmatchedCalls: number; /** Data-access and auth calls whose REQUEST_ID was shared with other rows, so never joined. */
|
|
37
|
+
ambiguousCalls: number;
|
|
38
|
+
/** Empty-reply size learned per action from the reference set (smallest recurring size). */
|
|
39
|
+
emptySizeByAction: Record<string, number>;
|
|
40
|
+
/** Data-access actions seen on baseline days from blocks absent on the wave days: ordinary traffic. */
|
|
41
|
+
baselineActions: string[];
|
|
42
|
+
/** Joined data-access calls whose action had no empty size of its own, judged against emptySize. */
|
|
43
|
+
judgedAgainstFallback: number;
|
|
44
|
+
/** Guest controller calls on the wave days with an ACTION_MESSAGE that parsed to no action. */
|
|
45
|
+
unparsedCalls: number;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Joins AuraRequest to Sites on REQUEST_ID to get reply sizes. Sites repeats a request id
|
|
49
|
+
* across rows, so it is reduced to one size (the maximum) per non-blank id BEFORE the join;
|
|
50
|
+
* joining the raw rows multiplied one action's count into the millions. Blank ids are dropped:
|
|
51
|
+
* as a key they would match every other blank (timeline/JoinKeys.ts rule).
|
|
52
|
+
*
|
|
53
|
+
* Every data-access call the wave guest made on the wave days is tested, whichever IP sent it:
|
|
54
|
+
* an actor that rotates addresses never forms a block, and must still be assessed. The empty
|
|
55
|
+
* size is learned from a separate reference set so a scanner receiving identical non-empty
|
|
56
|
+
* replies cannot set its own "empty" baseline.
|
|
57
|
+
*/
|
|
58
|
+
export declare function analyseResponses(b: Bundle, wave: Wave, actors: Actor[]): Promise<ResponseSummary>;
|
|
59
|
+
//# sourceMappingURL=responses.d.ts.map
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { DEFAULTS, id15 } from '../model.js';
|
|
2
|
+
import { baselineDaysFor, blockOf, normaliseIp } from './actors.js';
|
|
3
|
+
import { classesOf, classifyAction, parseActions } from './actions.js';
|
|
4
|
+
/** Smallest size seen at least max(3, 5% of the action's replies) times. */
|
|
5
|
+
function smallestRecurring(freq) {
|
|
6
|
+
const total = [...freq.values()].reduce((a, n) => a + n, 0);
|
|
7
|
+
const min = Math.max(3, Math.ceil(total * 0.05));
|
|
8
|
+
return [...freq.entries()].filter(([, n]) => n >= min).map(([size]) => size).sort((a, c) => a - c)[0];
|
|
9
|
+
}
|
|
10
|
+
const modeOf = (freq) => [...freq.entries()].sort((a, c) => c[1] - a[1] || a[0] - c[0])[0]?.[0];
|
|
11
|
+
/**
|
|
12
|
+
* Joins AuraRequest to Sites on REQUEST_ID to get reply sizes. Sites repeats a request id
|
|
13
|
+
* across rows, so it is reduced to one size (the maximum) per non-blank id BEFORE the join;
|
|
14
|
+
* joining the raw rows multiplied one action's count into the millions. Blank ids are dropped:
|
|
15
|
+
* as a key they would match every other blank (timeline/JoinKeys.ts rule).
|
|
16
|
+
*
|
|
17
|
+
* Every data-access call the wave guest made on the wave days is tested, whichever IP sent it:
|
|
18
|
+
* an actor that rotates addresses never forms a block, and must still be assessed. The empty
|
|
19
|
+
* size is learned from a separate reference set so a scanner receiving identical non-empty
|
|
20
|
+
* replies cannot set its own "empty" baseline.
|
|
21
|
+
*/
|
|
22
|
+
export async function analyseResponses(b, wave, actors) {
|
|
23
|
+
const actorOf = new Map(actors.flatMap((a) => a.ips.map((ip) => [normaliseIp(ip), a.id])));
|
|
24
|
+
const out = {
|
|
25
|
+
emptySize: null, emptySizeInferred: false, band: DEFAULTS.emptyBandBytes, dataAccessCalls: 0, joined: 0, dataAccessJoined: 0,
|
|
26
|
+
referenceReplies: 0, returnedContent: [], blankRequestIdsDropped: 0, unmatchedCalls: 0, unparsedCalls: 0,
|
|
27
|
+
ambiguousCalls: 0, emptySizeByAction: {}, baselineActions: [], judgedAgainstFallback: 0,
|
|
28
|
+
};
|
|
29
|
+
const isGuest = (r) => id15(r.USER_ID) === wave.guestId15 || id15(r.USER_ID_DERIVED) === wave.guestId15;
|
|
30
|
+
/**
|
|
31
|
+
* Reply sizes by REQUEST_ID for one day, joinable only when unambiguous: the id belongs to
|
|
32
|
+
* exactly one AuraRequest row and at most two Sites rows. On some sites one id is reused
|
|
33
|
+
* across hundreds of calls, and taking its largest Sites row would give every call that size.
|
|
34
|
+
*/
|
|
35
|
+
const repliesFor = async (day, countBlanks) => {
|
|
36
|
+
const sizes = new Map();
|
|
37
|
+
const sitesRows = new Map();
|
|
38
|
+
for await (const s of b.rows('Sites', day)) {
|
|
39
|
+
const rid = (s.REQUEST_ID ?? '').trim();
|
|
40
|
+
if (!rid) {
|
|
41
|
+
if (countBlanks)
|
|
42
|
+
out.blankRequestIdsDropped++;
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
sitesRows.set(rid, (sitesRows.get(rid) ?? 0) + 1);
|
|
46
|
+
const n = Number(s.RESPONSE_SIZE || 0);
|
|
47
|
+
if (n > (sizes.get(rid) ?? -1))
|
|
48
|
+
sizes.set(rid, n);
|
|
49
|
+
}
|
|
50
|
+
const auraRows = new Map();
|
|
51
|
+
for await (const r of b.rows('AuraRequest', day)) {
|
|
52
|
+
const rid = (r.REQUEST_ID ?? '').trim();
|
|
53
|
+
if (rid)
|
|
54
|
+
auraRows.set(rid, (auraRows.get(rid) ?? 0) + 1);
|
|
55
|
+
}
|
|
56
|
+
return (rid) => {
|
|
57
|
+
const size = sizes.get(rid);
|
|
58
|
+
if (size === undefined)
|
|
59
|
+
return undefined;
|
|
60
|
+
return (auraRows.get(rid) ?? 0) > 1 || (sitesRows.get(rid) ?? 0) > 2 ? 'ambiguous' : size;
|
|
61
|
+
};
|
|
62
|
+
};
|
|
63
|
+
const bump = (m, key, size) => {
|
|
64
|
+
let f = m.get(key);
|
|
65
|
+
if (!f) {
|
|
66
|
+
f = new Map();
|
|
67
|
+
m.set(key, f);
|
|
68
|
+
}
|
|
69
|
+
f.set(size, (f.get(size) ?? 0) + 1);
|
|
70
|
+
};
|
|
71
|
+
/** The action a reply is attributed to: the batch's first data-access action, else its first auth action. */
|
|
72
|
+
const primaryAction = (msg, dataAccess) => parseActions(msg).find((n) => classifyAction(n) === (dataAccess ? 'data-access' : 'auth')) ?? '';
|
|
73
|
+
// Streamed: size frequency maps plus compact data-access candidates; no whole rows are kept.
|
|
74
|
+
const reference = new Map();
|
|
75
|
+
// Reference sizes per action from replies NOT under test: wave-day auth replies and baseline reads.
|
|
76
|
+
const referenceByAction = new Map();
|
|
77
|
+
// Ordinary visitors' wave-day replies per action, with the blocks each size came from. These ARE
|
|
78
|
+
// under test, so they may only LOWER an action's empty size, never set it.
|
|
79
|
+
const visitorByAction = new Map();
|
|
80
|
+
const testedFreq = new Map();
|
|
81
|
+
const tested = [];
|
|
82
|
+
// Blocks that sent guest controller calls on the wave days. Their baseline-day replies are
|
|
83
|
+
// left out of the reference set: a scan that runs past midnight would otherwise set its own
|
|
84
|
+
// "empty" size from the next day's replies.
|
|
85
|
+
const waveBlocks = new Set();
|
|
86
|
+
for (const day of wave.days) {
|
|
87
|
+
const replyOf = await repliesFor(day, true);
|
|
88
|
+
for await (const r of b.rows('AuraRequest', day)) {
|
|
89
|
+
if (!r.ACTION_MESSAGE || !isGuest(r))
|
|
90
|
+
continue;
|
|
91
|
+
waveBlocks.add(blockOf(normaliseIp(r.CLIENT_IP ?? '')));
|
|
92
|
+
if (parseActions(r.ACTION_MESSAGE).length === 0) {
|
|
93
|
+
out.unparsedCalls++;
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
const cls = classesOf(r.ACTION_MESSAGE);
|
|
97
|
+
const dataAccess = cls.has('data-access');
|
|
98
|
+
if (dataAccess)
|
|
99
|
+
out.dataAccessCalls++;
|
|
100
|
+
if (!dataAccess && !cls.has('auth'))
|
|
101
|
+
continue;
|
|
102
|
+
const requestId = (r.REQUEST_ID ?? '').trim();
|
|
103
|
+
const size = replyOf(requestId);
|
|
104
|
+
if (size === undefined) {
|
|
105
|
+
out.unmatchedCalls++;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (size === 'ambiguous') {
|
|
109
|
+
out.ambiguousCalls++;
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
out.joined++;
|
|
113
|
+
const action = primaryAction(r.ACTION_MESSAGE, dataAccess);
|
|
114
|
+
if (!dataAccess) {
|
|
115
|
+
reference.set(size, (reference.get(size) ?? 0) + 1);
|
|
116
|
+
bump(referenceByAction, action, size);
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
out.dataAccessJoined++;
|
|
120
|
+
testedFreq.set(size, (testedFreq.get(size) ?? 0) + 1);
|
|
121
|
+
const ip = normaliseIp(r.CLIENT_IP ?? '');
|
|
122
|
+
if (!actorOf.has(ip)) {
|
|
123
|
+
let bySize = visitorByAction.get(action);
|
|
124
|
+
if (!bySize) {
|
|
125
|
+
bySize = new Map();
|
|
126
|
+
visitorByAction.set(action, bySize);
|
|
127
|
+
}
|
|
128
|
+
let blocks = bySize.get(size);
|
|
129
|
+
if (!blocks) {
|
|
130
|
+
blocks = new Set();
|
|
131
|
+
bySize.set(size, blocks);
|
|
132
|
+
}
|
|
133
|
+
blocks.add(blockOf(ip));
|
|
134
|
+
}
|
|
135
|
+
tested.push({ requestId, timestamp: r.TIMESTAMP_DERIVED, ip, size, actions: parseActions(r.ACTION_MESSAGE), actorId: actorOf.get(ip) ?? '', action });
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
// Blocks absent on the wave days that called each data-access action on baseline days. An action
|
|
139
|
+
// is ordinary traffic only when 3+ such blocks used it: one call from a fresh IP proves nothing.
|
|
140
|
+
const baselineBlocks = new Map();
|
|
141
|
+
for (const day of baselineDaysFor(b.manifest, wave.guestId15)) {
|
|
142
|
+
const replyOf = await repliesFor(day, false);
|
|
143
|
+
for await (const r of b.rows('AuraRequest', day)) {
|
|
144
|
+
if (!r.ACTION_MESSAGE || !isGuest(r) || !classesOf(r.ACTION_MESSAGE).has('data-access'))
|
|
145
|
+
continue;
|
|
146
|
+
if (waveBlocks.has(blockOf(normaliseIp(r.CLIENT_IP ?? ''))))
|
|
147
|
+
continue;
|
|
148
|
+
const size = replyOf((r.REQUEST_ID ?? '').trim());
|
|
149
|
+
if (typeof size !== 'number')
|
|
150
|
+
continue;
|
|
151
|
+
reference.set(size, (reference.get(size) ?? 0) + 1);
|
|
152
|
+
const action = primaryAction(r.ACTION_MESSAGE, true);
|
|
153
|
+
bump(referenceByAction, action, size);
|
|
154
|
+
let blocks = baselineBlocks.get(action);
|
|
155
|
+
if (!blocks) {
|
|
156
|
+
blocks = new Set();
|
|
157
|
+
baselineBlocks.set(action, blocks);
|
|
158
|
+
}
|
|
159
|
+
blocks.add(blockOf(normaliseIp(r.CLIENT_IP ?? '')));
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
out.baselineActions = [...baselineBlocks].filter(([, blocks]) => blocks.size >= 3).map(([action]) => action).sort();
|
|
163
|
+
out.referenceReplies = [...reference.values()].reduce((s, n) => s + n, 0);
|
|
164
|
+
let mode = modeOf(reference);
|
|
165
|
+
if (mode === undefined && tested.length > 0) {
|
|
166
|
+
mode = modeOf(testedFreq);
|
|
167
|
+
out.emptySizeInferred = true;
|
|
168
|
+
}
|
|
169
|
+
if (mode === undefined)
|
|
170
|
+
return out;
|
|
171
|
+
out.emptySize = mode;
|
|
172
|
+
const fallback = mode;
|
|
173
|
+
// Per action, the empty reply is the SMALLEST size that recurs (an empty list), not the most
|
|
174
|
+
// common one: legitimate reads return data, so their mode is not "empty". Visitors' wave-day
|
|
175
|
+
// replies may lower it, but only to a size that recurs across 3+ blocks and is no larger than
|
|
176
|
+
// the reference would allow, so replies under test can never set their own baseline.
|
|
177
|
+
for (const action of new Set([...referenceByAction.keys(), ...visitorByAction.keys()])) {
|
|
178
|
+
if (!action)
|
|
179
|
+
continue;
|
|
180
|
+
const base = referenceByAction.has(action) ? smallestRecurring(referenceByAction.get(action)) : undefined;
|
|
181
|
+
// An inferred fallback came from the replies under test, so it can never be a cap.
|
|
182
|
+
const cap = base ?? (out.emptySizeInferred ? undefined : fallback);
|
|
183
|
+
const visitorSizes = visitorByAction.get(action);
|
|
184
|
+
let lowered;
|
|
185
|
+
if (visitorSizes) {
|
|
186
|
+
const counts = new Map([...visitorSizes].map(([size, blocks]) => [size, blocks.size]));
|
|
187
|
+
const candidate = smallestRecurring(counts);
|
|
188
|
+
if (candidate !== undefined && cap !== undefined && (visitorSizes.get(candidate)?.size ?? 0) >= 3 && candidate <= cap)
|
|
189
|
+
lowered = candidate;
|
|
190
|
+
}
|
|
191
|
+
const size = base === undefined ? lowered : lowered === undefined ? base : Math.min(base, lowered);
|
|
192
|
+
if (size !== undefined)
|
|
193
|
+
out.emptySizeByAction[action] = size;
|
|
194
|
+
}
|
|
195
|
+
out.judgedAgainstFallback = tested.filter((s) => out.emptySizeByAction[s.action] === undefined).length;
|
|
196
|
+
out.returnedContent = tested
|
|
197
|
+
.filter((s) => s.size > (out.emptySizeByAction[s.action] ?? fallback) + out.band)
|
|
198
|
+
.sort((a, c) => a.timestamp.localeCompare(c.timestamp));
|
|
199
|
+
return out;
|
|
200
|
+
}
|
|
201
|
+
//# sourceMappingURL=responses.js.map
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { BundleManifest, Classification, FollowUp, Wave, WaveOutcome } from '../model.js';
|
|
2
|
+
import type { Actor } from './actors.js';
|
|
3
|
+
import type { SpikeAssessment } from './baseline.js';
|
|
4
|
+
import type { Asymmetry } from './configDelta.js';
|
|
5
|
+
import type { Outcomes } from './outcomes.js';
|
|
6
|
+
import type { ResponseSummary } from './responses.js';
|
|
7
|
+
export interface VerdictInput {
|
|
8
|
+
wave: Wave;
|
|
9
|
+
spikes: SpikeAssessment[];
|
|
10
|
+
actors: Actor[];
|
|
11
|
+
responses: ResponseSummary;
|
|
12
|
+
outcomes: Outcomes;
|
|
13
|
+
requiredLogsPresent: boolean;
|
|
14
|
+
}
|
|
15
|
+
export declare const FORBIDDEN_WORDS: RegExp;
|
|
16
|
+
export declare function isScannerLike(a: Actor): boolean;
|
|
17
|
+
export declare function classify(v: VerdictInput): Classification;
|
|
18
|
+
/**
|
|
19
|
+
* Why a wave cannot be called no-evidence. No-evidence is allowed only when this is empty:
|
|
20
|
+
* the required logs are present, the empty-reply size came from a reference set, every
|
|
21
|
+
* controller call parsed, EVERY data-access call was joined to a reply size (any unjoined read
|
|
22
|
+
* could be the one that returned content), and every wave day has a baseline median.
|
|
23
|
+
* The join test uses data-access joins, not all joins: a joined auth reply says nothing about
|
|
24
|
+
* what a data read returned (stricter than `joined > 0`).
|
|
25
|
+
*/
|
|
26
|
+
export declare function notAssessedReasons(v: VerdictInput): string[];
|
|
27
|
+
/**
|
|
28
|
+
* Returned content that decides the result. With an isolated actor, only the actor's replies
|
|
29
|
+
* count: a public site returns content to its ordinary visitors all the time. Without one,
|
|
30
|
+
* every guest reply counts, because nothing separates the actor from the visitors.
|
|
31
|
+
*/
|
|
32
|
+
export declare function decisiveReturned(v: VerdictInput): ResponseSummary['returnedContent'];
|
|
33
|
+
/** Precedence: access-gained > content-returned > not-assessed > no-evidence. */
|
|
34
|
+
export declare function outcomeOf(v: VerdictInput): WaveOutcome;
|
|
35
|
+
/** Org-wide limits: they hold for the whole report, including one with zero waves. */
|
|
36
|
+
export declare function globalLimitsFor(m: BundleManifest, followUp?: FollowUp): string[];
|
|
37
|
+
/** Limits specific to one wave. Org-wide limits live in globalLimitsFor. */
|
|
38
|
+
export declare function limitsFor(v: VerdictInput): string[];
|
|
39
|
+
export declare function nextStepsFor(v: VerdictInput, asymmetries: Asymmetry[], m: BundleManifest): string[];
|
|
40
|
+
//# sourceMappingURL=verdict.d.ts.map
|