@agent-beam/beam 0.2.30 → 0.2.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -4
- package/Skills/agentbeam-app-monitoring/SKILL.md +5 -4
- package/dist/cli.js +110 -1
- package/dist/cli.js.map +1 -1
- package/dist/cve.js +1 -1
- package/dist/cve.js.map +1 -1
- package/dist/monitor-client.d.ts +45 -1
- package/dist/monitor-client.js +3 -3
- package/dist/monitor-client.js.map +1 -1
- package/dist/monitor-pipeline.d.ts +60 -9
- package/dist/monitor-pipeline.js +325 -5
- package/dist/monitor-pipeline.js.map +1 -1
- package/dist/monitor-rules.d.ts +58 -0
- package/dist/monitor-rules.js +130 -1
- package/dist/monitor-rules.js.map +1 -1
- package/dist/monitor-signals.d.ts +63 -0
- package/dist/monitor-signals.js +125 -0
- package/dist/monitor-signals.js.map +1 -0
- package/dist/monitor-stacks.d.ts +41 -0
- package/dist/monitor-stacks.js +129 -0
- package/dist/monitor-stacks.js.map +1 -0
- package/dist/monitor.d.ts +2 -0
- package/dist/monitor.js +12 -4
- package/dist/monitor.js.map +1 -1
- package/dist/package-risk-rules.d.ts +27 -0
- package/dist/package-risk-rules.js +137 -0
- package/dist/package-risk-rules.js.map +1 -0
- package/dist/package-scan.d.ts +50 -0
- package/dist/package-scan.js +279 -0
- package/dist/package-scan.js.map +1 -0
- package/dist/packages.d.ts +43 -0
- package/dist/packages.js +228 -0
- package/dist/packages.js.map +1 -0
- package/dist/sdk.d.ts +61 -1
- package/dist/sdk.js +93 -8
- package/dist/sdk.js.map +1 -1
- package/dist/share-cli.d.ts +2 -0
- package/dist/share-cli.js +185 -0
- package/dist/share-cli.js.map +1 -0
- package/dist/share.d.ts +138 -0
- package/dist/share.js +384 -0
- package/dist/share.js.map +1 -0
- package/dist/stack-detect.d.ts +3 -0
- package/dist/stack-detect.js +62 -0
- package/dist/stack-detect.js.map +1 -0
- package/package.json +3 -1
- package/stress-report.json +78 -38
package/dist/monitor-client.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { type Identity } from "./enroll.js";
|
|
2
2
|
import type { InstalledPackage } from "./lockfiles.js";
|
|
3
|
+
import type { RequestMeta, RequestOutcome } from "./monitor-rules.js";
|
|
3
4
|
/** What the App monitoring API needs: where to send, and the secret to send with. */
|
|
4
5
|
export type MonitorAuth = Pick<Identity, "apiBase" | "deviceSecret">;
|
|
5
6
|
/**
|
|
6
|
-
* An app token (BEAM_APP_TOKEN, created per app
|
|
7
|
+
* An app token (BEAM_APP_TOKEN, created per app under App monitoring > Settings, apps and alerts) for
|
|
7
8
|
* servers, containers and CI; otherwise this machine's own connection from 'beam connect'.
|
|
8
9
|
*/
|
|
9
10
|
export declare function monitorAuth(env?: NodeJS.ProcessEnv): Promise<MonitorAuth | null>;
|
|
@@ -42,6 +43,49 @@ export interface MonitorFinding {
|
|
|
42
43
|
reason: string;
|
|
43
44
|
model: string;
|
|
44
45
|
};
|
|
46
|
+
/** The flagged request (traffic findings): client, agent, path, response status and what became of it. Redacted. */
|
|
47
|
+
request?: RequestMeta & {
|
|
48
|
+
ip?: string;
|
|
49
|
+
outcome: RequestOutcome;
|
|
50
|
+
};
|
|
51
|
+
/** Set on a roll-up: many requests that matched a rule but couldn't have worked, sent as one finding. */
|
|
52
|
+
rollup?: MonitorRollup;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Requests that matched one rule but were refused (3xx/4xx) or aimed at software the app doesn't
|
|
56
|
+
* run, over a few minutes. Sent as one low-severity finding instead of one per address, so a
|
|
57
|
+
* botnet's scan shows up as one campaign and doesn't spend the month's allowance or page anyone.
|
|
58
|
+
*/
|
|
59
|
+
export interface MonitorRollup {
|
|
60
|
+
outcome: "blocked" | "not_applicable";
|
|
61
|
+
requests: number;
|
|
62
|
+
/** Distinct client addresses (counted up to 10,000). */
|
|
63
|
+
addresses: number;
|
|
64
|
+
from: string;
|
|
65
|
+
to: string;
|
|
66
|
+
statuses: Record<string, number>;
|
|
67
|
+
/** The busiest addresses, with what each sent last. */
|
|
68
|
+
clients: {
|
|
69
|
+
ip: string;
|
|
70
|
+
count: number;
|
|
71
|
+
status?: number;
|
|
72
|
+
path?: string;
|
|
73
|
+
userAgent?: string;
|
|
74
|
+
country?: string;
|
|
75
|
+
asn?: number;
|
|
76
|
+
}[];
|
|
77
|
+
paths: {
|
|
78
|
+
path: string;
|
|
79
|
+
count: number;
|
|
80
|
+
}[];
|
|
81
|
+
userAgents: {
|
|
82
|
+
userAgent: string;
|
|
83
|
+
count: number;
|
|
84
|
+
}[];
|
|
85
|
+
countries: {
|
|
86
|
+
country: string;
|
|
87
|
+
count: number;
|
|
88
|
+
}[];
|
|
45
89
|
}
|
|
46
90
|
export interface Vulnerability {
|
|
47
91
|
id: string;
|
package/dist/monitor-client.js
CHANGED
|
@@ -4,14 +4,14 @@
|
|
|
4
4
|
import { readIdentity } from "./enroll.js";
|
|
5
5
|
import { getDashboardUrl } from "./config.js";
|
|
6
6
|
/**
|
|
7
|
-
* An app token (BEAM_APP_TOKEN, created per app
|
|
7
|
+
* An app token (BEAM_APP_TOKEN, created per app under App monitoring > Settings, apps and alerts) for
|
|
8
8
|
* servers, containers and CI; otherwise this machine's own connection from 'beam connect'.
|
|
9
9
|
*/
|
|
10
10
|
export async function monitorAuth(env = process.env) {
|
|
11
11
|
const token = env.BEAM_APP_TOKEN?.trim();
|
|
12
12
|
if (token) {
|
|
13
13
|
if (!/^beam_app_[\w-]{40,}$/.test(token))
|
|
14
|
-
throw new Error("BEAM_APP_TOKEN doesn't look like an app token (beam_app_…). Copy it again from
|
|
14
|
+
throw new Error("BEAM_APP_TOKEN doesn't look like an app token (beam_app_…). Copy it again from App monitoring > Settings, apps and alerts in the dashboard.");
|
|
15
15
|
const apiBase = getDashboardUrl();
|
|
16
16
|
// A bearer token over plain HTTP can be read on the wire; only loopback (local testing) is exempt.
|
|
17
17
|
const url = new URL(apiBase);
|
|
@@ -50,7 +50,7 @@ async function call(identity, path, init = {}) {
|
|
|
50
50
|
ok: false,
|
|
51
51
|
status: "revoked",
|
|
52
52
|
message: identity.deviceSecret.startsWith("beam_app_")
|
|
53
|
-
? `This app token was rotated or removed, or its app was deleted. Copy the current token from ${getDashboardUrl()}/dashboard/monitoring
|
|
53
|
+
? `This app token was rotated or removed, or its app was deleted. Copy the current token from ${getDashboardUrl()}/dashboard/monitoring/settings#apps into BEAM_APP_TOKEN.`
|
|
54
54
|
: "This device was revoked or is unknown to the workspace. Run 'beam connect' again.",
|
|
55
55
|
};
|
|
56
56
|
if (response.status === 404 || response.status === 405)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"monitor-client.js","sourceRoot":"","sources":["../src/monitor-client.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,8FAA8F;AAC9F,kFAAkF;AAClF,OAAO,EAAE,YAAY,EAAiB,MAAM,aAAa,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"monitor-client.js","sourceRoot":"","sources":["../src/monitor-client.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,8FAA8F;AAC9F,kFAAkF;AAClF,OAAO,EAAE,YAAY,EAAiB,MAAM,aAAa,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAO9C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAyB,OAAO,CAAC,GAAG;IACpE,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC;IACzC,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6IAA6I,CAAC,CAAC;QACzM,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;QAClC,mGAAmG;QACnG,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC1F,MAAM,IAAI,KAAK,CAAC,sCAAsC,OAAO,uDAAuD,CAAC,CAAC;QACxH,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC1C,CAAC;IACD,OAAO,YAAY,EAAE,CAAC;AACxB,CAAC;AAuDD,KAAK,UAAU,IAAI,CAAC,QAAqB,EAAE,IAAY,EAAE,OAAoB,EAAE;IAC7E,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,QAAQ,CAAC,OAAO,GAAG,IAAI,EAAE,EAAE;YACnD,GAAG,IAAI;YACP,OAAO,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,aAAa,EAAE,UAAU,QAAQ,CAAC,YAAY,EAAE,EAAE;YAC3H,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,mBAAmB,QAAQ,CAAC,OAAO,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;IACzJ,CAAC;IACD,IAAI,IAAI,GAAmE,EAAE,CAAC;IAC9E,IAAI,CAAC;QAAC,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAgB,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,4BAA4B,CAAC,CAAC;IAC7F,IAAI,QAAQ,CAAC,EAAE;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,OAAO,wBAAwB,QAAQ,CAAC,MAAM,GAAG,CAAC;IAC5H,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;IAC/E,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAClJ,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;QACzB,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,QAAQ,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW,CAAC;gBACpD,CAAC,CAAC,8FAA8F,eAAe,EAAE,0DAA0D;gBAC3K,CAAC,CAAC,mFAAmF;SACxF,CAAC;IACJ,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;QACpD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,OAAO,sCAAsC,EAAE,CAAC;IAClH,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAqB;IAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;IAC1D,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAqB,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AACjF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAqB,EAAE,QAA0B;IAClF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,sBAAsB,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IACpH,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,MAAM,CAAC;IAC9B,MAAM,IAAI,GAAG,MAAM,CAAC,IAA+C,CAAC;IACpE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAqB,EAAE,QAA4B;IACrF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7G,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAsB,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AAClF,CAAC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { MonitorFinding, MonitorLevel, MonitorSeverity } from "./monitor-client.js";
|
|
2
2
|
import type { CustomDetector } from "./data-detectors.js";
|
|
3
3
|
import { type LineSource, type MonitorRule } from "./monitor-rules.js";
|
|
4
|
+
import { type Stack } from "./monitor-stacks.js";
|
|
4
5
|
/** A short, stable fingerprint (two seeded 53-bit hashes); not cryptographic, only used for dedupe ids. */
|
|
5
6
|
export declare function fingerprint(text: string): string;
|
|
6
7
|
/** light: critical only · standard: high and up · strict: medium and up (same thresholds as the server). */
|
|
@@ -9,6 +10,7 @@ export declare const DEDUPE_WINDOW_MS: Record<MonitorLevel, number>;
|
|
|
9
10
|
export declare const BRUTE_FORCE_THRESHOLD: Record<MonitorLevel, number>;
|
|
10
11
|
export declare const LLM_FLOOD_THRESHOLD: Record<MonitorLevel, number>;
|
|
11
12
|
export declare const MAX_NEW_FINDINGS_PER_10_MIN = 200;
|
|
13
|
+
export declare const ROLLUP_EVERY_MS: number;
|
|
12
14
|
export type SendOutcome = "sent" | "retry" | "drop";
|
|
13
15
|
export interface IngestOptions {
|
|
14
16
|
/** What the text is. Default: worked out from the line (an access-log line or an app log line). */
|
|
@@ -29,6 +31,18 @@ export interface PipelineOptions {
|
|
|
29
31
|
cpuBudgetMsPerMinute?: number;
|
|
30
32
|
/** Or a budget shared with other work (the SDK's prompt and output checks). */
|
|
31
33
|
budget?: CpuBudget;
|
|
34
|
+
/** LLM tokens one user may use in an hour before it's raised. Default: SIGNAL_THRESHOLDS.tokensPerHour for the level. */
|
|
35
|
+
tokensPerUserHour?: number;
|
|
36
|
+
/**
|
|
37
|
+
* The app is JavaScript (the SDK): a request for a PHP, WordPress, ASP or Java path can't reach
|
|
38
|
+
* code that would run it, so it's rolled up like a refused one even without a response status.
|
|
39
|
+
*/
|
|
40
|
+
javascriptApp?: boolean;
|
|
41
|
+
/**
|
|
42
|
+
* The app's stack (nextjs, django, laravel, spring...). Attacks aimed at software it doesn't run
|
|
43
|
+
* are rolled up as not applicable, and its own framework's rules count in full.
|
|
44
|
+
*/
|
|
45
|
+
stack?: Stack;
|
|
32
46
|
}
|
|
33
47
|
/**
|
|
34
48
|
* A hard cap on detection time per minute, so monitoring can never take more than a fixed share of
|
|
@@ -57,6 +71,9 @@ export declare class WindowCounter {
|
|
|
57
71
|
get size(): number;
|
|
58
72
|
/** Records one event and returns the count over the window, this one included. */
|
|
59
73
|
add(key: string, now: number): number;
|
|
74
|
+
/** The count over the window, without recording anything. */
|
|
75
|
+
peek(key: string, now: number): number;
|
|
76
|
+
clear(key: string): void;
|
|
60
77
|
/** Drops keys with nothing in the window. */
|
|
61
78
|
prune(now: number): void;
|
|
62
79
|
}
|
|
@@ -70,19 +87,24 @@ export declare class MonitorPipeline {
|
|
|
70
87
|
private floodBucket;
|
|
71
88
|
private floodCount;
|
|
72
89
|
private suppressed;
|
|
90
|
+
private rollups;
|
|
91
|
+
private misses;
|
|
92
|
+
private errors;
|
|
93
|
+
private walks;
|
|
94
|
+
private loginFailures;
|
|
95
|
+
private tokens;
|
|
96
|
+
private knownHosts;
|
|
97
|
+
private reportedHosts;
|
|
73
98
|
private maintainedMinute;
|
|
74
99
|
readonly budget: CpuBudget | null;
|
|
75
100
|
constructor(opts: PipelineOptions);
|
|
76
101
|
private now;
|
|
102
|
+
private stack;
|
|
77
103
|
get pending(): MonitorFinding[];
|
|
104
|
+
/** Roll-ups still collecting; a forced flush sends them early. */
|
|
105
|
+
get rollingUp(): number;
|
|
78
106
|
/** Sizes of everything the pipeline keeps, for tests and diagnostics: all of them are bounded. */
|
|
79
|
-
get footprint():
|
|
80
|
-
seen: number;
|
|
81
|
-
queue: number;
|
|
82
|
-
queued: number;
|
|
83
|
-
authFailures: number;
|
|
84
|
-
llmCalls: number;
|
|
85
|
-
};
|
|
107
|
+
get footprint(): Record<"seen" | "queue" | "queued" | "authFailures" | "llmCalls" | "rollups" | "misses" | "errors" | "walks" | "loginFailures" | "tokens" | "hosts", number>;
|
|
86
108
|
/** Once a minute: forget dedupe entries and counters that can no longer matter. */
|
|
87
109
|
private maintain;
|
|
88
110
|
/**
|
|
@@ -91,11 +113,40 @@ export declare class MonitorPipeline {
|
|
|
91
113
|
*/
|
|
92
114
|
ingestLine(line: string, file?: string, options?: IngestOptions): string[];
|
|
93
115
|
private ingest;
|
|
116
|
+
/** The attack-in-progress signals for one traffic or log line. Returns the rule ids raised. */
|
|
117
|
+
private signals;
|
|
118
|
+
private loginSucceeded;
|
|
119
|
+
private privilegeChanged;
|
|
120
|
+
private noteTokens;
|
|
121
|
+
/**
|
|
122
|
+
* Structured security events from the app (the SDK's event() and usage()), checked like the log
|
|
123
|
+
* lines that say the same thing.
|
|
124
|
+
*/
|
|
125
|
+
event(type: "login.failed" | "login.succeeded" | "admin.granted" | "apikey.created", who: {
|
|
126
|
+
ip?: string;
|
|
127
|
+
user?: string;
|
|
128
|
+
}, detail?: string): string[];
|
|
129
|
+
/** LLM tokens used by one user (or address), as reported by the app. */
|
|
130
|
+
usage(user: string, tokens: number, detail?: string): string[];
|
|
131
|
+
/** Hosts the workspace has already seen this app call (from its config), so only new ones are raised. */
|
|
132
|
+
setKnownOutboundHosts(hosts: string[]): void;
|
|
133
|
+
/** The app called this URL or host. Raises the first call to a host the workspace hasn't seen for this app. */
|
|
134
|
+
outbound(target: string): boolean;
|
|
135
|
+
/** The request behind a traffic line, redacted like evidence, with what became of it. */
|
|
136
|
+
private describeRequest;
|
|
137
|
+
private rollUp;
|
|
138
|
+
/** Queues roll-ups that have collected for ROLLUP_EVERY_MS, or all of them when `force`. */
|
|
139
|
+
private emitRollups;
|
|
94
140
|
/** Queues a finding produced elsewhere (dependency checks) under its own stable id, once. */
|
|
95
141
|
addFinding(finding: MonitorFinding): boolean;
|
|
96
142
|
private raise;
|
|
97
143
|
private emitSuppressed;
|
|
98
144
|
private enqueue;
|
|
99
|
-
/**
|
|
100
|
-
|
|
145
|
+
/**
|
|
146
|
+
* Sends queued findings in batches; a "retry" outcome keeps the batch for the next flush.
|
|
147
|
+
* `force` (shutdown, the SDK's flush({ force })) also sends roll-ups still collecting.
|
|
148
|
+
*/
|
|
149
|
+
flush(send: (batch: MonitorFinding[]) => Promise<SendOutcome>, options?: {
|
|
150
|
+
force?: boolean;
|
|
151
|
+
}): Promise<void>;
|
|
101
152
|
}
|
package/dist/monitor-pipeline.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import { clientIp, isAuthFailure, isLlmRequest, lineSource, matchLine, redactLine } from "./monitor-rules.js";
|
|
1
|
+
import { clientIp, isAuthFailure, isAuthSuccess, isLlmRequest, lineSource, logUser, matchLine, privilegeChange, quickRequest, redactLine, requestMeta, requestOutcome, routeKey, tokenUsage, } from "./monitor-rules.js";
|
|
2
|
+
import { DistinctPerMinute, MinuteSeries, SIGNAL_THRESHOLDS, meaningfulPath, outboundHost } from "./monitor-signals.js";
|
|
3
|
+
import { foreignTo, ruleFamilies } from "./monitor-stacks.js";
|
|
2
4
|
/** A short, stable fingerprint (two seeded 53-bit hashes); not cryptographic, only used for dedupe ids. */
|
|
3
5
|
export function fingerprint(text) {
|
|
4
6
|
const h = (seed) => {
|
|
@@ -33,6 +35,22 @@ export const MAX_NEW_FINDINGS_PER_10_MIN = 200;
|
|
|
33
35
|
const FLOOD_WINDOW_MS = 10 * 60_000;
|
|
34
36
|
const MAX_QUEUE = 1000;
|
|
35
37
|
const BATCH_SIZE = 100;
|
|
38
|
+
// Attacks that were refused or can't work here are rolled up per rule and sent this often.
|
|
39
|
+
export const ROLLUP_EVERY_MS = 5 * 60_000;
|
|
40
|
+
const ROLLUP_LIMITS = { addresses: 10_000, clients: 2000, paths: 500, userAgents: 300, countries: 250 };
|
|
41
|
+
const ROLLUP_TOP = { clients: 25, paths: 10, userAgents: 10, countries: 10 };
|
|
42
|
+
/** Adds one to `key`, keeping at most `max` keys (new keys past it aren't tracked). */
|
|
43
|
+
function bump(map, key, max) {
|
|
44
|
+
if (!key)
|
|
45
|
+
return;
|
|
46
|
+
const n = map.get(key);
|
|
47
|
+
if (n !== undefined)
|
|
48
|
+
map.set(key, n + 1);
|
|
49
|
+
else if (map.size < max)
|
|
50
|
+
map.set(key, 1);
|
|
51
|
+
}
|
|
52
|
+
const top = (map, n) => [...map].sort((a, b) => b[1] - a[1]).slice(0, n);
|
|
53
|
+
const OUTCOME_TITLE = { blocked: "refused by the app", not_applicable: "aimed at software this app doesn't run" };
|
|
36
54
|
/** Groups lines that differ only in ids, numbers, addresses or timestamps. */
|
|
37
55
|
function normalizeForGrouping(line) {
|
|
38
56
|
return line.toLowerCase()
|
|
@@ -114,6 +132,19 @@ export class WindowCounter {
|
|
|
114
132
|
total += entry.counts[i];
|
|
115
133
|
return total;
|
|
116
134
|
}
|
|
135
|
+
/** The count over the window, without recording anything. */
|
|
136
|
+
peek(key, now) {
|
|
137
|
+
const entry = this.keys.get(key);
|
|
138
|
+
if (!entry)
|
|
139
|
+
return 0;
|
|
140
|
+
const minute = Math.floor(now / 60_000);
|
|
141
|
+
let total = 0;
|
|
142
|
+
for (let i = 0; i < this.windowMinutes; i++)
|
|
143
|
+
if (minute - entry.minutes[i] < this.windowMinutes)
|
|
144
|
+
total += entry.counts[i];
|
|
145
|
+
return total;
|
|
146
|
+
}
|
|
147
|
+
clear(key) { this.keys.delete(key); }
|
|
117
148
|
/** Drops keys with nothing in the window. */
|
|
118
149
|
prune(now) {
|
|
119
150
|
const minute = Math.floor(now / 60_000);
|
|
@@ -132,6 +163,15 @@ export class MonitorPipeline {
|
|
|
132
163
|
floodBucket = -1;
|
|
133
164
|
floodCount = 0;
|
|
134
165
|
suppressed = null;
|
|
166
|
+
rollups = new Map();
|
|
167
|
+
// Attack-in-progress signals (monitor-signals.ts).
|
|
168
|
+
misses = new MinuteSeries(10, 5000);
|
|
169
|
+
errors = new MinuteSeries(35, 1000);
|
|
170
|
+
walks = new DistinctPerMinute(SIGNAL_THRESHOLDS.pathWalk.light, 2000);
|
|
171
|
+
loginFailures = new WindowCounter(15, 10_000);
|
|
172
|
+
tokens = new MinuteSeries(60, 5000);
|
|
173
|
+
knownHosts = new Set();
|
|
174
|
+
reportedHosts = new Set();
|
|
135
175
|
maintainedMinute = -1;
|
|
136
176
|
budget;
|
|
137
177
|
constructor(opts) {
|
|
@@ -139,10 +179,17 @@ export class MonitorPipeline {
|
|
|
139
179
|
this.budget = opts.budget ?? (opts.cpuBudgetMsPerMinute ? new CpuBudget(opts.cpuBudgetMsPerMinute) : null);
|
|
140
180
|
}
|
|
141
181
|
now() { return this.opts.now?.() ?? Date.now(); }
|
|
182
|
+
stack() { return this.opts.stack ?? (this.opts.javascriptApp ? "node" : undefined); }
|
|
142
183
|
get pending() { return [...this.queue]; }
|
|
184
|
+
/** Roll-ups still collecting; a forced flush sends them early. */
|
|
185
|
+
get rollingUp() { return this.rollups.size; }
|
|
143
186
|
/** Sizes of everything the pipeline keeps, for tests and diagnostics: all of them are bounded. */
|
|
144
187
|
get footprint() {
|
|
145
|
-
return {
|
|
188
|
+
return {
|
|
189
|
+
seen: this.seen.size, queue: this.queue.length, queued: this.queued.size, authFailures: this.authFailures.size, llmCalls: this.llmCalls.size,
|
|
190
|
+
rollups: this.rollups.size, misses: this.misses.size, errors: this.errors.size, walks: this.walks.size, loginFailures: this.loginFailures.size,
|
|
191
|
+
tokens: this.tokens.size, hosts: this.knownHosts.size + this.reportedHosts.size,
|
|
192
|
+
};
|
|
146
193
|
}
|
|
147
194
|
/** Once a minute: forget dedupe entries and counters that can no longer matter. */
|
|
148
195
|
maintain(now) {
|
|
@@ -156,8 +203,14 @@ export class MonitorPipeline {
|
|
|
156
203
|
this.seen.delete(fp);
|
|
157
204
|
this.authFailures.prune(now);
|
|
158
205
|
this.llmCalls.prune(now);
|
|
206
|
+
this.misses.prune(now);
|
|
207
|
+
this.errors.prune(now);
|
|
208
|
+
this.walks.prune(now);
|
|
209
|
+
this.loginFailures.prune(now);
|
|
210
|
+
this.tokens.prune(now);
|
|
159
211
|
if (this.queued.size > 5000)
|
|
160
212
|
this.queued.clear(); // dependency ids; a repeat is deduped by the server too
|
|
213
|
+
this.emitRollups(now, false);
|
|
161
214
|
}
|
|
162
215
|
/**
|
|
163
216
|
* Matches one line and queues any findings at or above the level. Returns the rule ids raised.
|
|
@@ -179,16 +232,32 @@ export class MonitorPipeline {
|
|
|
179
232
|
const ip = options.client ? redactLine(options.client, custom).slice(0, 80) : source === "http" ? clientIp(line) : undefined;
|
|
180
233
|
this.maintain(this.now());
|
|
181
234
|
const excerpt = options.excerpt === false ? "(the app withheld the text)" : redactLine(line, custom);
|
|
235
|
+
const request = source === "http" ? this.describeRequest(line, ip, custom, options.excerpt !== false) : undefined;
|
|
182
236
|
const raised = [];
|
|
183
237
|
const seenRules = new Set();
|
|
238
|
+
const stack = this.stack();
|
|
184
239
|
for (const rule of matchLine(line, source, options.rules)) {
|
|
185
|
-
if (
|
|
240
|
+
if (seenRules.has(rule.id))
|
|
241
|
+
continue;
|
|
242
|
+
// An attack on a framework this app doesn't run (a Laravel exploit against a Next.js app),
|
|
243
|
+
// with no status to say otherwise, can't have worked.
|
|
244
|
+
const outcome = request?.outcome === "unknown" && stack && foreignTo(stack, ruleFamilies(rule.id)) ? "not_applicable" : request?.outcome;
|
|
245
|
+
// Refused, or aimed at software that isn't here: rolled up whatever the level, as context
|
|
246
|
+
// for the dashboard, never one finding (or alert) per address.
|
|
247
|
+
if (request && (outcome === "blocked" || outcome === "not_applicable")) {
|
|
248
|
+
seenRules.add(rule.id);
|
|
249
|
+
this.rollUp(rule, outcome, { ...request, outcome }, file, excerpt);
|
|
250
|
+
raised.push(rule.id);
|
|
251
|
+
continue;
|
|
252
|
+
}
|
|
253
|
+
if (!meetsLevel(rule.severity, level))
|
|
186
254
|
continue;
|
|
187
255
|
seenRules.add(rule.id);
|
|
188
256
|
// Traffic groups by client address (one scan = one finding); app logs group by line shape.
|
|
189
257
|
this.raise(level, ip ?? normalizeForGrouping(line), {
|
|
190
258
|
ruleId: rule.id, title: ip ? `${rule.title} from ${ip}` : rule.title, severity: rule.severity,
|
|
191
259
|
category: rule.category, source: wire, file, evidence: options.excerpt === false ? `Matched "${rule.title}"; the app withheld the text.` : excerpt,
|
|
260
|
+
...(request ? { request } : {}),
|
|
192
261
|
});
|
|
193
262
|
raised.push(rule.id);
|
|
194
263
|
}
|
|
@@ -199,6 +268,7 @@ export class MonitorPipeline {
|
|
|
199
268
|
this.raise(level, key, {
|
|
200
269
|
ruleId: "app.auth_bruteforce", title: `Repeated failed sign-ins from ${key}`, severity: "high", category: "recon",
|
|
201
270
|
source: wire, file, evidence: `${recent} failures in 5 minutes. Latest: ${redactLine(line, custom)}`.slice(0, 600),
|
|
271
|
+
...(request ? { request } : {}),
|
|
202
272
|
});
|
|
203
273
|
raised.push("app.auth_bruteforce");
|
|
204
274
|
}
|
|
@@ -210,12 +280,258 @@ export class MonitorPipeline {
|
|
|
210
280
|
this.raise(level, ip, {
|
|
211
281
|
ruleId: "llm.denial_of_wallet", title: `Unusual volume of LLM requests from ${ip}`, severity: "high", category: "llm",
|
|
212
282
|
source: wire, file, evidence: `${recent} ${source === "prompt" ? "prompts" : "requests to LLM endpoints"} in 5 minutes. Scripted use runs up the model bill. Latest: ${excerpt}`.slice(0, 600),
|
|
283
|
+
...(request ? { request } : {}),
|
|
213
284
|
});
|
|
214
285
|
raised.push("llm.denial_of_wallet");
|
|
215
286
|
}
|
|
216
287
|
}
|
|
288
|
+
if (source === "http" || source === "log")
|
|
289
|
+
raised.push(...this.signals(line, source, wire, file, ip, request, level, custom, excerpt));
|
|
217
290
|
return raised;
|
|
218
291
|
}
|
|
292
|
+
/** The attack-in-progress signals for one traffic or log line. Returns the rule ids raised. */
|
|
293
|
+
signals(line, source, wire, file, ip, request, level, custom, excerpt) {
|
|
294
|
+
const now = this.now();
|
|
295
|
+
const out = [];
|
|
296
|
+
const high = meetsLevel("high", level);
|
|
297
|
+
const base = { source: wire, file, ...(request ? { request } : {}) };
|
|
298
|
+
if (source === "http" && request) {
|
|
299
|
+
const path = request.path ?? quickRequest(line).path;
|
|
300
|
+
const status = request.status;
|
|
301
|
+
// A scan finding something: an address with a run of 4xx gets a 2xx on a real page.
|
|
302
|
+
if (ip && status !== undefined) {
|
|
303
|
+
if (status >= 400 && status < 500)
|
|
304
|
+
this.misses.add(ip, now);
|
|
305
|
+
else if (status >= 200 && status < 300 && path && meaningfulPath(path)) {
|
|
306
|
+
const misses = this.misses.sum(ip, now, 0, 9);
|
|
307
|
+
if (high && misses >= SIGNAL_THRESHOLDS.probeMisses[level]) {
|
|
308
|
+
const route = routeKey(redactLine(path, custom));
|
|
309
|
+
this.raise(level, ip, {
|
|
310
|
+
ruleId: "signal.probe_hit", title: `Scan from ${ip} found a live path`, severity: "high", category: "activity", ...base,
|
|
311
|
+
evidence: `${ip} got ${misses} 4xx answers in 10 minutes, then ${status} on ${route}. Latest: ${excerpt}`.slice(0, 600),
|
|
312
|
+
});
|
|
313
|
+
out.push("signal.probe_hit");
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
// A jump in server errors on one route, against its own last half hour.
|
|
318
|
+
if (status !== undefined && status >= 500 && path) {
|
|
319
|
+
const route = routeKey(redactLine(path, custom));
|
|
320
|
+
this.errors.add(route, now);
|
|
321
|
+
const recent = this.errors.sum(route, now, 0, 4);
|
|
322
|
+
const usual = this.errors.sum(route, now, 5, 34) / 6;
|
|
323
|
+
if (high && recent >= SIGNAL_THRESHOLDS.errorSpike[level] && recent >= 5 * Math.max(1, usual)) {
|
|
324
|
+
this.raise(level, route, {
|
|
325
|
+
ruleId: "signal.error_spike", title: `Server errors jumped on ${route}`, severity: "high", category: "activity", ...base,
|
|
326
|
+
evidence: `${recent} 5xx answers on ${route} in 5 minutes, against about ${Math.round(usual)} per 5 minutes before. Latest: ${excerpt}`.slice(0, 600),
|
|
327
|
+
});
|
|
328
|
+
out.push("signal.error_spike");
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
// One address walking the app: many distinct paths in a minute.
|
|
332
|
+
if (ip && path) {
|
|
333
|
+
const distinct = this.walks.add(ip, path.split(/[?#]/)[0], now);
|
|
334
|
+
if (high && distinct >= SIGNAL_THRESHOLDS.pathWalk[level]) {
|
|
335
|
+
this.raise(level, ip, {
|
|
336
|
+
ruleId: "signal.path_walk", title: `${ip} is walking the app's paths`, severity: "high", category: "activity", ...base,
|
|
337
|
+
evidence: `${distinct}+ different paths from ${ip} within a minute: a scanner or scraper mapping the app. Latest: ${excerpt}`.slice(0, 600),
|
|
338
|
+
});
|
|
339
|
+
out.push("signal.path_walk");
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
// A sign-in that succeeds right after a run of failures, by address or by user.
|
|
344
|
+
const who = [source === "http" ? ip : clientIp(line), logUser(line)];
|
|
345
|
+
const keys = [who[0] ? `ip:${who[0]}` : null, who[1] ? `user:${who[1]}` : null].filter((k) => Boolean(k));
|
|
346
|
+
if (keys.length && isAuthFailure(line, source))
|
|
347
|
+
for (const key of keys)
|
|
348
|
+
this.loginFailures.add(key, now);
|
|
349
|
+
else if (keys.length && isAuthSuccess(line, source))
|
|
350
|
+
out.push(...this.loginSucceeded(level, keys, who[0], wire, file, request, excerpt));
|
|
351
|
+
// A role or permission granted, or a new API key.
|
|
352
|
+
const change = privilegeChange(line, source);
|
|
353
|
+
if (change)
|
|
354
|
+
out.push(...this.privilegeChanged(level, change, ip ?? normalizeForGrouping(line), wire, file, request, excerpt));
|
|
355
|
+
// LLM token usage the app logs, per user.
|
|
356
|
+
const usage = source === "log" ? tokenUsage(line) : null;
|
|
357
|
+
if (usage)
|
|
358
|
+
out.push(...this.noteTokens(level, usage.user ?? who[0] ?? "unknown", usage.tokens, wire, file, excerpt, custom));
|
|
359
|
+
return out;
|
|
360
|
+
}
|
|
361
|
+
loginSucceeded(level, keys, ip, wire, file, request, excerpt) {
|
|
362
|
+
const now = this.now();
|
|
363
|
+
const failures = Math.max(...keys.map(k => this.loginFailures.peek(k, now)));
|
|
364
|
+
for (const key of keys)
|
|
365
|
+
this.loginFailures.clear(key);
|
|
366
|
+
if (failures < SIGNAL_THRESHOLDS.loginAfterFailures[level])
|
|
367
|
+
return [];
|
|
368
|
+
this.raise(level, keys[0], {
|
|
369
|
+
ruleId: "signal.login_after_failures", title: `Sign-in succeeded after ${failures} failed attempts${ip ? ` from ${ip}` : ""}`,
|
|
370
|
+
severity: "critical", category: "activity", source: wire, file, ...(request ? { request } : {}),
|
|
371
|
+
evidence: `${failures} failed sign-ins in 15 minutes, then a successful one: a guessed or stuffed password, possibly. Latest: ${excerpt}`.slice(0, 600),
|
|
372
|
+
});
|
|
373
|
+
return ["signal.login_after_failures"];
|
|
374
|
+
}
|
|
375
|
+
privilegeChanged(level, change, groupKey, wire, file, request, excerpt) {
|
|
376
|
+
const severity = change === "admin" ? "high" : "medium";
|
|
377
|
+
if (!meetsLevel(severity, level))
|
|
378
|
+
return [];
|
|
379
|
+
this.raise(level, `${change}\0${groupKey}`, {
|
|
380
|
+
ruleId: "signal.privilege_change", title: change === "admin" ? "Admin or role granted" : "New API key or access token created",
|
|
381
|
+
severity, category: "activity", source: wire, file, ...(request ? { request } : {}),
|
|
382
|
+
evidence: `${change === "admin" ? "Someone was given admin rights or a new role." : "A new API key or access token was created."} Latest: ${excerpt}`.slice(0, 600),
|
|
383
|
+
});
|
|
384
|
+
return ["signal.privilege_change"];
|
|
385
|
+
}
|
|
386
|
+
noteTokens(level, user, tokens, wire, file, excerpt, custom) {
|
|
387
|
+
const now = this.now();
|
|
388
|
+
this.tokens.add(user, now, tokens);
|
|
389
|
+
const total = this.tokens.sum(user, now, 0, 59);
|
|
390
|
+
const limit = this.opts.tokensPerUserHour ?? SIGNAL_THRESHOLDS.tokensPerHour[level];
|
|
391
|
+
if (!meetsLevel("high", level) || total < limit)
|
|
392
|
+
return [];
|
|
393
|
+
const shown = redactLine(user, custom).slice(0, 80);
|
|
394
|
+
this.raise(level, user, {
|
|
395
|
+
ruleId: "llm.token_spend", title: `Unusual LLM spend: ${Math.round(total).toLocaleString("en-US")} tokens in an hour for ${shown}`,
|
|
396
|
+
severity: "high", category: "llm", source: wire, file,
|
|
397
|
+
evidence: `${Math.round(total).toLocaleString("en-US")} tokens in the last hour for ${shown} (limit ${limit.toLocaleString("en-US")}). Scripted use or a runaway loop runs up the bill. Latest: ${excerpt}`.slice(0, 600),
|
|
398
|
+
});
|
|
399
|
+
return ["llm.token_spend"];
|
|
400
|
+
}
|
|
401
|
+
/**
|
|
402
|
+
* Structured security events from the app (the SDK's event() and usage()), checked like the log
|
|
403
|
+
* lines that say the same thing.
|
|
404
|
+
*/
|
|
405
|
+
event(type, who, detail = "") {
|
|
406
|
+
const level = this.opts.level();
|
|
407
|
+
if (!level)
|
|
408
|
+
return [];
|
|
409
|
+
const run = () => {
|
|
410
|
+
this.maintain(this.now());
|
|
411
|
+
const custom = this.opts.customDetectors?.() ?? [];
|
|
412
|
+
const ip = who.ip ? redactLine(who.ip, custom).slice(0, 80) : undefined;
|
|
413
|
+
const keys = [who.ip ? `ip:${who.ip}` : null, who.user ? `user:${who.user}` : null].filter((k) => Boolean(k));
|
|
414
|
+
const excerpt = redactLine(`${type}${ip ? ` ip=${ip}` : ""}${who.user ? ` user=${who.user}` : ""}${detail ? ` ${detail}` : ""}`, custom);
|
|
415
|
+
if (type === "login.failed") {
|
|
416
|
+
for (const key of keys)
|
|
417
|
+
this.loginFailures.add(key, this.now());
|
|
418
|
+
return [];
|
|
419
|
+
}
|
|
420
|
+
if (type === "login.succeeded")
|
|
421
|
+
return keys.length ? this.loginSucceeded(level, keys, ip, "log", "security events", undefined, excerpt) : [];
|
|
422
|
+
return this.privilegeChanged(level, type === "admin.granted" ? "admin" : "key", keys[1] ?? keys[0] ?? excerpt, "log", "security events", undefined, excerpt);
|
|
423
|
+
};
|
|
424
|
+
return (this.budget ? this.budget.run(run) : run()) ?? [];
|
|
425
|
+
}
|
|
426
|
+
/** LLM tokens used by one user (or address), as reported by the app. */
|
|
427
|
+
usage(user, tokens, detail = "") {
|
|
428
|
+
const level = this.opts.level();
|
|
429
|
+
if (!level || !(tokens > 0) || !user)
|
|
430
|
+
return [];
|
|
431
|
+
this.maintain(this.now());
|
|
432
|
+
const custom = this.opts.customDetectors?.() ?? [];
|
|
433
|
+
return this.noteTokens(level, user, tokens, "log", "llm usage", redactLine(`${tokens} tokens user=${user}${detail ? ` ${detail}` : ""}`, custom), custom);
|
|
434
|
+
}
|
|
435
|
+
/** Hosts the workspace has already seen this app call (from its config), so only new ones are raised. */
|
|
436
|
+
setKnownOutboundHosts(hosts) {
|
|
437
|
+
this.knownHosts = new Set(hosts.slice(0, 5000).map(h => outboundHost(h)).filter((h) => Boolean(h)));
|
|
438
|
+
}
|
|
439
|
+
/** The app called this URL or host. Raises the first call to a host the workspace hasn't seen for this app. */
|
|
440
|
+
outbound(target) {
|
|
441
|
+
const level = this.opts.level();
|
|
442
|
+
const host = outboundHost(target);
|
|
443
|
+
if (!level || !host || this.knownHosts.has(host) || this.reportedHosts.has(host))
|
|
444
|
+
return false;
|
|
445
|
+
if (this.reportedHosts.size >= 2000)
|
|
446
|
+
return false;
|
|
447
|
+
this.reportedHosts.add(host);
|
|
448
|
+
if (!meetsLevel("high", level))
|
|
449
|
+
return false;
|
|
450
|
+
this.maintain(this.now());
|
|
451
|
+
this.raise(level, host, {
|
|
452
|
+
ruleId: "signal.new_outbound_host", title: `First call to ${host}`, severity: "high", category: "activity", source: "log", file: "outbound calls",
|
|
453
|
+
evidence: `The app called ${host}, a host it hasn't called before. New destinations are how stolen data and stolen compute leave a server.`,
|
|
454
|
+
request: { outcome: "unknown", host },
|
|
455
|
+
});
|
|
456
|
+
return true;
|
|
457
|
+
}
|
|
458
|
+
/** The request behind a traffic line, redacted like evidence, with what became of it. */
|
|
459
|
+
describeRequest(line, ip, custom, withText) {
|
|
460
|
+
const meta = requestMeta(line);
|
|
461
|
+
const clip = (s, max) => (s === undefined ? undefined : redactLine(s, custom).slice(0, max));
|
|
462
|
+
const request = {
|
|
463
|
+
outcome: requestOutcome(meta, { stack: this.stack() }),
|
|
464
|
+
ip, method: meta.method, status: meta.status, bytes: meta.bytes, durationMs: meta.durationMs,
|
|
465
|
+
host: clip(meta.host, 200), country: meta.country, asn: meta.asn, userAgent: clip(meta.userAgent, 300),
|
|
466
|
+
// The path and referrer carry the payload; an app that withholds text withholds them too.
|
|
467
|
+
...(withText ? { path: clip(meta.path, 500), referer: clip(meta.referer, 300) } : {}),
|
|
468
|
+
};
|
|
469
|
+
for (const key of Object.keys(request))
|
|
470
|
+
if (request[key] === undefined)
|
|
471
|
+
delete request[key];
|
|
472
|
+
return request;
|
|
473
|
+
}
|
|
474
|
+
rollUp(rule, outcome, request, file, excerpt) {
|
|
475
|
+
const now = this.now();
|
|
476
|
+
const key = `${rule.id}\0${outcome}\0${file ?? ""}`;
|
|
477
|
+
let r = this.rollups.get(key);
|
|
478
|
+
if (!r) {
|
|
479
|
+
r = {
|
|
480
|
+
ruleId: rule.id, title: rule.title, category: rule.category, file, outcome, started: now, last: now, requests: 0, excerpt,
|
|
481
|
+
addresses: new Set(), statuses: new Map(), clients: new Map(), paths: new Map(), userAgents: new Map(), countries: new Map(),
|
|
482
|
+
};
|
|
483
|
+
this.rollups.set(key, r);
|
|
484
|
+
}
|
|
485
|
+
r.requests++;
|
|
486
|
+
r.last = now;
|
|
487
|
+
r.excerpt = excerpt;
|
|
488
|
+
const ip = request.ip ?? "unknown";
|
|
489
|
+
if (r.addresses.size < ROLLUP_LIMITS.addresses)
|
|
490
|
+
r.addresses.add(ip);
|
|
491
|
+
bump(r.statuses, request.status === undefined ? "none" : String(request.status), 100);
|
|
492
|
+
bump(r.paths, request.path, ROLLUP_LIMITS.paths);
|
|
493
|
+
bump(r.userAgents, request.userAgent, ROLLUP_LIMITS.userAgents);
|
|
494
|
+
bump(r.countries, request.country, ROLLUP_LIMITS.countries);
|
|
495
|
+
const client = r.clients.get(ip);
|
|
496
|
+
const latest = { status: request.status, path: request.path, userAgent: request.userAgent, country: request.country, asn: request.asn };
|
|
497
|
+
for (const k of Object.keys(latest))
|
|
498
|
+
if (latest[k] === undefined)
|
|
499
|
+
delete latest[k];
|
|
500
|
+
if (client)
|
|
501
|
+
Object.assign(client, latest, { count: client.count + 1 });
|
|
502
|
+
else if (r.clients.size < ROLLUP_LIMITS.clients)
|
|
503
|
+
r.clients.set(ip, { ip, count: 1, ...latest });
|
|
504
|
+
}
|
|
505
|
+
/** Queues roll-ups that have collected for ROLLUP_EVERY_MS, or all of them when `force`. */
|
|
506
|
+
emitRollups(now, force) {
|
|
507
|
+
for (const [key, r] of this.rollups) {
|
|
508
|
+
if (!force && now - r.started < ROLLUP_EVERY_MS)
|
|
509
|
+
continue;
|
|
510
|
+
this.rollups.delete(key);
|
|
511
|
+
const statuses = Object.fromEntries(r.statuses);
|
|
512
|
+
const answered = Object.entries(statuses).filter(([s]) => s !== "none").sort((a, b) => b[1] - a[1]).map(([s, n]) => `${n} × ${s}`).join(", ");
|
|
513
|
+
const [topPath] = top(r.paths, 1);
|
|
514
|
+
const rollup = {
|
|
515
|
+
outcome: r.outcome, requests: r.requests, addresses: r.addresses.size,
|
|
516
|
+
from: new Date(r.started).toISOString(), to: new Date(r.last).toISOString(), statuses,
|
|
517
|
+
clients: [...r.clients.values()].sort((a, b) => b.count - a.count).slice(0, ROLLUP_TOP.clients),
|
|
518
|
+
paths: top(r.paths, ROLLUP_TOP.paths).map(([path, count]) => ({ path, count })),
|
|
519
|
+
userAgents: top(r.userAgents, ROLLUP_TOP.userAgents).map(([userAgent, count]) => ({ userAgent, count })),
|
|
520
|
+
countries: top(r.countries, ROLLUP_TOP.countries).map(([country, count]) => ({ country, count })),
|
|
521
|
+
};
|
|
522
|
+
const plural = (n, word) => `${n.toLocaleString("en-US")} ${word}${n === 1 ? "" : word.endsWith("s") ? "es" : "s"}`;
|
|
523
|
+
this.enqueue({
|
|
524
|
+
id: `rollup-${fingerprint(key)}-${r.started}`, ruleId: r.ruleId, title: `${r.title}, ${OUTCOME_TITLE[r.outcome]}`,
|
|
525
|
+
severity: "low", category: r.category, source: "http", file: r.file, count: r.requests, firstSeen: rollup.from,
|
|
526
|
+
evidence: [
|
|
527
|
+
`${plural(r.requests, "request")} from ${plural(r.addresses.size, "address")}${answered ? `, answered ${answered}` : ""}.`,
|
|
528
|
+
topPath ? `Most sent: ${topPath[0]} (${topPath[1]}).` : "",
|
|
529
|
+
`Latest: ${r.excerpt}`,
|
|
530
|
+
].filter(Boolean).join(" ").slice(0, 900),
|
|
531
|
+
rollup,
|
|
532
|
+
});
|
|
533
|
+
}
|
|
534
|
+
}
|
|
219
535
|
/** Queues a finding produced elsewhere (dependency checks) under its own stable id, once. */
|
|
220
536
|
addFinding(finding) {
|
|
221
537
|
const level = this.opts.level();
|
|
@@ -274,10 +590,14 @@ export class MonitorPipeline {
|
|
|
274
590
|
}
|
|
275
591
|
this.queue.push(finding);
|
|
276
592
|
}
|
|
277
|
-
/**
|
|
278
|
-
|
|
593
|
+
/**
|
|
594
|
+
* Sends queued findings in batches; a "retry" outcome keeps the batch for the next flush.
|
|
595
|
+
* `force` (shutdown, the SDK's flush({ force })) also sends roll-ups still collecting.
|
|
596
|
+
*/
|
|
597
|
+
async flush(send, options = {}) {
|
|
279
598
|
if (this.suppressed && Math.floor(this.now() / FLOOD_WINDOW_MS) !== this.floodBucket)
|
|
280
599
|
this.emitSuppressed();
|
|
600
|
+
this.emitRollups(this.now(), Boolean(options.force));
|
|
281
601
|
while (this.queue.length) {
|
|
282
602
|
const batch = this.queue.splice(0, BATCH_SIZE);
|
|
283
603
|
if ((await send(batch)) === "retry") {
|