deepdebug-local-agent 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/la-protocol-v2.md +9 -3
- package/env.example +7 -0
- package/package.json +1 -1
- package/scripts/incident-parity/Gen.java +256 -0
- package/scripts/incident-parity/LogEntry.java +17 -0
- package/scripts/incident-parity/extract.py +51 -0
- package/scripts/incident-parity/gen.sh +32 -0
- package/src/incident-detector/README.md +51 -0
- package/src/incident-detector/aggregator.js +73 -0
- package/src/incident-detector/categorize.js +48 -0
- package/src/incident-detector/config.js +95 -0
- package/src/incident-detector/detector.js +92 -0
- package/src/incident-detector/emitter.js +154 -0
- package/src/incident-detector/index.js +44 -0
- package/src/incident-detector/java-compat.js +66 -0
- package/src/incident-detector/normalize.js +209 -0
- package/src/incident-detector/parse.js +114 -0
- package/src/incident-detector/sources/file-source.js +113 -0
- package/src/incident-detector/sources/loki-source.js +101 -0
- package/src/incident-detector/sources/stream-source.js +39 -0
- package/src/incident-detector/store.js +119 -0
- package/src/la-protocol-v2/runtime.js +10 -1
- package/test/incident-detector/parity.test.js +66 -0
- package/test/incident-detector/parity.vectors.json +2619 -0
- package/test/incident-detector/sources.test.js +179 -0
- package/test/incident-detector/units.test.js +430 -0
- package/test/incident-detector/wiring.test.js +165 -0
package/docs/la-protocol-v2.md
CHANGED
|
@@ -136,13 +136,19 @@ start-up. The mode is also read at each request: leaving `enforce` at run time s
|
|
|
136
136
|
## What it does NOT do
|
|
137
137
|
|
|
138
138
|
No handler for the other kinds (`incident.*`, `log.*`, `chat.session`, `wiki.page`, `job.*`, `approval.detail`, `report`),
|
|
139
|
-
no log/incident detector
|
|
140
|
-
(F3-11b), no `job.assign` runtime (F3-08). Those frames are acked `rejected protocol_unsupported`. `sha256` of
|
|
139
|
+
no `job.assign` runtime (F3-08) (the log/incident detector is F3-11b, see below). Those frames are acked `rejected protocol_unsupported`. `sha256` of
|
|
141
140
|
`content.response` is not sent (the contract does not say what it covers).
|
|
142
141
|
|
|
142
|
+
## Incident detector (F3-11b, `LOCAL_AGENT_INCIDENT_DETECTOR`, default false)
|
|
143
|
+
|
|
144
|
+
Reads the customer's logs (files, stdin, Loki; `LOCAL_AGENT_INCIDENT_LOG_FILES`, `..._STDIN`, `..._LOKI_URL` + `..._LOKI_QUERY`), detects errors with
|
|
145
|
+
the same rules as the Java monitor (errorHash, fingerprint, significance, noise; parity vectors generated from the Java code) and sends `incident.candidate`
|
|
146
|
+
(9 metadata fields, no text) per errorHash window. Announces the capability `incident.candidate`. The detail stays in a local JSONL store (0600, DLP-masked).
|
|
147
|
+
Needs `LOCAL_AGENT_PROTOCOL_V2=true`. Details, limits and hooks: `src/incident-detector/README.md`. The `incident.detail` content kind is NOT served yet.
|
|
148
|
+
|
|
143
149
|
## Tests
|
|
144
150
|
|
|
145
|
-
`npm test` (node:test)
|
|
151
|
+
`npm test` (node:test). The incident detector (F3-11b) is in `test/incident-detector/` (`parity.test.js` against `parity.vectors.json`, `units`, `sources`, `wiring`); the rest of v2 in `test/la-protocol-v2/`:
|
|
146
152
|
`contract.test.js` (all 176 frame vectors of F3-22, valid and invalid, the 54 `file.list` body vectors, plus the drift check), `traces.test.js` (the 26
|
|
147
153
|
traces), `session-inbound.test.js`, `session-outbound.test.js`, `link.test.js`, `tunnel-e2e.test.js` (the real
|
|
148
154
|
`src/server.js` against a reference CP; flag off = v1 frames), and for F3-07 `content-path-guard.test.js`,
|
package/env.example
CHANGED
|
@@ -85,4 +85,11 @@ LOG_LEVEL=info
|
|
|
85
85
|
# LOCAL_AGENT_CONTENT_ALLOW_HARDLINKS=true serve files with more than one hard link (refused by default)
|
|
86
86
|
# LOCAL_AGENT_CONTENT_INDEX_MAX_IDS=2048 live ids announced to the gateway in content.index (max 2048)
|
|
87
87
|
# LOCAL_AGENT_CONTENT_MAX_DIR_ENTRIES=20000 file.list: a folder with more entries is payload_too_large (100..200000)
|
|
88
|
+
# LOCAL_AGENT_INCIDENT_DETECTOR=true detects errors in the customer's logs and sends incident.candidate (metadata only; the text stays in a local store).
|
|
89
|
+
# Needs LOCAL_AGENT_PROTOCOL_V2=true and at least one source:
|
|
90
|
+
# LOCAL_AGENT_INCIDENT_LOG_FILES=/var/log/app/api.log,/var/log/app/worker.log (tailed; new lines only)
|
|
91
|
+
# LOCAL_AGENT_INCIDENT_LOKI_URL=http://loki:3100 + LOCAL_AGENT_INCIDENT_LOKI_QUERY='{app="api"}' (+ _LOKI_TOKEN, _LOKI_ORG)
|
|
92
|
+
# LOCAL_AGENT_INCIDENT_STDIN=true read log lines from the agent's own stdin
|
|
93
|
+
# LOCAL_AGENT_INCIDENT_STORE_DIR=~/.deepdebug/incidents LOCAL_AGENT_INCIDENT_WINDOW_MS=300000 LOCAL_AGENT_INCIDENT_FLUSH_MS=30000
|
|
94
|
+
# LOCAL_AGENT_INCIDENT_MAX_EMITS_PER_MINUTE=60 LOCAL_AGENT_INCIDENT_MAX_PENDING=500 LOCAL_AGENT_INCIDENT_MAX_TRACKED=5000
|
|
88
95
|
# LOCAL_AGENT_CONTENT_DENY_GLOBS=a/,*.x extra deny globs (they add to the built-in list: .env*, keys, credentials files, secrets/ ...)
|
package/package.json
CHANGED
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
import io.visionhammer.agents.monitor.detector.ErrorDetector;
|
|
2
|
+
import io.visionhammer.agents.monitor.model.LogEntry;
|
|
3
|
+
import io.visionhammer.agents.shared.logs.LogNoiseClassifier;
|
|
4
|
+
import io.visionhammer.agents.shared.service.IncidentFingerprintGenerator;
|
|
5
|
+
import java.lang.reflect.Method;
|
|
6
|
+
import java.time.Instant;
|
|
7
|
+
import java.util.*;
|
|
8
|
+
|
|
9
|
+
/** Runs the REAL Java (see extract.py) over a corpus and prints the parity vectors as JSON (ASCII only: every non-ASCII char is a \\uXXXX escape). */
|
|
10
|
+
public class Gen {
|
|
11
|
+
record C(String name, String service, String severity, String message, String stack) {}
|
|
12
|
+
|
|
13
|
+
static final List<C> CASES = new ArrayList<>();
|
|
14
|
+
static void c(String name, String service, String sev, String message, String stack) { CASES.add(new C(name, service, sev, message, stack)); }
|
|
15
|
+
|
|
16
|
+
static final String JAVA_STACK = "java.lang.IllegalStateException: order 4711 not found\n"
|
|
17
|
+
+ "\tat org.springframework.web.method.support.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:205)\n"
|
|
18
|
+
+ "\tat io.visionhammer.billing.OrderService.load(OrderService.java:88)\n"
|
|
19
|
+
+ "\tat io.visionhammer.billing.OrderController.get(OrderController.java:41)\n";
|
|
20
|
+
|
|
21
|
+
static {
|
|
22
|
+
c("java-npe", "billing-api", "ERROR", "java.lang.NullPointerException: Cannot invoke \"String.length()\" because \"s\" is null", "java.lang.NullPointerException: Cannot invoke \"String.length()\" because \"s\" is null\n\tat com.acme.Foo.bar(Foo.java:12)\n\tat com.acme.Main.main(Main.java:5)");
|
|
23
|
+
c("java-ids-4711", "billing-api", "ERROR", "java.lang.IllegalStateException: order 4711 not found", JAVA_STACK);
|
|
24
|
+
c("java-ids-9002", "billing-api", "ERROR", "java.lang.IllegalStateException: order 9002 not found", JAVA_STACK.replace("4711", "9002"));
|
|
25
|
+
c("java-ids-99999-long-int", "billing-api", "ERROR", "java.lang.IllegalStateException: order 123456 not found", JAVA_STACK);
|
|
26
|
+
c("framework-frames-only", "svc", "ERROR", "boom", "org.springframework.X: boom\n\tat org.springframework.a.B.c(B.java:1)\n\tat reactor.core.Y.z(Y.java:2)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)");
|
|
27
|
+
c("frame-line-numbers", "svc", "ERROR", "failed", "com.x.Err: failed\n\tat com.x.A.m(A.java:123)\n");
|
|
28
|
+
c("frame-kotlin-swift-py-js-ts-rb-go", "svc", "ERROR", "failed", "Err: failed\n\tat app/Foo.kt:77\n");
|
|
29
|
+
c("frame-swift", "ios", "ERROR", "crash", "Fatal error\n\tat Foo.swift:99 in bar\n");
|
|
30
|
+
c("frame-hex-addr", "svc", "ERROR", "failed", "Err: x\n\tat com.x.A.m(Native Method @0xdeadbeef)\n");
|
|
31
|
+
c("frame-over-200", "svc", "ERROR", "failed", "Err: x\n\tat " + "a.".repeat(150) + "B.m(B.java:1)\n");
|
|
32
|
+
c("caused-by-first", "svc", "ERROR", "wrapper failure", "Caused by: java.io.IOException: disk full\n\tat com.x.Disk.write(Disk.java:9)\n");
|
|
33
|
+
c("no-colon-type", "svc", "ERROR", "oops", "NullPointerException\n\tat com.x.A.m(A.java:1)\n");
|
|
34
|
+
c("type-colon-then-space", "svc", "ERROR", "oops", "a.b.Exc: text with spaces\n\tat com.x.A.m(A.java:1)\n");
|
|
35
|
+
c("type-space-then-colon", "svc", "ERROR", "oops", "Exception in thread main: x\n\tat com.x.A.m(A.java:1)\n");
|
|
36
|
+
c("crlf-stack", "svc", "ERROR", "oops", "a.b.Exc: text\r\n\tat com.x.A.m(A.java:1)\r\n\tat com.x.B.n(B.java:2)\r\n");
|
|
37
|
+
c("empty-stack", "svc", "ERROR", "com.acme.Thing: bad input 42", "");
|
|
38
|
+
c("null-stack", "svc", "ERROR", "com.acme.Thing: bad input 42", null);
|
|
39
|
+
c("blank-stack", "svc", "ERROR", "com.acme.Thing: bad input 42", " \n ");
|
|
40
|
+
c("msg-dotted-type", "svc", "ERROR", "io.netty.channel.ClosedChannelException: closed", null);
|
|
41
|
+
c("msg-undotted-type", "svc", "ERROR", "ClosedChannelException: closed", null);
|
|
42
|
+
c("msg-colon-after-space", "svc", "ERROR", "Error: something broke", null);
|
|
43
|
+
c("pg-error", "db", "ERROR", "2026-10-29 10:00:01.123 UTC [4242]: [3-1] db=app,user=u ERROR: duplicate key value violates unique constraint \"pk_orders\"", null);
|
|
44
|
+
c("pg-fatal", "db", "FATAL", "2026-10-29 10:00:01 UTC [4242]: [3-1] FATAL: terminating connection due to administrator command", null);
|
|
45
|
+
c("pg-warning", "db", "ERROR", "2026-10-29 10:00:01 UTC [77]: [1-1] WARNING: there is no transaction in progress", null);
|
|
46
|
+
c("pg-preamble-lowercase", "db", "ERROR", "2026-10-29 10:00:01 utc [77]: [1-1] error: relation \"x\" does not exist", null);
|
|
47
|
+
c("pg-preamble-no-keyword", "db", "ERROR", "2026-10-29 10:00:01 UTC [77]: [1-1] LOG: connection received", null);
|
|
48
|
+
c("iso-ts-variants", "svc", "ERROR", "failed at 2026-10-29T10:00:01.123456Z and 2026-10-29 10:00:01+01:00 and 2026-10-29T10:00:01 CEST done", null);
|
|
49
|
+
c("iso-ts-offset-nocolon", "svc", "ERROR", "failed at 2026-10-29T10:00:01-0300 end", null);
|
|
50
|
+
c("uuids-and-addrs", "svc", "ERROR", "obj 3f2b8c1e-5a4d-4e7b-9c21-0a1b2c3d4e5f at 0xDEADBEEF and @7f3a9c21 pid [12345] [7]", null);
|
|
51
|
+
c("uuid-uppercase", "svc", "ERROR", "obj 3F2B8C1E-5A4D-4E7B-9C21-0A1B2C3D4E5F failed", null);
|
|
52
|
+
c("bracket-pid", "svc", "ERROR", "worker [12] [123] [12345] crashed", null);
|
|
53
|
+
c("long-ints", "svc", "ERROR", "user 12345 order 1234 amount 99999999 x12345y id-12345 12345-id", null);
|
|
54
|
+
c("urls", "gw", "ERROR", "HTTP 500 POST https://svc-abc123-uc.a.run.app/api/v1/x?id=42 failed; retry http://localhost:8080/health", null);
|
|
55
|
+
c("urls-two-hostnames", "gw", "ERROR", "HTTP 500 POST https://svc-1234567890-uc.a.run.app/api/v1/x", null);
|
|
56
|
+
c("urls-two-hostnames-b", "gw", "ERROR", "HTTP 500 POST https://svc-1234567890.europe-west1.run.app/api/v1/x", null);
|
|
57
|
+
c("long-hex", "svc", "ERROR", "commit deadbeefcafe1234 and short beef12 and abcdef0123456789abcdef", null);
|
|
58
|
+
c("versions", "build", "ERROR", "maven-compiler-plugin:3.13.0 vs :3.1 and 1.2.3.4 and 10.0", null);
|
|
59
|
+
c("addr-0x", "svc", "ERROR", "segfault at 0x7ffee1234abc code 0X1F", null);
|
|
60
|
+
c("digits", "svc", "ERROR", "line 12 col 7 offset 0 size 100", null);
|
|
61
|
+
c("first-line-only", "svc", "ERROR", "first line 1\nsecond line 2\nthird 3", null);
|
|
62
|
+
c("first-line-crlf", "svc", "ERROR", "first line 1\r\nsecond line 2", null);
|
|
63
|
+
c("leading-newline", "svc", "ERROR", "\nreal text 5", null);
|
|
64
|
+
c("empty-message", "svc", "ERROR", "", null);
|
|
65
|
+
c("null-message", "svc", "ERROR", null, null);
|
|
66
|
+
c("only-ws-message", "svc", "ERROR", " \t ", null);
|
|
67
|
+
c("nbsp-message", "svc", "ERROR", " ", null);
|
|
68
|
+
c("nbsp-inside", "svc", "ERROR", "Error: connection refused to host", null);
|
|
69
|
+
c("unicode-space-inside", "svc", "ERROR", "Error: em space ideographic narrow
linesep done", null);
|
|
70
|
+
c("form-feed-vt", "svc", "ERROR", "Error:\u000bvt\u000cff\u001cfs done", null);
|
|
71
|
+
c("trim-control-chars", "svc", "ERROR", "\u0001\u0002 Error: ctl \u0003\u001f", null);
|
|
72
|
+
c("kelvin-and-dotted-i", "svc", "ERROR", "ERROR K İSTANBUL ÉCOLE Straße ΣΣ café 12", null);
|
|
73
|
+
c("greek-final-sigma", "svc", "ERROR", "ΟΔΥΣΣΕΥΣ failed 7", null);
|
|
74
|
+
c("non-ascii-adjacent-hex", "svc", "ERROR", "édeadbeefcafe12é and xé 123", null);
|
|
75
|
+
c("non-ascii-word-boundary", "svc", "ERROR", "café123456789abcdef and ü0123456789ab", null);
|
|
76
|
+
c("cjk-emoji", "svc", "ERROR", "错误: 连接失败 💥 code 500", null);
|
|
77
|
+
c("lone-surrogate", "svc", "ERROR", "bad \ud800 surrogate and \udc00 more 5", null);
|
|
78
|
+
c("surrogate-split-at-500", "svc", "ERROR", "x".repeat(499) + "💥 tail", null);
|
|
79
|
+
c("len-499", "svc", "ERROR", "y".repeat(499), null);
|
|
80
|
+
c("len-500", "svc", "ERROR", "y".repeat(500), null);
|
|
81
|
+
c("len-501", "svc", "ERROR", "y".repeat(501), null);
|
|
82
|
+
c("len-2000-with-numbers", "svc", "ERROR", ("error 12 abc ").repeat(160), null);
|
|
83
|
+
c("legacy-1000-boundary", "svc", "ERROR", "z".repeat(1005), null);
|
|
84
|
+
c("legacy-colon-numbers", "svc", "ERROR", "failed at Foo.java:123:45 and :99", null);
|
|
85
|
+
c("legacy-ts", "svc", "ERROR", "boom at 2026-10-29T10:00:01.123Z+01:00 done", null);
|
|
86
|
+
c("severity-fatal-no-pattern", "svc", "FATAL", "the process has quit", null);
|
|
87
|
+
c("severity-critical-no-pattern", "svc", "critical", "the process has quit", null);
|
|
88
|
+
c("severity-warn", "svc", "WARN", "java.lang.RuntimeException: x", "at a.B.c(B.java:1)");
|
|
89
|
+
c("severity-info", "svc", "INFO", "all good", null);
|
|
90
|
+
c("severity-error-plain", "svc", "ERROR", "plain failure without markers", null);
|
|
91
|
+
c("severity-null", "svc", null, "java.lang.RuntimeException: x", null);
|
|
92
|
+
c("severity-severe", "svc", "SEVERE", "java.lang.RuntimeException: x", null);
|
|
93
|
+
c("severity-emerg", "svc", "emerg", "something happened", null);
|
|
94
|
+
c("noise-healthcheck", "svc", "ERROR", "GET /actuator/health Exception", null);
|
|
95
|
+
c("noise-broken-pipe", "svc", "ERROR", "IOException: Broken pipe", null);
|
|
96
|
+
c("noise-ping", "svc", "ERROR", "GET /ping Exception", null);
|
|
97
|
+
c("noise-ping-nospace", "svc", "ERROR", "GET /ping Exception", null);
|
|
98
|
+
c("noise-favicon", "svc", "ERROR", "Error: favicon.ico missing", null);
|
|
99
|
+
c("noise-conn-reset", "svc", "ERROR", "Error: Connection reset by peer", null);
|
|
100
|
+
c("sig-java-frame-in-message", "svc", "ERROR", "failure\n\tat com.x.A.m(A.java:12)", null);
|
|
101
|
+
c("sig-java-frame-in-stack", "svc", "ERROR", "failure header", "x\n\tat com.x.A.m(A.java:12)");
|
|
102
|
+
c("sig-swift", "ios", "ERROR", "failure Foo.swift:12", null);
|
|
103
|
+
c("sig-node", "node", "ERROR", "failure (/app/dist/x.js:10:5)", null);
|
|
104
|
+
c("sig-python", "py", "ERROR", "Traceback\n File \"/app/main.py\", line 3, in x", null);
|
|
105
|
+
c("sig-deno", "edge", "ERROR", "failure at file:///src/fn/index.ts:12:3", null);
|
|
106
|
+
c("sig-generic-exception", "svc", "ERROR", "some Exception happened", null);
|
|
107
|
+
c("sig-generic-error-colon", "svc", "ERROR", "TypeError: x is not a function", null);
|
|
108
|
+
c("sig-http-500", "svc", "ERROR", "request done HTTP 503 upstream", null);
|
|
109
|
+
c("sig-http-404-lower", "svc", "ERROR", "http 404 not found", null);
|
|
110
|
+
c("sig-http-200", "svc", "ERROR", "HTTP 200 ok", null);
|
|
111
|
+
c("noise-infra-code", "run", "ERROR", "{code=9, message=Ready condition status changed to False}", null);
|
|
112
|
+
c("noise-infra-oom", "run", "ERROR", "Memory limit of 1024 MiB exceeded with 1100 MiB used", null);
|
|
113
|
+
c("noise-infra-signal", "run", "ERROR", "Uncaught signal: 13, pid=136, tid=136", null);
|
|
114
|
+
c("noise-infra-probe", "run", "ERROR", "Default STARTUP TCP probe failed 1 time consecutively for container x", null);
|
|
115
|
+
c("noise-not-probe", "svc", "ERROR", "Writer probe failed", null);
|
|
116
|
+
c("noise-infra-audit", "run", "ERROR", "AUDIT EVENT: method=/Services.ReplaceService resource=x", null);
|
|
117
|
+
c("noise-infra-auditlog-map", "run", "ERROR", "{@type=type.googleapis.com/google.cloud.audit.AuditLog, x=1}", null);
|
|
118
|
+
c("noise-infra-mongo", "svc", "ERROR", "com.mongodb.MongoSocketWriteException: Exception sending message", null);
|
|
119
|
+
c("noise-infra-sqlconn", "svc", "ERROR", "The request failed because either the HTTP response was malformed or", null);
|
|
120
|
+
c("noise-infra-leading-ws", "run", "ERROR", " \t{code=9, message=x}", null);
|
|
121
|
+
c("noise-infra-leading-nbsp", "run", "ERROR", " {code=9, message=x}", null);
|
|
122
|
+
c("noise-infra-leading-emsp", "run", "ERROR", " {code=9, message=x}", null);
|
|
123
|
+
c("noise-build-maven", "ci", "ERROR", "Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:3.13.0:compile", null);
|
|
124
|
+
c("noise-build-lifecycle", "ci", "ERROR", "org.apache.maven.lifecycle.LifecycleExecutionException: x", null);
|
|
125
|
+
c("noise-build-gradle", "ci", "ERROR", "Execution failed for task ':app:compileJava'.", null);
|
|
126
|
+
c("noise-build-gradle2", "ci", "ERROR", "FAILURE: Build failed with an exception.", null);
|
|
127
|
+
c("noise-blank", "svc", "ERROR", " ", null);
|
|
128
|
+
c("python-trace", "py", "ERROR", "ValueError: invalid literal for int() with base 10: 'abc'", "Traceback (most recent call last):\n File \"/app/main.py\", line 12, in <module>\n int('abc')\nValueError: invalid literal for int() with base 10: 'abc'");
|
|
129
|
+
c("node-trace", "node", "ERROR", "TypeError: Cannot read properties of undefined (reading 'id')", "TypeError: Cannot read properties of undefined (reading 'id')\n at handler (/app/src/h.js:10:15)\n at Layer.handle [as handle_request] (/app/node_modules/express/lib/router/layer.js:95:5)");
|
|
130
|
+
c("tab-indented-at", "svc", "ERROR", "x", "a.B: x\n at com.x.Y.z(Y.java:3)\n");
|
|
131
|
+
c("jdk-internal-skipped", "svc", "ERROR", "x", "a.B: x\n\tat jdk.internal.reflect.X.y(X.java:1)\n\tat com.x.Real.z(Real.java:3)\n");
|
|
132
|
+
c("proxy-skipped", "svc", "ERROR", "x", "a.B: x\n\tat com.sun.$Proxy12.run(Unknown Source)\n\tat com.x.Real.z(Real.java:3)\n");
|
|
133
|
+
c("apache-netty-r2dbc-skipped", "svc", "ERROR", "x", "a.B: x\n\tat org.apache.A.b(A.java:1)\n\tat io.netty.C.d(C.java:2)\n\tat io.r2dbc.E.f(E.java:3)\n\tat com.fasterxml.G.h(G.java:4)\n\tat javax.I.j(I.java:5)\n\tat jakarta.K.l(K.java:6)\n\tat com.x.Real.z(Real.java:3)\n");
|
|
134
|
+
c("fallback-first-at-when-all-skipped", "svc", "ERROR", "x", "a.B: x\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\n\tat org.springframework.Z.y(Z.java:9)\n");
|
|
135
|
+
c("frame-with-contains-reflect", "svc", "ERROR", "x", "a.B: x\n\tat com.x.java.lang.reflect.Foo.m(Foo.java:1)\n\tat com.x.Real.z(Real.java:3)\n");
|
|
136
|
+
c("long-frame-300", "svc", "ERROR", "x", "a.B: x\n\tat com.x." + "q".repeat(300) + ".m(Foo.java:1)\n");
|
|
137
|
+
c("service-differs-same-message", "svc-a", "ERROR", "java.lang.IllegalStateException: order 1 not found", null);
|
|
138
|
+
c("service-differs-same-message-b", "svc-b", "ERROR", "java.lang.IllegalStateException: order 1 not found", null);
|
|
139
|
+
c("linesep-only", "svc", "ERROR", "\u2028", null);
|
|
140
|
+
c("emsp-only", "svc", "ERROR", "\u2003\u3000\u1680\u205f", null);
|
|
141
|
+
c("narrow-nbsp-only", "svc", "ERROR", "\u202f", null);
|
|
142
|
+
c("figure-space-only", "svc", "ERROR", "\u2007", null);
|
|
143
|
+
c("fs-gs-rs-us-only", "svc", "ERROR", "\u001c\u001d\u001e\u001f", null);
|
|
144
|
+
c("stack-linesep-blank", "svc", "ERROR", "com.x.E: boom 1", "\u2028\u2003");
|
|
145
|
+
c("stack-nbsp-not-blank", "svc", "ERROR", "com.x.E: boom 1", "\u00a0Foo: bar\n\tat a.B.c(B.java:1)");
|
|
146
|
+
c("noise-leading-linesep", "run", "ERROR", "\u2028Memory limit of 1 MiB exceeded", null);
|
|
147
|
+
c("service-null", null, "ERROR", "java.lang.IllegalStateException: boom", null);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
static LogEntry cp(LogEntry x) { return new LogEntry(x.getService(), x.getSeverity(), x.getMessage(), x.getStacktrace(), x.getTimestamp()); }
|
|
151
|
+
static Method NORM_MSG, NORM_FRAME;
|
|
152
|
+
static String esc(String s) {
|
|
153
|
+
if (s == null) return "null";
|
|
154
|
+
StringBuilder b = new StringBuilder("\"");
|
|
155
|
+
for (int i = 0; i < s.length(); i++) {
|
|
156
|
+
char ch = s.charAt(i);
|
|
157
|
+
switch (ch) {
|
|
158
|
+
case '"' -> b.append("\\\"");
|
|
159
|
+
case '\\' -> b.append("\\\\");
|
|
160
|
+
case '\n' -> b.append("\\n");
|
|
161
|
+
case '\r' -> b.append("\\r");
|
|
162
|
+
case '\t' -> b.append("\\t");
|
|
163
|
+
default -> { if (ch < 0x20 || ch > 0x7e) b.append(String.format("\\u%04x", (int) ch)); else b.append(ch); }
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return b.append('"').toString();
|
|
167
|
+
}
|
|
168
|
+
static String nullable(Object o) { return o == null ? "null" : esc(o.toString()); }
|
|
169
|
+
|
|
170
|
+
public static void main(String[] a) throws Exception {
|
|
171
|
+
NORM_MSG = IncidentFingerprintGenerator.class.getDeclaredMethod("normaliseMessage", String.class); NORM_MSG.setAccessible(true);
|
|
172
|
+
NORM_FRAME = IncidentFingerprintGenerator.class.getDeclaredMethod("normaliseFrame", String.class); NORM_FRAME.setAccessible(true);
|
|
173
|
+
IncidentFingerprintGenerator fp = new IncidentFingerprintGenerator();
|
|
174
|
+
ErrorDetector det = new ErrorDetector();
|
|
175
|
+
StringBuilder o = new StringBuilder("{\n\"schema\": \"incident-parity-vectors/1\",\n\"cases\": [\n");
|
|
176
|
+
boolean first = true;
|
|
177
|
+
for (C k : CASES) {
|
|
178
|
+
LogEntry e = new LogEntry(k.service, k.severity, k.message, k.stack, null);
|
|
179
|
+
String msg = k.message;
|
|
180
|
+
String firstLine = msg == null ? "" : (msg.indexOf('\n') >= 0 ? msg.substring(0, msg.indexOf('\n')) : msg);
|
|
181
|
+
String errorType = fp.extractErrorType(msg, k.stack);
|
|
182
|
+
String frame = fp.extractFirstUserFrame(k.stack);
|
|
183
|
+
String fingerprint = fp.compute(errorType, frame, msg);
|
|
184
|
+
String legacy;
|
|
185
|
+
try { legacy = det.hash(e); } catch (Exception ex) { legacy = null; }
|
|
186
|
+
if (!first) o.append(",\n");
|
|
187
|
+
first = false;
|
|
188
|
+
o.append("{\"name\": ").append(esc(k.name))
|
|
189
|
+
.append(", \"service\": ").append(nullable(k.service))
|
|
190
|
+
.append(", \"severity\": ").append(nullable(k.severity))
|
|
191
|
+
.append(", \"message\": ").append(nullable(k.message))
|
|
192
|
+
.append(", \"stack\": ").append(nullable(k.stack))
|
|
193
|
+
.append(", \"expect\": {")
|
|
194
|
+
.append("\"significant\": ").append(det.isSignificant(e))
|
|
195
|
+
.append(", \"groupKey\": ").append(nullable(msg == null ? GroupHash.normalizeGroupKey("") : GroupHash.normalizeGroupKey(firstLine)))
|
|
196
|
+
.append(", \"groupHash\": ").append(nullable(GroupHash.computeGroupHash(e)))
|
|
197
|
+
.append(", \"errorType\": ").append(nullable(errorType))
|
|
198
|
+
.append(", \"firstFrame\": ").append(nullable(frame))
|
|
199
|
+
.append(", \"normalisedMessage\": ").append(nullable(NORM_MSG.invoke(fp, msg)))
|
|
200
|
+
.append(", \"normalisedFrame\": ").append(nullable(NORM_FRAME.invoke(fp, frame)))
|
|
201
|
+
.append(", \"fingerprint\": ").append(nullable(fingerprint))
|
|
202
|
+
.append(", \"noiseClass\": ").append(nullable(LogNoiseClassifier.classify(msg)))
|
|
203
|
+
.append(", \"legacyHash\": ").append(nullable(legacy))
|
|
204
|
+
.append("}}");
|
|
205
|
+
}
|
|
206
|
+
o.append("\n],\n\"dedup\": [\n");
|
|
207
|
+
// natural-key dedup (LogEntryDedup): same service|micros|md5(message) = same entry (an overlap re-read)
|
|
208
|
+
Instant t0 = Instant.parse("2026-10-29T10:00:00.123456789Z");
|
|
209
|
+
record D(String name, List<LogEntry> entries, List<String> known) {}
|
|
210
|
+
List<D> ds = new ArrayList<>();
|
|
211
|
+
LogEntry e1 = new LogEntry("api", "ERROR", "boom 1", null, t0);
|
|
212
|
+
LogEntry e1b = new LogEntry("api", "ERROR", "boom 1", null, t0);
|
|
213
|
+
LogEntry e2 = new LogEntry("api", "ERROR", "boom 1", null, t0.plusMillis(1));
|
|
214
|
+
LogEntry e3 = new LogEntry("api2", "ERROR", "boom 1", null, t0);
|
|
215
|
+
LogEntry e4 = new LogEntry("api", "ERROR", "boom 2", null, t0);
|
|
216
|
+
LogEntry e5 = new LogEntry("api", "ERROR", "no timestamp", null, null);
|
|
217
|
+
LogEntry e6 = new LogEntry(null, "ERROR", "null service", null, t0);
|
|
218
|
+
LogEntry e7 = new LogEntry("api", "ERROR", null, null, t0);
|
|
219
|
+
LogEntry e8 = new LogEntry("api", "ERROR", "x".repeat(9000), null, t0);
|
|
220
|
+
LogEntry e9 = new LogEntry("api", "ERROR", "x".repeat(8000) + "different tail", null, t0);
|
|
221
|
+
LogEntry e10 = new LogEntry("api", "ERROR", "café 💥 错误", null, Instant.parse("2026-10-29T10:00:00.999999Z"));
|
|
222
|
+
LogEntry e11 = new LogEntry("api", "ERROR", "boom 1", null, Instant.parse("1969-12-31T23:59:59.999999500Z"));
|
|
223
|
+
ds.add(new D("duplicates-in-batch", List.of(e1, e1b, e2, e3, e4), List.of()));
|
|
224
|
+
ds.add(new D("known-keys-skip", List.of(e1, e2, e3), List.of(LogEntryDedupPure.key(e1))));
|
|
225
|
+
ds.add(new D("no-timestamp-always-kept", List.of(e5, cp(e5), cp(e5)), List.of()));
|
|
226
|
+
ds.add(new D("null-service-and-message", List.of(e6, e7, cp(e6)), List.of()));
|
|
227
|
+
ds.add(new D("truncation-8000", List.of(e8, e9), List.of()));
|
|
228
|
+
ds.add(new D("unicode-and-micros", List.of(e10, cp(e10)), List.of()));
|
|
229
|
+
ds.add(new D("pre-epoch-micros", List.of(e11, cp(e11)), List.of()));
|
|
230
|
+
boolean f2 = true;
|
|
231
|
+
for (D d : ds) {
|
|
232
|
+
if (!f2) o.append(",\n");
|
|
233
|
+
f2 = false;
|
|
234
|
+
o.append("{\"name\": ").append(esc(d.name)).append(", \"entries\": [");
|
|
235
|
+
for (int i = 0; i < d.entries.size(); i++) {
|
|
236
|
+
LogEntry x = d.entries.get(i);
|
|
237
|
+
o.append(i > 0 ? ", " : "").append("{\"service\": ").append(nullable(x.getService())).append(", \"ts\": ").append(nullable(x.getTimestamp())).append(", \"message\": ").append(nullable(x.getMessage())).append("}");
|
|
238
|
+
}
|
|
239
|
+
o.append("], \"known\": [");
|
|
240
|
+
for (int i = 0; i < d.known.size(); i++) o.append(i > 0 ? ", " : "").append(esc(d.known.get(i)));
|
|
241
|
+
o.append("], \"keys\": [");
|
|
242
|
+
for (int i = 0; i < d.entries.size(); i++) o.append(i > 0 ? ", " : "").append(nullable(LogEntryDedupPure.key(d.entries.get(i))));
|
|
243
|
+
o.append("], \"kept\": [");
|
|
244
|
+
List<LogEntry> kept = LogEntryDedupPure.filterNew(d.entries, new HashSet<>(d.known));
|
|
245
|
+
boolean fk = true;
|
|
246
|
+
for (LogEntry x : kept) {
|
|
247
|
+
int idx = -1;
|
|
248
|
+
for (int i = 0; i < d.entries.size(); i++) if (d.entries.get(i) == x) { idx = i; break; }
|
|
249
|
+
o.append(fk ? "" : ", ").append(idx); fk = false;
|
|
250
|
+
}
|
|
251
|
+
o.append("]}");
|
|
252
|
+
}
|
|
253
|
+
o.append("\n]\n}\n");
|
|
254
|
+
System.out.print(o);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
package io.visionhammer.agents.monitor.model;
|
|
2
|
+
|
|
3
|
+
import java.time.Instant;
|
|
4
|
+
|
|
5
|
+
/** Test stub: only the getters the code under test reads (the real one is a Lombok builder). */
|
|
6
|
+
public class LogEntry {
|
|
7
|
+
private final String service, severity, message, stacktrace;
|
|
8
|
+
private final Instant timestamp;
|
|
9
|
+
public LogEntry(String service, String severity, String message, String stacktrace, Instant timestamp) {
|
|
10
|
+
this.service = service; this.severity = severity; this.message = message; this.stacktrace = stacktrace; this.timestamp = timestamp;
|
|
11
|
+
}
|
|
12
|
+
public String getService() { return service; }
|
|
13
|
+
public String getSeverity() { return severity; }
|
|
14
|
+
public String getMessage() { return message; }
|
|
15
|
+
public String getStacktrace() { return stacktrace; }
|
|
16
|
+
public Instant getTimestamp() { return timestamp; }
|
|
17
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Cuts the Java under test out of visionhammer-agents into <work>/ (annotations stripped, behaviour verbatim)."""
|
|
2
|
+
import re, sys
|
|
3
|
+
agents, work = sys.argv[1:3]
|
|
4
|
+
def read(p):
|
|
5
|
+
return open(f"{agents}/{p}", encoding="utf-8").read()
|
|
6
|
+
def method(src, header_regex):
|
|
7
|
+
"""Source of the method whose signature matches, balanced on braces."""
|
|
8
|
+
m = re.search(header_regex, src)
|
|
9
|
+
assert m, header_regex
|
|
10
|
+
i = src.index("{", m.end() - 1 if src[m.end() - 1] == "{" else m.end())
|
|
11
|
+
depth, j = 0, i
|
|
12
|
+
while True:
|
|
13
|
+
c = src[j]
|
|
14
|
+
if c == "{": depth += 1
|
|
15
|
+
elif c == "}":
|
|
16
|
+
depth -= 1
|
|
17
|
+
if depth == 0: break
|
|
18
|
+
j += 1
|
|
19
|
+
return src[m.start():j + 1]
|
|
20
|
+
def strip(src):
|
|
21
|
+
out = []
|
|
22
|
+
for line in src.splitlines():
|
|
23
|
+
s = line.strip()
|
|
24
|
+
if s in ("@Component", "@Slf4j", "@RequiredArgsConstructor") or s.startswith(("import lombok", "import org.springframework", "import reactor")):
|
|
25
|
+
continue
|
|
26
|
+
out.append(line)
|
|
27
|
+
return "\n".join(out) + "\n"
|
|
28
|
+
# 1. IncidentFingerprintGenerator and LogNoiseClassifier: whole files
|
|
29
|
+
open(f"{work}/IncidentFingerprintGenerator.java", "w").write(strip(read("shared/src/main/java/io/visionhammer/agents/shared/service/IncidentFingerprintGenerator.java")))
|
|
30
|
+
open(f"{work}/LogNoiseClassifier.java", "w").write(read("shared/src/main/java/io/visionhammer/agents/shared/logs/LogNoiseClassifier.java"))
|
|
31
|
+
# 2. ErrorDetector: whole file (needs the LogEntry stub, same package of the stub)
|
|
32
|
+
ed = strip(read("monitor-service/src/main/java/io/visionhammer/agents/monitor/detector/ErrorDetector.java"))
|
|
33
|
+
open(f"{work}/ErrorDetector.java", "w").write(ed)
|
|
34
|
+
# 3. the two pure statics of MonitorPollScheduler
|
|
35
|
+
sch = read("monitor-service/src/main/java/io/visionhammer/agents/monitor/scheduler/MonitorPollScheduler.java")
|
|
36
|
+
g1 = method(sch, r"static String computeGroupHash\(LogEntry entry\) \{")
|
|
37
|
+
g2 = method(sch, r"static String normalizeGroupKey\(String firstLine\) \{")
|
|
38
|
+
open(f"{work}/GroupHash.java", "w").write("import io.visionhammer.agents.monitor.model.LogEntry;\npublic final class GroupHash {\n" + g1 + "\n" + g2 + "\n}\n")
|
|
39
|
+
# 4. the pure statics of LogEntryDedup
|
|
40
|
+
dd = read("monitor-service/src/main/java/io/visionhammer/agents/monitor/service/LogEntryDedup.java")
|
|
41
|
+
parts = [
|
|
42
|
+
r"public static final int MAX_MESSAGE_CHARS = 8000;",
|
|
43
|
+
]
|
|
44
|
+
body = [" public static final int MAX_MESSAGE_CHARS = 8000;"]
|
|
45
|
+
for hdr in [r"public static String key\(LogEntry e\) \{", r"public static String storedMessage\(String message\) \{",
|
|
46
|
+
r"static String keyOf\(String source, Instant ts, String storedMessage\) \{", r"private static long micros\(Instant ts\) \{",
|
|
47
|
+
r"static String md5Hex\(String s\) \{", r"public static List<LogEntry> filterNew\(List<LogEntry> entries, Set<String> known\) \{"]:
|
|
48
|
+
body.append(method(dd, hdr))
|
|
49
|
+
open(f"{work}/LogEntryDedupPure.java", "w").write(
|
|
50
|
+
"import java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.time.Instant;\nimport java.time.temporal.ChronoUnit;\n"
|
|
51
|
+
"import java.util.*;\nimport io.visionhammer.agents.monitor.model.LogEntry;\npublic final class LogEntryDedupPure {\n" + "\n".join(body) + "\n}\n")
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# F3-11b: regenerates test/incident-detector/parity.vectors.json FROM THE JAVA CODE (the source of truth), so the Node
|
|
3
|
+
# detector can be proven byte-identical. Usage: scripts/incident-parity/gen.sh <visionhammer-agents checkout> [out.json]
|
|
4
|
+
# Needs JDK 21 (javac/java on PATH, or JAVA_HOME). The Java sources are used VERBATIM: only the Lombok/Spring annotations and
|
|
5
|
+
# the r2dbc imports are stripped (they are not behaviour), and the pure static methods are cut out of their classes.
|
|
6
|
+
set -euo pipefail
|
|
7
|
+
AGENTS="${1:?path to a visionhammer-agents checkout (release/qa)}"
|
|
8
|
+
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
9
|
+
OUT="${2:-$HERE/../../test/incident-detector/parity.vectors.json}"
|
|
10
|
+
WORK="$(mktemp -d)"
|
|
11
|
+
trap 'rm -rf "$WORK"' EXIT
|
|
12
|
+
JAVAC="${JAVA_HOME:+$JAVA_HOME/bin/}javac"; JAVA="${JAVA_HOME:+$JAVA_HOME/bin/}java"
|
|
13
|
+
python3 -I "$HERE/extract.py" "$AGENTS" "$WORK"
|
|
14
|
+
cp "$HERE"/Gen.java "$HERE"/LogEntry.java "$WORK/"
|
|
15
|
+
( cd "$WORK" && "$JAVAC" -d out *.java && "$JAVA" -cp out Gen > "$WORK/vectors.json" )
|
|
16
|
+
python3 -I - "$AGENTS" "$WORK/vectors.json" "$OUT" <<'PY'
|
|
17
|
+
import hashlib, json, subprocess, sys
|
|
18
|
+
agents, src, out = sys.argv[1:4]
|
|
19
|
+
commit = subprocess.run(["git", "-C", agents, "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip()
|
|
20
|
+
files = ["shared/src/main/java/io/visionhammer/agents/shared/service/IncidentFingerprintGenerator.java",
|
|
21
|
+
"shared/src/main/java/io/visionhammer/agents/shared/logs/LogNoiseClassifier.java",
|
|
22
|
+
"monitor-service/src/main/java/io/visionhammer/agents/monitor/detector/ErrorDetector.java",
|
|
23
|
+
"monitor-service/src/main/java/io/visionhammer/agents/monitor/scheduler/MonitorPollScheduler.java",
|
|
24
|
+
"monitor-service/src/main/java/io/visionhammer/agents/monitor/service/LogEntryDedup.java"]
|
|
25
|
+
sha = {f: hashlib.sha256(open(f"{agents}/{f}", "rb").read()).hexdigest() for f in files}
|
|
26
|
+
d = json.load(open(src))
|
|
27
|
+
d["source"] = {"repo": "VISIONHAMMER/visionhammer-agents", "commit": commit, "files": sha,
|
|
28
|
+
"note": "generated by scripts/incident-parity/gen.sh from the Java code; do not edit by hand"}
|
|
29
|
+
json.dump(d, open(out, "w"), indent=1, ensure_ascii=True)
|
|
30
|
+
open(out, "a").write("\n")
|
|
31
|
+
print("wrote", out, len(d["cases"]), "cases", len(d["dedup"]), "dedup cases")
|
|
32
|
+
PY
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Incident detector (F3-11b)
|
|
2
|
+
|
|
3
|
+
The Local Agent reads the customer's logs, detects errors with the **same rules as the Java monitor** and sends `incident.candidate`
|
|
4
|
+
(metadata only) on the protocol v2 session. The text of the error (message, stack trace) stays on the customer's machine.
|
|
5
|
+
|
|
6
|
+
Flag: `LOCAL_AGENT_INCIDENT_DETECTOR=true` (default **false**; needs `LOCAL_AGENT_PROTOCOL_V2=true`, the module is only started from
|
|
7
|
+
`la-protocol-v2/runtime.js`). With the flag off only `config.js` is loaded: no source is opened, nothing is read or sent.
|
|
8
|
+
|
|
9
|
+
## Flow
|
|
10
|
+
|
|
11
|
+
`source -> parse -> (natural-key dedup) -> isSignificant -> noise filter -> errorHash + fingerprint -> local store + aggregator -> emitter -> session`
|
|
12
|
+
|
|
13
|
+
| file | what |
|
|
14
|
+
|---|---|
|
|
15
|
+
| `java-compat.js`, `normalize.js` | the Java rules ported with the `java.lang.String` quirks (`\s`, `trim`, `isBlank`, lone surrogates). Proven by `test/incident-detector/parity.vectors.json` |
|
|
16
|
+
| `parse.js` | JSON lines and plain text (multi-line stack traces) -> entries |
|
|
17
|
+
| `sources/file-source.js` | tail of files (new lines only, rotation/truncation, bounded reads) |
|
|
18
|
+
| `sources/loki-source.js` | polls a Loki (`query_range`), cursor, byte and time limits |
|
|
19
|
+
| `sources/stream-source.js` | any Readable (the agent's stdin); a slow consumer pauses the stream |
|
|
20
|
+
| `aggregator.js` | per-`errorHash` windows: leading edge at once, batched updates, final update at window close, LRU bound |
|
|
21
|
+
| `store.js` | local JSONL detail store (0600, rotates, DLP-masked). `getDetail(errorHash)` is the hook for `incident.detail` |
|
|
22
|
+
| `emitter.js` | the frame: 9 contract fields, DLP on labels, token bucket, one frame in flight, backoff, coalescing |
|
|
23
|
+
| `detector.js`, `index.js` | glue |
|
|
24
|
+
|
|
25
|
+
## What goes out, what stays
|
|
26
|
+
|
|
27
|
+
Out: `errorHash` (32 hex = sha256 prefix of the normalised FIRST line, Java `computeGroupHash`), `fingerprint` (64 hex, Java
|
|
28
|
+
`IncidentFingerprintGenerator`), `severity`, `service`, `environment`, `count`, `firstSeen`, `lastSeen` (the window), `logCategory`.
|
|
29
|
+
The contract has no field for the stack size or for text: the size is in the local store (`stackBytes`).
|
|
30
|
+
Stays: message, stack, everything else, in `LOCAL_AGENT_INCIDENT_STORE_DIR` (default `~/.deepdebug/incidents`).
|
|
31
|
+
|
|
32
|
+
## Window semantics (for the control plane, F3-11c)
|
|
33
|
+
|
|
34
|
+
One window per `errorHash`, `LOCAL_AGENT_INCIDENT_WINDOW_MS` (5 min) from its first occurrence. Every candidate of a window has the same
|
|
35
|
+
`firstSeen`; `count` only grows. `(errorHash, firstSeen)` identifies the window: upsert on it. The first occurrence is sent at once
|
|
36
|
+
(count 1); later ones in batches at most every `LOCAL_AGENT_INCIDENT_FLUSH_MS` (30 s), plus a final update when the window closes.
|
|
37
|
+
|
|
38
|
+
## Hooks that are NOT built
|
|
39
|
+
|
|
40
|
+
- **Alloy / OTLP receiver**: the LA has no OTLP receiver today. A source is an object `{name, start(onEntries), stop()}` where
|
|
41
|
+
`onEntries(entries)` takes `{severity, message, stacktrace, service, tsMs}`; an OTLP/Alloy source plugs in the same way (`buildSources` in `index.js`).
|
|
42
|
+
- **Cloud providers (CloudWatch/GCP/Azure)**: F3-11d.
|
|
43
|
+
- **`content.request` kind `incident.detail`**: the contract/gateway define the kind but not the `ref` scheme; `link.incidents.getDetail(errorHash, {limit})` is ready
|
|
44
|
+
to be called by a handler (the DLP of the content handlers then applies on the way out).
|
|
45
|
+
|
|
46
|
+
## Parity vectors
|
|
47
|
+
|
|
48
|
+
`scripts/incident-parity/gen.sh <visionhammer-agents checkout>` runs the real Java (ErrorDetector, MonitorPollScheduler.computeGroupHash,
|
|
49
|
+
IncidentFingerprintGenerator, LogNoiseClassifier, LogEntryDedup) over a corpus and rewrites `test/incident-detector/parity.vectors.json`
|
|
50
|
+
(with the commit and sha256 of the Java files). Regenerate when the Java rules change; `npm test` then says what diverged.
|
|
51
|
+
`ErrorDetector.hash` (legacy `service|message` hash) is in the vectors but not ported: Java only falls back to it when `computeGroupHash` throws.
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* aggregator.js - dedup and windows of detected errors (F3-11b).
|
|
3
|
+
*
|
|
4
|
+
* Java dedups on (tenant, errorHash) in incident_dedup_log: one row per hash with occurrences/first_seen/last_seen, and the
|
|
5
|
+
* cooldown lives in the control plane. Here the same key (errorHash) is aggregated in WINDOWS so what is sent is bounded:
|
|
6
|
+
*
|
|
7
|
+
* window from the first occurrence of an errorHash, `windowMs` long (processing time). Every candidate of that window has
|
|
8
|
+
* the same firstSeen, so (errorHash, firstSeen) identifies the window and `count` only grows: the control plane can
|
|
9
|
+
* upsert on it.
|
|
10
|
+
* emission leading edge: the FIRST occurrence of a window is emitted at once (count 1). Later occurrences are batched: one
|
|
11
|
+
* update per `flushMs` while the count changes, and a final one when the window closes with something not yet sent.
|
|
12
|
+
* bounds at most `maxTracked` hashes at once. Beyond it the least recently seen one is closed (final update if it has one
|
|
13
|
+
* pending) and counted as evicted. A hash that comes back after its window closed opens a new window.
|
|
14
|
+
* severity the highest seen in the window. fingerprint/service/logCategory: those of the first occurrence.
|
|
15
|
+
* Pure: no timers, no I/O. The detector calls tick() on an interval.
|
|
16
|
+
*/
|
|
17
|
+
import { maxSeverity } from "./categorize.js";
|
|
18
|
+
|
|
19
|
+
export function createAggregator({ windowMs, flushMs, maxTracked, now = Date.now, onCandidate }) {
|
|
20
|
+
const states = new Map(); // errorHash -> state (insertion order = least recently seen first)
|
|
21
|
+
const stats = { observed: 0, windows: 0, emitted: 0, evicted: 0 };
|
|
22
|
+
|
|
23
|
+
const snapshot = (s) => ({
|
|
24
|
+
errorHash: s.errorHash, fingerprint: s.fingerprint, severity: s.severity, service: s.service, environment: s.environment,
|
|
25
|
+
count: s.count, firstSeen: s.firstSeen, lastSeen: s.lastSeen, logCategory: s.logCategory, windowStart: s.windowStart,
|
|
26
|
+
});
|
|
27
|
+
const send = (s, t) => { s.sentCount = s.count; s.lastEmitAt = t; stats.emitted++; onCandidate(snapshot(s)); };
|
|
28
|
+
const close = (s, t) => { states.delete(s.errorHash); if (s.count > s.sentCount) send(s, t); };
|
|
29
|
+
|
|
30
|
+
return {
|
|
31
|
+
stats,
|
|
32
|
+
get tracked() { return states.size; },
|
|
33
|
+
/** @returns {{opened: boolean, windowStart: number}} opened = this occurrence opened a new window. */
|
|
34
|
+
observe({ errorHash, fingerprint, severity, service, environment, logCategory, tsMs }) {
|
|
35
|
+
const t = now();
|
|
36
|
+
stats.observed++;
|
|
37
|
+
const seenAt = Number.isFinite(tsMs) && tsMs >= 0 ? Math.min(tsMs, t) : t; // never in the future, never before the epoch (the log line is untrusted)
|
|
38
|
+
let s = states.get(errorHash);
|
|
39
|
+
if (s && t - s.windowStart >= windowMs) { close(s, t); s = undefined; }
|
|
40
|
+
if (!s) {
|
|
41
|
+
while (states.size >= maxTracked) {
|
|
42
|
+
const oldest = states.values().next().value;
|
|
43
|
+
stats.evicted++;
|
|
44
|
+
close(oldest, t);
|
|
45
|
+
}
|
|
46
|
+
s = { errorHash, fingerprint, severity, service, environment, logCategory, count: 0, sentCount: 0, firstSeen: seenAt, lastSeen: seenAt, windowStart: t, lastEmitAt: 0 };
|
|
47
|
+
stats.windows++;
|
|
48
|
+
s.count = 1;
|
|
49
|
+
states.set(errorHash, s);
|
|
50
|
+
send(s, t);
|
|
51
|
+
return { opened: true, windowStart: s.windowStart };
|
|
52
|
+
}
|
|
53
|
+
states.delete(errorHash); // move to the most recent position
|
|
54
|
+
states.set(errorHash, s);
|
|
55
|
+
s.count++;
|
|
56
|
+
s.severity = maxSeverity(s.severity, severity);
|
|
57
|
+
// firstSeen never changes inside a window (it is part of the window identity), even for an out-of-order line
|
|
58
|
+
if (seenAt > s.lastSeen) s.lastSeen = seenAt;
|
|
59
|
+
if (t - s.lastEmitAt >= flushMs) send(s, t);
|
|
60
|
+
return { opened: false, windowStart: s.windowStart };
|
|
61
|
+
},
|
|
62
|
+
/** Sends the updates that are due and closes the windows that ended. */
|
|
63
|
+
tick() {
|
|
64
|
+
const t = now();
|
|
65
|
+
for (const s of [...states.values()]) {
|
|
66
|
+
if (t - s.windowStart >= windowMs) close(s, t);
|
|
67
|
+
else if (s.count > s.sentCount && t - s.lastEmitAt >= flushMs) send(s, t);
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
/** Sends everything not sent yet and forgets the windows (shutdown). */
|
|
71
|
+
drain() { const t = now(); for (const s of [...states.values()]) close(s, t); },
|
|
72
|
+
};
|
|
73
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* categorize.js - the small mappings between a log entry and the metadata of `incident.candidate` (F3-11b).
|
|
3
|
+
* Nothing here returns text from the entry: only enum values and sanitised labels.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** Gateway SeverityNormalizer for the levels ErrorDetector lets through (SEVERE is not in the gateway table; it counts as an error). */
|
|
7
|
+
export function severityOf(level) {
|
|
8
|
+
switch (String(level ?? "").trim().toUpperCase()) {
|
|
9
|
+
case "CRITICAL": case "ALERT": case "EMERG": case "EMERGENCY": case "FATAL": return "critical";
|
|
10
|
+
case "ERROR": case "ERR": case "SEVERE": return "high";
|
|
11
|
+
default: return "medium";
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
const RANK = { info: 0, low: 1, medium: 2, high: 3, critical: 4 };
|
|
15
|
+
export const maxSeverity = (a, b) => (RANK[b] > RANK[a] ? b : a);
|
|
16
|
+
|
|
17
|
+
/** AutoIncidentCreator.inferEnvironment: the platform convention for the service name suffix. */
|
|
18
|
+
export function inferEnvironment(service) {
|
|
19
|
+
if (service == null || String(service).trim() === "") return "production";
|
|
20
|
+
const s = String(service).toLowerCase();
|
|
21
|
+
if (s.endsWith("-qa") || s.includes("-qa-")) return "qa";
|
|
22
|
+
if (s.endsWith("-demo") || s.includes("-demo-")) return "demo";
|
|
23
|
+
if (s.endsWith("-staging") || s.includes("-staging-") || s.endsWith("-stg")) return "staging";
|
|
24
|
+
return "production";
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Schema: ^[A-Za-z0-9._:/-]{1,128}$ */
|
|
28
|
+
export function sanitizeService(raw) {
|
|
29
|
+
const s = String(raw ?? "").replace(/[^A-Za-z0-9._:/-]/g, "_").slice(0, 128);
|
|
30
|
+
return s === "" ? "unknown" : s;
|
|
31
|
+
}
|
|
32
|
+
/** Schema: ^[a-z0-9][a-z0-9_-]{0,31}$ */
|
|
33
|
+
export function sanitizeEnvironment(raw) {
|
|
34
|
+
const s = String(raw ?? "").toLowerCase().replace(/[^a-z0-9_-]/g, "-").replace(/^[^a-z0-9]+/, "").slice(0, 32);
|
|
35
|
+
return s === "" ? "production" : s;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** logCategory enum of the contract. The order is the priority: an OOM that mentions a timeout is an oom. */
|
|
39
|
+
export function logCategoryOf(message, stacktrace, errorType) {
|
|
40
|
+
const t = `${message ?? ""}\n${stacktrace ?? ""}`;
|
|
41
|
+
if (/OutOfMemoryError|out of memory|Memory limit of|\bOOM\b/i.test(t)) return "oom";
|
|
42
|
+
if (/TimeoutException|timed? ?out|deadline exceeded/i.test(t)) return "timeout";
|
|
43
|
+
if (/SQLException|PSQLException|SQLSTATE|duplicate key|deadlock|MongoException|R2dbc|JDBC|relation "[^"]*" does not exist/i.test(t) || /^PG_/.test(errorType ?? "")) return "db";
|
|
44
|
+
if (/AccessDenied|Unauthorized|Forbidden|SecurityException|authentication failed|invalid token|\bCSRF\b/i.test(t)) return "security";
|
|
45
|
+
if (/\bHTTP\s+5\d\d\b|\bstatus[= :]+5\d\d\b/i.test(t)) return "http_5xx";
|
|
46
|
+
if (errorType || /Exception|Traceback|Error:/.test(t)) return "exception";
|
|
47
|
+
return "other";
|
|
48
|
+
}
|