mergen-server 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +57 -0
- package/README.md +96 -0
- package/dist/__stubs__/calibration.js +471 -0
- package/dist/__stubs__/causal.js +601 -0
- package/dist/__stubs__/closed-source.js +150 -0
- package/dist/__stubs__/license.js +35 -0
- package/dist/__stubs__/plans.js +197 -0
- package/dist/app.js +467 -0
- package/dist/cli.js +250 -0
- package/dist/commands/agent-identity.js +79 -0
- package/dist/commands/gate.js +107 -0
- package/dist/commands/github.js +632 -0
- package/dist/commands/incident.js +644 -0
- package/dist/commands/policy.js +710 -0
- package/dist/commands/scan.js +243 -0
- package/dist/commands/setup.js +1218 -0
- package/dist/commands/shared.js +109 -0
- package/dist/commands/team.js +809 -0
- package/dist/datadog/blame-attribution.js +228 -0
- package/dist/datadog/client.js +159 -0
- package/dist/datadog/compactor.js +169 -0
- package/dist/datadog/fingerprinter.js +34 -0
- package/dist/datadog/incident-state.js +21 -0
- package/dist/datadog/line-matcher.js +56 -0
- package/dist/datadog/memory-store.js +399 -0
- package/dist/datadog/otel-trace.js +67 -0
- package/dist/index.js +505 -0
- package/dist/intelligence/action-risk.js +132 -0
- package/dist/intelligence/activity-feed.js +40 -0
- package/dist/intelligence/agent-identity.js +115 -0
- package/dist/intelligence/agent-pipeline.js +329 -0
- package/dist/intelligence/agent-profiles.js +81 -0
- package/dist/intelligence/ai-commit.js +16 -0
- package/dist/intelligence/approval-analytics.js +42 -0
- package/dist/intelligence/approval-events.js +5 -0
- package/dist/intelligence/arch-boundaries.js +94 -0
- package/dist/intelligence/arch-graph.js +117 -0
- package/dist/intelligence/autonomy.js +397 -0
- package/dist/intelligence/azure-broker.js +57 -0
- package/dist/intelligence/azure-proxy-sessions.js +49 -0
- package/dist/intelligence/baseline.js +1 -0
- package/dist/intelligence/behavior-baseline.js +134 -0
- package/dist/intelligence/billing.js +1 -0
- package/dist/intelligence/blast-radius.js +224 -0
- package/dist/intelligence/bypass.js +157 -0
- package/dist/intelligence/calibration-classifier.js +77 -0
- package/dist/intelligence/calibration-git-sync.js +74 -0
- package/dist/intelligence/calibration.js +1 -0
- package/dist/intelligence/cascade-detector.js +58 -0
- package/dist/intelligence/case-study-generator.js +126 -0
- package/dist/intelligence/causal-graph.js +126 -0
- package/dist/intelligence/causal.js +1 -0
- package/dist/intelligence/change-risk.js +102 -0
- package/dist/intelligence/compliance-report.js +134 -0
- package/dist/intelligence/confidence-report.js +85 -0
- package/dist/intelligence/corpus-to-policy.js +140 -0
- package/dist/intelligence/credential-broker.js +112 -0
- package/dist/intelligence/credential-misuse.js +39 -0
- package/dist/intelligence/debug-sessions.js +108 -0
- package/dist/intelligence/decision-escalation.js +36 -0
- package/dist/intelligence/decision-memory.js +47 -0
- package/dist/intelligence/decision-similarity.js +85 -0
- package/dist/intelligence/default-safety-tests.js +53 -0
- package/dist/intelligence/degradation-watcher.js +108 -0
- package/dist/intelligence/detector-plugins.js +70 -0
- package/dist/intelligence/detectors.js +1 -0
- package/dist/intelligence/device-auth.js +138 -0
- package/dist/intelligence/diff-size.js +42 -0
- package/dist/intelligence/ebpf-verify.js +45 -0
- package/dist/intelligence/enterprise-policy-engine.js +1122 -0
- package/dist/intelligence/error-fingerprint.js +1 -0
- package/dist/intelligence/execution-gate.js +100 -0
- package/dist/intelligence/execution-mode.js +22 -0
- package/dist/intelligence/gate-analytics.js +256 -0
- package/dist/intelligence/gate-decision.js +64 -0
- package/dist/intelligence/gcp-broker.js +50 -0
- package/dist/intelligence/git-adr-sync.js +207 -0
- package/dist/intelligence/hitl-hold.js +460 -0
- package/dist/intelligence/hypothesis-history.js +93 -0
- package/dist/intelligence/idp-identity.js +83 -0
- package/dist/intelligence/impl-critic.js +216 -0
- package/dist/intelligence/incident-autopilot.js +622 -0
- package/dist/intelligence/incident-replay.js +162 -0
- package/dist/intelligence/incident-result-cache.js +33 -0
- package/dist/intelligence/incident-similarity.js +58 -0
- package/dist/intelligence/infra-detectors.js +312 -0
- package/dist/intelligence/license.js +185 -0
- package/dist/intelligence/llm-spokesperson.js +62 -0
- package/dist/intelligence/mcp-prompts.js +320 -0
- package/dist/intelligence/mcp-resources.js +171 -0
- package/dist/intelligence/normalize.js +108 -0
- package/dist/intelligence/notifications.js +83 -0
- package/dist/intelligence/oidc-issuer.js +87 -0
- package/dist/intelligence/override-corpus.js +597 -0
- package/dist/intelligence/planning-gate.js +84 -0
- package/dist/intelligence/plans.js +223 -0
- package/dist/intelligence/platt-scaling.js +164 -0
- package/dist/intelligence/policy-proposals.js +84 -0
- package/dist/intelligence/policy-suggester.js +161 -0
- package/dist/intelligence/policy-sync.js +103 -0
- package/dist/intelligence/policy-test-runner.js +35 -0
- package/dist/intelligence/postmortem-parser.js +181 -0
- package/dist/intelligence/postmortem-retrieval.js +224 -0
- package/dist/intelligence/postmortem-store.js +596 -0
- package/dist/intelligence/pr-commenter.js +81 -0
- package/dist/intelligence/pr-policy-gate.js +38 -0
- package/dist/intelligence/pr-shadow-analyzer.js +229 -0
- package/dist/intelligence/process-tree.js +108 -0
- package/dist/intelligence/prompts.js +1 -0
- package/dist/intelligence/replay.js +38 -0
- package/dist/intelligence/repro-steps.js +1 -0
- package/dist/intelligence/resource-extractors.js +0 -0
- package/dist/intelligence/resource-file-context.js +138 -0
- package/dist/intelligence/risk-score.js +22 -0
- package/dist/intelligence/rollback.js +137 -0
- package/dist/intelligence/runbook-updater.js +143 -0
- package/dist/intelligence/sandbox.js +194 -0
- package/dist/intelligence/script-trust.js +104 -0
- package/dist/intelligence/session-metrics.js +67 -0
- package/dist/intelligence/session-threat-tracker.js +231 -0
- package/dist/intelligence/shadow-digest-cron.js +81 -0
- package/dist/intelligence/shadow-log.js +284 -0
- package/dist/intelligence/shell-ast.js +145 -0
- package/dist/intelligence/siem-forward.js +104 -0
- package/dist/intelligence/slack-digest.js +151 -0
- package/dist/intelligence/slack-override-loop.js +217 -0
- package/dist/intelligence/slack-routing.js +67 -0
- package/dist/intelligence/slack.js +900 -0
- package/dist/intelligence/sql-ast.js +31 -0
- package/dist/intelligence/team.js +1 -0
- package/dist/intelligence/telemetry.js +76 -0
- package/dist/intelligence/threshold-optimizer.js +75 -0
- package/dist/intelligence/token-budget.js +66 -0
- package/dist/intelligence/tool-guard.js +708 -0
- package/dist/intelligence/tool-manifest 2.js +74 -0
- package/dist/intelligence/tool-manifest.js +125 -0
- package/dist/intelligence/tools-analysis.js +857 -0
- package/dist/intelligence/tools-arch.js +152 -0
- package/dist/intelligence/tools-autonomy.js +495 -0
- package/dist/intelligence/tools-blast-radius.js +140 -0
- package/dist/intelligence/tools-browser.js +431 -0
- package/dist/intelligence/tools-change-timeline.js +154 -0
- package/dist/intelligence/tools-credentials.js +117 -0
- package/dist/intelligence/tools-datadog.js +233 -0
- package/dist/intelligence/tools-debug-sessions.js +209 -0
- package/dist/intelligence/tools-discovery.js +284 -0
- package/dist/intelligence/tools-infra.js +612 -0
- package/dist/intelligence/tools-intent.js +128 -0
- package/dist/intelligence/tools-memory.js +514 -0
- package/dist/intelligence/tools-runbook.js +951 -0
- package/dist/intelligence/tools-sessions.js +107 -0
- package/dist/intelligence/tools-state.js +232 -0
- package/dist/intelligence/tools-utility.js +430 -0
- package/dist/intelligence/tools-validate.js +215 -0
- package/dist/intelligence/tools.js +1 -0
- package/dist/intelligence/trace-context.js +11 -0
- package/dist/intelligence/unclassified-clusters.js +87 -0
- package/dist/intelligence/usage.js +195 -0
- package/dist/routes/active-authors.js +63 -0
- package/dist/routes/activity-feed.js +21 -0
- package/dist/routes/adr.js +117 -0
- package/dist/routes/agent-activity.js +184 -0
- package/dist/routes/agent-blunders.js +163 -0
- package/dist/routes/agents.js +175 -0
- package/dist/routes/api-keys.js +65 -0
- package/dist/routes/arch.js +75 -0
- package/dist/routes/audit-export.js +93 -0
- package/dist/routes/billing-dashboard.js +158 -0
- package/dist/routes/billing-outcome.js +77 -0
- package/dist/routes/calibration.js +305 -0
- package/dist/routes/ci-gate-history.js +20 -0
- package/dist/routes/ci-gate.js +232 -0
- package/dist/routes/ci.js +293 -0
- package/dist/routes/compliance-report.js +24 -0
- package/dist/routes/confidence.js +30 -0
- package/dist/routes/credentials.js +119 -0
- package/dist/routes/dashboard.js +1407 -0
- package/dist/routes/decision-memory.js +42 -0
- package/dist/routes/demo.js +844 -0
- package/dist/routes/device-auth-stub.js +173 -0
- package/dist/routes/explain-why.js +39 -0
- package/dist/routes/gate-analytics.js +213 -0
- package/dist/routes/gate.js +28 -0
- package/dist/routes/github-webhook.js +364 -0
- package/dist/routes/habituation.js +30 -0
- package/dist/routes/health-integrations.js +243 -0
- package/dist/routes/heartbeats.js +45 -0
- package/dist/routes/hitl.js +364 -0
- package/dist/routes/impact-report.js +645 -0
- package/dist/routes/incident-webhook.js +63 -0
- package/dist/routes/incidents.js +366 -0
- package/dist/routes/layers.js +62 -0
- package/dist/routes/license.js +187 -0
- package/dist/routes/oidc-issuer.js +30 -0
- package/dist/routes/onboarding.js +170 -0
- package/dist/routes/otel.js +67 -0
- package/dist/routes/otlp-receiver.js +179 -0
- package/dist/routes/overrides.js +241 -0
- package/dist/routes/pagerduty.js +258 -0
- package/dist/routes/policies.js +809 -0
- package/dist/routes/policy-nl.js +169 -0
- package/dist/routes/postmortem.js +102 -0
- package/dist/routes/pr-shadow.js +47 -0
- package/dist/routes/rbac.js +61 -0
- package/dist/routes/replay.js +17 -0
- package/dist/routes/risk-report.js +164 -0
- package/dist/routes/runbooks.js +125 -0
- package/dist/routes/safety-test.js +174 -0
- package/dist/routes/sdk.js +222 -0
- package/dist/routes/sensor.js +819 -0
- package/dist/routes/sentry.js +140 -0
- package/dist/routes/sessions.js +43 -0
- package/dist/routes/setup-ui.js +594 -0
- package/dist/routes/shadow-report.js +107 -0
- package/dist/routes/slack-routing.js +171 -0
- package/dist/routes/team-usage.js +71 -0
- package/dist/routes/telemetry.js +28 -0
- package/dist/routes/tenants.js +199 -0
- package/dist/routes/tickets.js +167 -0
- package/dist/routes/validate.js +13 -0
- package/dist/routes/war-room.js +113 -0
- package/dist/scripts/check-arch.js +22 -0
- package/dist/seeds/community-corpus.js +176 -0
- package/dist/seeds/corpus.js +1224 -0
- package/dist/sensor/action-ledger.js +327 -0
- package/dist/sensor/adr-store.js +140 -0
- package/dist/sensor/agent-blunder-store.js +370 -0
- package/dist/sensor/agent-context-store.js +225 -0
- package/dist/sensor/agent-memory-store.js +231 -0
- package/dist/sensor/audit-fetch.js +23 -0
- package/dist/sensor/audit-log.js +273 -0
- package/dist/sensor/buffer-schemas.js +225 -0
- package/dist/sensor/buffer.js +599 -0
- package/dist/sensor/bypass-tracker.js +93 -0
- package/dist/sensor/ci-gate-history.js +70 -0
- package/dist/sensor/cloud-auth.js +183 -0
- package/dist/sensor/commit-context-store.js +235 -0
- package/dist/sensor/docker-log-stream.js +164 -0
- package/dist/sensor/docker-monitor.js +122 -0
- package/dist/sensor/extended-buffer.js +46 -0
- package/dist/sensor/feedback-token.js +41 -0
- package/dist/sensor/file-lock.js +6 -0
- package/dist/sensor/fs-watcher.js +106 -0
- package/dist/sensor/gate-heartbeat.js +122 -0
- package/dist/sensor/git-suspect.js +136 -0
- package/dist/sensor/habituation-store.js +77 -0
- package/dist/sensor/heartbeat-monitor.js +133 -0
- package/dist/sensor/incident-store.js +340 -0
- package/dist/sensor/infra-normalizer.js +513 -0
- package/dist/sensor/ingest.js +339 -0
- package/dist/sensor/jest-reporter.js +52 -0
- package/dist/sensor/k8s-events.js +132 -0
- package/dist/sensor/layer2-store.js +111 -0
- package/dist/sensor/layer3-store.js +230 -0
- package/dist/sensor/layer4-store.js +147 -0
- package/dist/sensor/logger.js +13 -0
- package/dist/sensor/otel-exporter.js +161 -0
- package/dist/sensor/paths.js +71 -0
- package/dist/sensor/policy-history.js +147 -0
- package/dist/sensor/pr-shadow-store.js +83 -0
- package/dist/sensor/process-watcher.js +162 -0
- package/dist/sensor/rbac.js +92 -0
- package/dist/sensor/redact.js +114 -0
- package/dist/sensor/redis-store.js +191 -0
- package/dist/sensor/route-reachability.js +64 -0
- package/dist/sensor/security-utils.js +18 -0
- package/dist/sensor/service-graph.js +170 -0
- package/dist/sensor/service-topology.js +227 -0
- package/dist/sensor/session-history.js +74 -0
- package/dist/sensor/session-persist.js +38 -0
- package/dist/sensor/shadow-promote.js +122 -0
- package/dist/sensor/sourcemap.js +281 -0
- package/dist/sensor/sqlite-store.js +205 -0
- package/dist/sensor/sso.js +164 -0
- package/dist/sensor/vitest-reporter.js +65 -0
- package/dist/sensor/watcher.js +48 -0
- package/dist/storage/interfaces.js +0 -0
- package/dist/storage/pg/pg-action-ledger.js +138 -0
- package/dist/storage/pg/pg-approval-store.js +107 -0
- package/dist/storage/pg/pg-blunder-store.js +193 -0
- package/dist/storage/pg/pg-ci-gate-history.js +83 -0
- package/dist/storage/pg/pg-client.js +24 -0
- package/dist/storage/pg/pg-event-store.js +63 -0
- package/dist/storage/pg/pg-incident-store.js +190 -0
- package/dist/storage/pg/pg-migrations.js +26 -0
- package/dist/storage/pg/pg-override-corpus.js +447 -0
- package/dist/storage/pg/pg-shadow-log.js +129 -0
- package/dist/storage/sqlite/sqlite-action-ledger.js +18 -0
- package/dist/storage/sqlite/sqlite-approval-store.js +37 -0
- package/dist/storage/sqlite/sqlite-blunder-store.js +27 -0
- package/dist/storage/sqlite/sqlite-ci-gate-history.js +19 -0
- package/dist/storage/sqlite/sqlite-event-store.js +29 -0
- package/dist/storage/sqlite/sqlite-incident-store.js +34 -0
- package/dist/storage/sqlite/sqlite-override-corpus.js +75 -0
- package/dist/storage/sqlite/sqlite-shadow-log.js +36 -0
- package/dist/storage/store-factory.js +55 -0
- package/dist/storage/store-registry.js +31 -0
- package/dist/update-checker.js +107 -0
- package/dist/workers/autopilot-worker.js +28 -0
- package/dist/workers/notification-worker.js +28 -0
- package/dist/workers/queues.js +58 -0
- package/dist/workers/validation-worker.js +24 -0
- package/dist/workers/worker-registry.js +18 -0
- package/package.json +123 -0
- package/sdk/mergen-inject.js +260 -0
- package/sdk/node.js +313 -0
- package/sdk/vite-plugin.ts +57 -0
- package/sdk/webpack-plugin.js +106 -0
|
@@ -0,0 +1,1224 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { DATA_DIR } from "../sensor/paths.js";
|
|
4
|
+
import logger from "../sensor/logger.js";
|
|
5
|
+
const REPLAY_DIR = path.join(DATA_DIR, "replay-snapshots");
|
|
6
|
+
const T0 = 17040672e5;
|
|
7
|
+
const GAP = 3 * 60 * 60 * 1e3;
|
|
8
|
+
const t = (i) => T0 + i * GAP;
|
|
9
|
+
const cap = (i) => t(i) + 35e3;
|
|
10
|
+
function ie(kind, service, severity, message, attrs = {}) {
|
|
11
|
+
return { kind, timestamp: 0, service, severity, message, attributes: attrs, source: "otlp" };
|
|
12
|
+
}
|
|
13
|
+
function ce(level, msg, url, ts) {
|
|
14
|
+
return { type: "console", level, args: [msg], url, timestamp: ts };
|
|
15
|
+
}
|
|
16
|
+
function ne(method, url, status, duration, ts, err) {
|
|
17
|
+
return {
|
|
18
|
+
type: "network",
|
|
19
|
+
method,
|
|
20
|
+
url,
|
|
21
|
+
status,
|
|
22
|
+
statusText: status >= 500 ? "Internal Server Error" : status === 429 ? "Too Many Requests" : status === 503 ? "Service Unavailable" : "OK",
|
|
23
|
+
duration,
|
|
24
|
+
timestamp: ts,
|
|
25
|
+
...err ? { error: err } : {}
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
const SNAPSHOTS = [
|
|
29
|
+
// ── 001–005 infra_db_connection_pool ─────────────────────────────────────
|
|
30
|
+
{
|
|
31
|
+
pid: "seed-001",
|
|
32
|
+
capturedAt: cap(0),
|
|
33
|
+
firedAt: t(0),
|
|
34
|
+
logs: [
|
|
35
|
+
ce("error", "[api] database connection pool exhausted \u2014 0/20 connections available after 30000ms", "http://api:8080/api/users", t(0) - 28e3),
|
|
36
|
+
ce("error", "[api] Error: connect ETIMEDOUT 10.12.0.5:5432\n at Pool.acquire (/app/node_modules/pg-pool/index.js:192:15)", "http://api:8080/api/users", t(0) - 27800),
|
|
37
|
+
ce("warn", "[api] pg-pool attempting emergency drain and reconnect \u2014 attempt 1/3", "http://api:8080", t(0) - 15e3)
|
|
38
|
+
],
|
|
39
|
+
network: [
|
|
40
|
+
ne("GET", "http://api:8080/api/users", 503, 30421, t(0) - 27500, "upstream connect error or disconnect/reset before headers"),
|
|
41
|
+
ne("GET", "http://api:8080/api/orders", 503, 30389, t(0) - 27200, "upstream connect error"),
|
|
42
|
+
ne("GET", "http://api:8080/api/checkout", 503, 142, t(0) - 2e4, "circuit breaker open")
|
|
43
|
+
],
|
|
44
|
+
contexts: [],
|
|
45
|
+
terminal: [],
|
|
46
|
+
processExits: [],
|
|
47
|
+
ciEvents: [],
|
|
48
|
+
deployments: [],
|
|
49
|
+
infraEvents: [
|
|
50
|
+
{ ...ie("db_connection_pool_exhausted", "api-service", "critical", "pg-pool exhausted \u2014 0/20 connections available after 30s timeout. DB host: 10.12.0.5:5432"), timestamp: t(0) - 28e3, attributes: { endpoint: "postgres://10.12.0.5:5432/app", traceId: "a1b2c3d4e5f60001a1b2c3d4e5f60001", poolMax: 20, poolUsed: 20 } }
|
|
51
|
+
],
|
|
52
|
+
originalTag: "infra_db_connection_pool",
|
|
53
|
+
originalConfidenceScore: 0.8,
|
|
54
|
+
originalFixHint: "Increase pool size: set `DB_POOL_MAX` (or `pool.max` in your ORM config). Check for connection leaks \u2014 unclosed transactions hold pool slots indefinitely. Add a connection pool metrics query to validate utilisation."
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
pid: "seed-002",
|
|
58
|
+
capturedAt: cap(1),
|
|
59
|
+
firedAt: t(1),
|
|
60
|
+
logs: [
|
|
61
|
+
ce("error", "[payments] mysql2: too many connections (max_connections=151 reached)", "http://payments:8082/charge", t(1) - 22e3),
|
|
62
|
+
ce("error", "[payments] SequelizeConnectionAcquireTimeoutError: timeout of 30000ms exceeded while acquiring a connection", "http://payments:8082/charge", t(1) - 21800)
|
|
63
|
+
],
|
|
64
|
+
network: [
|
|
65
|
+
ne("POST", "http://payments:8082/charge", 503, 30012, t(1) - 21600, "connection acquire timeout")
|
|
66
|
+
],
|
|
67
|
+
contexts: [],
|
|
68
|
+
terminal: [],
|
|
69
|
+
processExits: [],
|
|
70
|
+
ciEvents: [],
|
|
71
|
+
deployments: [],
|
|
72
|
+
infraEvents: [
|
|
73
|
+
{ ...ie("db_connection_pool_exhausted", "payments-service", "critical", "MySQL max_connections=151 reached \u2014 Sequelize pool cannot acquire connection"), timestamp: t(1) - 22e3, attributes: { endpoint: "mysql://10.12.0.8:3306/payments", maxConnections: 151 } }
|
|
74
|
+
],
|
|
75
|
+
originalTag: "infra_db_connection_pool",
|
|
76
|
+
originalConfidenceScore: 0.8,
|
|
77
|
+
originalFixHint: "Increase pool size: set `DB_POOL_MAX` (or `pool.max` in your ORM config). Check for connection leaks \u2014 unclosed transactions hold pool slots indefinitely. Add a connection pool metrics query to validate utilisation."
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
pid: "seed-003",
|
|
81
|
+
capturedAt: cap(2),
|
|
82
|
+
firedAt: t(2),
|
|
83
|
+
logs: [
|
|
84
|
+
ce("error", "[cache] ioredis: max retries per request limit exceeded \u2014 Redis connection pool saturated", "http://cache:8083/get", t(2) - 18e3),
|
|
85
|
+
ce("error", "[cache] ReplyError: ERR max number of clients reached", "http://cache:8083/get", t(2) - 17500)
|
|
86
|
+
],
|
|
87
|
+
network: [
|
|
88
|
+
ne("GET", "http://cache:8083/api/session", 503, 18200, t(2) - 17e3, "Redis pool saturated")
|
|
89
|
+
],
|
|
90
|
+
contexts: [],
|
|
91
|
+
terminal: [],
|
|
92
|
+
processExits: [],
|
|
93
|
+
ciEvents: [],
|
|
94
|
+
deployments: [],
|
|
95
|
+
infraEvents: [
|
|
96
|
+
{ ...ie("db_connection_pool_exhausted", "cache-service", "high", "Redis: max number of clients reached \u2014 ioredis connection pool exhausted"), timestamp: t(2) - 18e3, attributes: { endpoint: "redis://10.12.0.10:6379", maxClients: 1e4 } }
|
|
97
|
+
],
|
|
98
|
+
originalTag: "infra_db_connection_pool",
|
|
99
|
+
originalConfidenceScore: 0.8,
|
|
100
|
+
originalFixHint: "Increase pool size: set `DB_POOL_MAX` (or `pool.max` in your ORM config). Check for connection leaks \u2014 unclosed transactions hold pool slots indefinitely. Add a connection pool metrics query to validate utilisation."
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
pid: "seed-004",
|
|
104
|
+
capturedAt: cap(3),
|
|
105
|
+
firedAt: t(3),
|
|
106
|
+
logs: [
|
|
107
|
+
ce("error", "[api] PgBouncer: remaining connection slots are reserved for non-replication superuser connections", "http://api:8080/api/products", t(3) - 25e3),
|
|
108
|
+
ce("error", "[api] FATAL: remaining connection slots are reserved (pgbouncer pool_size=30, server_pool_size=35)", "http://api:8080/api/products", t(3) - 24500),
|
|
109
|
+
ce("warn", "[api] retrying DB connection in 5s (attempt 2/5)", "http://api:8080", t(3) - 2e4)
|
|
110
|
+
],
|
|
111
|
+
network: [
|
|
112
|
+
ne("GET", "http://api:8080/api/products", 503, 25100, t(3) - 24200, "upstream timeout"),
|
|
113
|
+
ne("GET", "http://api:8080/api/cart", 503, 25050, t(3) - 23800, "upstream timeout")
|
|
114
|
+
],
|
|
115
|
+
contexts: [],
|
|
116
|
+
terminal: [],
|
|
117
|
+
processExits: [],
|
|
118
|
+
ciEvents: [],
|
|
119
|
+
deployments: [],
|
|
120
|
+
infraEvents: [
|
|
121
|
+
{ ...ie("db_connection_pool_exhausted", "api-service", "critical", "PgBouncer pool_size=30 exhausted \u2014 server_pool_size=35 slots reserved for superuser"), timestamp: t(3) - 25e3, attributes: { endpoint: "postgres://pgbouncer:6432/app", pgbouncerPoolSize: 30 } }
|
|
122
|
+
],
|
|
123
|
+
originalTag: "infra_db_connection_pool",
|
|
124
|
+
originalConfidenceScore: 0.8,
|
|
125
|
+
originalFixHint: "Increase pool size: set `DB_POOL_MAX` (or `pool.max` in your ORM config). Check for connection leaks \u2014 unclosed transactions hold pool slots indefinitely. Add a connection pool metrics query to validate utilisation."
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
pid: "seed-005",
|
|
129
|
+
capturedAt: cap(4),
|
|
130
|
+
firedAt: t(4),
|
|
131
|
+
logs: [
|
|
132
|
+
ce("error", "[django-api] django.db.utils.OperationalError: FATAL: remaining connection slots are reserved for non-replication superuser connections", "http://django-api:8000/api/v1/users", t(4) - 2e4),
|
|
133
|
+
ce("error", "[django-api] sqlalchemy.exc.TimeoutError: QueuePool limit of size 10 overflow 20 reached, connection timed out, timeout 30", "http://django-api:8000/api/v1/orders", t(4) - 19500)
|
|
134
|
+
],
|
|
135
|
+
network: [
|
|
136
|
+
ne("GET", "http://django-api:8000/api/v1/users", 503, 3e4, t(4) - 19e3, "database connection timeout"),
|
|
137
|
+
ne("POST", "http://django-api:8000/api/v1/orders", 503, 3e4, t(4) - 18500, "database connection timeout")
|
|
138
|
+
],
|
|
139
|
+
contexts: [],
|
|
140
|
+
terminal: [],
|
|
141
|
+
processExits: [],
|
|
142
|
+
ciEvents: [],
|
|
143
|
+
deployments: [],
|
|
144
|
+
infraEvents: [
|
|
145
|
+
{ ...ie("db_connection_pool_exhausted", "django-api", "critical", "SQLAlchemy QueuePool limit size=10 overflow=20 reached \u2014 connection timed out after 30s"), timestamp: t(4) - 2e4, attributes: { endpoint: "postgres://10.12.0.5:5432/django_prod", queuePoolSize: 10, overflow: 20 } }
|
|
146
|
+
],
|
|
147
|
+
originalTag: "infra_db_connection_pool",
|
|
148
|
+
originalConfidenceScore: 0.8,
|
|
149
|
+
originalFixHint: "Increase pool size: set `DB_POOL_MAX` (or `pool.max` in your ORM config). Check for connection leaks \u2014 unclosed transactions hold pool slots indefinitely. Add a connection pool metrics query to validate utilisation."
|
|
150
|
+
},
|
|
151
|
+
// ── 006–010 infra_oom_kill ───────────────────────────────────────────────
|
|
152
|
+
{
|
|
153
|
+
pid: "seed-006",
|
|
154
|
+
capturedAt: cap(5),
|
|
155
|
+
firedAt: t(5),
|
|
156
|
+
logs: [
|
|
157
|
+
ce("error", "[worker] FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory", "http://worker:8084", t(5) - 12e3),
|
|
158
|
+
ce("error", "[k8s] OOMKilled: container worker-service exceeded memory limit 512Mi (exit code 137)", "http://worker:8084", t(5) - 11500)
|
|
159
|
+
],
|
|
160
|
+
network: [],
|
|
161
|
+
contexts: [],
|
|
162
|
+
terminal: [],
|
|
163
|
+
processExits: [
|
|
164
|
+
{ type: "process_exit", process: "worker-service", exitCode: 137, reason: "oom", signal: "SIGKILL", memoryLimitBytes: 536870912, timestamp: t(5) - 11500 }
|
|
165
|
+
],
|
|
166
|
+
ciEvents: [],
|
|
167
|
+
deployments: [],
|
|
168
|
+
infraEvents: [
|
|
169
|
+
{ ...ie("oom_kill", "worker-service", "critical", "OOMKilled: container exceeded memory limit 512Mi \u2014 Node.js heap exhausted"), timestamp: t(5) - 12e3, attributes: { exitCode: 137, memoryLimitMb: 512, process: "worker-service" } }
|
|
170
|
+
],
|
|
171
|
+
originalTag: "infra_oom_kill",
|
|
172
|
+
originalConfidenceScore: 0.9,
|
|
173
|
+
originalFixHint: "Increase container memory limit in k8s manifest / docker-compose. Heap-profile with `node --inspect` or `py-spy top` to find the leak. Check for unbounded caches or retained buffers in recent deploys."
|
|
174
|
+
},
|
|
175
|
+
{
|
|
176
|
+
pid: "seed-007",
|
|
177
|
+
capturedAt: cap(6),
|
|
178
|
+
firedAt: t(6),
|
|
179
|
+
logs: [
|
|
180
|
+
ce("error", "[ml-service] MemoryError: Unable to allocate 2.50 GiB for an array with shape (336000000,) and data type float64", "http://ml-service:8085/predict", t(6) - 9e3),
|
|
181
|
+
ce("error", "[k8s] OOMKilled: ml-service exceeded memory limit 2Gi (RSS: 2.1Gi, limit: 2.0Gi)", "http://ml-service:8085", t(6) - 8500)
|
|
182
|
+
],
|
|
183
|
+
network: [
|
|
184
|
+
ne("POST", "http://ml-service:8085/predict", 503, 9100, t(6) - 8200, "service killed during request")
|
|
185
|
+
],
|
|
186
|
+
contexts: [],
|
|
187
|
+
terminal: [],
|
|
188
|
+
processExits: [
|
|
189
|
+
{ type: "process_exit", process: "ml-service", exitCode: 137, reason: "oom", signal: "SIGKILL", memoryLimitBytes: 2147483648, timestamp: t(6) - 8500 }
|
|
190
|
+
],
|
|
191
|
+
ciEvents: [],
|
|
192
|
+
deployments: [],
|
|
193
|
+
infraEvents: [
|
|
194
|
+
{ ...ie("oom_kill", "ml-service", "critical", "OOMKilled: Python ML service exceeded 2Gi limit \u2014 NumPy allocation failed"), timestamp: t(6) - 9e3, attributes: { exitCode: 137, memoryLimitMb: 2048, process: "ml-service" } }
|
|
195
|
+
],
|
|
196
|
+
originalTag: "infra_oom_kill",
|
|
197
|
+
originalConfidenceScore: 0.9,
|
|
198
|
+
originalFixHint: "Increase container memory limit in k8s manifest / docker-compose. Heap-profile with `node --inspect` or `py-spy top` to find the leak. Check for unbounded caches or retained buffers in recent deploys."
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
pid: "seed-008",
|
|
202
|
+
capturedAt: cap(7),
|
|
203
|
+
firedAt: t(7),
|
|
204
|
+
logs: [
|
|
205
|
+
ce("error", "[go-service] runtime: out of memory: cannot allocate 1073741824-byte block (1536 MB in use)", "http://go-service:8086/process", t(7) - 7e3),
|
|
206
|
+
ce("error", "[k8s] OOMKilled: go-service container killed by OOM killer \u2014 memory limit 1.5Gi exceeded", "http://go-service:8086", t(7) - 6500)
|
|
207
|
+
],
|
|
208
|
+
network: [],
|
|
209
|
+
contexts: [],
|
|
210
|
+
terminal: [],
|
|
211
|
+
processExits: [
|
|
212
|
+
{ type: "process_exit", process: "go-service", exitCode: 137, reason: "oom", signal: "SIGKILL", memoryLimitBytes: 1610612736, timestamp: t(7) - 6500 }
|
|
213
|
+
],
|
|
214
|
+
ciEvents: [],
|
|
215
|
+
deployments: [],
|
|
216
|
+
infraEvents: [
|
|
217
|
+
{ ...ie("oom_kill", "go-service", "critical", "Go runtime OOM: cannot allocate 1GB block \u2014 RSS 1536MB exceeded 1.5Gi limit"), timestamp: t(7) - 7e3, attributes: { exitCode: 137, memoryLimitMb: 1536 } }
|
|
218
|
+
],
|
|
219
|
+
originalTag: "infra_oom_kill",
|
|
220
|
+
originalConfidenceScore: 0.9,
|
|
221
|
+
originalFixHint: "Increase container memory limit in k8s manifest / docker-compose. Heap-profile with `node --inspect` or `py-spy top` to find the leak. Check for unbounded caches or retained buffers in recent deploys."
|
|
222
|
+
},
|
|
223
|
+
{
|
|
224
|
+
pid: "seed-009",
|
|
225
|
+
capturedAt: cap(8),
|
|
226
|
+
firedAt: t(8),
|
|
227
|
+
logs: [
|
|
228
|
+
ce("error", "[inference] java.lang.OutOfMemoryError: Java heap space\n at com.company.inference.ModelLoader.loadModel(ModelLoader.java:142)", "http://inference:8087/infer", t(8) - 15e3),
|
|
229
|
+
ce("error", "[k8s] OOMKilled: inference-service \u2014 Java heap 4Gi exceeded JVM -Xmx limit", "http://inference:8087", t(8) - 14500)
|
|
230
|
+
],
|
|
231
|
+
network: [
|
|
232
|
+
ne("POST", "http://inference:8087/infer", 503, 15100, t(8) - 14200, "OOMKilled")
|
|
233
|
+
],
|
|
234
|
+
contexts: [],
|
|
235
|
+
terminal: [],
|
|
236
|
+
processExits: [
|
|
237
|
+
{ type: "process_exit", process: "inference-service", exitCode: 137, reason: "oom", signal: "SIGKILL", memoryLimitBytes: 4294967296, timestamp: t(8) - 14500 }
|
|
238
|
+
],
|
|
239
|
+
ciEvents: [],
|
|
240
|
+
deployments: [],
|
|
241
|
+
infraEvents: [
|
|
242
|
+
{ ...ie("oom_kill", "inference-service", "critical", "Java OutOfMemoryError: heap space \u2014 JVM Xmx 4Gi limit exceeded during model load"), timestamp: t(8) - 15e3, attributes: { exitCode: 137, memoryLimitMb: 4096 } }
|
|
243
|
+
],
|
|
244
|
+
originalTag: "infra_oom_kill",
|
|
245
|
+
originalConfidenceScore: 0.9,
|
|
246
|
+
originalFixHint: "Increase container memory limit in k8s manifest / docker-compose. Heap-profile with `node --inspect` or `py-spy top` to find the leak. Check for unbounded caches or retained buffers in recent deploys."
|
|
247
|
+
},
|
|
248
|
+
{
|
|
249
|
+
pid: "seed-010",
|
|
250
|
+
capturedAt: cap(9),
|
|
251
|
+
firedAt: t(9),
|
|
252
|
+
logs: [
|
|
253
|
+
ce("warn", "[report-worker] RSS growing: 380MB/512MB limit \u2014 possible memory leak in CSV serializer", "http://report-worker:8088", t(9) - 45e3),
|
|
254
|
+
ce("warn", "[report-worker] RSS: 490MB/512MB \u2014 GC pressure increasing, major GC pauses > 2s", "http://report-worker:8088", t(9) - 3e4),
|
|
255
|
+
ce("error", "[report-worker] RSS: 511MB/512MB \u2014 CrashLoopBackOff imminent", "http://report-worker:8088", t(9) - 1e4)
|
|
256
|
+
],
|
|
257
|
+
network: [],
|
|
258
|
+
contexts: [],
|
|
259
|
+
terminal: [],
|
|
260
|
+
processExits: [],
|
|
261
|
+
ciEvents: [],
|
|
262
|
+
deployments: [],
|
|
263
|
+
infraEvents: [
|
|
264
|
+
{ ...ie("memory_pressure", "report-worker", "high", "RSS 511MB/512MB \u2014 sustained memory pressure, GC pauses > 2s, CrashLoopBackOff imminent"), timestamp: t(9) - 1e4, attributes: { memoryLimitMb: 512, rssUsedMb: 511 } }
|
|
265
|
+
],
|
|
266
|
+
originalTag: "infra_oom_kill",
|
|
267
|
+
originalConfidenceScore: 0.65,
|
|
268
|
+
originalFixHint: "Profile heap allocation. Check for unbounded in-memory caches or event-listener leaks. Review recent deploys for large data structure growth."
|
|
269
|
+
},
|
|
270
|
+
// ── 011–015 infra_rate_limit_cascade ────────────────────────────────────
|
|
271
|
+
{
|
|
272
|
+
pid: "seed-011",
|
|
273
|
+
capturedAt: cap(10),
|
|
274
|
+
firedAt: t(10),
|
|
275
|
+
logs: [
|
|
276
|
+
ce("error", "[payments] Stripe API error: rate limit exceeded (429) \u2014 100 req/s limit hit on /v1/charges endpoint", "http://payments:8082/charge", t(10) - 18e3),
|
|
277
|
+
ce("error", "[payments] Unhandled StripeRateLimitError: too many requests in 1s \u2014 retry backoff required", "http://payments:8082/charge", t(10) - 17500),
|
|
278
|
+
ce("warn", "[payments] payment queue backing up \u2014 847 pending charges, Stripe rate limited", "http://payments:8082", t(10) - 1e4)
|
|
279
|
+
],
|
|
280
|
+
network: [
|
|
281
|
+
ne("POST", "https://api.stripe.com/v1/charges", 429, 120, t(10) - 18e3, "Rate limit exceeded"),
|
|
282
|
+
ne("POST", "https://api.stripe.com/v1/charges", 429, 115, t(10) - 17e3, "Rate limit exceeded"),
|
|
283
|
+
ne("POST", "https://api.stripe.com/v1/charges", 429, 118, t(10) - 16e3, "Rate limit exceeded")
|
|
284
|
+
],
|
|
285
|
+
contexts: [],
|
|
286
|
+
terminal: [],
|
|
287
|
+
processExits: [],
|
|
288
|
+
ciEvents: [],
|
|
289
|
+
deployments: [],
|
|
290
|
+
infraEvents: [
|
|
291
|
+
{ ...ie("rate_limit_cascade", "payments-service", "high", "Stripe API 429: 100 req/s limit exceeded \u2014 retry storm amplifying throttling"), timestamp: t(10) - 18e3, attributes: { endpoint: "https://api.stripe.com/v1/charges", retryAfterSec: 1 } }
|
|
292
|
+
],
|
|
293
|
+
originalTag: "infra_rate_limit_cascade",
|
|
294
|
+
originalConfidenceScore: 0.75,
|
|
295
|
+
originalFixHint: "Add exponential backoff with jitter to the retry loop. Honour the `Retry-After` response header. Consider a token-bucket circuit breaker to shed load before hitting the upstream limit."
|
|
296
|
+
},
|
|
297
|
+
{
|
|
298
|
+
pid: "seed-012",
|
|
299
|
+
capturedAt: cap(11),
|
|
300
|
+
firedAt: t(11),
|
|
301
|
+
logs: [
|
|
302
|
+
ce("error", "[notifications] Twilio API: 429 Too Many Requests \u2014 messaging rate limit of 1 message/sec exceeded for +1555XXXXXXXX", "http://notifications:8090/send", t(11) - 12e3),
|
|
303
|
+
ce("warn", "[notifications] SMS queue depth: 2400 messages \u2014 Twilio rate-limited, throughput reduced to 1 msg/s", "http://notifications:8090", t(11) - 8e3)
|
|
304
|
+
],
|
|
305
|
+
network: [
|
|
306
|
+
ne("POST", "https://api.twilio.com/2010-04-01/Accounts/ACxxx/Messages.json", 429, 95, t(11) - 12e3, "Rate limit exceeded")
|
|
307
|
+
],
|
|
308
|
+
contexts: [],
|
|
309
|
+
terminal: [],
|
|
310
|
+
processExits: [],
|
|
311
|
+
ciEvents: [],
|
|
312
|
+
deployments: [],
|
|
313
|
+
infraEvents: [
|
|
314
|
+
{ ...ie("rate_limit_cascade", "notifications-service", "high", "Twilio 429: 1 msg/s rate limit exceeded \u2014 SMS queue depth 2400, throughput degraded"), timestamp: t(11) - 12e3, attributes: { endpoint: "https://api.twilio.com/Messages", queueDepth: 2400 } }
|
|
315
|
+
],
|
|
316
|
+
originalTag: "infra_rate_limit_cascade",
|
|
317
|
+
originalConfidenceScore: 0.75,
|
|
318
|
+
originalFixHint: "Add exponential backoff with jitter to the retry loop. Honour the `Retry-After` response header. Consider a token-bucket circuit breaker to shed load before hitting the upstream limit."
|
|
319
|
+
},
|
|
320
|
+
{
|
|
321
|
+
pid: "seed-013",
|
|
322
|
+
capturedAt: cap(12),
|
|
323
|
+
firedAt: t(12),
|
|
324
|
+
logs: [
|
|
325
|
+
ce("error", "[ci-service] GitHub API secondary rate limit exceeded: You have exceeded a secondary rate limit and have been temporarily blocked. Please retry after 60 seconds.", "http://ci-service:8091/sync", t(12) - 2e4),
|
|
326
|
+
ce("warn", "[ci-service] Backing off 60s \u2014 GitHub API rate limited (5000 req/hr primary limit also at 4987/5000)", "http://ci-service:8091", t(12) - 15e3)
|
|
327
|
+
],
|
|
328
|
+
network: [
|
|
329
|
+
ne("GET", "https://api.github.com/repos/company/app/commits", 429, 200, t(12) - 2e4, "secondary rate limit exceeded")
|
|
330
|
+
],
|
|
331
|
+
contexts: [],
|
|
332
|
+
terminal: [],
|
|
333
|
+
processExits: [],
|
|
334
|
+
ciEvents: [],
|
|
335
|
+
deployments: [],
|
|
336
|
+
infraEvents: [
|
|
337
|
+
{ ...ie("rate_limit_cascade", "ci-service", "medium", "GitHub API secondary rate limit \u2014 5000 req/hr primary at 4987/5000, secondary limit triggered"), timestamp: t(12) - 2e4, attributes: { endpoint: "https://api.github.com", retryAfterSec: 60 } }
|
|
338
|
+
],
|
|
339
|
+
originalTag: "infra_rate_limit_cascade",
|
|
340
|
+
originalConfidenceScore: 0.75,
|
|
341
|
+
originalFixHint: "Add exponential backoff with jitter to the retry loop. Honour the `Retry-After` response header. Consider a token-bucket circuit breaker to shed load before hitting the upstream limit."
|
|
342
|
+
},
|
|
343
|
+
{
|
|
344
|
+
pid: "seed-014",
|
|
345
|
+
capturedAt: cap(13),
|
|
346
|
+
firedAt: t(13),
|
|
347
|
+
logs: [
|
|
348
|
+
ce("error", "[email-service] AWS SES: 454 Throttling \u2014 Maximum sending rate exceeded. Daily sending quota: 50000, used: 50000", "http://email-service:8092/send", t(13) - 16e3),
|
|
349
|
+
ce("error", "[email-service] MessageRejected: Daily message quota exceeded \u2014 SES sending suspended until 00:00 UTC", "http://email-service:8092/send", t(13) - 15500),
|
|
350
|
+
ce("warn", "[email-service] 1247 emails queued \u2014 SES quota exhausted for today. Earliest send: midnight UTC", "http://email-service:8092", t(13) - 1e4)
|
|
351
|
+
],
|
|
352
|
+
network: [
|
|
353
|
+
ne("POST", "https://email.us-east-1.amazonaws.com/", 400, 350, t(13) - 16e3, "Daily message quota exceeded")
|
|
354
|
+
],
|
|
355
|
+
contexts: [],
|
|
356
|
+
terminal: [],
|
|
357
|
+
processExits: [],
|
|
358
|
+
ciEvents: [],
|
|
359
|
+
deployments: [],
|
|
360
|
+
infraEvents: [
|
|
361
|
+
{ ...ie("rate_limit_cascade", "email-service", "high", "AWS SES daily quota 50000/50000 exhausted \u2014 MessageRejected until midnight UTC"), timestamp: t(13) - 16e3, attributes: { endpoint: "https://email.us-east-1.amazonaws.com", dailyLimit: 5e4, dailyUsed: 5e4 } }
|
|
362
|
+
],
|
|
363
|
+
originalTag: "infra_rate_limit_cascade",
|
|
364
|
+
originalConfidenceScore: 0.75,
|
|
365
|
+
originalFixHint: "Add exponential backoff with jitter to the retry loop. Honour the `Retry-After` response header. Consider a token-bucket circuit breaker to shed load before hitting the upstream limit."
|
|
366
|
+
},
|
|
367
|
+
{
|
|
368
|
+
pid: "seed-015",
|
|
369
|
+
capturedAt: cap(14),
|
|
370
|
+
firedAt: t(14),
|
|
371
|
+
logs: [
|
|
372
|
+
ce("error", "[crm-sync] Salesforce API: REQUEST_LIMIT_EXCEEDED \u2014 Total API calls: 15000/15000 for 24-hour period", "http://crm-sync:8093/sync", t(14) - 22e3),
|
|
373
|
+
ce("warn", "[crm-sync] Salesforce daily API limit exhausted \u2014 sync paused until quota resets at 00:00 GMT", "http://crm-sync:8093", t(14) - 18e3)
|
|
374
|
+
],
|
|
375
|
+
network: [
|
|
376
|
+
ne("POST", "https://company.my.salesforce.com/services/data/v57.0/sobjects/Contact", 403, 280, t(14) - 22e3, "REQUEST_LIMIT_EXCEEDED")
|
|
377
|
+
],
|
|
378
|
+
contexts: [],
|
|
379
|
+
terminal: [],
|
|
380
|
+
processExits: [],
|
|
381
|
+
ciEvents: [],
|
|
382
|
+
deployments: [],
|
|
383
|
+
infraEvents: [
|
|
384
|
+
{ ...ie("rate_limit_cascade", "crm-sync", "high", "Salesforce daily API calls exhausted 15000/15000 \u2014 REQUEST_LIMIT_EXCEEDED until quota reset"), timestamp: t(14) - 22e3, attributes: { endpoint: "https://company.my.salesforce.com/services/data", dailyLimit: 15e3 } }
|
|
385
|
+
],
|
|
386
|
+
originalTag: "infra_rate_limit_cascade",
|
|
387
|
+
originalConfidenceScore: 0.75,
|
|
388
|
+
originalFixHint: "Add exponential backoff with jitter to the retry loop. Honour the `Retry-After` response header. Consider a token-bucket circuit breaker to shed load before hitting the upstream limit."
|
|
389
|
+
},
|
|
390
|
+
// ── 016–020 infra_slow_query ─────────────────────────────────────────────
|
|
391
|
+
{
|
|
392
|
+
pid: "seed-016",
|
|
393
|
+
capturedAt: cap(15),
|
|
394
|
+
firedAt: t(15),
|
|
395
|
+
logs: [
|
|
396
|
+
ce("warn", "[api] slow query detected: SELECT * FROM orders JOIN order_items ON ... WHERE user_id=$1 \u2014 4821ms (threshold: 500ms)", "http://api:8080/api/orders", t(15) - 15e3),
|
|
397
|
+
ce("error", "[api] StatementTimeout: canceling statement due to statement_timeout (5000ms) \u2014 N+1 query pattern detected on orders.order_items", "http://api:8080/api/orders", t(15) - 14500)
|
|
398
|
+
],
|
|
399
|
+
network: [
|
|
400
|
+
ne("GET", "http://api:8080/api/orders", 504, 5200, t(15) - 14200, "query timeout")
|
|
401
|
+
],
|
|
402
|
+
contexts: [],
|
|
403
|
+
terminal: [],
|
|
404
|
+
processExits: [],
|
|
405
|
+
ciEvents: [],
|
|
406
|
+
deployments: [],
|
|
407
|
+
infraEvents: [
|
|
408
|
+
{ ...ie("slow_query", "api-service", "high", "N+1 SELECT on orders.order_items \u2014 4821ms, exceeds 500ms threshold. statement_timeout triggered."), timestamp: t(15) - 15e3, attributes: { endpoint: "postgres://10.12.0.5:5432/app", durationMs: 4821, table: "order_items", pattern: "N+1" } }
|
|
409
|
+
],
|
|
410
|
+
originalTag: "infra_slow_query",
|
|
411
|
+
originalConfidenceScore: 0.7,
|
|
412
|
+
originalFixHint: "Run EXPLAIN ANALYZE on the slow query. Add indexes on WHERE / JOIN columns. Check recent schema migrations for unindexed columns."
|
|
413
|
+
},
|
|
414
|
+
{
|
|
415
|
+
pid: "seed-017",
|
|
416
|
+
capturedAt: cap(16),
|
|
417
|
+
firedAt: t(16),
|
|
418
|
+
logs: [
|
|
419
|
+
ce("warn", "[search-api] slow query: SELECT id, payload FROM user_events WHERE event_type=$1 AND created_at > $2 \u2014 12340ms (Seq Scan, 0 index)", "http://search-api:8094/search", t(16) - 2e4),
|
|
420
|
+
ce("error", "[search-api] query cancelled: statement_timeout 15000ms on user_events (rows examined: 48000000, no index on event_type)", "http://search-api:8094/search", t(16) - 17500)
|
|
421
|
+
],
|
|
422
|
+
network: [
|
|
423
|
+
ne("GET", "http://search-api:8094/search", 504, 15200, t(16) - 17e3, "gateway timeout")
|
|
424
|
+
],
|
|
425
|
+
contexts: [],
|
|
426
|
+
terminal: [],
|
|
427
|
+
processExits: [],
|
|
428
|
+
ciEvents: [],
|
|
429
|
+
deployments: [],
|
|
430
|
+
infraEvents: [
|
|
431
|
+
{ ...ie("slow_query", "search-api", "high", "Full table scan on user_events (48M rows) \u2014 missing index on event_type column, 12340ms query"), timestamp: t(16) - 2e4, attributes: { endpoint: "postgres://10.12.0.5:5432/app", durationMs: 12340, table: "user_events", rowsExamined: 48e6 } }
|
|
432
|
+
],
|
|
433
|
+
originalTag: "infra_slow_query",
|
|
434
|
+
originalConfidenceScore: 0.7,
|
|
435
|
+
originalFixHint: "Run EXPLAIN ANALYZE on the slow query. Add indexes on WHERE / JOIN columns. Check recent schema migrations for unindexed columns."
|
|
436
|
+
},
|
|
437
|
+
{
|
|
438
|
+
pid: "seed-018",
|
|
439
|
+
capturedAt: cap(17),
|
|
440
|
+
firedAt: t(17),
|
|
441
|
+
logs: [
|
|
442
|
+
ce("error", "[api] Lock wait timeout exceeded \u2014 UPDATE payments SET status=$1 WHERE id=$2 waiting for lock held by txn 4821 (running 180s)", "http://api:8080/api/payments", t(17) - 8e3),
|
|
443
|
+
ce("warn", "[api] transaction 4821 blocking 14 other transactions \u2014 payments table row lock contention", "http://api:8080", t(17) - 7e3)
|
|
444
|
+
],
|
|
445
|
+
network: [
|
|
446
|
+
ne("PUT", "http://api:8080/api/payments/pay_abc123", 504, 8200, t(17) - 7800, "lock timeout")
|
|
447
|
+
],
|
|
448
|
+
contexts: [],
|
|
449
|
+
terminal: [],
|
|
450
|
+
processExits: [],
|
|
451
|
+
ciEvents: [],
|
|
452
|
+
deployments: [],
|
|
453
|
+
infraEvents: [
|
|
454
|
+
{ ...ie("slow_query", "api-service", "high", "Lock wait timeout on payments table \u2014 txn 4821 holding row lock for 180s, blocking 14 transactions"), timestamp: t(17) - 8e3, attributes: { endpoint: "postgres://10.12.0.5:5432/app", durationMs: 8e3, table: "payments", blockingTxns: 14 } }
|
|
455
|
+
],
|
|
456
|
+
originalTag: "infra_slow_query",
|
|
457
|
+
originalConfidenceScore: 0.7,
|
|
458
|
+
originalFixHint: "Run EXPLAIN ANALYZE on the slow query. Add indexes on WHERE / JOIN columns. Check recent schema migrations for unindexed columns."
|
|
459
|
+
},
|
|
460
|
+
{
|
|
461
|
+
pid: "seed-019",
|
|
462
|
+
capturedAt: cap(18),
|
|
463
|
+
firedAt: t(18),
|
|
464
|
+
logs: [
|
|
465
|
+
ce("warn", "[products-api] slow query: SELECT * FROM products WHERE name LIKE $1 \u2014 8920ms (Seq Scan 2.1M rows, no trigram index)", "http://products-api:8095/search", t(18) - 12e3),
|
|
466
|
+
ce("error", "[products-api] statement_timeout: query cancelled after 10000ms \u2014 LIKE prefix scan without pg_trgm index", "http://products-api:8095/search", t(18) - 10100)
|
|
467
|
+
],
|
|
468
|
+
network: [
|
|
469
|
+
ne("GET", "http://products-api:8095/search?q=phone", 504, 10200, t(18) - 1e4, "gateway timeout")
|
|
470
|
+
],
|
|
471
|
+
contexts: [],
|
|
472
|
+
terminal: [],
|
|
473
|
+
processExits: [],
|
|
474
|
+
ciEvents: [],
|
|
475
|
+
deployments: [],
|
|
476
|
+
infraEvents: [
|
|
477
|
+
{ ...ie("slow_query", "products-api", "medium", "LIKE prefix scan on products.name \u2014 8920ms, missing pg_trgm GIN index, 2.1M row full scan"), timestamp: t(18) - 12e3, attributes: { endpoint: "postgres://10.12.0.5:5432/app", durationMs: 8920, table: "products", rowsExamined: 21e5 } }
|
|
478
|
+
],
|
|
479
|
+
originalTag: "infra_slow_query",
|
|
480
|
+
originalConfidenceScore: 0.7,
|
|
481
|
+
originalFixHint: "Run EXPLAIN ANALYZE on the slow query. Add indexes on WHERE / JOIN columns. Check recent schema migrations for unindexed columns."
|
|
482
|
+
},
|
|
483
|
+
{
|
|
484
|
+
pid: "seed-020",
|
|
485
|
+
capturedAt: cap(19),
|
|
486
|
+
firedAt: t(19),
|
|
487
|
+
logs: [
|
|
488
|
+
ce("warn", "[api] slow query: SELECT * FROM events WHERE metadata @> $1::jsonb \u2014 6740ms (Seq Scan, no GIN index on metadata column)", "http://api:8080/api/events", t(19) - 1e4),
|
|
489
|
+
ce("error", "[api] canceling statement due to statement_timeout \u2014 jsonb @> operator on unindexed metadata column (events table: 12M rows)", "http://api:8080/api/events", t(19) - 8100)
|
|
490
|
+
],
|
|
491
|
+
network: [
|
|
492
|
+
ne("GET", "http://api:8080/api/events", 504, 8200, t(19) - 8e3, "query timeout")
|
|
493
|
+
],
|
|
494
|
+
contexts: [],
|
|
495
|
+
terminal: [],
|
|
496
|
+
processExits: [],
|
|
497
|
+
ciEvents: [],
|
|
498
|
+
deployments: [],
|
|
499
|
+
infraEvents: [
|
|
500
|
+
{ ...ie("slow_query", "api-service", "high", "JSONB @> containment query on unindexed events.metadata \u2014 6740ms, Seq Scan 12M rows"), timestamp: t(19) - 1e4, attributes: { endpoint: "postgres://10.12.0.5:5432/app", durationMs: 6740, table: "events", rowsExamined: 12e6 } }
|
|
501
|
+
],
|
|
502
|
+
originalTag: "infra_slow_query",
|
|
503
|
+
originalConfidenceScore: 0.7,
|
|
504
|
+
originalFixHint: "Run EXPLAIN ANALYZE on the slow query. Add indexes on WHERE / JOIN columns. Check recent schema migrations for unindexed columns."
|
|
505
|
+
},
|
|
506
|
+
// ── 021–025 infra_certificate_expiry ────────────────────────────────────
|
|
507
|
+
{
|
|
508
|
+
pid: "seed-021",
|
|
509
|
+
capturedAt: cap(20),
|
|
510
|
+
firedAt: t(20),
|
|
511
|
+
logs: [
|
|
512
|
+
ce("error", "[ingress] SSL_ERROR_RX_RECORD_TOO_LONG: TLS handshake failed for api.company.com \u2014 certificate expired 2024-01-01T00:00:00Z (3 days ago)", "https://api.company.com", t(20) - 5e3),
|
|
513
|
+
ce("error", "[nginx] SSL certificate error: api.company.com: certificate has expired (notAfter=Jan 1 00:00:00 2024 GMT)", "https://api.company.com", t(20) - 4500)
|
|
514
|
+
],
|
|
515
|
+
network: [
|
|
516
|
+
ne("GET", "https://api.company.com/health", 526, 200, t(20) - 4800, "certificate expired")
|
|
517
|
+
],
|
|
518
|
+
contexts: [],
|
|
519
|
+
terminal: [],
|
|
520
|
+
processExits: [],
|
|
521
|
+
ciEvents: [],
|
|
522
|
+
deployments: [],
|
|
523
|
+
infraEvents: [
|
|
524
|
+
{ ...ie("certificate_expiry", "api-gateway", "critical", "TLS certificate expired for api.company.com (notAfter: 2024-01-01 00:00:00 UTC) \u2014 Let's Encrypt auto-renewal failed"), timestamp: t(20) - 5e3, attributes: { domain: "api.company.com", expiredAt: "2024-01-01T00:00:00Z", daysPastExpiry: 3 } }
|
|
525
|
+
],
|
|
526
|
+
originalTag: "infra_certificate_expiry",
|
|
527
|
+
originalConfidenceScore: 0.85,
|
|
528
|
+
originalFixHint: "Check expiry: `openssl s_client -connect <host>:443 2>/dev/null | openssl x509 -noout -dates` Renew via Let's Encrypt (`certbot renew`) or your CA. Verify the cert covers the target hostname (SAN list)."
|
|
529
|
+
},
|
|
530
|
+
{
|
|
531
|
+
pid: "seed-022",
|
|
532
|
+
capturedAt: cap(21),
|
|
533
|
+
firedAt: t(21),
|
|
534
|
+
logs: [
|
|
535
|
+
ce("error", "[webhooks] CERT_HAS_EXPIRED: unable to verify the first certificate for webhooks.company.com \u2014 notAfter Jan 12 00:00:00 2024 GMT", "https://webhooks.company.com/events", t(21) - 8e3),
|
|
536
|
+
ce("warn", "[webhooks] 847 webhook deliveries failing \u2014 all HTTPS connections rejected by clients (expired TLS cert)", "https://webhooks.company.com", t(21) - 5e3)
|
|
537
|
+
],
|
|
538
|
+
network: [
|
|
539
|
+
ne("POST", "https://webhooks.company.com/events", 525, 180, t(21) - 7800, "SSL certificate has expired")
|
|
540
|
+
],
|
|
541
|
+
contexts: [],
|
|
542
|
+
terminal: [],
|
|
543
|
+
processExits: [],
|
|
544
|
+
ciEvents: [],
|
|
545
|
+
deployments: [],
|
|
546
|
+
infraEvents: [
|
|
547
|
+
{ ...ie("certificate_expiry", "webhooks-service", "critical", "TLS cert for webhooks.company.com expired Jan 12 2024 \u2014 847 webhook deliveries rejected by clients"), timestamp: t(21) - 8e3, attributes: { domain: "webhooks.company.com", expiredAt: "2024-01-12T00:00:00Z" } }
|
|
548
|
+
],
|
|
549
|
+
originalTag: "infra_certificate_expiry",
|
|
550
|
+
originalConfidenceScore: 0.85,
|
|
551
|
+
originalFixHint: "Check expiry: `openssl s_client -connect <host>:443 2>/dev/null | openssl x509 -noout -dates` Renew via Let's Encrypt (`certbot renew`) or your CA. Verify the cert covers the target hostname (SAN list)."
|
|
552
|
+
},
|
|
553
|
+
{
|
|
554
|
+
pid: "seed-023",
|
|
555
|
+
capturedAt: cap(22),
|
|
556
|
+
firedAt: t(22),
|
|
557
|
+
logs: [
|
|
558
|
+
ce("error", "[auth-service] mTLS handshake failed: certificate verify failed (unable to get local issuer certificate) \u2014 peer cert signed by expired intermediate CA", "http://auth-service:8081/validate", t(22) - 6e3),
|
|
559
|
+
ce("error", "[auth-service] service mesh mTLS broken \u2014 all sidecar-to-sidecar connections failing: intermediate CA cert expired", "http://auth-service:8081", t(22) - 5500)
|
|
560
|
+
],
|
|
561
|
+
network: [
|
|
562
|
+
ne("POST", "http://auth-service:8081/validate", 503, 6100, t(22) - 5700, "mTLS certificate verification failed")
|
|
563
|
+
],
|
|
564
|
+
contexts: [],
|
|
565
|
+
terminal: [],
|
|
566
|
+
processExits: [],
|
|
567
|
+
ciEvents: [],
|
|
568
|
+
deployments: [],
|
|
569
|
+
infraEvents: [
|
|
570
|
+
{ ...ie("certificate_expiry", "auth-service", "critical", "Service mesh mTLS broken \u2014 intermediate CA cert expired, all sidecar connections failing"), timestamp: t(22) - 6e3, attributes: { domain: "internal-ca.company.local", certType: "intermediate-ca" } }
|
|
571
|
+
],
|
|
572
|
+
originalTag: "infra_certificate_expiry",
|
|
573
|
+
originalConfidenceScore: 0.85,
|
|
574
|
+
originalFixHint: "Check expiry: `openssl s_client -connect <host>:443 2>/dev/null | openssl x509 -noout -dates` Renew via Let's Encrypt (`certbot renew`) or your CA. Verify the cert covers the target hostname (SAN list)."
|
|
575
|
+
},
|
|
576
|
+
{
|
|
577
|
+
pid: "seed-024",
|
|
578
|
+
capturedAt: cap(23),
|
|
579
|
+
firedAt: t(23),
|
|
580
|
+
logs: [
|
|
581
|
+
ce("error", "[integration] Error: unable to verify the first certificate \u2014 downstream partner API cert hostname mismatch: CN=old.partner.com, expected new.partner.com", "http://integration:8096/sync", t(23) - 7e3),
|
|
582
|
+
ce("warn", "[integration] 240 outbound webhook deliveries failing \u2014 partner changed their TLS cert hostname without notice", "http://integration:8096", t(23) - 4e3)
|
|
583
|
+
],
|
|
584
|
+
network: [
|
|
585
|
+
ne("POST", "https://new.partner.com/webhook", 525, 190, t(23) - 6800, "hostname mismatch in certificate")
|
|
586
|
+
],
|
|
587
|
+
contexts: [],
|
|
588
|
+
terminal: [],
|
|
589
|
+
processExits: [],
|
|
590
|
+
ciEvents: [],
|
|
591
|
+
deployments: [],
|
|
592
|
+
infraEvents: [
|
|
593
|
+
{ ...ie("certificate_expiry", "integration-service", "high", "TLS hostname mismatch: partner changed cert CN from old.partner.com to new.partner.com \u2014 240 deliveries failing"), timestamp: t(23) - 7e3, attributes: { domain: "new.partner.com", certType: "hostname-mismatch" } }
|
|
594
|
+
],
|
|
595
|
+
originalTag: "infra_certificate_expiry",
|
|
596
|
+
originalConfidenceScore: 0.85,
|
|
597
|
+
originalFixHint: "Check expiry: `openssl s_client -connect <host>:443 2>/dev/null | openssl x509 -noout -dates` Renew via Let's Encrypt (`certbot renew`) or your CA. Verify the cert covers the target hostname (SAN list)."
|
|
598
|
+
},
|
|
599
|
+
{
|
|
600
|
+
pid: "seed-025",
|
|
601
|
+
capturedAt: cap(24),
|
|
602
|
+
firedAt: t(24),
|
|
603
|
+
logs: [
|
|
604
|
+
ce("error", "[certbot] Certbot renewal failed for api.internal.company.com: Challenge failed for domain \u2014 HTTP-01 challenge response returned 404 (port 80 not reachable)", "http://certbot-cron:8097", t(24) - 864e5),
|
|
605
|
+
ce("error", "[nginx] SSL certificate will expire in 2 days \u2014 api.internal.company.com. Certbot renewal has been failing for 28 days.", "https://api.internal.company.com", t(24) - 36e5),
|
|
606
|
+
ce("error", "[nginx] SSL certificate EXPIRED: api.internal.company.com notAfter reached", "https://api.internal.company.com", t(24) - 1e3)
|
|
607
|
+
],
|
|
608
|
+
network: [
|
|
609
|
+
ne("GET", "https://api.internal.company.com/health", 525, 150, t(24) - 800, "certificate expired")
|
|
610
|
+
],
|
|
611
|
+
contexts: [],
|
|
612
|
+
terminal: [],
|
|
613
|
+
processExits: [],
|
|
614
|
+
ciEvents: [],
|
|
615
|
+
deployments: [],
|
|
616
|
+
infraEvents: [
|
|
617
|
+
{ ...ie("certificate_expiry", "internal-api", "critical", "Let's Encrypt renewal failed for 28 days \u2014 HTTP-01 challenge blocked (port 80 not reachable). Cert expired."), timestamp: t(24) - 1e3, attributes: { domain: "api.internal.company.com", certType: "letsencrypt", renewalFailureDays: 28 } }
|
|
618
|
+
],
|
|
619
|
+
originalTag: "infra_certificate_expiry",
|
|
620
|
+
originalConfidenceScore: 0.85,
|
|
621
|
+
originalFixHint: "Check expiry: `openssl s_client -connect <host>:443 2>/dev/null | openssl x509 -noout -dates` Renew via Let's Encrypt (`certbot renew`) or your CA. Verify the cert covers the target hostname (SAN list)."
|
|
622
|
+
},
|
|
623
|
+
// ── 026–030 infra_downstream_latency ────────────────────────────────────
|
|
624
|
+
{
|
|
625
|
+
pid: "seed-026",
|
|
626
|
+
capturedAt: cap(25),
|
|
627
|
+
firedAt: t(25),
|
|
628
|
+
logs: [
|
|
629
|
+
ce("warn", "[api] Redis latency spike: GET session:* \u2014 p99 latency 1240ms (normal: 3ms). redis-cluster node redis-2 under GC pressure.", "http://api:8080/api/session", t(25) - 25e3),
|
|
630
|
+
ce("warn", "[api] Redis connection timeout: GET user:profile:12345 took 5001ms \u2014 exceeds 5000ms client timeout", "http://api:8080/api/user", t(25) - 2e4)
|
|
631
|
+
],
|
|
632
|
+
network: [],
|
|
633
|
+
contexts: [],
|
|
634
|
+
terminal: [],
|
|
635
|
+
processExits: [],
|
|
636
|
+
ciEvents: [],
|
|
637
|
+
deployments: [],
|
|
638
|
+
infraEvents: [
|
|
639
|
+
{ ...ie("downstream_latency_spike", "api-service", "high", "Redis p99 latency 1240ms (normal 3ms) \u2014 cluster node under GC pressure, client timeouts cascading"), timestamp: t(25) - 25e3, attributes: { endpoint: "redis://redis-cluster:6379", p99Ms: 1240, normalP99Ms: 3 } }
|
|
640
|
+
],
|
|
641
|
+
originalTag: "infra_downstream_latency",
|
|
642
|
+
originalConfidenceScore: 0.6,
|
|
643
|
+
originalFixHint: "Add circuit breaker around the slow downstream. Increase timeout if the dependency is legitimately slow, or reduce p99 of the dependency."
|
|
644
|
+
},
|
|
645
|
+
{
|
|
646
|
+
pid: "seed-027",
|
|
647
|
+
capturedAt: cap(26),
|
|
648
|
+
firedAt: t(26),
|
|
649
|
+
logs: [
|
|
650
|
+
ce("warn", "[search-api] Elasticsearch response time spike: POST /_search \u2014 p99 8400ms (normal: 200ms). Cluster health: yellow (1 shard unassigned).", "http://search-api:8094/search", t(26) - 3e4),
|
|
651
|
+
ce("warn", "[search-api] search request timeout after 10000ms \u2014 Elasticsearch cluster degraded (yellow status, 3 relocating shards)", "http://search-api:8094/search", t(26) - 25e3)
|
|
652
|
+
],
|
|
653
|
+
network: [],
|
|
654
|
+
contexts: [],
|
|
655
|
+
terminal: [],
|
|
656
|
+
processExits: [],
|
|
657
|
+
ciEvents: [],
|
|
658
|
+
deployments: [],
|
|
659
|
+
infraEvents: [
|
|
660
|
+
{ ...ie("downstream_latency_spike", "search-api", "high", "Elasticsearch p99 8400ms (normal 200ms) \u2014 cluster yellow: 1 unassigned shard, 3 relocating"), timestamp: t(26) - 3e4, attributes: { endpoint: "http://elasticsearch:9200/_search", p99Ms: 8400, clusterStatus: "yellow" } }
|
|
661
|
+
],
|
|
662
|
+
originalTag: "infra_downstream_latency",
|
|
663
|
+
originalConfidenceScore: 0.6,
|
|
664
|
+
originalFixHint: "Add circuit breaker around the slow downstream. Increase timeout if the dependency is legitimately slow, or reduce p99 of the dependency."
|
|
665
|
+
},
|
|
666
|
+
{
|
|
667
|
+
pid: "seed-028",
|
|
668
|
+
capturedAt: cap(27),
|
|
669
|
+
firedAt: t(27),
|
|
670
|
+
logs: [
|
|
671
|
+
ce("warn", "[asset-service] S3 GetObject latency spike: us-east-1 \u2014 p99 4800ms (normal: 120ms). AWS status page: increased error rates in us-east-1.", "http://asset-service:8098/asset", t(27) - 35e3),
|
|
672
|
+
ce("warn", "[asset-service] S3 request timeout after 5000ms \u2014 asset download failing for 23% of requests during us-east-1 partial outage", "http://asset-service:8098/asset", t(27) - 3e4)
|
|
673
|
+
],
|
|
674
|
+
network: [],
|
|
675
|
+
contexts: [],
|
|
676
|
+
terminal: [],
|
|
677
|
+
processExits: [],
|
|
678
|
+
ciEvents: [],
|
|
679
|
+
deployments: [],
|
|
680
|
+
infraEvents: [
|
|
681
|
+
{ ...ie("downstream_latency_spike", "asset-service", "high", "S3 GetObject p99 4800ms (normal 120ms) \u2014 us-east-1 partial degradation, 23% request timeout rate"), timestamp: t(27) - 35e3, attributes: { endpoint: "https://s3.amazonaws.com", p99Ms: 4800, region: "us-east-1" } }
|
|
682
|
+
],
|
|
683
|
+
originalTag: "infra_downstream_latency",
|
|
684
|
+
originalConfidenceScore: 0.6,
|
|
685
|
+
originalFixHint: "Add circuit breaker around the slow downstream. Increase timeout if the dependency is legitimately slow, or reduce p99 of the dependency."
|
|
686
|
+
},
|
|
687
|
+
{
|
|
688
|
+
pid: "seed-029",
|
|
689
|
+
capturedAt: cap(28),
|
|
690
|
+
firedAt: t(28),
|
|
691
|
+
logs: [
|
|
692
|
+
ce("warn", "[event-processor] Kafka consumer poll latency: 3200ms (normal: 50ms) \u2014 broker kafka-1 under disk I/O pressure, fetch.max.wait.ms exceeded", "http://event-processor:8099", t(28) - 4e4),
|
|
693
|
+
ce("error", "[event-processor] KafkaConsumer: fetch timed out after 5000ms \u2014 broker not responding, consumer group lag growing: 180000 messages behind", "http://event-processor:8099", t(28) - 35e3)
|
|
694
|
+
],
|
|
695
|
+
network: [],
|
|
696
|
+
contexts: [],
|
|
697
|
+
terminal: [],
|
|
698
|
+
processExits: [],
|
|
699
|
+
ciEvents: [],
|
|
700
|
+
deployments: [],
|
|
701
|
+
infraEvents: [
|
|
702
|
+
{ ...ie("downstream_latency_spike", "event-processor", "high", "Kafka broker fetch latency 3200ms (normal 50ms) \u2014 disk I/O pressure on kafka-1, consumer lag 180k messages"), timestamp: t(28) - 4e4, attributes: { endpoint: "kafka-1:9092", p99Ms: 3200, consumerLag: 18e4 } }
|
|
703
|
+
],
|
|
704
|
+
originalTag: "infra_downstream_latency",
|
|
705
|
+
originalConfidenceScore: 0.6,
|
|
706
|
+
originalFixHint: "Add circuit breaker around the slow downstream. Increase timeout if the dependency is legitimately slow, or reduce p99 of the dependency."
|
|
707
|
+
},
|
|
708
|
+
{
|
|
709
|
+
pid: "seed-030",
|
|
710
|
+
capturedAt: cap(29),
|
|
711
|
+
firedAt: t(29),
|
|
712
|
+
logs: [
|
|
713
|
+
ce("warn", "[checkout] payment gateway p99 latency: 7200ms (normal: 800ms) \u2014 Adyen gateway experiencing elevated response times (incident #ADY-2024-01)", "http://checkout:8100/checkout", t(29) - 5e4),
|
|
714
|
+
ce("warn", "[checkout] payment processing timeout after 8000ms \u2014 Adyen gateway degraded, 41% of payment attempts timing out", "http://checkout:8100/checkout", t(29) - 45e3)
|
|
715
|
+
],
|
|
716
|
+
network: [],
|
|
717
|
+
contexts: [],
|
|
718
|
+
terminal: [],
|
|
719
|
+
processExits: [],
|
|
720
|
+
ciEvents: [],
|
|
721
|
+
deployments: [],
|
|
722
|
+
infraEvents: [
|
|
723
|
+
{ ...ie("downstream_latency_spike", "checkout-service", "high", "Adyen payment gateway p99 7200ms (normal 800ms) \u2014 41% timeout rate, vendor incident active"), timestamp: t(29) - 5e4, attributes: { endpoint: "https://checkout.adyen.com", p99Ms: 7200, timeoutRate: 0.41 } }
|
|
724
|
+
],
|
|
725
|
+
originalTag: "infra_downstream_latency",
|
|
726
|
+
originalConfidenceScore: 0.6,
|
|
727
|
+
originalFixHint: "Add circuit breaker around the slow downstream. Increase timeout if the dependency is legitimately slow, or reduce p99 of the dependency."
|
|
728
|
+
},
|
|
729
|
+
// ── 031–035 infra_service_unavailable ───────────────────────────────────
|
|
730
|
+
{
|
|
731
|
+
pid: "seed-031",
|
|
732
|
+
capturedAt: cap(30),
|
|
733
|
+
firedAt: t(30),
|
|
734
|
+
logs: [
|
|
735
|
+
ce("error", "[api] Redis connection refused \u2014 ECONNREFUSED 10.12.0.10:6379. Primary node failed, failover in progress (estimated 90s)", "http://api:8080/api/session", t(30) - 18e3),
|
|
736
|
+
ce("warn", "[api] session store unavailable \u2014 Redis Sentinel failover in progress. Returning 503 for all authenticated requests", "http://api:8080", t(30) - 15e3)
|
|
737
|
+
],
|
|
738
|
+
network: [
|
|
739
|
+
ne("GET", "http://api:8080/api/user/profile", 503, 120, t(30) - 17e3, "session store unavailable"),
|
|
740
|
+
ne("GET", "http://api:8080/api/dashboard", 503, 115, t(30) - 14e3, "session store unavailable")
|
|
741
|
+
],
|
|
742
|
+
contexts: [],
|
|
743
|
+
terminal: [],
|
|
744
|
+
processExits: [],
|
|
745
|
+
ciEvents: [],
|
|
746
|
+
deployments: [],
|
|
747
|
+
infraEvents: [
|
|
748
|
+
{ ...ie("service_unavailable", "api-service", "critical", "Redis primary node ECONNREFUSED \u2014 Sentinel failover in progress (~90s), all session operations failing"), timestamp: t(30) - 18e3, attributes: { endpoint: "redis://10.12.0.10:6379", reason: "redis-sentinel-failover" } }
|
|
749
|
+
],
|
|
750
|
+
originalTag: "infra_service_unavailable",
|
|
751
|
+
originalConfidenceScore: 0.7,
|
|
752
|
+
originalFixHint: "Check k8s pod status: `kubectl get pods -n <namespace>`. Look for CrashLoopBackOff or Pending pods. Review readiness probe failures and recent deploy events."
|
|
753
|
+
},
|
|
754
|
+
{
|
|
755
|
+
pid: "seed-032",
|
|
756
|
+
capturedAt: cap(31),
|
|
757
|
+
firedAt: t(31),
|
|
758
|
+
logs: [
|
|
759
|
+
ce("error", "[search-api] Elasticsearch cluster status RED \u2014 shard [search_index-0][0] unassigned, no copies available. All search requests failing.", "http://search-api:8094/search", t(31) - 25e3),
|
|
760
|
+
ce("error", "[search-api] ClusterBlockException: index [search_index] blocked by cluster block FORBIDDEN/8/index write \u2014 disk usage 95.2%, watermark exceeded", "http://search-api:8094/search", t(31) - 22e3)
|
|
761
|
+
],
|
|
762
|
+
network: [
|
|
763
|
+
ne("GET", "http://search-api:8094/search", 503, 200, t(31) - 22e3, "cluster RED \u2014 no available shards")
|
|
764
|
+
],
|
|
765
|
+
contexts: [],
|
|
766
|
+
terminal: [],
|
|
767
|
+
processExits: [],
|
|
768
|
+
ciEvents: [],
|
|
769
|
+
deployments: [],
|
|
770
|
+
infraEvents: [
|
|
771
|
+
{ ...ie("service_unavailable", "search-api", "critical", "Elasticsearch cluster RED: shard [search_index-0][0] unassigned. Disk 95.2%, high watermark write-blocked"), timestamp: t(31) - 25e3, attributes: { endpoint: "http://elasticsearch:9200", clusterStatus: "red", diskUsagePct: 95 } }
|
|
772
|
+
],
|
|
773
|
+
originalTag: "infra_service_unavailable",
|
|
774
|
+
originalConfidenceScore: 0.7,
|
|
775
|
+
originalFixHint: "Check k8s pod status: `kubectl get pods -n <namespace>`. Look for CrashLoopBackOff or Pending pods. Review readiness probe failures and recent deploy events."
|
|
776
|
+
},
|
|
777
|
+
{
|
|
778
|
+
pid: "seed-033",
|
|
779
|
+
capturedAt: cap(32),
|
|
780
|
+
firedAt: t(32),
|
|
781
|
+
logs: [
|
|
782
|
+
ce("error", "[asset-service] S3 NoSuchBucket: The specified bucket does not exist \u2014 company-assets-prod bucket missing from us-east-1 (region migration in progress?)", "http://asset-service:8098/upload", t(32) - 12e3),
|
|
783
|
+
ce("error", "[asset-service] All S3 operations returning NoSuchBucket \u2014 CDN origin pulling from non-existent bucket. 100% asset upload failure.", "http://asset-service:8098", t(32) - 1e4)
|
|
784
|
+
],
|
|
785
|
+
network: [
|
|
786
|
+
ne("PUT", "https://company-assets-prod.s3.amazonaws.com/uploads/img_abc.jpg", 404, 350, t(32) - 11800, "NoSuchBucket")
|
|
787
|
+
],
|
|
788
|
+
contexts: [],
|
|
789
|
+
terminal: [],
|
|
790
|
+
processExits: [],
|
|
791
|
+
ciEvents: [],
|
|
792
|
+
deployments: [],
|
|
793
|
+
infraEvents: [
|
|
794
|
+
{ ...ie("service_unavailable", "asset-service", "critical", "S3 bucket company-assets-prod missing in us-east-1 \u2014 100% upload failure rate, possible misconfigured region migration"), timestamp: t(32) - 12e3, attributes: { endpoint: "https://s3.amazonaws.com/company-assets-prod", reason: "NoSuchBucket" } }
|
|
795
|
+
],
|
|
796
|
+
originalTag: "infra_service_unavailable",
|
|
797
|
+
originalConfidenceScore: 0.7,
|
|
798
|
+
originalFixHint: "Check k8s pod status: `kubectl get pods -n <namespace>`. Look for CrashLoopBackOff or Pending pods. Review readiness probe failures and recent deploy events."
|
|
799
|
+
},
|
|
800
|
+
{
|
|
801
|
+
pid: "seed-034",
|
|
802
|
+
capturedAt: cap(33),
|
|
803
|
+
firedAt: t(33),
|
|
804
|
+
logs: [
|
|
805
|
+
ce("error", "[event-service] RabbitMQ connection closed unexpectedly: CONNECTION_FORCED - broker forced connection closure with reason 'shutdown'", "http://event-service:8101/publish", t(33) - 22e3),
|
|
806
|
+
ce("error", "[event-service] amqplib: Channel lost \u2014 RabbitMQ broker unreachable at 10.12.0.15:5672. 1847 unacked messages.", "http://event-service:8101", t(33) - 2e4),
|
|
807
|
+
ce("warn", "[event-service] reconnection attempt 5/10 \u2014 RabbitMQ still unavailable", "http://event-service:8101", t(33) - 1e4)
|
|
808
|
+
],
|
|
809
|
+
network: [],
|
|
810
|
+
contexts: [],
|
|
811
|
+
terminal: [],
|
|
812
|
+
processExits: [],
|
|
813
|
+
ciEvents: [],
|
|
814
|
+
deployments: [],
|
|
815
|
+
infraEvents: [
|
|
816
|
+
{ ...ie("service_unavailable", "event-service", "critical", "RabbitMQ broker unreachable at 10.12.0.15:5672 \u2014 broker shutdown, 1847 unacked messages, reconnection failing"), timestamp: t(33) - 22e3, attributes: { endpoint: "amqp://10.12.0.15:5672", unackedMessages: 1847 } }
|
|
817
|
+
],
|
|
818
|
+
originalTag: "infra_service_unavailable",
|
|
819
|
+
originalConfidenceScore: 0.7,
|
|
820
|
+
originalFixHint: "Check k8s pod status: `kubectl get pods -n <namespace>`. Look for CrashLoopBackOff or Pending pods. Review readiness probe failures and recent deploy events."
|
|
821
|
+
},
|
|
822
|
+
{
|
|
823
|
+
pid: "seed-035",
|
|
824
|
+
capturedAt: cap(34),
|
|
825
|
+
firedAt: t(34),
|
|
826
|
+
logs: [
|
|
827
|
+
ce("error", "[api-gateway] upstream connect error or disconnect/reset before headers: auth-service \u2014 0/3 pods passing healthcheck", "http://api-gateway:8080/api/auth", t(34) - 15e3),
|
|
828
|
+
ce("error", "[api-gateway] circuit breaker OPEN for auth-service: 100% failure rate in 60s window (0/3 healthy upstream pods)", "http://api-gateway:8080", t(34) - 12e3)
|
|
829
|
+
],
|
|
830
|
+
network: [
|
|
831
|
+
ne("POST", "http://api-gateway:8080/api/auth/login", 503, 130, t(34) - 14e3, "circuit breaker open \u2014 no healthy upstream")
|
|
832
|
+
],
|
|
833
|
+
contexts: [],
|
|
834
|
+
terminal: [],
|
|
835
|
+
processExits: [],
|
|
836
|
+
ciEvents: [],
|
|
837
|
+
deployments: [],
|
|
838
|
+
infraEvents: [
|
|
839
|
+
{ ...ie("service_unavailable", "api-gateway", "critical", "auth-service 0/3 pods healthy \u2014 circuit breaker OPEN, all login requests failing, CrashLoopBackOff on auth pods"), timestamp: t(34) - 15e3, attributes: { endpoint: "http://auth-service:8081", healthyPods: 0, totalPods: 3 } }
|
|
840
|
+
],
|
|
841
|
+
originalTag: "infra_service_unavailable",
|
|
842
|
+
originalConfidenceScore: 0.7,
|
|
843
|
+
originalFixHint: "Check k8s pod status: `kubectl get pods -n <namespace>`. Look for CrashLoopBackOff or Pending pods. Review readiness probe failures and recent deploy events."
|
|
844
|
+
},
|
|
845
|
+
// ── 036–040 infra_disk_pressure ─────────────────────────────────────────
|
|
846
|
+
{
|
|
847
|
+
pid: "seed-036",
|
|
848
|
+
capturedAt: cap(35),
|
|
849
|
+
firedAt: t(35),
|
|
850
|
+
logs: [
|
|
851
|
+
ce("error", "[api] ENOSPC: no space left on device \u2014 write to /var/log/app/access.log failed (disk 98% full)", "http://api:8080", t(35) - 12e3),
|
|
852
|
+
ce("error", "[api] logger failed: write /var/log/app/error.log: no space left on device \u2014 rotating logs failed, disk full", "http://api:8080", t(35) - 11500),
|
|
853
|
+
ce("warn", "[api] disk usage: /var/log 98% (19.6GB/20GB) \u2014 log rotation has been failing for 6 hours (logrotate misconfigured)", "http://api:8080", t(35) - 1e4)
|
|
854
|
+
],
|
|
855
|
+
network: [],
|
|
856
|
+
contexts: [],
|
|
857
|
+
terminal: [],
|
|
858
|
+
processExits: [],
|
|
859
|
+
ciEvents: [],
|
|
860
|
+
deployments: [],
|
|
861
|
+
infraEvents: [
|
|
862
|
+
{ ...ie("disk_pressure", "api-service", "critical", "/var/log filesystem 98% full (19.6GB/20GB) \u2014 log writes failing ENOSPC, logrotate misconfigured"), timestamp: t(35) - 12e3, attributes: { filesystem: "/var/log", usedPct: 98, usedGB: 19.6, totalGB: 20 } }
|
|
863
|
+
],
|
|
864
|
+
originalTag: "infra_disk_pressure",
|
|
865
|
+
originalConfidenceScore: 0.85,
|
|
866
|
+
originalFixHint: "Check disk usage: `df -h` and `du -sh /var/log/* /tmp/*`. Rotate logs immediately: `journalctl --vacuum-size=500M`. Increase PV size or add log retention policy to prevent recurrence."
|
|
867
|
+
},
|
|
868
|
+
{
|
|
869
|
+
pid: "seed-037",
|
|
870
|
+
capturedAt: cap(36),
|
|
871
|
+
firedAt: t(36),
|
|
872
|
+
logs: [
|
|
873
|
+
ce("error", "[build-worker] ENOSPC: no space left on device \u2014 docker build failed: cannot create layer /tmp/docker-build-abc123 (disk 100% full)", "http://build-worker:8102", t(36) - 8e3),
|
|
874
|
+
ce("error", "[build-worker] /tmp filesystem full (4.0GB/4.0GB) \u2014 Docker layer cache accumulation over 30 days without pruning", "http://build-worker:8102", t(36) - 7500)
|
|
875
|
+
],
|
|
876
|
+
network: [],
|
|
877
|
+
contexts: [],
|
|
878
|
+
terminal: [],
|
|
879
|
+
processExits: [],
|
|
880
|
+
ciEvents: [],
|
|
881
|
+
deployments: [],
|
|
882
|
+
infraEvents: [
|
|
883
|
+
{ ...ie("disk_pressure", "build-worker", "critical", "/tmp filesystem 100% full (4.0GB/4.0GB) \u2014 Docker build layer cache accumulated without pruning for 30 days"), timestamp: t(36) - 8e3, attributes: { filesystem: "/tmp", usedPct: 100, usedGB: 4, totalGB: 4 } }
|
|
884
|
+
],
|
|
885
|
+
originalTag: "infra_disk_pressure",
|
|
886
|
+
originalConfidenceScore: 0.85,
|
|
887
|
+
originalFixHint: "Check disk usage: `df -h` and `du -sh /var/log/* /tmp/*`. Rotate logs immediately: `journalctl --vacuum-size=500M`. Increase PV size or add log retention policy to prevent recurrence."
|
|
888
|
+
},
|
|
889
|
+
{
|
|
890
|
+
pid: "seed-038",
|
|
891
|
+
capturedAt: cap(37),
|
|
892
|
+
firedAt: t(37),
|
|
893
|
+
logs: [
|
|
894
|
+
ce("error", '[postgres] PANIC: could not write to file "pg_wal/00000001000000050000000A": No space left on device', "http://postgres:5432", t(37) - 5e3),
|
|
895
|
+
ce("error", "[postgres] database system is shut down \u2014 WAL write failed, /data filesystem 100% full (PostgreSQL PANIC halt)", "http://postgres:5432", t(37) - 4500)
|
|
896
|
+
],
|
|
897
|
+
network: [],
|
|
898
|
+
contexts: [],
|
|
899
|
+
terminal: [],
|
|
900
|
+
processExits: [],
|
|
901
|
+
ciEvents: [],
|
|
902
|
+
deployments: [],
|
|
903
|
+
infraEvents: [
|
|
904
|
+
{ ...ie("disk_pressure", "postgres", "critical", "PostgreSQL PANIC: WAL write failed \u2014 /data filesystem 100% full, database halted. pg_wal accumulation."), timestamp: t(37) - 5e3, attributes: { filesystem: "/data", usedPct: 100, cause: "pg_wal_accumulation" } }
|
|
905
|
+
],
|
|
906
|
+
originalTag: "infra_disk_pressure",
|
|
907
|
+
originalConfidenceScore: 0.85,
|
|
908
|
+
originalFixHint: "Check disk usage: `df -h` and `du -sh /var/log/* /tmp/*`. Rotate logs immediately: `journalctl --vacuum-size=500M`. Increase PV size or add log retention policy to prevent recurrence."
|
|
909
|
+
},
|
|
910
|
+
{
|
|
911
|
+
pid: "seed-039",
|
|
912
|
+
capturedAt: cap(38),
|
|
913
|
+
firedAt: t(38),
|
|
914
|
+
logs: [
|
|
915
|
+
ce("warn", "[prometheus] TSDB disk usage: 95% (47.5GB/50GB PV) \u2014 retention policy set to 30d but actual data growth exceeds expectation", "http://prometheus:9090", t(38) - 12e4),
|
|
916
|
+
ce("error", "[prometheus] storage compaction failed: no space left on device \u2014 TSDB cannot write new chunks. Metrics collection halted.", "http://prometheus:9090", t(38) - 1e4)
|
|
917
|
+
],
|
|
918
|
+
network: [],
|
|
919
|
+
contexts: [],
|
|
920
|
+
terminal: [],
|
|
921
|
+
processExits: [],
|
|
922
|
+
ciEvents: [],
|
|
923
|
+
deployments: [],
|
|
924
|
+
infraEvents: [
|
|
925
|
+
{ ...ie("disk_pressure", "prometheus", "high", "Prometheus TSDB 95% full (47.5GB/50GB PV) \u2014 compaction failing, metrics collection halted"), timestamp: t(38) - 1e4, attributes: { filesystem: "/prometheus", usedPct: 95, usedGB: 47.5, totalGB: 50 } }
|
|
926
|
+
],
|
|
927
|
+
originalTag: "infra_disk_pressure",
|
|
928
|
+
originalConfidenceScore: 0.85,
|
|
929
|
+
originalFixHint: "Check disk usage: `df -h` and `du -sh /var/log/* /tmp/*`. Rotate logs immediately: `journalctl --vacuum-size=500M`. Increase PV size or add log retention policy to prevent recurrence."
|
|
930
|
+
},
|
|
931
|
+
{
|
|
932
|
+
pid: "seed-040",
|
|
933
|
+
capturedAt: cap(39),
|
|
934
|
+
firedAt: t(39),
|
|
935
|
+
logs: [
|
|
936
|
+
ce("error", "[worker] ENOSPC: no space left on device \u2014 job output file /data/exports/export_20240101.csv cannot be created", "http://worker:8084/export", t(39) - 15e3),
|
|
937
|
+
ce("error", "[worker] /data PersistentVolume 97% full (9.7GB/10GB) \u2014 export jobs failing. 847 failed exports in last hour.", "http://worker:8084", t(39) - 12e3)
|
|
938
|
+
],
|
|
939
|
+
network: [],
|
|
940
|
+
contexts: [],
|
|
941
|
+
terminal: [],
|
|
942
|
+
processExits: [],
|
|
943
|
+
ciEvents: [],
|
|
944
|
+
deployments: [],
|
|
945
|
+
infraEvents: [
|
|
946
|
+
{ ...ie("disk_pressure", "worker-service", "critical", "/data PV 97% full (9.7GB/10GB) \u2014 export job file creation failing ENOSPC, 847 failed exports in last hour"), timestamp: t(39) - 15e3, attributes: { filesystem: "/data", usedPct: 97, usedGB: 9.7, totalGB: 10 } }
|
|
947
|
+
],
|
|
948
|
+
originalTag: "infra_disk_pressure",
|
|
949
|
+
originalConfidenceScore: 0.85,
|
|
950
|
+
originalFixHint: "Check disk usage: `df -h` and `du -sh /var/log/* /tmp/*`. Rotate logs immediately: `journalctl --vacuum-size=500M`. Increase PV size or add log retention policy to prevent recurrence."
|
|
951
|
+
},
|
|
952
|
+
// ── 041–045 infra_queue_backlog ─────────────────────────────────────────
|
|
953
|
+
{
|
|
954
|
+
pid: "seed-041",
|
|
955
|
+
capturedAt: cap(40),
|
|
956
|
+
firedAt: t(40),
|
|
957
|
+
logs: [
|
|
958
|
+
ce("warn", "[order-processor] SQS queue depth: 52847 messages (normal: <100) \u2014 order-processor consumers crashed after deploy a3f8c12, queue backing up", "http://order-processor:8103", t(40) - 36e5),
|
|
959
|
+
ce("error", "[order-processor] SQS ApproximateNumberOfMessages: 52847. Oldest message: 58 minutes. SLA breach imminent (60 min max latency).", "http://order-processor:8103", t(40) - 18e5)
|
|
960
|
+
],
|
|
961
|
+
network: [],
|
|
962
|
+
contexts: [],
|
|
963
|
+
terminal: [],
|
|
964
|
+
processExits: [],
|
|
965
|
+
ciEvents: [],
|
|
966
|
+
deployments: [],
|
|
967
|
+
infraEvents: [
|
|
968
|
+
{ ...ie("queue_backlog", "order-processor", "critical", "SQS order-processing-prod queue depth 52847 (normal <100) \u2014 consumer crash after deploy a3f8c12, oldest message 58min"), timestamp: t(40) - 18e5, attributes: { queue: "order-processing-prod", depth: 52847, oldestMessageAgeSec: 3480 } }
|
|
969
|
+
],
|
|
970
|
+
originalTag: "infra_queue_backlog",
|
|
971
|
+
originalConfidenceScore: 0.65,
|
|
972
|
+
originalFixHint: "Scale up consumers (increase replica count or partition count). Check for slow message processing \u2014 add timing metrics around handler logic. Verify no consumer is in an error loop silently dropping messages."
|
|
973
|
+
},
|
|
974
|
+
{
|
|
975
|
+
pid: "seed-042",
|
|
976
|
+
capturedAt: cap(41),
|
|
977
|
+
firedAt: t(41),
|
|
978
|
+
logs: [
|
|
979
|
+
ce("warn", "[event-service] Kafka consumer group event-processor-prod lag: 2147483 messages \u2014 partition 0-7 all growing. Consumers alive but processing rate fallen 95%.", "http://event-service:8099", t(41) - 72e5),
|
|
980
|
+
ce("error", "[event-service] Kafka consumer lag exceeds alert threshold 500000: current lag 2147483. Message age on partition 0: 4.2 hours.", "http://event-service:8099", t(41) - 36e5)
|
|
981
|
+
],
|
|
982
|
+
network: [],
|
|
983
|
+
contexts: [],
|
|
984
|
+
terminal: [],
|
|
985
|
+
processExits: [],
|
|
986
|
+
ciEvents: [],
|
|
987
|
+
deployments: [],
|
|
988
|
+
infraEvents: [
|
|
989
|
+
{ ...ie("queue_backlog", "event-service", "critical", "Kafka consumer group event-processor-prod lag 2.1M messages \u2014 8 partitions, processing rate -95%, oldest message 4.2h"), timestamp: t(41) - 36e5, attributes: { topic: "application-events", consumerGroup: "event-processor-prod", lag: 2147483, oldestMessageAgeSec: 15120 } }
|
|
990
|
+
],
|
|
991
|
+
originalTag: "infra_queue_backlog",
|
|
992
|
+
originalConfidenceScore: 0.65,
|
|
993
|
+
originalFixHint: "Scale up consumers (increase replica count or partition count). Check for slow message processing \u2014 add timing metrics around handler logic. Verify no consumer is in an error loop silently dropping messages."
|
|
994
|
+
},
|
|
995
|
+
{
|
|
996
|
+
pid: "seed-043",
|
|
997
|
+
capturedAt: cap(42),
|
|
998
|
+
firedAt: t(42),
|
|
999
|
+
logs: [
|
|
1000
|
+
ce("warn", "[celery] task queue depth: 84240 tasks in send_email queue (normal: <500). Workers processing at 12 tasks/min (normal: 2400/min).", "http://celery-worker:8104", t(42) - 54e5),
|
|
1001
|
+
ce("error", "[celery] send_email queue: ETA breach \u2014 84240 tasks pending, 97% will miss 10-minute delivery SLA at current throughput", "http://celery-worker:8104", t(42) - 18e5)
|
|
1002
|
+
],
|
|
1003
|
+
network: [],
|
|
1004
|
+
contexts: [],
|
|
1005
|
+
terminal: [],
|
|
1006
|
+
processExits: [],
|
|
1007
|
+
ciEvents: [],
|
|
1008
|
+
deployments: [],
|
|
1009
|
+
infraEvents: [
|
|
1010
|
+
{ ...ie("queue_backlog", "celery-worker", "high", "Celery send_email queue 84240 tasks (normal <500) \u2014 processing rate 12/min vs normal 2400/min, SLA breach imminent"), timestamp: t(42) - 18e5, attributes: { queue: "send_email", depth: 84240, processingRatePerMin: 12, normalRatePerMin: 2400 } }
|
|
1011
|
+
],
|
|
1012
|
+
originalTag: "infra_queue_backlog",
|
|
1013
|
+
originalConfidenceScore: 0.65,
|
|
1014
|
+
originalFixHint: "Scale up consumers (increase replica count or partition count). Check for slow message processing \u2014 add timing metrics around handler logic. Verify no consumer is in an error loop silently dropping messages."
|
|
1015
|
+
},
|
|
1016
|
+
{
|
|
1017
|
+
pid: "seed-044",
|
|
1018
|
+
capturedAt: cap(43),
|
|
1019
|
+
firedAt: t(43),
|
|
1020
|
+
logs: [
|
|
1021
|
+
ce("warn", "[payment-events] RabbitMQ dead letter queue growing: payment-events.dlq has 12847 messages (normal: 0). Consumer rejecting messages without requeue.", "http://payment-service:8082", t(43) - 72e5),
|
|
1022
|
+
ce("error", "[payment-events] DLQ alarm: 12847 dead-lettered payment events \u2014 consumer throwing unhandled DeserializationException on schema change in v2.1.0 deploy", "http://payment-service:8082", t(43) - 36e5)
|
|
1023
|
+
],
|
|
1024
|
+
network: [],
|
|
1025
|
+
contexts: [],
|
|
1026
|
+
terminal: [],
|
|
1027
|
+
processExits: [],
|
|
1028
|
+
ciEvents: [],
|
|
1029
|
+
deployments: [],
|
|
1030
|
+
infraEvents: [
|
|
1031
|
+
{ ...ie("queue_backlog", "payment-service", "high", "RabbitMQ payment-events DLQ: 12847 messages \u2014 consumer DeserializationException from schema change in v2.1.0, messages unprocessable"), timestamp: t(43) - 36e5, attributes: { queue: "payment-events.dlq", depth: 12847, reason: "DeserializationException" } }
|
|
1032
|
+
],
|
|
1033
|
+
originalTag: "infra_queue_backlog",
|
|
1034
|
+
originalConfidenceScore: 0.65,
|
|
1035
|
+
originalFixHint: "Scale up consumers (increase replica count or partition count). Check for slow message processing \u2014 add timing metrics around handler logic. Verify no consumer is in an error loop silently dropping messages."
|
|
1036
|
+
},
|
|
1037
|
+
{
|
|
1038
|
+
pid: "seed-045",
|
|
1039
|
+
capturedAt: cap(44),
|
|
1040
|
+
firedAt: t(44),
|
|
1041
|
+
logs: [
|
|
1042
|
+
ce("warn", "[background-jobs] Redis queue depth: RPOPLPUSH background_jobs 47820 keys \u2014 6 workers running but not draining", "http://background-jobs:8105", t(44) - 27e5),
|
|
1043
|
+
ce("error", "[background-jobs] Redis background_jobs queue: 47820 pending jobs, oldest job 45min. Workers appear stuck \u2014 no heartbeat for 2 of 6 workers.", "http://background-jobs:8105", t(44) - 18e5)
|
|
1044
|
+
],
|
|
1045
|
+
network: [],
|
|
1046
|
+
contexts: [],
|
|
1047
|
+
terminal: [],
|
|
1048
|
+
processExits: [],
|
|
1049
|
+
ciEvents: [],
|
|
1050
|
+
deployments: [],
|
|
1051
|
+
infraEvents: [
|
|
1052
|
+
{ ...ie("queue_backlog", "background-jobs", "high", "Redis background_jobs queue: 47820 pending jobs, oldest 45min. 2/6 workers not heartbeating \u2014 likely deadlocked"), timestamp: t(44) - 18e5, attributes: { queue: "background_jobs", depth: 47820, stuckWorkers: 2, totalWorkers: 6 } }
|
|
1053
|
+
],
|
|
1054
|
+
originalTag: "infra_queue_backlog",
|
|
1055
|
+
originalConfidenceScore: 0.65,
|
|
1056
|
+
originalFixHint: "Scale up consumers (increase replica count or partition count). Check for slow message processing \u2014 add timing metrics around handler logic. Verify no consumer is in an error loop silently dropping messages."
|
|
1057
|
+
},
|
|
1058
|
+
// ── 046–050 Multi-signal cascade incidents ───────────────────────────────
|
|
1059
|
+
// These represent real incidents where multiple failure modes compound.
|
|
1060
|
+
{
|
|
1061
|
+
pid: "seed-046",
|
|
1062
|
+
capturedAt: cap(45),
|
|
1063
|
+
firedAt: t(45),
|
|
1064
|
+
logs: [
|
|
1065
|
+
ce("error", "[api] database connection pool exhausted \u2014 0/20 connections (pool leak suspected in long-running report query)", "http://api:8080/api/reports", t(45) - 3e4),
|
|
1066
|
+
ce("warn", "[worker] memory pressure: RSS 490MB/512MB \u2014 report worker accumulating large result sets before pool exhaustion", "http://worker:8084/report", t(45) - 25e3),
|
|
1067
|
+
ce("error", "[api] ENOSPC writing report cache to /tmp \u2014 /tmp filesystem at 98% (report files not cleaned up)", "http://api:8080", t(45) - 2e4)
|
|
1068
|
+
],
|
|
1069
|
+
network: [
|
|
1070
|
+
ne("GET", "http://api:8080/api/reports", 503, 30100, t(45) - 29500, "connection pool exhausted")
|
|
1071
|
+
],
|
|
1072
|
+
contexts: [],
|
|
1073
|
+
terminal: [],
|
|
1074
|
+
processExits: [],
|
|
1075
|
+
ciEvents: [],
|
|
1076
|
+
deployments: [],
|
|
1077
|
+
infraEvents: [
|
|
1078
|
+
{ ...ie("db_connection_pool_exhausted", "api-service", "critical", "pg-pool exhausted 0/20 \u2014 long-running report query holding connections"), timestamp: t(45) - 3e4, attributes: { endpoint: "postgres://10.12.0.5:5432/app", poolMax: 20 } },
|
|
1079
|
+
{ ...ie("memory_pressure", "worker-service", "high", "RSS 490MB/512MB \u2014 report worker building large result sets in memory"), timestamp: t(45) - 25e3, attributes: { memoryLimitMb: 512, rssUsedMb: 490 } },
|
|
1080
|
+
{ ...ie("disk_pressure", "api-service", "high", "/tmp 98% full \u2014 report cache files not cleaned up after pool exhaustion"), timestamp: t(45) - 2e4, attributes: { filesystem: "/tmp", usedPct: 98 } }
|
|
1081
|
+
],
|
|
1082
|
+
originalTag: "infra_disk_pressure",
|
|
1083
|
+
originalConfidenceScore: 0.85,
|
|
1084
|
+
originalFixHint: "Check disk usage: `df -h` and `du -sh /var/log/* /tmp/*`. Rotate logs immediately: `journalctl --vacuum-size=500M`. Increase PV size or add log retention policy to prevent recurrence."
|
|
1085
|
+
},
|
|
1086
|
+
{
|
|
1087
|
+
pid: "seed-047",
|
|
1088
|
+
capturedAt: cap(46),
|
|
1089
|
+
firedAt: t(46),
|
|
1090
|
+
logs: [
|
|
1091
|
+
ce("error", "[k8s] OOMKilled: search-service container (2Gi limit) \u2014 Elasticsearch query loading full index segment into heap", "http://search-service:8094", t(46) - 2e4),
|
|
1092
|
+
ce("error", "[search-api] Elasticsearch cluster status degraded: search-service pod OOMKilled, 1 of 3 replicas unavailable", "http://search-api:8094/search", t(46) - 15e3),
|
|
1093
|
+
ce("warn", "[search-api] search latency degraded: 2 replicas serving traffic \u2014 p99 latency 4200ms (normal: 200ms)", "http://search-api:8094/search", t(46) - 1e4)
|
|
1094
|
+
],
|
|
1095
|
+
network: [
|
|
1096
|
+
ne("GET", "http://search-api:8094/search", 503, 5e3, t(46) - 14e3, "upstream OOMKilled")
|
|
1097
|
+
],
|
|
1098
|
+
contexts: [],
|
|
1099
|
+
terminal: [],
|
|
1100
|
+
processExits: [
|
|
1101
|
+
{ type: "process_exit", process: "search-service", exitCode: 137, reason: "oom", signal: "SIGKILL", memoryLimitBytes: 2147483648, timestamp: t(46) - 2e4 }
|
|
1102
|
+
],
|
|
1103
|
+
ciEvents: [],
|
|
1104
|
+
deployments: [],
|
|
1105
|
+
infraEvents: [
|
|
1106
|
+
{ ...ie("oom_kill", "search-service", "critical", "OOMKilled: search-service 2Gi limit exceeded \u2014 Elasticsearch heap loading full index segment for large query"), timestamp: t(46) - 2e4, attributes: { exitCode: 137, memoryLimitMb: 2048 } },
|
|
1107
|
+
{ ...ie("downstream_latency_spike", "search-api", "high", "p99 latency 4200ms (normal 200ms) \u2014 2/3 replicas serving after OOMKill"), timestamp: t(46) - 1e4, attributes: { endpoint: "http://search-service:8094", p99Ms: 4200 } }
|
|
1108
|
+
],
|
|
1109
|
+
originalTag: "infra_oom_kill",
|
|
1110
|
+
originalConfidenceScore: 0.9,
|
|
1111
|
+
originalFixHint: "Increase container memory limit in k8s manifest / docker-compose. Heap-profile with `node --inspect` or `py-spy top` to find the leak. Check for unbounded caches or retained buffers in recent deploys."
|
|
1112
|
+
},
|
|
1113
|
+
{
|
|
1114
|
+
pid: "seed-048",
|
|
1115
|
+
capturedAt: cap(47),
|
|
1116
|
+
firedAt: t(47),
|
|
1117
|
+
logs: [
|
|
1118
|
+
ce("error", "[ingress] TLS certificate expired for api.company.com \u2014 all HTTPS traffic returning 526 (SSL handshake failed)", "https://api.company.com", t(47) - 1e4),
|
|
1119
|
+
ce("error", "[payments] Stripe webhook delivery failing: upstream 526 SSL error from api.company.com/webhooks/stripe \u2014 Stripe retrying", "http://payments:8082", t(47) - 8e3),
|
|
1120
|
+
ce("warn", "[payments] 284 Stripe webhooks queued for retry \u2014 cert expiry causing webhook backlog. Stripe retry window: 72 hours.", "http://payments:8082", t(47) - 5e3)
|
|
1121
|
+
],
|
|
1122
|
+
network: [
|
|
1123
|
+
ne("GET", "https://api.company.com/health", 526, 150, t(47) - 9500, "SSL certificate expired"),
|
|
1124
|
+
ne("POST", "https://api.company.com/webhooks/stripe", 526, 145, t(47) - 8500, "SSL certificate expired")
|
|
1125
|
+
],
|
|
1126
|
+
contexts: [],
|
|
1127
|
+
terminal: [],
|
|
1128
|
+
processExits: [],
|
|
1129
|
+
ciEvents: [],
|
|
1130
|
+
deployments: [],
|
|
1131
|
+
infraEvents: [
|
|
1132
|
+
{ ...ie("certificate_expiry", "api-gateway", "critical", "TLS cert expired for api.company.com \u2014 all HTTPS traffic failing 526, Stripe webhook backlog building"), timestamp: t(47) - 1e4, attributes: { domain: "api.company.com", sideEffect: "stripe-webhook-backlog", webhookBacklog: 284 } },
|
|
1133
|
+
{ ...ie("queue_backlog", "payments-service", "high", "284 Stripe webhooks queued for retry \u2014 cert expiry blocking delivery, 72h retry window"), timestamp: t(47) - 5e3, attributes: { queue: "stripe-webhooks", depth: 284 } }
|
|
1134
|
+
],
|
|
1135
|
+
originalTag: "infra_certificate_expiry",
|
|
1136
|
+
originalConfidenceScore: 0.85,
|
|
1137
|
+
originalFixHint: "Check expiry: `openssl s_client -connect <host>:443 2>/dev/null | openssl x509 -noout -dates` Renew via Let's Encrypt (`certbot renew`) or your CA. Verify the cert covers the target hostname (SAN list)."
|
|
1138
|
+
},
|
|
1139
|
+
{
|
|
1140
|
+
pid: "seed-049",
|
|
1141
|
+
capturedAt: cap(48),
|
|
1142
|
+
firedAt: t(48),
|
|
1143
|
+
logs: [
|
|
1144
|
+
ce("warn", "[checkout] Adyen rate limit approaching: 820/1000 requests in current 60s window", "http://checkout:8100/checkout", t(48) - 12e4),
|
|
1145
|
+
ce("error", "[checkout] Adyen 429 Too Many Requests \u2014 payment processing rate limited. Retry storm in progress: 3 workers retrying same requests.", "http://checkout:8100/checkout", t(48) - 6e4),
|
|
1146
|
+
ce("warn", "[checkout] Kafka payment-events queue backing up: 8420 messages \u2014 rate-limited Adyen responses queuing for retry", "http://checkout:8100", t(48) - 3e4)
|
|
1147
|
+
],
|
|
1148
|
+
network: [
|
|
1149
|
+
ne("POST", "https://checkout.adyen.com/v68/payments", 429, 95, t(48) - 6e4, "Too Many Requests"),
|
|
1150
|
+
ne("POST", "https://checkout.adyen.com/v68/payments", 429, 88, t(48) - 55e3, "Too Many Requests")
|
|
1151
|
+
],
|
|
1152
|
+
contexts: [],
|
|
1153
|
+
terminal: [],
|
|
1154
|
+
processExits: [],
|
|
1155
|
+
ciEvents: [],
|
|
1156
|
+
deployments: [],
|
|
1157
|
+
infraEvents: [
|
|
1158
|
+
{ ...ie("rate_limit_cascade", "checkout-service", "high", "Adyen 429: 1000 req/60s limit hit \u2014 retry storm amplifying rate limiting, 3 workers retrying same requests"), timestamp: t(48) - 6e4, attributes: { endpoint: "https://checkout.adyen.com", rateLimit: 1e3, rateLimitWindowSec: 60 } },
|
|
1159
|
+
{ ...ie("queue_backlog", "checkout-service", "medium", "Kafka payment-events backlog 8420 messages \u2014 rate-limited Adyen responses accumulating"), timestamp: t(48) - 3e4, attributes: { queue: "payment-events", depth: 8420 } }
|
|
1160
|
+
],
|
|
1161
|
+
originalTag: "infra_rate_limit_cascade",
|
|
1162
|
+
originalConfidenceScore: 0.75,
|
|
1163
|
+
originalFixHint: "Add exponential backoff with jitter to the retry loop. Honour the `Retry-After` response header. Consider a token-bucket circuit breaker to shed load before hitting the upstream limit."
|
|
1164
|
+
},
|
|
1165
|
+
{
|
|
1166
|
+
pid: "seed-050",
|
|
1167
|
+
capturedAt: cap(49),
|
|
1168
|
+
firedAt: t(49),
|
|
1169
|
+
logs: [
|
|
1170
|
+
ce("error", "[postgres] autovacuum: found unexpected data in FSM header of relation 16423/app/orders \u2014 disk pressure causing vacuum failures", "http://postgres:5432", t(49) - 72e5),
|
|
1171
|
+
ce("error", '[postgres] ERROR: could not extend file "base/16423/1294784": No space left on device. HINT: Check free disk space.', "http://postgres:5432", t(49) - 36e5),
|
|
1172
|
+
ce("error", "[api] connection pool exhausted \u2014 postgres cannot extend relation, all queries failing with ENOSPC", "http://api:8080/api", t(49) - 35e5)
|
|
1173
|
+
],
|
|
1174
|
+
network: [
|
|
1175
|
+
ne("GET", "http://api:8080/api/users", 503, 30100, t(49) - 34e5, "postgres ENOSPC"),
|
|
1176
|
+
ne("POST", "http://api:8080/api/orders", 503, 30050, t(49) - 33e5, "postgres ENOSPC")
|
|
1177
|
+
],
|
|
1178
|
+
contexts: [],
|
|
1179
|
+
terminal: [],
|
|
1180
|
+
processExits: [],
|
|
1181
|
+
ciEvents: [],
|
|
1182
|
+
deployments: [],
|
|
1183
|
+
infraEvents: [
|
|
1184
|
+
{ ...ie("disk_pressure", "postgres", "critical", "PostgreSQL ENOSPC: cannot extend relation \u2014 /data PV full, table growth exceeded PV capacity"), timestamp: t(49) - 36e5, attributes: { filesystem: "/data", cause: "relation_extension_failed", table: "orders" } },
|
|
1185
|
+
{ ...ie("db_connection_pool_exhausted", "api-service", "critical", "pg-pool exhausted \u2014 postgres cannot extend relation, all query slots blocked by ENOSPC errors"), timestamp: t(49) - 35e5, attributes: { endpoint: "postgres://10.12.0.5:5432/app", poolMax: 20 } }
|
|
1186
|
+
],
|
|
1187
|
+
originalTag: "infra_disk_pressure",
|
|
1188
|
+
originalConfidenceScore: 0.85,
|
|
1189
|
+
originalFixHint: "Check disk usage: `df -h` and `du -sh /var/log/* /tmp/*`. Rotate logs immediately: `journalctl --vacuum-size=500M`. Increase PV size or add log retention policy to prevent recurrence."
|
|
1190
|
+
}
|
|
1191
|
+
];
|
|
1192
|
+
function loadSeedCorpus() {
|
|
1193
|
+
let loaded = 0;
|
|
1194
|
+
let skipped = 0;
|
|
1195
|
+
try {
|
|
1196
|
+
fs.mkdirSync(REPLAY_DIR, { recursive: true });
|
|
1197
|
+
} catch (err) {
|
|
1198
|
+
logger.warn({ err }, "seed-corpus: cannot create replay dir");
|
|
1199
|
+
return { loaded: 0, skipped: SNAPSHOTS.length };
|
|
1200
|
+
}
|
|
1201
|
+
for (const snap of SNAPSHOTS) {
|
|
1202
|
+
const dest = path.join(REPLAY_DIR, `${snap.pid}.json`);
|
|
1203
|
+
if (fs.existsSync(dest)) {
|
|
1204
|
+
skipped++;
|
|
1205
|
+
continue;
|
|
1206
|
+
}
|
|
1207
|
+
try {
|
|
1208
|
+
fs.writeFileSync(dest, JSON.stringify(snap), "utf8");
|
|
1209
|
+
loaded++;
|
|
1210
|
+
} catch (err) {
|
|
1211
|
+
logger.warn({ err, pid: snap.pid }, "seed-corpus: failed to write snapshot");
|
|
1212
|
+
skipped++;
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
if (loaded > 0) {
|
|
1216
|
+
logger.info({ loaded, skipped }, "seed-corpus: loaded");
|
|
1217
|
+
}
|
|
1218
|
+
return { loaded, skipped };
|
|
1219
|
+
}
|
|
1220
|
+
const SEED_COUNT = SNAPSHOTS.length;
|
|
1221
|
+
export {
|
|
1222
|
+
SEED_COUNT,
|
|
1223
|
+
loadSeedCorpus
|
|
1224
|
+
};
|