@tellagen/mcp-server 0.1.3 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +208 -12
- package/SKILL.md +395 -15
- package/contracts/runbooks/manifest.json +47 -0
- package/contracts/runbooks/v1/fixtures/global-runbook.json +7 -0
- package/contracts/runbooks/v1/runbook-definition.schema.json +25 -0
- package/contracts/runbooks/v2/compiled-plan.schema.json +40 -0
- package/contracts/runbooks/v2/fixtures/filesystem-pressure.json +181 -0
- package/contracts/runbooks/v2/fixtures/grafana-error-rate-plan.json +50 -0
- package/contracts/runbooks/v2/fixtures/grafana-error-rate.json +58 -0
- package/contracts/runbooks/v2/fixtures/timeline-event-blocks-rejected.json +90 -0
- package/contracts/runbooks/v2/fixtures/timeline-event-with-blocks.json +61 -0
- package/contracts/runbooks/v2/runbook-definition.schema.json +377 -0
- package/contracts/runbooks/v2/timeline-event-blocks.schema.json +480 -0
- package/contracts/tellagen-runbook-contracts.json +8 -0
- package/dist/auth.d.ts +2 -1
- package/dist/auth.js +2 -1
- package/dist/auth.js.map +1 -1
- package/dist/client.d.ts +20 -0
- package/dist/client.js +81 -12
- package/dist/client.js.map +1 -1
- package/dist/diagnosis-structure.d.ts +1416 -0
- package/dist/diagnosis-structure.js +656 -0
- package/dist/diagnosis-structure.js.map +1 -0
- package/dist/errors.d.ts +2 -1
- package/dist/errors.js +4 -1
- package/dist/errors.js.map +1 -1
- package/dist/figures.d.ts +2336 -0
- package/dist/figures.js +1288 -0
- package/dist/figures.js.map +1 -0
- package/dist/image-info.d.ts +14 -0
- package/dist/image-info.js +65 -0
- package/dist/image-info.js.map +1 -0
- package/dist/index.js +2 -6
- package/dist/index.js.map +1 -1
- package/dist/server.js +66 -7
- package/dist/server.js.map +1 -1
- package/dist/source-links.d.ts +101 -0
- package/dist/source-links.js +374 -0
- package/dist/source-links.js.map +1 -0
- package/dist/start.d.ts +11 -0
- package/dist/start.js +14 -0
- package/dist/start.js.map +1 -0
- package/dist/tools/attachments.d.ts +19 -0
- package/dist/tools/attachments.js +177 -0
- package/dist/tools/attachments.js.map +1 -0
- package/dist/tools/causes.d.ts +50 -0
- package/dist/tools/causes.js +97 -0
- package/dist/tools/causes.js.map +1 -0
- package/dist/tools/datasources.d.ts +3 -0
- package/dist/tools/datasources.js +325 -0
- package/dist/tools/datasources.js.map +1 -0
- package/dist/tools/diagnostics.d.ts +3 -0
- package/dist/tools/diagnostics.js +113 -0
- package/dist/tools/diagnostics.js.map +1 -0
- package/dist/tools/figures.d.ts +8 -0
- package/dist/tools/figures.js +209 -0
- package/dist/tools/figures.js.map +1 -0
- package/dist/tools/findings.js +85 -7
- package/dist/tools/findings.js.map +1 -1
- package/dist/tools/incidents.js +44 -4
- package/dist/tools/incidents.js.map +1 -1
- package/dist/tools/index.js +14 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/investigations.js +546 -21
- package/dist/tools/investigations.js.map +1 -1
- package/dist/tools/postmortem-runbooks.d.ts +786 -0
- package/dist/tools/postmortem-runbooks.js +330 -0
- package/dist/tools/postmortem-runbooks.js.map +1 -0
- package/dist/tools/source-links.d.ts +47 -0
- package/dist/tools/source-links.js +302 -0
- package/dist/tools/source-links.js.map +1 -0
- package/dist/tools/timeline.js +178 -13
- package/dist/tools/timeline.js.map +1 -1
- package/dist/version.d.ts +2 -0
- package/dist/version.js +12 -0
- package/dist/version.js.map +1 -0
- package/package.json +3 -2
- package/server.json +15 -1
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://tellagen.dev/contracts/runbooks/v1/runbook-definition.schema.json",
|
|
4
|
+
"title": "Tellagen legacy investigation runbook definition",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": [
|
|
8
|
+
"schema_version",
|
|
9
|
+
"applies_to",
|
|
10
|
+
"required_sources",
|
|
11
|
+
"safety_level",
|
|
12
|
+
"body_markdown"
|
|
13
|
+
],
|
|
14
|
+
"properties": {
|
|
15
|
+
"schema_version": {"const": "v1_markdown"},
|
|
16
|
+
"applies_to": {"type": "object"},
|
|
17
|
+
"required_sources": {
|
|
18
|
+
"type": "array",
|
|
19
|
+
"items": {"type": "string", "minLength": 1, "maxLength": 256},
|
|
20
|
+
"maxItems": 64
|
|
21
|
+
},
|
|
22
|
+
"safety_level": {"type": "string", "minLength": 1, "maxLength": 64},
|
|
23
|
+
"body_markdown": {"type": "string", "minLength": 1, "maxLength": 131072}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://tellagen.dev/contracts/runbooks/v2/compiled-plan.schema.json",
|
|
4
|
+
"title": "Tellagen compiled investigation runbook plan",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": [
|
|
8
|
+
"schema_version",
|
|
9
|
+
"entry_check_ids",
|
|
10
|
+
"source_requirements",
|
|
11
|
+
"checks",
|
|
12
|
+
"escalation",
|
|
13
|
+
"completion"
|
|
14
|
+
],
|
|
15
|
+
"properties": {
|
|
16
|
+
"schema_version": {"const": "v2"},
|
|
17
|
+
"entry_check_ids": {
|
|
18
|
+
"type": "array",
|
|
19
|
+
"minItems": 1,
|
|
20
|
+
"maxItems": 16,
|
|
21
|
+
"uniqueItems": true,
|
|
22
|
+
"items": {"$ref": "runbook-definition.schema.json#/$defs/stableID"}
|
|
23
|
+
},
|
|
24
|
+
"source_requirements": {
|
|
25
|
+
"type": "array",
|
|
26
|
+
"minItems": 1,
|
|
27
|
+
"maxItems": 16,
|
|
28
|
+
"items": {"$ref": "runbook-definition.schema.json#/$defs/sourceRequirement"}
|
|
29
|
+
},
|
|
30
|
+
"checks": {
|
|
31
|
+
"type": "array",
|
|
32
|
+
"minItems": 1,
|
|
33
|
+
"maxItems": 64,
|
|
34
|
+
"items": {"$ref": "runbook-definition.schema.json#/$defs/check"}
|
|
35
|
+
},
|
|
36
|
+
"escalation": {"$ref": "runbook-definition.schema.json#/$defs/escalation"},
|
|
37
|
+
"remediation": {"$ref": "runbook-definition.schema.json#/$defs/remediation"},
|
|
38
|
+
"completion": {"$ref": "runbook-definition.schema.json#/$defs/completion"}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "v2",
|
|
3
|
+
"match": {
|
|
4
|
+
"scope": "context",
|
|
5
|
+
"symptoms": ["disk_pressure", "low disk alert"],
|
|
6
|
+
"priority": 95
|
|
7
|
+
},
|
|
8
|
+
"source_requirements": [
|
|
9
|
+
{
|
|
10
|
+
"id": "filesystem-metrics",
|
|
11
|
+
"source_key": "grafana",
|
|
12
|
+
"capabilities": ["prometheus:query"],
|
|
13
|
+
"required": true,
|
|
14
|
+
"on_unavailable": {"action": "record_gap"}
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"id": "infrastructure-docs",
|
|
18
|
+
"source_key": "github",
|
|
19
|
+
"capabilities": ["repos:read"],
|
|
20
|
+
"required": true,
|
|
21
|
+
"on_unavailable": {"action": "record_gap"}
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"id": "host-filesystem",
|
|
25
|
+
"source_key": "local_host",
|
|
26
|
+
"capabilities": [
|
|
27
|
+
"host:filesystem_inventory",
|
|
28
|
+
"host:path_mount_mapping",
|
|
29
|
+
"host:filesystem_consumers"
|
|
30
|
+
],
|
|
31
|
+
"required": true,
|
|
32
|
+
"on_unavailable": {"action": "record_gap"}
|
|
33
|
+
}
|
|
34
|
+
],
|
|
35
|
+
"entry_check_ids": ["identify-alert-filesystem"],
|
|
36
|
+
"checks": [
|
|
37
|
+
{
|
|
38
|
+
"id": "identify-alert-filesystem",
|
|
39
|
+
"intent": "Identify the exact host, device, mountpoint, filesystem type, capacity, available bytes, and trend that triggered the alert",
|
|
40
|
+
"source_requirement_id": "filesystem-metrics",
|
|
41
|
+
"safety": "read_only",
|
|
42
|
+
"invocation": {
|
|
43
|
+
"kind": "query",
|
|
44
|
+
"capability": "prometheus:query",
|
|
45
|
+
"template": "(node_filesystem_avail_bytes{fstype=~\"ext4|xfs\"} / node_filesystem_size_bytes{fstype=~\"ext4|xfs\"}) * 100",
|
|
46
|
+
"parameters": {
|
|
47
|
+
"time_range": {"type": "time_range", "source": "incident.time_range", "target": "request.range"}
|
|
48
|
+
},
|
|
49
|
+
"bounds": {"lookback": "6h", "step": "1m", "max_points": 360}
|
|
50
|
+
},
|
|
51
|
+
"evidence_contract": {
|
|
52
|
+
"kind": "timeseries",
|
|
53
|
+
"required_fields": ["artifact", "query", "time_range", "resource_identity", "result_summary"]
|
|
54
|
+
},
|
|
55
|
+
"confirms_when": "A series identifies one alerting host, device, and mountpoint with less than the configured free-space threshold",
|
|
56
|
+
"falsifies_when": "No matching filesystem series breached the alert threshold during the incident window",
|
|
57
|
+
"on_confirm": ["read-infrastructure-topology"],
|
|
58
|
+
"on_falsify": ["read-infrastructure-topology"],
|
|
59
|
+
"on_error": {"action": "record_gap"}
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"id": "read-infrastructure-topology",
|
|
63
|
+
"intent": "Read the configured infrastructure repository to identify the approved host route and candidate service storage paths",
|
|
64
|
+
"source_requirement_id": "infrastructure-docs",
|
|
65
|
+
"safety": "read_only",
|
|
66
|
+
"invocation": {
|
|
67
|
+
"kind": "query",
|
|
68
|
+
"capability": "repos:read",
|
|
69
|
+
"template": "Read the configured infrastructure repository metadata, monitoring stack README, host inventory policy, Compose topology, and production inventory. Return the approved host identity and candidate service storage paths without reading credentials.",
|
|
70
|
+
"parameters": {},
|
|
71
|
+
"bounds": {"lookback": "5m", "step": "5m", "max_points": 1}
|
|
72
|
+
},
|
|
73
|
+
"evidence_contract": {
|
|
74
|
+
"kind": "document",
|
|
75
|
+
"required_fields": ["artifact", "query", "result_summary"]
|
|
76
|
+
},
|
|
77
|
+
"confirms_when": "Reviewed documentation identifies an approved host route and candidate storage paths for the alerted service",
|
|
78
|
+
"falsifies_when": "Reviewed documentation explicitly shows the suspected service does not run on the alerting host",
|
|
79
|
+
"on_confirm": ["map-runtime-filesystems"],
|
|
80
|
+
"on_falsify": ["map-runtime-filesystems"],
|
|
81
|
+
"on_error": {"action": "record_gap"}
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"id": "map-runtime-filesystems",
|
|
85
|
+
"intent": "Map the alerted filesystem and every suspected service storage path to runtime host, device, and mountpoint identities",
|
|
86
|
+
"source_requirement_id": "host-filesystem",
|
|
87
|
+
"safety": "read_only",
|
|
88
|
+
"invocation": {
|
|
89
|
+
"kind": "query",
|
|
90
|
+
"capability": "host:path_mount_mapping",
|
|
91
|
+
"template": "Use the reviewed bounded host evidence wrapper to map root, monitoring, and container storage paths to host, device, mountpoint, and filesystem identities. Compare candidates only with explicit resource identity keys.",
|
|
92
|
+
"parameters": {},
|
|
93
|
+
"bounds": {"lookback": "5m", "step": "5m", "max_points": 1}
|
|
94
|
+
},
|
|
95
|
+
"evidence_contract": {
|
|
96
|
+
"kind": "command_output",
|
|
97
|
+
"required_fields": ["artifact", "query", "resource_identity", "join_basis", "result_summary"]
|
|
98
|
+
},
|
|
99
|
+
"confirms_when": "The suspected consumer maps to the same host, device, and mountpoint as the alert, with the join keys recorded",
|
|
100
|
+
"falsifies_when": "The suspected consumer explicitly maps to a different device or mountpoint on the alerting host",
|
|
101
|
+
"on_confirm": ["find-alerted-filesystem-consumers"],
|
|
102
|
+
"on_falsify": ["find-alerted-filesystem-consumers"],
|
|
103
|
+
"on_error": {"action": "record_gap"}
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
"id": "find-alerted-filesystem-consumers",
|
|
107
|
+
"intent": "Identify bounded top-level consumers on the alerted filesystem without crossing mount boundaries",
|
|
108
|
+
"source_requirement_id": "host-filesystem",
|
|
109
|
+
"safety": "read_only",
|
|
110
|
+
"invocation": {
|
|
111
|
+
"kind": "query",
|
|
112
|
+
"capability": "host:filesystem_consumers",
|
|
113
|
+
"template": "Use the reviewed bounded host evidence wrapper to list top-level consumers on the alerted filesystem only. Do not follow other mounts and do not read file contents, configuration, environment, or credentials.",
|
|
114
|
+
"parameters": {},
|
|
115
|
+
"bounds": {"lookback": "5m", "step": "5m", "max_points": 1}
|
|
116
|
+
},
|
|
117
|
+
"evidence_contract": {
|
|
118
|
+
"kind": "command_output",
|
|
119
|
+
"required_fields": ["artifact", "query", "resource_identity", "result_sample", "result_summary"]
|
|
120
|
+
},
|
|
121
|
+
"confirms_when": "One or more bounded consumers on the alerted filesystem account for material usage and are identified by path and bytes",
|
|
122
|
+
"falsifies_when": "No inspected consumer on the alerted filesystem accounts for material usage within the bounded evidence",
|
|
123
|
+
"on_confirm": ["verify-filesystem-headroom"],
|
|
124
|
+
"on_falsify": ["verify-filesystem-headroom"],
|
|
125
|
+
"on_error": {"action": "record_gap"}
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
"id": "verify-filesystem-headroom",
|
|
129
|
+
"intent": "Verify current free-space headroom for the original alert host, device, and mountpoint",
|
|
130
|
+
"source_requirement_id": "filesystem-metrics",
|
|
131
|
+
"safety": "read_only",
|
|
132
|
+
"invocation": {
|
|
133
|
+
"kind": "query",
|
|
134
|
+
"capability": "prometheus:query",
|
|
135
|
+
"template": "(node_filesystem_avail_bytes{fstype=~\"ext4|xfs\"} / node_filesystem_size_bytes{fstype=~\"ext4|xfs\"}) * 100",
|
|
136
|
+
"parameters": {
|
|
137
|
+
"time_range": {"type": "time_range", "source": "incident.time_range", "target": "request.range"}
|
|
138
|
+
},
|
|
139
|
+
"bounds": {"lookback": "30m", "step": "30s", "max_points": 60}
|
|
140
|
+
},
|
|
141
|
+
"evidence_contract": {
|
|
142
|
+
"kind": "timeseries",
|
|
143
|
+
"required_fields": ["artifact", "query", "time_range", "resource_identity", "result_summary"]
|
|
144
|
+
},
|
|
145
|
+
"confirms_when": "The original host, device, and mountpoint have recovered above the responder-approved free-space threshold",
|
|
146
|
+
"falsifies_when": "The original filesystem remains below the threshold or the returned resource identity differs",
|
|
147
|
+
"on_confirm": [],
|
|
148
|
+
"on_falsify": [],
|
|
149
|
+
"on_error": {"action": "record_gap"}
|
|
150
|
+
}
|
|
151
|
+
],
|
|
152
|
+
"escalation": {
|
|
153
|
+
"when": "Runtime topology is unavailable, resource identities do not match, or no bounded consumer explains the alert",
|
|
154
|
+
"target": "infrastructure owner and incident commander"
|
|
155
|
+
},
|
|
156
|
+
"remediation": {
|
|
157
|
+
"mode": "suggest_only",
|
|
158
|
+
"proposed_steps": [
|
|
159
|
+
"Ask the infrastructure owner to review evidence-linked consumers on the alerted filesystem and approve a reversible cleanup target.",
|
|
160
|
+
"Treat service storage or WAL failures on a different filesystem as a separate remediation task with its own owner and verification."
|
|
161
|
+
],
|
|
162
|
+
"approval_required": true,
|
|
163
|
+
"approval_expectation": "The incident commander or infrastructure owner approves the exact target and rollback before any cleanup or service recovery action.",
|
|
164
|
+
"rollback_steps": [
|
|
165
|
+
"Stop the approved cleanup if free space, service health, or evidence integrity worsens.",
|
|
166
|
+
"Restore only responder-approved reversible moves and re-run the original filesystem identity check."
|
|
167
|
+
],
|
|
168
|
+
"verification_check_id": "verify-filesystem-headroom"
|
|
169
|
+
},
|
|
170
|
+
"completion": {
|
|
171
|
+
"required_check_ids": [
|
|
172
|
+
"identify-alert-filesystem",
|
|
173
|
+
"read-infrastructure-topology",
|
|
174
|
+
"map-runtime-filesystems",
|
|
175
|
+
"find-alerted-filesystem-consumers"
|
|
176
|
+
],
|
|
177
|
+
"allow_inconclusive": true,
|
|
178
|
+
"require_gap_reason_for_skips": true
|
|
179
|
+
},
|
|
180
|
+
"notes": "Never infer filesystem causality by dividing or comparing metrics with different host, device, or mountpoint identities. Unknown topology is an evidence gap, not a root cause."
|
|
181
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "v2",
|
|
3
|
+
"entry_check_ids": ["check-error-rate"],
|
|
4
|
+
"source_requirements": [
|
|
5
|
+
{
|
|
6
|
+
"id": "runtime-errors",
|
|
7
|
+
"source_key": "grafana",
|
|
8
|
+
"capabilities": ["prometheus:query"],
|
|
9
|
+
"required": true,
|
|
10
|
+
"on_unavailable": {"action": "record_gap"}
|
|
11
|
+
}
|
|
12
|
+
],
|
|
13
|
+
"checks": [
|
|
14
|
+
{
|
|
15
|
+
"id": "check-error-rate",
|
|
16
|
+
"intent": "Determine whether user-facing 5xx increased",
|
|
17
|
+
"source_requirement_id": "runtime-errors",
|
|
18
|
+
"safety": "read_only",
|
|
19
|
+
"invocation": {
|
|
20
|
+
"kind": "query",
|
|
21
|
+
"capability": "prometheus:query",
|
|
22
|
+
"template": "sum(rate(http_requests_total{service=\"{{service}}\",status_class=\"5xx\"}[{{rate_window}}]))",
|
|
23
|
+
"parameters": {
|
|
24
|
+
"rate_window": {"type": "duration", "value": "5m"},
|
|
25
|
+
"service": {"type": "string", "source": "incident.service"},
|
|
26
|
+
"time_range": {"type": "time_range", "source": "incident.time_range", "target": "request.range"}
|
|
27
|
+
},
|
|
28
|
+
"bounds": {"lookback": "30m", "step": "30s", "max_points": 1000}
|
|
29
|
+
},
|
|
30
|
+
"evidence_contract": {
|
|
31
|
+
"kind": "timeseries",
|
|
32
|
+
"required_fields": ["query", "result_summary", "time_range"]
|
|
33
|
+
},
|
|
34
|
+
"confirms_when": "5xx exceeds baseline and aligns with incident start",
|
|
35
|
+
"falsifies_when": "5xx remains at baseline",
|
|
36
|
+
"on_confirm": [],
|
|
37
|
+
"on_falsify": [],
|
|
38
|
+
"on_error": {"action": "record_gap"}
|
|
39
|
+
}
|
|
40
|
+
],
|
|
41
|
+
"escalation": {
|
|
42
|
+
"when": "Required evidence is unavailable or customer impact increases",
|
|
43
|
+
"target": "service owner"
|
|
44
|
+
},
|
|
45
|
+
"completion": {
|
|
46
|
+
"required_check_ids": ["check-error-rate"],
|
|
47
|
+
"allow_inconclusive": true,
|
|
48
|
+
"require_gap_reason_for_skips": true
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "v2",
|
|
3
|
+
"match": {
|
|
4
|
+
"scope": "service",
|
|
5
|
+
"services": ["checkout-api"],
|
|
6
|
+
"tags": ["payments"],
|
|
7
|
+
"symptoms": ["5xx", "latency"],
|
|
8
|
+
"severities": ["sev1", "sev2"],
|
|
9
|
+
"priority": 80
|
|
10
|
+
},
|
|
11
|
+
"source_requirements": [
|
|
12
|
+
{
|
|
13
|
+
"id": "runtime-errors",
|
|
14
|
+
"source_key": "grafana",
|
|
15
|
+
"capabilities": ["prometheus:query"],
|
|
16
|
+
"required": true,
|
|
17
|
+
"on_unavailable": {"action": "record_gap"}
|
|
18
|
+
}
|
|
19
|
+
],
|
|
20
|
+
"entry_check_ids": ["check-error-rate"],
|
|
21
|
+
"checks": [
|
|
22
|
+
{
|
|
23
|
+
"id": "check-error-rate",
|
|
24
|
+
"intent": "Determine whether user-facing 5xx increased",
|
|
25
|
+
"source_requirement_id": "runtime-errors",
|
|
26
|
+
"safety": "read_only",
|
|
27
|
+
"invocation": {
|
|
28
|
+
"kind": "query",
|
|
29
|
+
"capability": "prometheus:query",
|
|
30
|
+
"template": "sum(rate(http_requests_total{service=\"{{service}}\",status_class=\"5xx\"}[{{rate_window}}]))",
|
|
31
|
+
"parameters": {
|
|
32
|
+
"service": {"type": "string", "source": "incident.service"},
|
|
33
|
+
"rate_window": {"type": "duration", "value": "5m"},
|
|
34
|
+
"time_range": {"type": "time_range", "source": "incident.time_range", "target": "request.range"}
|
|
35
|
+
},
|
|
36
|
+
"bounds": {"lookback": "30m", "step": "30s", "max_points": 1000}
|
|
37
|
+
},
|
|
38
|
+
"evidence_contract": {
|
|
39
|
+
"kind": "timeseries",
|
|
40
|
+
"required_fields": ["query", "time_range", "result_summary"]
|
|
41
|
+
},
|
|
42
|
+
"confirms_when": "5xx exceeds baseline and aligns with incident start",
|
|
43
|
+
"falsifies_when": "5xx remains at baseline",
|
|
44
|
+
"on_confirm": [],
|
|
45
|
+
"on_falsify": [],
|
|
46
|
+
"on_error": {"action": "record_gap"}
|
|
47
|
+
}
|
|
48
|
+
],
|
|
49
|
+
"escalation": {
|
|
50
|
+
"when": "Required evidence is unavailable or customer impact increases",
|
|
51
|
+
"target": "service owner"
|
|
52
|
+
},
|
|
53
|
+
"completion": {
|
|
54
|
+
"required_check_ids": ["check-error-rate"],
|
|
55
|
+
"allow_inconclusive": true,
|
|
56
|
+
"require_gap_reason_for_skips": true
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
{
|
|
2
|
+
"description": "Block sets the API rejects with 400 invalid_block. Each case lists the blocks a client might send, the field the API names and the problem it reports. The full list of bad fields is in details.errors.",
|
|
3
|
+
"cases": [
|
|
4
|
+
{
|
|
5
|
+
"name": "unknown block type",
|
|
6
|
+
"blocks": [{"type": "heatmap", "cells": [[1, 2], [3, 4]]}],
|
|
7
|
+
"field": "blocks[0].type",
|
|
8
|
+
"message": "unknown block type \"heatmap\" (use stats, compare, trend, table, checks, changes, range_overlap, histogram, run_history or diff)"
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"name": "a number sent as text",
|
|
12
|
+
"blocks": [{"type": "compare", "unit": "ms", "rows": [{"label": "GET /v1/me", "before": 900, "after": "slow"}]}],
|
|
13
|
+
"field": "blocks[0].rows[0].after",
|
|
14
|
+
"message": "must be a number"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"name": "five key figures",
|
|
18
|
+
"blocks": [{"type": "stats", "items": [
|
|
19
|
+
{"label": "a", "value": "1"}, {"label": "b", "value": "2"}, {"label": "c", "value": "3"},
|
|
20
|
+
{"label": "d", "value": "4"}, {"label": "e", "value": "5"}
|
|
21
|
+
]}],
|
|
22
|
+
"field": "blocks[0].items",
|
|
23
|
+
"message": "needs 1 to 4 items (got 5)"
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"name": "a table row with a missing cell",
|
|
27
|
+
"blocks": [{"type": "table", "columns": [{"label": "Query"}, {"label": "Calls / min"}], "rows": [{"cells": ["orders by customer", "1,840"]}, {"cells": ["update inventory"]}]}],
|
|
28
|
+
"field": "blocks[0].rows[1].cells",
|
|
29
|
+
"message": "needs exactly 2 cells (got 1)"
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"name": "a mark outside the series",
|
|
33
|
+
"blocks": [{"type": "trend", "series": [38, 41, 88, 91], "marks": [{"i": 4, "label": "TTL change shipped"}]}],
|
|
34
|
+
"field": "blocks[0].marks[0].i",
|
|
35
|
+
"message": "must be a point index from 0 to 3"
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"name": "start without step_seconds",
|
|
39
|
+
"blocks": [{"type": "trend", "series": [38, 41], "start": "2026-08-12T09:00:00Z"}],
|
|
40
|
+
"field": "blocks[0].step_seconds",
|
|
41
|
+
"message": "is required when start is set"
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"name": "a tone that is a colour",
|
|
45
|
+
"blocks": [{"type": "stats", "items": [{"label": "Peak CPU", "value": "91%", "tone": "red"}]}],
|
|
46
|
+
"field": "blocks[0].items[0].tone",
|
|
47
|
+
"message": "must be good, warn, bad or neutral"
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"name": "slot and anchor together",
|
|
51
|
+
"blocks": [{"type": "stats", "slot": "summary", "anchor": "Verified outcome", "items": [{"label": "Peak CPU", "value": "91%"}]}],
|
|
52
|
+
"field": "blocks[0].anchor",
|
|
53
|
+
"message": "cannot be combined with slot"
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"name": "a field the contract does not have",
|
|
57
|
+
"blocks": [{"type": "checks", "items": [{"label": "Alert cleared", "state": "pass", "owner": "ana"}]}],
|
|
58
|
+
"field": "blocks[0].items[0].owner",
|
|
59
|
+
"message": "is not a known field (known: label, state, detail)"
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"name": "a run history mark between runs that are not neighbours",
|
|
63
|
+
"blocks": [{"type": "run_history", "runs": [{"ref": "1", "result": "success"}, {"ref": "2", "result": "failure"}, {"ref": "3", "result": "failure"}], "marks": [{"between": ["1", "3"], "change_ref": "regression"}]}],
|
|
64
|
+
"field": "blocks[0].marks[0].between",
|
|
65
|
+
"message": "must name two neighbouring runs, the second right after the first"
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"name": "histogram counts that do not match the edges",
|
|
69
|
+
"blocks": [{"type": "histogram", "edges": [10, 100, 1000], "before": [5], "after": [2, 8]}],
|
|
70
|
+
"field": "blocks[0].before",
|
|
71
|
+
"message": "needs exactly 2 counts (got 1)"
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"name": "a diff with more than twelve lines",
|
|
75
|
+
"blocks": [{"type": "diff", "path": "a.go", "ref": "abc", "additions": 13, "deletions": 0, "hunks": [[
|
|
76
|
+
{"op": "+", "text": "1"}, {"op": "+", "text": "2"}, {"op": "+", "text": "3"}, {"op": "+", "text": "4"}, {"op": "+", "text": "5"},
|
|
77
|
+
{"op": "+", "text": "6"}, {"op": "+", "text": "7"}, {"op": "+", "text": "8"}, {"op": "+", "text": "9"}, {"op": "+", "text": "10"},
|
|
78
|
+
{"op": "+", "text": "11"}, {"op": "+", "text": "12"}, {"op": "+", "text": "13"}
|
|
79
|
+
]]}],
|
|
80
|
+
"field": "blocks[0].hunks",
|
|
81
|
+
"message": "at most 12 lines in all (got 13)"
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"name": "two ranges with only one in the whole range",
|
|
85
|
+
"blocks": [{"type": "range_overlap", "min": 0, "max": 100, "rows": [{"label": "a", "from": 10, "to": 20}, {"label": "b", "from": 50, "to": 150}]}],
|
|
86
|
+
"field": "blocks[0].rows[1].to",
|
|
87
|
+
"message": "must be within min and max (0 to 100)"
|
|
88
|
+
}
|
|
89
|
+
]
|
|
90
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
{
|
|
2
|
+
"title": "Reclaimed disk space on the monitoring host",
|
|
3
|
+
"type": "change",
|
|
4
|
+
"is_key": true,
|
|
5
|
+
"started_at": "2026-08-03T12:55:21Z",
|
|
6
|
+
"tags": ["disk", "monitoring", "mitigation", "recovery"],
|
|
7
|
+
"external_reference": "disk-remediation-20260803",
|
|
8
|
+
"body": "Cleared disposable Docker state on the standalone monitoring and CI host and confirmed that nothing durable was touched.\n\nInspection output:\n- Docker data root is /mnt/HC_Volume_104395459/docker.\n- Unused anonymous Docker volumes and Loki's active Docker JSON stdout log were the two large consumers.\n- No GitHub Actions Runner.Worker process was active.\n\nActions performed:\n- Pruned only unused anonymous volumes; active named Prometheus, Loki and Grafana volumes were preserved.\n- Pruned unused images.\n- Truncated only Loki's Docker stdout log after capturing a recent tail. Durable Loki data was not touched.\n\nVerified outcome:\n- All monitoring containers remain running and their health checks pass.\n- The DiskSpaceLow alert no longer fires.\n\nPermanent follow-up: recreate the monitoring containers in a maintenance window so the existing Docker JSON log limits apply to them.",
|
|
9
|
+
"blocks": [
|
|
10
|
+
{
|
|
11
|
+
"type": "stats",
|
|
12
|
+
"slot": "summary",
|
|
13
|
+
"items": [
|
|
14
|
+
{"label": "Volumes pruned", "value": "26.48 GB", "caption": "526 unused anonymous"},
|
|
15
|
+
{"label": "Loki log truncated", "value": "12.95 GB", "caption": "Docker stdout log only"},
|
|
16
|
+
{"label": "Images reclaimed", "value": "596.8 MB", "caption": "unused images"},
|
|
17
|
+
{"label": "DiskSpaceLow alert", "value": "0 firing", "caption": "ALERTS query is empty", "tone": "good"}
|
|
18
|
+
]
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"type": "compare",
|
|
22
|
+
"anchor": "Verified outcome",
|
|
23
|
+
"title": "Disk used",
|
|
24
|
+
"unit": "%",
|
|
25
|
+
"better": "lower",
|
|
26
|
+
"metric": "used",
|
|
27
|
+
"threshold": {"value": 85, "label": "Alert line: 85% used (15% free)"},
|
|
28
|
+
"rows": [
|
|
29
|
+
{"label": "/", "mono": true, "before": 86, "after": 80, "details": [{"label": "free", "before": "13.77%", "after": "19.45%"}]},
|
|
30
|
+
{"label": "/mnt/HC_Volume_104395459", "mono": true, "before": 93, "after": 20, "details": [{"label": "free", "before": "7.15%", "after": "76.07%"}]}
|
|
31
|
+
]
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"type": "trend",
|
|
35
|
+
"anchor": "Verified outcome",
|
|
36
|
+
"title": "Disk used on /mnt/HC_Volume_104395459",
|
|
37
|
+
"unit": "%",
|
|
38
|
+
"series": [86, 88, 91, 93, 93, 41, 20],
|
|
39
|
+
"start": "2026-08-03T10:00:00Z",
|
|
40
|
+
"step_seconds": 600,
|
|
41
|
+
"threshold": {"value": 85, "label": "Alert line: 85% used"},
|
|
42
|
+
"marks": [{"i": 5, "label": "Volumes pruned"}],
|
|
43
|
+
"source": {
|
|
44
|
+
"kind": "prometheus",
|
|
45
|
+
"datasource": "prometheus-prod",
|
|
46
|
+
"query": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/mnt/HC_Volume_104395459\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/HC_Volume_104395459\"})",
|
|
47
|
+
"step": "10m"
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"type": "checks",
|
|
52
|
+
"title": "Recovery checks",
|
|
53
|
+
"items": [
|
|
54
|
+
{"label": "Both mounts above 15% free", "state": "pass", "detail": "root 19.45%, volume 76.07% free"},
|
|
55
|
+
{"label": "DiskSpaceLow alert not firing", "state": "pass"},
|
|
56
|
+
{"label": "All ten monitoring containers running", "state": "pass"},
|
|
57
|
+
{"label": "Prometheus, Grafana and Loki health checks", "state": "pass", "detail": "Loki reports ready"}
|
|
58
|
+
]
|
|
59
|
+
}
|
|
60
|
+
]
|
|
61
|
+
}
|