@nuvlore/extension-bds 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +3 -0
  3. package/agents/bds-airflow-job-status-readonly.yaml +123 -0
  4. package/agents/bds-disk-investigator.yaml +112 -0
  5. package/agents/bds-inventory-readonly.yaml +60 -0
  6. package/agents/bds-oncall-reporter.yaml +113 -0
  7. package/agents/bds-prism-service-proxy-timeout-readonly.yaml +105 -0
  8. package/agents/bds-scylla-tenant-bucket-readonly.yaml +104 -0
  9. package/agents/bds-tenant-data-operation-log-readonly.yaml +98 -0
  10. package/agents/bds-yarn-app-health-readonly.yaml +114 -0
  11. package/agents/pharos-alert-investigator.yaml +81 -0
  12. package/package.json +60 -0
  13. package/skills/bds-airflow-job-status-investigation/SKILL.md +104 -0
  14. package/skills/bds-airflow-job-status-investigation/references/airflow-access-runbook.md +35 -0
  15. package/skills/bds-airflow-job-status-investigation/references/airflow-bas-dr-runbook.md +42 -0
  16. package/skills/bds-airflow-job-status-investigation/references/airflow-copy-and-migration-runbook.md +63 -0
  17. package/skills/bds-airflow-job-status-investigation/references/airflow-job-status-runbook.md +15 -0
  18. package/skills/bds-airflow-job-status-investigation/references/airflow-output-runbook.md +28 -0
  19. package/skills/bds-airflow-job-status-investigation/references/airflow-snapshot-runbook.md +38 -0
  20. package/skills/bds-disk-io-alert-investigation/SKILL.md +116 -0
  21. package/skills/bds-hadoop-review/SKILL.md +66 -0
  22. package/skills/bds-machine-inventory/SKILL.md +48 -0
  23. package/skills/bds-oncall-followup/SKILL.md +71 -0
  24. package/skills/bds-oncall-followup/references/oncall-followup-runbook.md +58 -0
  25. package/skills/bds-prism-service-proxy-timeout-investigation/SKILL.md +73 -0
  26. package/skills/bds-prism-service-proxy-timeout-investigation/references/service-proxy-timeout-runbook.md +48 -0
  27. package/skills/bds-scylla-tenant-bucket-investigation/SKILL.md +73 -0
  28. package/skills/bds-scylla-tenant-bucket-investigation/references/tenant-bucket-runbook.md +91 -0
  29. package/skills/bds-tenant-data-operation-log-investigation/SKILL.md +89 -0
  30. package/skills/bds-yarn-app-health-investigation/SKILL.md +77 -0
  31. package/skills/bds-yarn-app-health-investigation/references/yarn-app-health-runbook.md +53 -0
  32. package/skills/pharos-alert-investigation/SKILL.md +80 -0
  33. package/src/bds-atomic-log-tools.mjs +300 -0
  34. package/src/bds-team-membership-claude-output.mjs +36 -0
  35. package/src/bds-team-membership-quality.mjs +90 -0
  36. package/tools/bds_airflow_dag_log_read.mjs +24 -0
  37. package/tools/bds_airflow_job_status_command_pack.mjs +587 -0
  38. package/tools/bds_bas_log_read.mjs +23 -0
  39. package/tools/bds_bdm_httpd_log_read.mjs +24 -0
  40. package/tools/bds_chef_client_log_read.mjs +23 -0
  41. package/tools/bds_dashboard_links.mjs +178 -0
  42. package/tools/bds_dashboard_query_knowledge.mjs +258 -0
  43. package/tools/bds_dashboard_query_read.mjs +120 -0
  44. package/tools/bds_disk_io_alert_command_pack.mjs +287 -0
  45. package/tools/bds_environment_host_lookup.mjs +741 -0
  46. package/tools/bds_hdfs_datanode_log_read.mjs +23 -0
  47. package/tools/bds_hdfs_journalnode_log_read.mjs +23 -0
  48. package/tools/bds_hdfs_namenode_log_read.mjs +23 -0
  49. package/tools/bds_hdfs_path_knowledge.mjs +324 -0
  50. package/tools/bds_information_source_knowledge.mjs +356 -0
  51. package/tools/bds_kms_log_read.mjs +23 -0
  52. package/tools/bds_log_path_knowledge.mjs +267 -0
  53. package/tools/bds_machine_inventory.mjs +434 -0
  54. package/tools/bds_mapreduce_historyserver_log_read.mjs +23 -0
  55. package/tools/bds_migration_service_operation_command_pack.mjs +233 -0
  56. package/tools/bds_oncall_followup_payload_draft.mjs +285 -0
  57. package/tools/bds_pharos_query_pack.mjs +218 -0
  58. package/tools/bds_portal_operation_knowledge.mjs +212 -0
  59. package/tools/bds_prism_clone_status_links.mjs +129 -0
  60. package/tools/bds_prism_service_proxy_timeout_query_pack.mjs +224 -0
  61. package/tools/bds_public_cloud_pod_log_read.mjs +18 -0
  62. package/tools/bds_scylla_tenant_bucket_command_pack.mjs +323 -0
  63. package/tools/bds_service_proxy_log_read.mjs +23 -0
  64. package/tools/bds_spark_historyserver_log_read.mjs +32 -0
  65. package/tools/bds_spas_mt_log_read.mjs +23 -0
  66. package/tools/bds_tenant_data_operation_log_command_pack.mjs +321 -0
  67. package/tools/bds_yarn_app_health_command_pack.mjs +529 -0
  68. package/tools/bds_yarn_nodemanager_log_read.mjs +23 -0
  69. package/tools/bds_yarn_resourcemanager_log_read.mjs +23 -0
  70. package/tools/bds_zookeeper_log_read.mjs +23 -0
  71. package/tools/confluence_handoff_page_draft.mjs +173 -0
  72. package/vendor/dashboard/src/grafana-dashboard-query.mjs +61 -0
  73. package/vendor/extension-read-only-toolkit/LICENSE +202 -0
  74. package/vendor/extension-read-only-toolkit/README.md +29 -0
  75. package/vendor/extension-read-only-toolkit/package.json +46 -0
  76. package/vendor/extension-read-only-toolkit/src/devops-data-paths.mjs +222 -0
  77. package/vendor/extension-read-only-toolkit/src/devops-diagnostics.mjs +440 -0
  78. package/vendor/extension-read-only-toolkit/src/enterprise-api-read.mjs +180 -0
  79. package/vendor/extension-read-only-toolkit/src/read-only-command-pack.mjs +1 -0
  80. package/vendor/extension-read-only-toolkit/src/read-only-plan.mjs +45 -0
  81. package/vendor/extension-read-only-toolkit/src/read-only-shell-policy.d.mts +26 -0
  82. package/vendor/extension-read-only-toolkit/src/read-only-shell-policy.mjs +170 -0
  83. package/vendor/extension-workday-infra-access/LICENSE +202 -0
  84. package/vendor/extension-workday-infra-access/README.md +3 -0
  85. package/vendor/extension-workday-infra-access/agents/workday-infra-access-readonly.yaml +183 -0
  86. package/vendor/extension-workday-infra-access/package.json +49 -0
  87. package/vendor/extension-workday-infra-access/skills/workday-infra-access/SKILL.md +102 -0
  88. package/vendor/extension-workday-infra-access/skills/workday-infra-access/references/access-boundary-runbook.md +43 -0
  89. package/vendor/extension-workday-infra-access/skills/workday-infra-access/references/access-runbook.md +15 -0
  90. package/vendor/extension-workday-infra-access/skills/workday-infra-access/references/access-scylla-cloud-runbook.md +39 -0
  91. package/vendor/extension-workday-infra-access/skills/workday-infra-access/references/access-spc-argo-runbook.md +99 -0
  92. package/vendor/extension-workday-infra-access/src/workday-access-profiles.mjs +194 -0
  93. package/vendor/extension-workday-infra-access/tools/aws_cust_federated_readonly_command_pack.mjs +234 -0
  94. package/vendor/extension-workday-infra-access/tools/gcp_readonly_command_pack.mjs +167 -0
  95. package/vendor/extension-workday-infra-access/tools/public_cloud_access_status.mjs +48 -0
  96. package/vendor/extension-workday-infra-access/tools/workday_boundary_login.mjs +460 -0
  97. package/vendor/extension-workday-infra-access/tools/workday_private_cloud_sudo_readonly_command.mjs +386 -0
  98. package/vendor/extension-workday-infra-access/tools/workday_public_cloud_environment_map.mjs +189 -0
  99. package/vendor/extension-workday-infra-access/tools/workday_public_cloud_kubernetes_read.mjs +297 -0
  100. package/vendor/extension-workday-infra-access/tools/workday_public_cloud_spc_login.mjs +505 -0
  101. package/vendor/extension-workday-infra-access/tools/workday_spc_argo_readonly_command_pack.mjs +215 -0
  102. package/vendor/extension-workday-infra-access/workflows/public-cloud-readonly-investigation.js +99 -0
  103. package/workflows/bds-airflow-job-status-investigation.js +168 -0
  104. package/workflows/bds-disk-io-alert-investigation.js +173 -0
  105. package/workflows/bds-oncall-followup.js +112 -0
  106. package/workflows/bds-prism-service-proxy-timeout-investigation.js +69 -0
  107. package/workflows/bds-scylla-tenant-bucket-investigation.js +139 -0
  108. package/workflows/bds-tenant-data-operation-log-investigation.js +189 -0
  109. package/workflows/bds-yarn-app-health-investigation.js +217 -0
@@ -0,0 +1,529 @@
1
+ export const description = "Generate a atomic-helper, read-only BDS YARN application command pack for app timeout requests, aggregated log host discovery, NodeManager health, and DataNode cross-checks; does not SSH, run YARN, or inspect HDFS.";
2
+
3
+ import { z } from "zod";
4
+ import { defineReadOnlyTool, fail, ok } from "@nuvlore/extension-read-only-toolkit/devops-diagnostics";
5
+ import { ReadOnlyStep, remoteCommandInput, shellQuote } from "@nuvlore/extension-read-only-toolkit/read-only-command-pack";
6
+
7
+ const DEFAULT_LOG_ROOT = "hdfs://bdsashcustaz1/var/log/hadoop-yarn/apps";
8
+
9
+ const SAFETY_RULES = [
10
+ "read-only only: yarn application -status, yarn logs metadata reads, hdfs dfs -ls/-du/-find, hdfs dfsadmin -report, and yarn node -status",
11
+ "ResourceManager ApplicationSummary reads are allowed for historical app state and failed-app lists when RM/Timeline retention has expired",
12
+ "historical incident-window checks must use read-only journalctl/grep/zgrep over kernel, NodeManager, and DataNode logs only",
13
+ "never run yarn application -kill, yarn node -decommission, hdfs dfs -rm, hdfs dfs -put, hdfs dfs -get, hdfs dfs -cp, hdfs dfs -mv, chmod, chown, service restart, or host mutation",
14
+ "do not fetch full aggregated yarn logs first; use -show_application_log_info, precise HDFS path discovery, and bounded grep/head/tail",
15
+ "treat RM/Timeline absence separately from filesystem log availability",
16
+ "never print LDAP passwords, Keychain payloads, tokens, or sudo stdin",
17
+ "pass sudo credentials through stdin only when the operator explicitly approved it"
18
+ ];
19
+
20
+ const HISTORICAL_NOISE_FILTERS = [
21
+ "DFSUtil.*duration=0",
22
+ "Scheduling blk_",
23
+ "Deleted BP-",
24
+ "DataNode.clienttrace",
25
+ "PacketResponder:.*terminating",
26
+ "short GC pause detail lines; call out only long pauses, for example >= 1000 ms",
27
+ "minor NodeManager process-accounting WARNs after the incident window unless paired with health failures"
28
+ ];
29
+
30
+ const hostSchema = z
31
+ .string()
32
+ .min(1)
33
+ .max(255)
34
+ .regex(/^(?!-)[a-zA-Z0-9._:@%-]+$/);
35
+
36
+ const appIdSchema = z
37
+ .string()
38
+ .min(1)
39
+ .max(128)
40
+ .regex(/^application_[0-9]+_[0-9]+$/);
41
+
42
+ const clusterSchema = z
43
+ .string()
44
+ .min(1)
45
+ .max(128)
46
+ .regex(/^[A-Za-z0-9_.:-]+$/)
47
+ .optional();
48
+
49
+ const principalSchema = z
50
+ .string()
51
+ .min(1)
52
+ .max(128)
53
+ .regex(/^[A-Za-z0-9_.@/-]+$/)
54
+ .default("workdayadmin");
55
+
56
+ const keytabSchema = z
57
+ .string()
58
+ .min(1)
59
+ .max(512)
60
+ .regex(/^\/[A-Za-z0-9._/-]+$/)
61
+ .default("/etc/hadoop/conf/workdayadmin.keytab");
62
+
63
+ const pathSchema = z
64
+ .string()
65
+ .min(1)
66
+ .max(512)
67
+ .regex(/^[A-Za-z0-9:._@%/-]+$/)
68
+ .default(DEFAULT_LOG_ROOT);
69
+
70
+ const tenantSchema = z
71
+ .string()
72
+ .min(1)
73
+ .max(160)
74
+ .regex(/^[A-Za-z0-9_.-]+$/)
75
+ .optional();
76
+
77
+ const ownerSchema = z
78
+ .string()
79
+ .min(1)
80
+ .max(160)
81
+ .regex(/^[A-Za-z0-9_.-]+$/)
82
+ .optional();
83
+
84
+ const hostListSchema = z
85
+ .array(hostSchema)
86
+ .max(80)
87
+ .default([]);
88
+
89
+ const utcTimestampSchema = z
90
+ .string()
91
+ .min(1)
92
+ .max(64)
93
+ .regex(/^[0-9]{4}-[0-9]{2}-[0-9]{2}[ T][0-9]{2}:[0-9]{2}(:[0-9]{2}(\\.[0-9]{1,6})?)?(Z| UTC)?$/)
94
+ .optional();
95
+
96
+ const minuteWindowSchema = z.number().int().min(1).max(30).default(2);
97
+
98
+ function appBucket(appId) {
99
+ return appId.split("_").at(-1);
100
+ }
101
+
102
+ function hdfsGlobForKnownOwner({ logRoot, appOwner, appId }) {
103
+ const bucket = appBucket(appId);
104
+ return `${logRoot}/${appOwner}/{logs,logs-tfile,bucket-logs-tfile/${bucket}}/${appId}`;
105
+ }
106
+
107
+ function hdfsFindCommand(logRoot, appId) {
108
+ return `timeout 120s hdfs dfs -find ${shellQuote(logRoot)} -name ${shellQuote(`*${appId}*`)} 2>&1 | head -300`;
109
+ }
110
+
111
+ function parseUtcTimestamp(value) {
112
+ if (!value) return undefined;
113
+ const normalized = value
114
+ .replace(" ", "T")
115
+ .replace(/ UTC$/, "Z")
116
+ .replace(/(?<!Z)$/, value.endsWith("Z") || value.endsWith(" UTC") ? "" : "Z");
117
+ const date = new Date(normalized);
118
+ if (Number.isNaN(date.getTime())) {
119
+ return undefined;
120
+ }
121
+ return date;
122
+ }
123
+
124
+ function formatUtc(date) {
125
+ return date.toISOString().replace(/\.\d{3}Z$/, "Z");
126
+ }
127
+
128
+ function historicalWindow(timestamp, minutes) {
129
+ const date = parseUtcTimestamp(timestamp);
130
+ if (!date) {
131
+ return {
132
+ since: "<incident UTC minus 2 minutes>",
133
+ until: "<incident UTC plus 2 minutes>",
134
+ grepPattern: "<YYYY-MM-DD[ T]HH:MM range for incident window>"
135
+ };
136
+ }
137
+ const since = new Date(date.getTime() - minutes * 60 * 1000);
138
+ const until = new Date(date.getTime() + minutes * 60 * 1000);
139
+ const yyyyMmDd = date.toISOString().slice(0, 10);
140
+ const hour = date.toISOString().slice(11, 13);
141
+ const minute = date.toISOString().slice(14, 16);
142
+ const minuteInt = Number(minute);
143
+ const low = Math.max(0, minuteInt - minutes);
144
+ const high = Math.min(59, minuteInt + minutes);
145
+ const minuteAlternation = Array.from({ length: high - low + 1 }, (_, index) => String(low + index).padStart(2, "0")).join("|");
146
+ return {
147
+ since: formatUtc(since),
148
+ until: formatUtc(until),
149
+ grepPattern: `${yyyyMmDd}[ T]${hour}:(${minuteAlternation})`
150
+ };
151
+ }
152
+
153
+ class BdsYarnAppHealthCommandPack {
154
+ constructor(input) {
155
+ this.cluster = input.cluster;
156
+ this.driverHost = input.driverHost;
157
+ this.applicationId = input.applicationId;
158
+ this.tenant = input.tenant;
159
+ this.appOwner = input.appOwner;
160
+ this.logRoot = input.logRoot ?? DEFAULT_LOG_ROOT;
161
+ this.kerberosPrincipal = input.kerberosPrincipal ?? "workdayadmin";
162
+ this.kerberosKeytab = input.kerberosKeytab ?? "/etc/hadoop/conf/workdayadmin.keytab";
163
+ this.nodeHosts = input.nodeHosts ?? [];
164
+ this.incidentTimestampUtc = input.incidentTimestampUtc;
165
+ this.historicalWindowMinutes = input.historicalWindowMinutes ?? 2;
166
+ }
167
+
168
+ ssh(command) {
169
+ return `ssh -o BatchMode=yes -o ConnectTimeout=20 -- ${this.driverHost} ${JSON.stringify(command)}`;
170
+ }
171
+
172
+ kinitPrefix() {
173
+ return `kinit -kt ${shellQuote(this.kerberosKeytab)} ${shellQuote(this.kerberosPrincipal)}`;
174
+ }
175
+
176
+ sudoWrapperExample(command) {
177
+ return `mcp__devops-tool__workday_private_cloud_sudo_readonly_command ${JSON.stringify({
178
+ action: "run",
179
+ host: this.driverHost,
180
+ command,
181
+ humanApprovedSudoRead: true
182
+ })}`;
183
+ }
184
+
185
+ appPathCandidates() {
186
+ const candidates = [];
187
+ if (this.appOwner) {
188
+ candidates.push(hdfsGlobForKnownOwner({
189
+ logRoot: this.logRoot,
190
+ appOwner: this.appOwner,
191
+ appId: this.applicationId
192
+ }));
193
+ }
194
+ if (this.tenant) {
195
+ candidates.push(`${this.logRoot}/${this.tenant}/{logs,logs-tfile}/${this.applicationId}`);
196
+ }
197
+ candidates.push(`${this.logRoot}/<appOwner>/bucket-logs-tfile/${appBucket(this.applicationId)}/${this.applicationId}`);
198
+ return Array.from(new Set(candidates));
199
+ }
200
+
201
+ nodeStatusCommands() {
202
+ const hosts = this.nodeHosts.length ? this.nodeHosts : ["<host-from-aggregated-log-file-name>"];
203
+ return hosts.map((host) => `timeout 20s yarn node -status ${host}:45454 2>&1 | egrep 'Node-State|Node-Http-Address|Last-Health-Update|Health-Report|Containers|Memory|CPU|Node-Labels|Resource Utilization'`).join("\n");
204
+ }
205
+
206
+ hdfsMatchCommands() {
207
+ const hosts = this.nodeHosts.length ? this.nodeHosts : ["<host-from-aggregated-log-file-name>"];
208
+ return hosts.map((host) => `timeout 20s hdfs dfsadmin -report 2>&1 | grep -F -A18 '(${host})' || true`).join("\n");
209
+ }
210
+
211
+ mcpSafeCommandInputs() {
212
+ const driver = this.driverHost;
213
+ const appPath = this.appOwner
214
+ ? hdfsGlobForKnownOwner({ logRoot: this.logRoot, appOwner: this.appOwner, appId: this.applicationId })
215
+ : undefined;
216
+ const window = this.historicalWindow();
217
+ const kinit = `kinit -kt ${this.kerberosKeytab} ${this.kerberosPrincipal}`;
218
+ const items = [
219
+ { label: "driver-hostname", input: remoteCommandInput({ host: driver, command: "hostname -f" }) },
220
+ { label: "driver-utc", input: remoteCommandInput({ host: driver, command: "date -u" }) },
221
+ { label: "driver-user", input: remoteCommandInput({ host: driver, command: "id -un" }) },
222
+ { label: "check-yarn", input: remoteCommandInput({ host: driver, command: "command -v yarn" }) },
223
+ { label: "check-hdfs", input: remoteCommandInput({ host: driver, command: "command -v hdfs" }) },
224
+ { label: "check-kinit", input: remoteCommandInput({ host: driver, command: "command -v kinit" }) },
225
+ { label: "kerberos", input: remoteCommandInput({ host: driver, command: kinit }) },
226
+ { label: "rm-state", input: remoteCommandInput({ host: driver, command: `yarn application -status ${this.applicationId}` }) },
227
+ { label: "rm-application-summary", input: remoteCommandInput({ host: driver, command: `grep -h 'ApplicationSummary' /var/log/hadoop-yarn/yarn-yarn-resourcemanager-*.log* | grep '${this.applicationId}' | grep 'exitCode: 0' | head -20` }) },
228
+ {
229
+ label: "log-info",
230
+ input: remoteCommandInput({
231
+ host: driver,
232
+ command: `yarn logs -applicationId ${this.applicationId}${this.appOwner ? ` -appOwner ${this.appOwner}` : ""} -show_application_log_info | head -300`
233
+ })
234
+ },
235
+ {
236
+ label: "hdfs-find-log-path",
237
+ input: remoteCommandInput({
238
+ host: driver,
239
+ command: `hdfs dfs -find ${shellQuote(this.logRoot)} -name ${shellQuote(`*${this.applicationId}*`)} | head -300`,
240
+ timeoutMs: 120_000
241
+ })
242
+ },
243
+ { label: "hdfs-report-summary", input: remoteCommandInput({ host: driver, command: "hdfs dfsadmin -report | head -35" }) },
244
+ ];
245
+ if (appPath) {
246
+ items.push(
247
+ { label: "list-log-host-files", input: remoteCommandInput({ host: driver, command: `hdfs dfs -ls ${shellQuote(appPath)} | head -300` }) },
248
+ { label: "size-log-host-files", input: remoteCommandInput({ host: driver, command: `hdfs dfs -du -h ${shellQuote(appPath)} | head -300` }) },
249
+ );
250
+ }
251
+ if (this.nodeHosts.length) {
252
+ items.push(...this.nodeHosts.flatMap((host) => [
253
+ {
254
+ label: `yarn-node-${host}`,
255
+ input: remoteCommandInput({
256
+ host: driver,
257
+ command: `yarn node -status ${host}:45454 | egrep 'Node-State|Node-Http-Address|Last-Health-Update|Health-Report|Containers|Memory|CPU|Node-Labels|Resource Utilization'`
258
+ })
259
+ },
260
+ {
261
+ label: `hdfs-datanode-${host}`,
262
+ input: remoteCommandInput({
263
+ host: driver,
264
+ command: `hdfs dfsadmin -report | grep -F -A18 '(${host})'`
265
+ })
266
+ },
267
+ {
268
+ label: `historical-kernel-${host}`,
269
+ input: remoteCommandInput({
270
+ host,
271
+ command: `journalctl -k --since ${shellQuote(window.since)} --until ${shellQuote(window.until)} --no-pager | egrep -i 'error|warn|fail|failed|timeout|timed out|reset|I/O|blk_update_request|medium error|disk|volume|network|hung|blocked|oom|killed|read-only|corrupt|stale|dead|unhealthy' | head -20`
272
+ })
273
+ },
274
+ {
275
+ label: `historical-yarn-hdfs-${host}`,
276
+ input: remoteCommandInput({
277
+ host,
278
+ command: `grep -hE ${shellQuote(window.grepPattern)} /var/log/hadoop-yarn/yarn-yarn-nodemanager-*.log* /var/log/hadoop-hdfs/hadoop-hdfs-datanode-*.log* | egrep -i 'ERROR|WARN|Exception|timeout|timed out|Lost|UNHEALTHY|unhealthy|failed|Failed|network|Network|Connection|refused|reset|slow|Slow|health|I/O error|corrupt|stale|dead|OutOfMemory|Killed' | egrep -v 'DFSUtil.*duration=0|Scheduling blk_|Deleted BP-|DataNode.clienttrace|PacketResponder:.*terminating' | head -40`
279
+ })
280
+ },
281
+ {
282
+ label: `historical-long-gc-${host}`,
283
+ input: remoteCommandInput({
284
+ host,
285
+ command: `grep -hE ${shellQuote(`${window.grepPattern}.*Pause`)} /var/log/hadoop-yarn/gc-nodemanager-*.log* /var/log/hadoop-hdfs/gc-datanode-*.log* | egrep '([1-9][0-9]{3,}ms|[0-9]+\\.[0-9]+s)' | head -10`
286
+ })
287
+ }
288
+ ]));
289
+ }
290
+ return items;
291
+ }
292
+
293
+ mcpSafeReadiness() {
294
+ const missing = [];
295
+ if (!this.appOwner) {
296
+ missing.push("appOwner is unknown, so list-log-host-files and size-log-host-files are omitted until hdfs-find-log-path or yarn log metadata reveals the owner/path.");
297
+ }
298
+ if (!this.nodeHosts.length) {
299
+ missing.push("nodeHosts are unknown, so yarn-node, hdfs-datanode, and historical host-log commands are omitted until aggregated log filenames reveal the app hosts.");
300
+ }
301
+ return {
302
+ concreteOnly: true,
303
+ omittedUntilDiscovered: missing,
304
+ nextStep: missing.length
305
+ ? "Run only the concrete MCP-safe commands shown, then call bds_yarn_app_health_command_pack again with discovered appOwner and nodeHosts."
306
+ : "All app-owner and node-host dependent MCP-safe commands are concrete."
307
+ };
308
+ }
309
+
310
+ historicalWindow() {
311
+ return historicalWindow(this.incidentTimestampUtc, this.historicalWindowMinutes);
312
+ }
313
+
314
+ historicalHealthCommands() {
315
+ const hosts = this.nodeHosts.length ? this.nodeHosts : ["<host-from-aggregated-log-file-name>"];
316
+ const window = this.historicalWindow();
317
+ const remoteScript = [
318
+ "set +e",
319
+ "export LC_ALL=C",
320
+ "host=$(hostname -f 2>/dev/null || hostname)",
321
+ "echo \"HOST $host\"",
322
+ `echo "WINDOW ${window.since} to ${window.until}"`,
323
+ "echo KERNEL_MATCHES",
324
+ `journalctl -k --since ${shellQuote(window.since)} --until ${shellQuote(window.until)} --no-pager 2>/dev/null \\`,
325
+ " | egrep -i 'error|warn|fail|failed|timeout|timed out|reset|I/O|blk_update_request|medium error|disk|volume|network|hung|blocked|oom|killed|read-only|corrupt|stale|dead|unhealthy' \\",
326
+ " | head -20",
327
+ "echo NM_DN_MATCHES",
328
+ `PAT=${shellQuote(window.grepPattern)}`,
329
+ "ERR='ERROR|WARN|Exception|timeout|timed out|Lost|UNHEALTHY|unhealthy|failed|Failed|network|Network|Connection|refused|reset|slow|Slow|health|I/O error|corrupt|stale|dead|OutOfMemory|Killed'",
330
+ "for f in /var/log/hadoop-yarn/yarn-yarn-nodemanager-*.log* /var/log/hadoop-hdfs/hadoop-hdfs-datanode-*.log*; do",
331
+ " [ -f \"$f\" ] || continue",
332
+ " case \"$f\" in *.gz) cmd=zgrep ;; *) cmd=grep ;; esac",
333
+ " matches=$($cmd -E \"$PAT\" \"$f\" 2>/dev/null | egrep -i \"$ERR\" | egrep -v 'DFSUtil.*duration=0|Scheduling blk_|Deleted BP-|DataNode.clienttrace|PacketResponder:.*terminating' | head -10)",
334
+ " if [ -n \"$matches\" ]; then echo \"FILE:$f\"; printf '%s\\n' \"$matches\"; fi",
335
+ "done",
336
+ "echo LONG_GC_PAUSES",
337
+ "for f in /var/log/hadoop-yarn/gc-nodemanager-*.log* /var/log/hadoop-hdfs/gc-datanode-*.log*; do",
338
+ " [ -f \"$f\" ] || continue",
339
+ " case \"$f\" in *.gz) cmd=zgrep ;; *) cmd=grep ;; esac",
340
+ " $cmd -E \"$PAT.*Pause\" \"$f\" 2>/dev/null | awk '{line=$0; dur=$(NF); sub(/ms$/, \"\", dur); if (dur+0 >= 1000) print line}' | head -5 | sed \"s#^#FILE:$f #\"",
341
+ "done"
342
+ ].join("\n");
343
+ return hosts.map((host) => [
344
+ `echo ===== ${host} =====`,
345
+ `ssh -o BatchMode=yes -o ConnectTimeout=20 -o ControlMaster=no -- ${host} ${shellQuote(remoteScript)}`
346
+ ].join("\n")).join("\n\n");
347
+ }
348
+
349
+ commands() {
350
+ const appPath = this.appOwner
351
+ ? hdfsGlobForKnownOwner({ logRoot: this.logRoot, appOwner: this.appOwner, appId: this.applicationId })
352
+ : `<resolved HDFS app log path for ${this.applicationId}>`;
353
+
354
+ return [
355
+ new ReadOnlyStep({
356
+ id: "01-access-and-identity",
357
+ purpose: "Confirm the driver host, UTC clock, available Hadoop/YARN commands, and sudo timestamp without changing state.",
358
+ command: this.ssh("hostname -f; date -u; id -un; sudo -n true >/dev/null 2>&1; echo SUDO_N_STATUS_$?; command -v yarn; command -v hdfs; command -v kinit"),
359
+ expectedEvidence: "driver host identity, time anchor, current user, sudo availability, yarn/hdfs/kinit paths"
360
+ }),
361
+ new ReadOnlyStep({
362
+ id: "02-kerberos-and-rm-state",
363
+ purpose: "Initialize Kerberos from the existing keytab and separate RM/Timeline state from filesystem log state.",
364
+ command: `${this.kinitPrefix()}; timeout 30s yarn application -status ${this.applicationId} 2>&1; echo STATUS_RC:$?`,
365
+ expectedEvidence: "application state if still in RM/Timeline, or explicit not-found evidence"
366
+ }),
367
+ new ReadOnlyStep({
368
+ id: "02b-rm-application-summary",
369
+ purpose: "Recover historical RM completion evidence when RM/Timeline no longer retains the app, and support failed Spark app list investigations.",
370
+ command: `grep -h 'ApplicationSummary' /var/log/hadoop-yarn/yarn-yarn-resourcemanager-*.log* | grep ${shellQuote(this.applicationId)} | grep 'exitCode: 0' | head -20`,
371
+ expectedEvidence: "ApplicationSummary line with finalStatus, diagnostics, finish time, queue, user, and appMasterHost when the app is no longer available through yarn application -status"
372
+ }),
373
+ new ReadOnlyStep({
374
+ id: "03-log-info-without-full-log",
375
+ purpose: "Ask YARN for aggregated log metadata before pulling full logs; this avoids hanging on very large driver logs.",
376
+ command: `${this.kinitPrefix()}; timeout 60s yarn logs -applicationId ${this.applicationId}${this.appOwner ? ` -appOwner ${this.appOwner}` : ""} -show_application_log_info 2>&1 | head -300`,
377
+ expectedEvidence: "Application State, log aggregation fallback message, app owner hints, container or node log metadata"
378
+ }),
379
+ new ReadOnlyStep({
380
+ id: "04-discover-aggregated-log-path",
381
+ purpose: "Find the precise HDFS aggregated log path when owner or suffix is unknown; support bucket-logs-tfile/<last4> layout.",
382
+ command: [
383
+ `${this.kinitPrefix()};`,
384
+ `# Candidate paths: ${this.appPathCandidates().join(" ; ")}`,
385
+ hdfsFindCommand(this.logRoot, this.applicationId)
386
+ ].join(" "),
387
+ expectedEvidence: "exact HDFS app log directory, app owner, suffix such as logs-tfile or bucket-logs-tfile, and bucket number"
388
+ }),
389
+ new ReadOnlyStep({
390
+ id: "05-list-log-host-files",
391
+ purpose: "List host-level aggregated log files and sizes to identify the YARN nodes used by the app and unusually large logs.",
392
+ command: [
393
+ `${this.kinitPrefix()};`,
394
+ `timeout 20s hdfs dfs -ls ${shellQuote(appPath)} 2>&1 | head -300;`,
395
+ `timeout 20s hdfs dfs -du -h ${shellQuote(appPath)} 2>&1 | head -300`
396
+ ].join(" "),
397
+ expectedEvidence: "hostnames from file names, file sizes, replication size, driver log size outliers"
398
+ }),
399
+ new ReadOnlyStep({
400
+ id: "06-yarn-node-health",
401
+ purpose: "Cross-check every host from aggregated logs against current YARN NodeManager health.",
402
+ command: `${this.kinitPrefix()}\n${this.nodeStatusCommands()}`,
403
+ expectedEvidence: "Node-State, Health-Report, Last-Health-Update, resource usage, and container counts for every app host"
404
+ }),
405
+ new ReadOnlyStep({
406
+ id: "07-hdfs-cluster-health",
407
+ purpose: "Check current HDFS cluster health counters and determine which app hosts are also DataNodes.",
408
+ command: [
409
+ `${this.kinitPrefix()}`,
410
+ "timeout 45s hdfs dfsadmin -report 2>&1 | head -35",
411
+ this.hdfsMatchCommands()
412
+ ].join("\n"),
413
+ expectedEvidence: "live DataNode count, under-replicated blocks, missing blocks, corrupt replicas, and per-host DataNode status where applicable"
414
+ }),
415
+ new ReadOnlyStep({
416
+ id: "08-historical-incident-window",
417
+ purpose: "Check the exact incident window for kernel, NodeManager, DataNode, disk, network, timeout, unhealthy, and long-GC signals without changing host state.",
418
+ command: this.historicalHealthCommands(),
419
+ expectedEvidence: "per-host historical kernel/YARN/HDFS health matches around the incident timestamp, with known noise filtered and failed SSH hosts retried narrowly"
420
+ })
421
+ ];
422
+ }
423
+
424
+ toMarkdown() {
425
+ return [
426
+ "# BDS YARN App Host Health Read-Only Command Pack",
427
+ "",
428
+ this.cluster ? `Cluster: ${this.cluster}` : "",
429
+ `Driver host: ${this.driverHost}`,
430
+ `Application ID: ${this.applicationId}`,
431
+ this.incidentTimestampUtc ? `Incident UTC: ${this.incidentTimestampUtc}` : "",
432
+ this.tenant ? `Tenant: ${this.tenant}` : "",
433
+ this.appOwner ? `App owner: ${this.appOwner}` : "",
434
+ `Log root: ${this.logRoot}`,
435
+ "",
436
+ "## Safety Boundary",
437
+ "",
438
+ ...SAFETY_RULES.map((rule) => `- ${rule}`),
439
+ "",
440
+ "## Credential Handling",
441
+ "",
442
+ "- Use Boundary/SSH access to the driver host.",
443
+ "- Use the existing Kerberos keytab via `kinit -kt`; do not print keytab material.",
444
+ "- If sudo is needed to become root, pass the saved LDAP token through stdin only.",
445
+ "",
446
+ "```bash",
447
+ this.sudoWrapperExample(`${this.kinitPrefix()}; yarn application -status ${this.applicationId}`),
448
+ "```",
449
+ "",
450
+ "## Read-Only Commands",
451
+ "",
452
+ ...this.commands().map((command) => command.toMarkdown()),
453
+ "",
454
+ "## MCP-Safe Remote Execution",
455
+ "",
456
+ "When using `remote_readonly_command`, run one concrete JSON input at a time from the structured list below. This list intentionally omits commands that still need discovered appOwner or nodeHosts. Do not copy commands from the earlier bash examples into MCP tool calls. Do not combine commands with `;`, `&&`, command substitution, sudo, shell redirection, nested ssh, or local file writes. If one of these read-only commands is denied or unavailable, report that exact evidence gap and stay within this PRISM/YARN/DataNode investigation.",
457
+ "",
458
+ `Concrete-only status: ${this.mcpSafeReadiness().nextStep}`,
459
+ "",
460
+ "```json",
461
+ JSON.stringify(this.mcpSafeCommandInputs(), null, 2),
462
+ "```",
463
+ "",
464
+ "Procedure, interpretation, and response shape live in the `bds-yarn-app-health-investigation` skill and runbook."
465
+ ].filter(Boolean).join("\n");
466
+ }
467
+
468
+ toJson() {
469
+ return {
470
+ cluster: this.cluster,
471
+ driverHost: this.driverHost,
472
+ applicationId: this.applicationId,
473
+ tenant: this.tenant,
474
+ appOwner: this.appOwner,
475
+ logRoot: this.logRoot,
476
+ appBucket: appBucket(this.applicationId),
477
+ incidentTimestampUtc: this.incidentTimestampUtc,
478
+ historicalWindow: this.historicalWindow(),
479
+ candidatePaths: this.appPathCandidates(),
480
+ nodeHosts: this.nodeHosts,
481
+ safetyBoundary: SAFETY_RULES,
482
+ historicalNoiseFilters: HISTORICAL_NOISE_FILTERS,
483
+ mcpSafeReadiness: this.mcpSafeReadiness(),
484
+ mcpSafeRemoteCommands: this.mcpSafeCommandInputs(),
485
+ commands: this.commands().map((command) => command.toJson())
486
+ };
487
+ }
488
+ }
489
+
490
+ function buildResponse(input) {
491
+ const commandPack = new BdsYarnAppHealthCommandPack(input);
492
+ if (input.format === "json") {
493
+ return JSON.stringify(commandPack.toJson(), null, 2);
494
+ }
495
+ return commandPack.toMarkdown();
496
+ }
497
+
498
+ export default defineReadOnlyTool({
499
+ name: "bds_yarn_app_health_command_pack",
500
+ description: "Generate a atomic-helper, read-only BDS YARN application command pack for app timeout requests, aggregated log host discovery, NodeManager health, and DataNode cross-checks; does not SSH, run YARN, or inspect HDFS.",
501
+ meta: {
502
+ permissions: { rule: "ask" },
503
+ secret_provider: "keychain"
504
+ },
505
+ inputSchema: {
506
+ cluster: clusterSchema.describe("Optional BDS cluster name, for example BDS_ASH_CUST_AZ1."),
507
+ driverHost: hostSchema.describe("Driver or cluster host used for read-only SSH/YARN/HDFS checks."),
508
+ applicationId: appIdSchema.describe("YARN application id, for example application_1778917335863_0195."),
509
+ tenant: tenantSchema.describe("Optional tenant from the request log, for example WD1-PROD.clearskyhealth."),
510
+ appOwner: ownerSchema.describe("Optional YARN app owner if already known, for example wd1-prod_prism."),
511
+ logRoot: pathSchema.describe("YARN aggregated log root."),
512
+ nodeHosts: hostListSchema.describe("Optional app host list discovered from aggregated log filenames."),
513
+ incidentTimestampUtc: utcTimestampSchema.describe("Optional incident timestamp in UTC, for example 2026-05-22 02:46:56.993 UTC."),
514
+ historicalWindowMinutes: minuteWindowSchema.describe("Minutes before and after incidentTimestampUtc to inspect in historical host logs."),
515
+ kerberosPrincipal: principalSchema,
516
+ kerberosKeytab: keytabSchema,
517
+ format: z.enum(["markdown", "json"]).default("markdown")
518
+ },
519
+ annotations: { openWorldHint: true },
520
+ async handler(input) {
521
+ try {
522
+ return ok(buildResponse(input));
523
+ } catch (error) {
524
+ return fail(error);
525
+ }
526
+ }
527
+ });
528
+
529
+ export const __test = { appBucket, hdfsGlobForKnownOwner, hdfsFindCommand, historicalWindow, BdsYarnAppHealthCommandPack };
@@ -0,0 +1,23 @@
1
+ export const description = "Read YARN NodeManager logs from a private-cloud NM/DataNode host through the approved Boundary sudo read tool.";
2
+
3
+ import { createPrivateCloudLogReadTool, globPrivateLogSpec } from "../src/bds-atomic-log-tools.mjs";
4
+
5
+ const spec = globPrivateLogSpec({
6
+ toolName: "bds_yarn_nodemanager_log_read",
7
+ description: "Read YARN NodeManager logs from a private-cloud NM/DataNode host through the approved Boundary sudo read tool.",
8
+ glob: "/var/log/hadoop-yarn/yarn-yarn-nodemanager-*.log"
9
+ });
10
+
11
+ export const __test = spec;
12
+
13
+ const sharedTool = createPrivateCloudLogReadTool(spec);
14
+
15
+ export default {
16
+ name: sharedTool.name,
17
+ description,
18
+ inputSchema: sharedTool.inputSchema,
19
+ meta: sharedTool.meta,
20
+ annotations: sharedTool.annotations,
21
+ invoke: sharedTool.invoke,
22
+ handler: sharedTool.handler
23
+ };
@@ -0,0 +1,23 @@
1
+ export const description = "Read YARN ResourceManager logs from a private-cloud RM host through the approved Boundary sudo read tool.";
2
+
3
+ import { createPrivateCloudLogReadTool, globPrivateLogSpec } from "../src/bds-atomic-log-tools.mjs";
4
+
5
+ const spec = globPrivateLogSpec({
6
+ toolName: "bds_yarn_resourcemanager_log_read",
7
+ description: "Read YARN ResourceManager logs from a private-cloud RM host through the approved Boundary sudo read tool.",
8
+ glob: "/var/log/hadoop-yarn/yarn-yarn-resourcemanager-*.log"
9
+ });
10
+
11
+ export const __test = spec;
12
+
13
+ const sharedTool = createPrivateCloudLogReadTool(spec);
14
+
15
+ export default {
16
+ name: sharedTool.name,
17
+ description,
18
+ inputSchema: sharedTool.inputSchema,
19
+ meta: sharedTool.meta,
20
+ annotations: sharedTool.annotations,
21
+ invoke: sharedTool.invoke,
22
+ handler: sharedTool.handler
23
+ };
@@ -0,0 +1,23 @@
1
+ export const description = "Read ZooKeeper server logs from a private-cloud ZooKeeper host through the approved Boundary sudo read tool.";
2
+
3
+ import { createPrivateCloudLogReadTool, globPrivateLogSpec } from "../src/bds-atomic-log-tools.mjs";
4
+
5
+ const spec = globPrivateLogSpec({
6
+ toolName: "bds_zookeeper_log_read",
7
+ description: "Read ZooKeeper server logs from a private-cloud ZooKeeper host through the approved Boundary sudo read tool.",
8
+ glob: "/var/log/zookeeper/zookeeper-zookeeper-server-*.log"
9
+ });
10
+
11
+ export const __test = spec;
12
+
13
+ const sharedTool = createPrivateCloudLogReadTool(spec);
14
+
15
+ export default {
16
+ name: sharedTool.name,
17
+ description,
18
+ inputSchema: sharedTool.inputSchema,
19
+ meta: sharedTool.meta,
20
+ annotations: sharedTool.annotations,
21
+ invoke: sharedTool.invoke,
22
+ handler: sharedTool.handler
23
+ };