@nuvlore/extension-bds 0.1.1

package/workflows/bds-yarn-app-health-investigation.js

@@ -0,0 +1,217 @@
+export const meta = {
+  name: "bds-yarn-app-health-investigation",
+  description: "Investigate BDS YARN application host health for Prism timeout requests with read-only aggregated-log discovery, NodeManager checks, and HDFS DataNode cross-checks.",
+  whenToUse: "Use when Slack/Jira asks to cross-check data nodes or YARN nodes used by a BDS application ID, especially Prism request timeout cases with a driver host.",
+  phases: ["Plan", "ResolveHost", "Access", "Logs", "Nodes", "HDFS", "Historical", "Reply"],
+};
+
+phase("Plan");
+const plan = await agent(
+  [
+    "Extract BDS cluster, driver host, application ID, tenant, prid/cid, and incident UTC window from the user-provided request.",
+    "Use the skill flow as the procedure; call only mcp__devops_tool__bds_yarn_app_health_command_pack to normalize inputs and produce the read-only command pack.",
+    "No Bash, browser access, WebFetch, local reproduction, or mutating Hadoop/YARN commands.",
+    "Do not switch to disk alerts, Jira, Bamboo, Pharos, local repository search, or local cron files for this PRISM/YARN/DataNode host-health workflow.",
+    "Do not spawn Agent, Task, TaskCreate, background agents, or worktrees. Keep execution in this workflow and use only the phase tools.",
+    "Return parsed fields, missing fields, and command-pack output.",
+  ].join("\n"),
+  {
+    label: "bds-yarn-app-plan",
+    phase: "Plan",
+    tools: "mcp__devops_tool__bds_yarn_app_health_command_pack",
+    disallowedTools: "Agent,Task,TaskCreate,Bash,Read,Glob,Grep,WebFetch,WebSearch,Write,Edit,NotebookEdit,AskUserQuestion",
+  },
+);
+
+phase("ResolveHost");
+const resolvedHost = await agent(
+  [
+    "Resolve the concrete BDS driver/control host before Boundary login or remote commands.",
+    "If the request provides a driver host, confirm it against BDS host lookup/inventory instead of assuming it is valid for the cluster.",
+    "If no driver host is provided, use mcp__devops_tool__bds_environment_host_lookup with roles=[\"resourcemanager\",\"namenode\",\"nodemanager\",\"datanode\"] to choose a read-only host that can run YARN/HDFS status commands.",
+    "Keep driverHost, cluster, applicationId, tenant, incidentUtc, and later app node hosts as separate fields.",
+    "Do not invent host tools such as workday_host_info, host_info, host_query, or bds_enterprise_host_lookup.",
+    "Do not use grep, ls, Read, Glob, repository search, local /hadoop paths, local yarn, or local hdfs as a substitute for host lookup.",
+    "If the exact host lookup MCP tool is unavailable or fails, stop and report the exact tool error as a tooling/access gap.",
+    "Return driverHost, cluster, lookup evidence, role used, stale host exclusions, and ambiguity.",
+    "",
+    "Plan:",
+    plan,
+  ].join("\n"),
+  {
+    label: "bds-yarn-app-resolve-host",
+    phase: "ResolveHost",
+    tools: "mcp__devops_tool__bds_environment_host_lookup,mcp__devops_tool__bds_machine_inventory",
+    disallowedTools: "Agent,Task,TaskCreate,Bash,Read,Glob,Grep,WebFetch,WebSearch,Write,Edit,NotebookEdit,AskUserQuestion,mcp__devops_tool__workday_boundary_login,mcp__devops_tool__remote_readonly_command",
+  },
+);
+
+phase("Access");
+const access = await agent(
+  [
+    "Check Boundary/DC readiness for the resolved driverHost; use workday_boundary_login action=status unless login was explicitly approved.",
+    "If sudo needs a password, use only mcp__devops_tool__workday_private_cloud_sudo_readonly_command; never ask for, print, or directly read the LDAP password.",
+    "Confirm host identity, UTC clock, and yarn/hdfs/kinit availability with remote_readonly_command.",
+    "Use only concrete MCP-Safe Remote Execution JSON inputs from the plan; do not copy bash examples, placeholder inputs, ssh wrappers, semicolons, sudo, or local shell wrappers.",
+    "Call remote_readonly_command with exactly the copied JSON object. Do not put ssh in the command field; the tool performs SSH from targetType and host.",
+    "If remote_readonly_command is denied or unavailable, report the exact evidence gap and continue only with available PRISM/YARN/DataNode evidence.",
+    "Do not modify SSH config, write files, or print passwords, tokens, Keychain JSON, keytabs, or sudo stdin.",
+    "Do not switch to disk alerts, Jira, Bamboo, Pharos, local repository search, or local cron files when access is blocked.",
+    "",
+    "Plan:",
+    plan,
+    "",
+    "ResolvedHost:",
+    resolvedHost,
+  ].join("\n"),
+  {
+    label: "bds-yarn-app-access",
+    phase: "Access",
+    tools: "mcp__devops_tool__workday_boundary_login,mcp__devops_tool__remote_readonly_command",
+    disallowedTools: "Agent,Task,TaskCreate,Bash,Read,Glob,Grep,WebFetch,WebSearch,Write,Edit,NotebookEdit,AskUserQuestion",
+  },
+);
+
+phase("Logs");
+const logs = await agent(
+  [
+    "Collect read-only YARN and HDFS aggregated-log evidence.",
+    "Use Kerberos keytab only as required for reads; run yarn application -status and yarn logs -show_application_log_info before bounded log reads.",
+    "If yarn application -status cannot find a historical app, treat that as possible RM retention expiry and read ResourceManager ApplicationSummary logs under /var/log/hadoop-yarn/yarn-yarn-resourcemanager-*.log*.",
+    "For full failed-app-id requests, filter ApplicationSummary lines in the incident UTC window by applicationType=SPARK, finalStatus=FAILED, and diagnostics containing exitCode: 0, then return finishTimeUtc, appId, name, user, queue, finalStatus, and appMasterHost.",
+    "If RM/Timeline cannot find the app, use hdfs dfs -find for the exact app ID; resolve owner/suffix layouts such as wd1-prod_prism/bucket-logs-tfile/<last4>/<applicationId>.",
+    "List and size the resolved app log directory with hdfs dfs -ls and hdfs dfs -du -h.",
+    "Use only one concrete MCP-safe remote_readonly_command input at a time from the plan.",
+    "If the current plan has no concrete list/size/node/historical MCP-safe inputs because owner or hosts are unknown, first use concrete discovery inputs, then call bds_yarn_app_health_command_pack again with discovered appOwner and nodeHosts.",
+    "Do not run hdfs dfs -get, hdfs dfs -rm, hdfs dfs -put, full unbounded yarn logs, or any write command.",
+    "If evidence collection is blocked, report the blocked command and do not pivot to unrelated observability or local files.",
+    "",
+    "Plan:",
+    plan,
+    "",
+    "ResolvedHost:",
+    resolvedHost,
+    "",
+    "Access:",
+    access,
+  ].join("\n"),
+  {
+    label: "bds-yarn-app-logs",
+    phase: "Logs",
+    tools: "mcp__devops_tool__remote_readonly_command",
+    disallowedTools: "Agent,Task,TaskCreate,Bash,Read,Glob,Grep,WebFetch,WebSearch,Write,Edit,NotebookEdit,AskUserQuestion",
+  },
+);
+
+phase("Nodes");
+const nodes = await agent(
+  [
+    "Derive YARN hosts from aggregated log file names; if newly discovered, call bds_yarn_app_health_command_pack again with nodeHosts for concrete node/historical commands.",
+    "Run yarn node -status <host>:45454 for every app host.",
+    "Capture Node-State, Health-Report, Last-Health-Update, containers, memory, CPU, and utilization.",
+    "If blocked or denied, mark that node evidence-incomplete and do not change scope.",
+    "Return a compact node table and identify non-RUNNING or non-empty health reports.",
+    "",
+    "Logs:",
+    logs,
+  ].join("\n"),
+  {
+    label: "bds-yarn-app-nodes",
+    phase: "Nodes",
+    tools: "mcp__devops_tool__bds_yarn_app_health_command_pack,mcp__devops_tool__remote_readonly_command",
+    disallowedTools: "Agent,Task,TaskCreate,Bash,Read,Glob,Grep,WebFetch,WebSearch,Write,Edit,NotebookEdit,AskUserQuestion",
+  },
+);
+
+phase("HDFS");
+const hdfs = await agent(
+  [
+    "Collect current HDFS health from hdfs dfsadmin -report: live DataNodes, under-replicated/missing/low-redundancy blocks, corrupt replicas, and pending deletion counts.",
+    "For each app host, note DataNode presence, decommission status, and last contact.",
+    "Separate NodeManager-only hosts from DataNode hosts.",
+    "If HDFS checks are blocked or denied, report the exact evidence gap and do not use disk-alert, Pharos, Jira, Bamboo, or local-file evidence as a substitute.",
+    "",
+    "Nodes:",
+    nodes,
+  ].join("\n"),
+  {
+    label: "bds-yarn-app-hdfs",
+    phase: "HDFS",
+    tools: "mcp__devops_tool__remote_readonly_command",
+    disallowedTools: "Agent,Task,TaskCreate,Bash,Read,Glob,Grep,WebFetch,WebSearch,Write,Edit,NotebookEdit,AskUserQuestion",
+  },
+);
+
+phase("Historical");
+const historical = await agent(
+  [
+    "If an incident UTC timestamp is available, check the historical incident window across the app hosts.",
+    "Use only read-only host log reads: journalctl -k plus bounded grep/zgrep over YARN NodeManager and HDFS DataNode logs.",
+    "Filter routine noise: DFSUtil duration=0, block deletion INFO, DataNode.clienttrace, PacketResponder terminating, and short GC pause detail lines.",
+    "Call out only relevant kernel, disk, network, timeout, unhealthy, DataNode, NodeManager, or long-GC signals.",
+    "If broad scans drop SSH or time out, retry incomplete hosts with narrower file targets and no SSH multiplexing before concluding.",
+    "Do not mutate hosts, services, HDFS, YARN, SSH config, or local files.",
+    "If historical checks are blocked or denied, state historical evidence is incomplete and do not pivot to disk alerts, Jira, Bamboo, Pharos, local repository search, or local cron files.",
+    "",
+    "Logs:",
+    logs,
+    "",
+    "Nodes:",
+    nodes,
+    "",
+    "HDFS:",
+    hdfs,
+  ].join("\n"),
+  {
+    label: "bds-yarn-app-historical",
+    phase: "Historical",
+    tools: "mcp__devops_tool__remote_readonly_command",
+    disallowedTools: "Agent,Task,TaskCreate,Bash,Read,Glob,Grep,WebFetch,WebSearch,Write,Edit,NotebookEdit,AskUserQuestion",
+  },
+);
+
+phase("Reply");
+const reply = await agent(
+  [
+    "Draft a concise requester-ready answer from collected evidence only.",
+    "If a full ResourceManager failed-app list was requested, include timestamps and state the ApplicationSummary filter criteria used.",
+    "Include app hosts, current YARN health, current HDFS counters, DataNode matches, and historical-window findings when checked.",
+    "If historical logs were checked, state the exact window and whether kernel/YARN NodeManager/HDFS DataNode, disk/network, unhealthy, or long-GC signals were found.",
+    "If only current-state checks were performed, state clearly that current-state checks do not prove historical incident-window health.",
+    "Do not mention minor unrelated WARNs in the requester-facing reply unless they are plausibly tied to the incident window or HDFS/DataNode path.",
+    "Do not overstate RCA or claim there was no issue at the incident time without historical-window evidence.",
+    "If any step was blocked by read-only policy or access, include the exact blocked evidence gap and do not replace it with unrelated disk/Jira/Bamboo/Pharos/local evidence.",
+    "",
+    "Plan:",
+    plan,
+    "",
+    "Logs:",
+    logs,
+    "",
+    "Nodes:",
+    nodes,
+    "",
+    "HDFS:",
+    hdfs,
+    "",
+    "Historical:",
+    historical,
+  ].join("\n"),
+  {
+    label: "bds-yarn-app-reply",
+    phase: "Reply",
+    tools: "",
+    disallowedTools: "Agent,Task,TaskCreate,Bash,Read,Glob,Grep,WebFetch,WebSearch,Write,Edit,NotebookEdit,AskUserQuestion",
+  },
+);
+
+return {
+  plan,
+  resolvedHost,
+  access,
+  logs,
+  nodes,
+  hdfs,
+  historical,
+  reply,
+};