@nuvlore/extension-bds 0.1.1

package/tools/bds_airflow_job_status_command_pack.mjs

@@ -0,0 +1,587 @@
+export const description = "Generate a atomic-helper, read-only BDS Airflow command pack from environment Kubernetes control-plane JSON, airflow namespace checks, and S3 VPCE probes; does not SSH, run kubectl, or inspect Airflow.";
+
+import { readFile } from "node:fs/promises";
+import { z } from "zod";
+import { defineReadOnlyTool, fail, ok } from "@nuvlore/extension-read-only-toolkit/devops-diagnostics";
+import { ReadOnlyStep, remoteCommandInput, shellQuote } from "@nuvlore/extension-read-only-toolkit/read-only-command-pack";
+import {
+  DEFAULT_BOUNDARY_BINARY_DIR,
+  DEFAULT_BOUNDARY_PROXY_JUMP_HOST,
+  DEFAULT_DC_BOUNDARY_ADDR,
+  DcLoginProfile,
+  SudoEnvironmentProfile
+} from "@nuvlore/extension-workday-infra-access/src/workday-access-profiles.mjs";
+
+const DEFAULT_NAMESPACE = "airflow";
+const DEFAULT_BOUNDARY_ADDR = DEFAULT_DC_BOUNDARY_ADDR;
+const DEFAULT_BAS_DR_SLA_SOURCE_HOST = "s-11rdzb3.sys.az1.cust.ash.wd";
+const DEFAULT_BAS_DR_SLA_CLUSTER = "BDS_ASH_CUST_AZ1";
+
+const SAFETY_RULES = [
+  "read-only only: kubectl get, describe, logs, events, config current-context, and bounded airflow CLI list/state commands",
+  "before parsing local BDS config or Airflow code, refresh the relevant local repos with git pull --ff-only; if a repo is dirty, report the pull gap and do not stash or overwrite local changes",
+  "S3 VPCE connectivity checks must stay read-only: DNS lookup, route lookup, and curl connect probes with no local file output and no aws s3 cp/copy/write operations",
+  "use a Kubernetes control-plane host from the environment repo JSON kube_control_planes list",
+  "for ATL, prefer the latest BDS_ATL_CUST_AZ1 config host list; old control-plane hostnames can disappear after config refresh",
+  "use Boundary with BOUNDARY_SKIP_CACHE_DAEMON=true and PATH including /opt/homebrew/bin when SSH config proxy commands cannot find or reach boundary",
+  "if default SSH config routes ATL through a stale Atl-Nprd-Mgmt bastion or banner exchange times out, explicitly use the Boundary proxy jump host proxy-jump.atl.bnd with -J for read-only SSH",
+  "kubectl as a normal SSH user may fall back to localhost:8080 and fail; use the root kubeconfig on the control-plane host for cluster reads",
+  "when sudo requires a password, use mcp__devops-tool__workday_private_cloud_sudo_readonly_command; command packs and agents must not read Keychain directly",
+  "default namespace is usually empty for BDS Airflow; query the airflow namespace explicitly",
+  "for bas_dr_sla_miss or DR copy SLA alerts, collect pod/node/runtime evidence first; do not delete running pods or stop host processes from this command pack",
+  "force-delete is only a post-approval remediation candidate for stale completed or terminating DAG task pods after logs prove task success and the pod has no useful live progress",
+  "for snapshot-cleanup latency investigations, read the schedule from cookbook_bitbucket bds_kube/recipes/helm_deploy.rb and keep all Airflow, HDFS, and NameNode checks read-only",
+  "for snapshot-cleanup lock investigations, bound log scans by cleaner run time and hourly NameNode/audit files; avoid unbounded all-day zgrep over multi-GB audit directories",
+  "never run kubectl delete/apply/patch/scale/rollout restart/exec with mutation, helm writes, airflow clear, airflow backfill, or DAG/task state mutation",
+  "never print LDAP passwords, Keychain JSON, tokens, kubeconfig, or sudo stdin"
+];
+
+const INTERPRETATION_RULES = [
+  "Ready Airflow scheduler/web pods with zero restarts and no pod events support core Airflow component health.",
+  "Repeated normal namespace events for short-lived DAG pods can indicate DAG activity rather than failure when reason is Scheduled/Pulled/Created/Started only.",
+  "HTTP 200 responses on /airflow/health support webserver liveness/readiness.",
+  "Airflow DagBag Failed to import plus FileNotFoundError for /etc/hadoop/conf/bds_config.json or /data/workday-bds-airflow/dr_to_dr_config/dr_to_dr_config.json is a DAG/config risk even when scheduler/web pods are Running.",
+  "A long-running tenant-data namespace migration pod can be healthy if Ready with zero restarts and its DistCp/YARN progress is still advancing; report the tenant, process, src/dst clusters, source snapshot, and HDFS source/destination paths separately from Airflow component health.",
+  "For bas_dr_sla_miss on tenant-data DR copy, treat mirror snapshot lag as the alert symptom and distinguish DistCp/copy failure from Kubernetes node runtime cleanup failure using pod logs, pod events, deletionTimestamp/finalizers, and node process/runtime evidence.",
+  "A DAG task pod stuck in Terminating or ContainerCreating on one worker with no finalizers and no pod events, while task logs already show Command exited with return code 0 and Marking task as SUCCESS, supports stale kubelet/container runtime cleanup failure rather than Airflow DAG or tenant configuration failure.",
+  "A Kubernetes worker with very high load, many D/Z state or zombie processes, kubelet/containerd systemctl timeouts, and stuck runc/mount/cgroup cleanup activity should stay cordoned pending DC/node recovery; do not uncordon until node runtime health is confirmed.",
+  "After stale pod cleanup by an approved operator, verify recovery by confirming new DAG task pods schedule on healthy workers, affected tenants create newer DR mirror snapshots, and BAS lag drops below the 3600 second threshold.",
+  "For BDS_ASH_CUST_AZ1 bas_dr_sla_miss, the live BAS lag source is the Pharos script exporter check on s-11rdzb3.sys.az1.cust.ash.wd: sudo /data/pharos_script_exporter/scripts/bas_dr_sla_miss_metric.rb BDS_ASH_CUST_AZ1.",
+  "For BDS migration S3 copy failures, a curl result with http=000 and a TCP connect timeout to the VPCE hostname indicates the request did not reach HTTP/S3 authorization; a reachable alternate VPCE returning HTTP 307 or another HTTP status proves the same worker can reach the S3 endpoint path.",
+  "When switching from a failed direct-connect VPCE to an alternate S3 VPCE, verify old and new endpoint hostnames from the same ATL worker/source IP before rerunning the migration copy.",
+  "For snapshot cleanup incidents, deleteSnapshot is expected to hold the NameNode write lock; HDFS metadata reads such as getFileInfo/getfileinfo, listStatus, open, and getBlockLocations require the NameNode read lock and wait behind the write lock.",
+  "A cleaner run with many repeated snapshot deletions for the same tenant, elevated delete durations, and matching NameNode deleteSnapshot long write-locks supports snapshot backlog or large diff cleanup as the cause; direct diff size remains unproven unless logs or an approved bounded diff/count command show it.",
+  "For PRISM/Spark CreateDataFrame latency during snapshot cleanup, use fast-case NameNode audit logs such as target _delta_log getfileinfo before or after the lock window to verify the path issues read-lock metadata RPCs.",
+  "For BDS_ASH_CUST_AZ1, the snapshot cleanup schedule is configured in cookbook_bitbucket bds_kube/recipes/helm_deploy.rb as SCHEDULE_ASH_CUST = '0 2,3,5 * * 3-5' with a 3600 second timeout.",
+  "A BDS control-panel DAG identifier like dag_tenant_data_copy_on_demand--1780713387370568 can encode epoch microseconds; convert the suffix to UTC and inspect the matching Airflow log directory manual__YYYY-MM-DDTHH:MM:SS.micros+00:00 before concluding the run is missing.",
+  "For tenant on-demand copy status, the scheduler container log path /data/workday-bds-airflow/logs/dag_tenant_data_copy_on_demand/<manual_run>/run_tenant_data_ondemand_restore_copy/1.log gives the source/target tenant, snapshots, YARN apps, DistCp progress, and final task success or failure.",
+  "Prometheus statsd-exporter superfluous response.WriteHeader lines are noisy unless paired with pod restarts or scrape failures.",
+  "sensu-client chown read-only filesystem messages are sidecar noise unless the Airflow containers are unhealthy."
+];
+
+const hostSchema = z
+  .string()
+  .min(1)
+  .max(255)
+  .regex(/^(?!-)[a-zA-Z0-9._:@%-]+$/);
+
+const namespaceSchema = z
+  .string()
+  .min(1)
+  .max(63)
+  .regex(/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/)
+  .default(DEFAULT_NAMESPACE);
+
+const envNameSchema = z
+  .string()
+  .min(1)
+  .max(160)
+  .regex(/^[A-Za-z0-9_.:-]+$/)
+  .optional();
+
+const pathSchema = z.string().min(1).max(1024).optional();
+const urlSchema = z.string().trim().url().max(2048).default(DEFAULT_BOUNDARY_ADDR);
+const localPathSchema = z.string().trim().min(1).max(2048);
+const s3EndpointUrlSchema = z.string().trim().url().max(2048).regex(/^https:\/\/[^/]+\.s3\.[A-Za-z0-9-]+\.vpce\.amazonaws\.com\/?$/);
+
+const dagSchema = z
+  .string()
+  .min(1)
+  .max(255)
+  .regex(/^[A-Za-z0-9_.:@/-]+$/)
+  .optional();
+
+const runSchema = z
+  .string()
+  .min(1)
+  .max(255)
+  .regex(/^[A-Za-z0-9_.:@+=,~%/-]+$/)
+  .optional();
+
+function urlHostname(value) {
+  return new URL(value).hostname;
+}
+
+function uniq(values) {
+  return Array.from(new Set(values.filter(Boolean)));
+}
+
+function collectStringArrays(value, path = [], output = []) {
+  if (!value || typeof value !== "object") return output;
+  if (Array.isArray(value)) {
+    if (value.every((item) => typeof item === "string")) {
+      output.push({ path: path.join("."), values: value });
+    }
+    for (let index = 0; index < value.length; index += 1) {
+      collectStringArrays(value[index], [...path, String(index)], output);
+    }
+    return output;
+  }
+  for (const [key, nested] of Object.entries(value)) {
+    collectStringArrays(nested, [...path, key], output);
+  }
+  return output;
+}
+
+function discoverKubernetesHosts(doc) {
+  const arrays = collectStringArrays(doc);
+  const byPath = new Map(arrays.map((entry) => [entry.path, entry.values]));
+  const controlPlane =
+    byPath.get("kubernetes.kube_control_planes") ??
+    arrays.find((entry) => /kubernetes.*kube_control_planes$/i.test(entry.path))?.values ??
+    [];
+  const workers =
+    byPath.get("kubernetes.kube_workers") ??
+    arrays.find((entry) => /kubernetes.*kube_workers$/i.test(entry.path))?.values ??
+    [];
+  const etcd =
+    byPath.get("kubernetes.etcd") ??
+    arrays.find((entry) => /kubernetes.*etcd$/i.test(entry.path))?.values ??
+    [];
+  return {
+    controlPlaneHosts: uniq(controlPlane),
+    workerHosts: uniq(workers),
+    etcdHosts: uniq(etcd)
+  };
+}
+
+async function loadEnvironmentJson(configJsonPath) {
+  if (!configJsonPath) return {};
+  const parsed = JSON.parse(await readFile(configJsonPath, "utf8"));
+  return discoverKubernetesHosts(parsed);
+}
+
+class BdsAirflowJobStatusCommandPack {
+  constructor(input) {
+    this.environmentName = input.environmentName;
+    this.configJsonPath = input.configJsonPath;
+    this.controlPlaneHosts = input.controlPlaneHosts ?? [];
+    this.workerHosts = input.workerHosts ?? [];
+    this.etcdHosts = input.etcdHosts ?? [];
+    this.namespace = input.namespace ?? DEFAULT_NAMESPACE;
+    this.dagId = input.dagId;
+    this.runId = input.runId;
+    this.s3VpceEndpointUrls = input.s3VpceEndpointUrls ?? [];
+    this.alternateS3VpceEndpointUrls = input.alternateS3VpceEndpointUrls ?? [];
+    this.boundaryAddr = input.boundaryAddr ?? DEFAULT_BOUNDARY_ADDR;
+    this.boundaryBinaryDir = input.boundaryBinaryDir ?? DEFAULT_BOUNDARY_BINARY_DIR;
+    this.boundaryProxyJumpHost = input.boundaryProxyJumpHost ?? DEFAULT_BOUNDARY_PROXY_JUMP_HOST;
+    this.useBoundaryBnd = input.useBoundaryBnd ?? false;
+    this.dcProfile = new DcLoginProfile({
+      boundaryAddr: this.boundaryAddr,
+      boundaryBinaryDir: this.boundaryBinaryDir,
+      boundaryProxyJumpHost: this.boundaryProxyJumpHost,
+      useBoundaryBnd: this.useBoundaryBnd
+    });
+    this.sudoProfile = new SudoEnvironmentProfile();
+    this.basDrSlaSourceHost = input.basDrSlaSourceHost ?? DEFAULT_BAS_DR_SLA_SOURCE_HOST;
+    this.basDrSlaCluster = input.basDrSlaCluster ?? DEFAULT_BAS_DR_SLA_CLUSTER;
+  }
+
+  preferredControlPlane() {
+    return this.controlPlaneHosts[0] ?? "<kube_control_plane_host_from_environment_json>";
+  }
+
+  boundaryHost() {
+    return this.boundaryHostFor(this.preferredControlPlane());
+  }
+
+  boundaryHostFor(host) {
+    return this.dcProfile.boundaryHostFor(host);
+  }
+
+  sshEnvironmentPrefix() {
+    return this.dcProfile.environmentPrefix();
+  }
+
+  sshCommand(remoteCommand) {
+    return this.sshCommandFor(this.preferredControlPlane(), remoteCommand);
+  }
+
+  sshCommandFor(host, remoteCommand) {
+    return this.dcProfile.sshCommandFor(host, remoteCommand);
+  }
+
+  sshProxyJumpCommandFor(host, remoteCommand) {
+    return this.dcProfile.proxyJumpCommandFor(host, remoteCommand);
+  }
+
+  rootSsh(command) {
+    return this.sshCommand(`sudo -n bash -lc ${JSON.stringify(command)}`);
+  }
+
+  sudoWrapperExample(command) {
+    return this.sudoWrapperExampleFor(this.preferredControlPlane(), command);
+  }
+
+  keychainCandidates() {
+    return this.sudoProfile.keychainCandidates();
+  }
+
+  keychainReadCommand() {
+    return this.sudoProfile.keychainReadCommand();
+  }
+
+  sudoWrapperExampleFor(host, command) {
+    return this.sudoProfile.sudoWrapperFor({
+      dcProfile: this.dcProfile,
+      host,
+      command
+    });
+  }
+
+  airflowSelector() {
+    const items = [];
+    if (this.dagId) items.push(`dag_id=${this.dagId}`);
+    if (this.runId) items.push(`run_id=${this.runId}`);
+    return items.length ? items.join(" ") : "<dag_id/run_id if provided by requester>";
+  }
+
+  s3EndpointUrls() {
+    return Array.from(new Set([
+      ...this.s3VpceEndpointUrls,
+      ...this.alternateS3VpceEndpointUrls
+    ]));
+  }
+
+  s3VpceConnectivityScript() {
+    const endpoints = this.s3EndpointUrls();
+    if (!endpoints.length) {
+      return [
+        "set -eu",
+        "endpoint='<s3_vpce_https_endpoint_from_failure_log>'",
+        "host=${endpoint#https://}",
+        "host=${host%%/*}",
+        "echo ENDPOINT $endpoint",
+        "getent hosts \"$host\"",
+        "getent hosts \"$host\" | awk '{print $1}' | while read -r ip; do ip route get \"$ip\"; done",
+        "curl -k -sS -o /dev/null --connect-timeout 10 --max-time 15 -w 'host=%{local_ip} remote=%{remote_ip} connect=%{time_connect} http=%{http_code} err=%{errormsg}\\n' \"$endpoint\""
+      ].join("\n");
+    }
+
+    return [
+      "set -eu",
+      ...endpoints.flatMap((endpoint) => {
+        const label = this.s3VpceEndpointUrls.includes(endpoint) ? "current-or-failing" : "alternate";
+        const host = urlHostname(endpoint);
+        return [
+          `echo ENDPOINT ${label} ${shellQuote(endpoint)}`,
+          `getent hosts ${shellQuote(host)}`,
+          `getent hosts ${shellQuote(host)} | awk '{print $1}' | while read -r ip; do ip route get "$ip"; done`,
+          `curl -k -sS -o /dev/null --connect-timeout 10 --max-time 15 -w 'host=%{local_ip} remote=%{remote_ip} connect=%{time_connect} http=%{http_code} err=%{errormsg}\\n' ${shellQuote(endpoint)}`
+        ];
+      })
+    ].join("\n");
+  }
+
+  mcpSafeCommandInputs() {
+    const host = this.preferredControlPlane();
+    const ns = this.namespace;
+    return [
+      { label: "control-plane-hostname", input: remoteCommandInput({ host, command: "hostname -f" }) },
+      { label: "control-plane-user", input: remoteCommandInput({ host, command: "id -un" }) },
+      { label: "normal-user-kubectl-context", input: remoteCommandInput({ host, command: "kubectl config current-context" }) },
+      { label: "normal-user-default-pods", input: remoteCommandInput({ host, command: "kubectl get pod" }) },
+      { label: "root-kubectl-context", input: remoteCommandInput({ host, command: "sudo -n kubectl config current-context" }) },
+      { label: "airflow-pods", input: remoteCommandInput({ host, command: `sudo -n kubectl get pod -n ${ns} -o wide` }) },
+      { label: "airflow-events", input: remoteCommandInput({ host, command: `sudo -n kubectl get events -n ${ns} --sort-by=.lastTimestamp | tail -80` }) },
+      { label: "airflow-deployments", input: remoteCommandInput({ host, command: `sudo -n kubectl get deploy,statefulset,job,cronjob -n ${ns} -o wide` }) }
+    ];
+  }
+
+  commands() {
+    const ns = this.namespace;
+    return [
+      new ReadOnlyStep({
+        id: "01-environment-json",
+        purpose: "Use the environment repo JSON as the authoritative source for Kubernetes control-plane hosts.",
+        command: [
+          "# Refresh relevant local repos first. Use --ff-only and report dirty repos instead of stashing or overwriting local changes.",
+          "git -C /Users/boqiang.liang/500-work/bds-config-nprd pull --ff-only",
+          "git -C /Users/boqiang.liang/500-work/bds-k8-dc-airflow pull --ff-only",
+          `jq -r '(.bds.kubernetes.kube_control_planes // .kubernetes.kube_control_planes // [])[]' ${this.configJsonPath ? shellQuote(this.configJsonPath) : "<environment-json-file>"}`,
+          `# control planes discovered: ${this.controlPlaneHosts.length ? this.controlPlaneHosts.join(", ") : "<none provided yet>"}`,
+          `# workers discovered: ${this.workerHosts.length ? this.workerHosts.join(", ") : "<none provided yet>"}`
+        ].join("\n"),
+        expectedEvidence: "kubernetes.kube_control_planes, kube_workers, and etcd host lists for the selected environment"
+      }),
+      new ReadOnlyStep({
+        id: "02-control-plane-login",
+        purpose: "Run bauth init, then ordinary SSH to one control-plane host and confirm host identity before running cluster reads.",
+        command: [
+          this.sshCommand("hostname -f; date -u; id -un"),
+          "",
+          "# If the local SSH config uses a stale ATL bastion or times out during banner exchange, explicitly use the Boundary proxy jump host:",
+          this.sshProxyJumpCommandFor(this.preferredControlPlane(), "hostname -f; date -u; id -un")
+        ].join("\n"),
+        expectedEvidence: "control-plane hostname, UTC timestamp, and current user; for ATL, proxy-jump.atl.bnd can bypass a stale Atl-Nprd-Mgmt bastion path"
+      }),
+      new ReadOnlyStep({
+        id: "03-normal-user-kubectl-baseline",
+        purpose: "Document whether the normal SSH user lacks kubeconfig; localhost:8080 refusal is expected evidence, not the cluster state.",
+        command: this.sshCommand("kubectl get pod 2>&1 | head -40"),
+        expectedEvidence: "either normal-user context output or localhost:8080 refused evidence"
+      }),
+      new ReadOnlyStep({
+        id: "04-root-context-with-sudo-stdin",
+        purpose: "When sudo -n requires a password, use the approved PROD LDAP keychain item through stdin only and verify the root kubeconfig context.",
+        command: this.sudoWrapperExample("kubectl config current-context"),
+        expectedEvidence: "root kubeconfig context such as kubernetes-admin@kubernetes; no password or token output"
+      }),
+      new ReadOnlyStep({
+        id: "05-root-kubectl-airflow-pods",
+        purpose: "Use the root kubeconfig on the control-plane host and query the airflow namespace explicitly.",
+        command: this.sudoWrapperExample(`kubectl get pod -n ${ns} -o wide`),
+        expectedEvidence: "airflow scheduler and web pods, readiness, status, restarts, age, node placement, and pod IP"
+      }),
+      new ReadOnlyStep({
+        id: "06-airflow-workload-and-events",
+        purpose: "Read workload objects and namespace events to identify restarts, pending pods, failed jobs, or recent scheduling issues.",
+        command: this.sudoWrapperExample(`kubectl get deploy,statefulset,job,cronjob -n ${ns} -o wide; kubectl get events -n ${ns} --sort-by=.lastTimestamp | tail -80`),
+        expectedEvidence: "deployment/job shape and recent Airflow namespace warnings or normal scheduling events"
+      }),
+      new ReadOnlyStep({
+        id: "07-airflow-component-logs",
+        purpose: "Read bounded scheduler and web logs for DAG parsing, executor, KubernetesPodOperator, or task-state errors.",
+        command: [
+          this.sudoWrapperExample(`kubectl describe pod -n ${ns} -l app=airflow-scheduler | egrep -A4 '^Name:|^Containers:|^Conditions:|^Events:'`),
+          "",
+          this.sudoWrapperExample(`kubectl describe pod -n ${ns} -l app=airflow-web | egrep -A4 '^Name:|^Containers:|^Conditions:|^Events:'`),
+          "",
+          this.sudoWrapperExample(`kubectl logs -n ${ns} deploy/airflow-scheduler --all-containers --tail=160 2>&1`),
+          "",
+          this.sudoWrapperExample(`kubectl logs -n ${ns} deploy/airflow-web --all-containers --tail=80 2>&1`)
+        ].join("\n"),
+        expectedEvidence: "pod conditions/events, recent Airflow scheduler/web errors, DAG parse failures, health probe responses, or clean logs"
+      }),
+      new ReadOnlyStep({
+        id: "08-target-dag-run-if-known",
+        purpose: "If DAG or run ID is known, inspect the Airflow metadata from the scheduler pod without changing task state.",
+        command: this.sudoWrapperExample(`pod=$(kubectl get pod -n ${ns} -o name | grep airflow-scheduler | head -1); kubectl exec -n ${ns} "$pod" -- airflow dags list-runs${this.dagId ? ` -d ${shellQuote(this.dagId)}` : ""} --no-backfill --output table | head -80`),
+        expectedEvidence: `DAG run rows for ${this.airflowSelector()}, including run id, state, start/end dates, and execution date`
+      }),
+      new ReadOnlyStep({
+        id: "09-running-task-copy-source-target",
+        purpose: "For a long-running DAG task pod, extract the tenant data copy process, src/dst clusters, source snapshot, and HDFS source/destination paths from pod metadata and logs; do not delete pods that are actively progressing.",
+        command: [
+          this.sudoWrapperExample(`kubectl get pod -n ${ns} -o wide | grep -E 'dag-.*tenant|namespace-migration|tenant-data-copy' || true`),
+          "",
+          this.sudoWrapperExample(`task_pod=<running_dag_task_pod>; kubectl describe pod -n ${ns} "$task_pod" | egrep -A8 'Labels:|Args:|State:|Started:|Restart Count:|Events:'`),
+          "",
+          this.sudoWrapperExample(`task_pod=<running_dag_task_pod>; kubectl logs -n ${ns} "$task_pod" --tail=-1 | grep -n -E 'tenant-data-copy|Copy tenant data from cluster|Obtained Source Snapshot|Starting: Building listing|hdfs://|DistCp|Submitted application|map [0-9]+%' | head -220`)
+        ].join("\n"),
+        expectedEvidence: "tenant/env, task/process name, src-cluster, dst-cluster, source snapshot id, HDFS source path, HDFS destination path, YARN app id, and latest progress"
+      }),
+      new ReadOnlyStep({
+        id: "09b-on-demand-copy-control-panel-log",
+        purpose: "When the control panel supplies a DAG identifier with an epoch-microsecond suffix, convert it to the Airflow manual run log directory and read the exact task log.",
+        command: this.sudoWrapperExample([
+          `pod=$(kubectl get pod -n ${ns} -o name | grep airflow-scheduler | head -1)`,
+          "control_panel_id='<dag_tenant_data_copy_on_demand--epoch_microseconds_from_request>'",
+          "micros=${control_panel_id##*--}",
+          "manual_run=$(MICROS=\"$micros\" /usr/bin/python3 - <<'PY'",
+          "import datetime, os",
+          "micros = int(os.environ['MICROS'])",
+          "dt = datetime.datetime.fromtimestamp(micros / 1_000_000, datetime.timezone.utc)",
+          "print('manual__' + dt.isoformat())",
+          "PY",
+          ")",
+          "kubectl exec -n airflow \"$pod\" -- sh -lc \"log=/data/workday-bds-airflow/logs/dag_tenant_data_copy_on_demand/$manual_run/run_tenant_data_ondemand_restore_copy/1.log; echo __TARGET_LOG__ \\\"$log\\\"; grep -n -E 'Running command:|--src-tenant|--dst-tenant|Copy tenant data|Obtained Source Snapshot|Latest remote matching Snapshot|Starting: Building listing|Submitted application|map [0-9]+%|ERROR|Exception|Traceback|Command exited|Marking task|Task exited|return code' \\\"$log\\\" | head -320\""
+        ].join("\n")),
+        expectedEvidence: "exact on-demand copy log path, src/dst tenant arguments, source snapshot, latest remote matching snapshot, YARN applications, DistCp progress, and final Command exited / Marking task state"
+      }),
+      new ReadOnlyStep({
+        id: "10-bas-dr-sla-miss-node-runtime-evidence",
+        purpose: "For bas_dr_sla_miss or DR copy SLA alerts, prove whether the lag is caused by stuck Airflow task pods on a degraded Kubernetes worker rather than DistCp, DAG, or tenant config failure.",
+        command: [
+          this.sudoWrapperExample(`kubectl get node -o wide`),
+          "",
+          this.sudoWrapperExample(`kubectl get pod -n ${ns} -o wide | grep -E 'dag-tenant-data-dr-pull|tenant-data-dr-restore-copy|DR|dr' || true`),
+          "",
+          this.sudoWrapperExample(`task_pod=<affected_dr_task_pod>; kubectl get pod -n ${ns} "$task_pod" -o jsonpath='{.metadata.name}{\" node=\"}{.spec.nodeName}{\" phase=\"}{.status.phase}{\" deletionTimestamp=\"}{.metadata.deletionTimestamp}{\" finalizers=\"}{.metadata.finalizers}{\"\\n\"}'`),
+          "",
+          this.sudoWrapperExample(`task_pod=<affected_dr_task_pod>; kubectl describe pod -n ${ns} "$task_pod" | egrep -A12 '^Name:|^Node:|^Status:|^Containers:|^Conditions:|^Events:'`),
+          "",
+          this.sudoWrapperExample(`task_pod=<affected_dr_task_pod>; kubectl logs -n ${ns} "$task_pod" --tail=-1 2>&1 | grep -n -E 'Command exited with return code 0|Marking task as SUCCESS|Task exited with return code 0|Submitted application|ERROR|Traceback|DistCp|Creating Source Snapshot|Creating Destination Snapshot|s[0-9]{12}r' | tail -160`),
+          "",
+          this.sshCommand("bad_node=<worker_with_stuck_dr_pods>; ssh -o BatchMode=yes -o ConnectTimeout=20 -- \"$bad_node\" \"hostname -f; date -u; uptime; ps -eo stat= | awk '{c[$1]++} END {for (s in c) print s,c[s]}' | sort; ps -eo pid,ppid,stat,comm,args | egrep '(^| )[DZ]|defunct|runc|containerd|kubelet|mount|cgroup' | head -120; timeout 8 systemctl is-active kubelet containerd 2>&1\"")
+        ].join("\n"),
+        expectedEvidence: "node scheduling status, affected DR pod placement/status/deletionTimestamp/finalizers/events, task success or failure markers, worker load, D/Z or zombie process counts, kubelet/containerd timeout evidence, and stuck runc/mount/cgroup cleanup lines"
+      }),
+      new ReadOnlyStep({
+        id: "11-bas-lag-recovery-verification",
+        purpose: "After approved operator remediation, verify that DR copy has caught up without relying only on Slack alert state.",
+        command: [
+          this.sudoWrapperExample(`kubectl get pod -n ${ns} -o wide | grep -E 'dag-tenant-data-dr-pull|tenant-data-dr-restore-copy' || true`),
+          "",
+          this.sudoWrapperExample(`kubectl get pod -n ${ns} -o wide | grep '<bad_worker_host>' || true`),
+          "",
+          this.sshCommand("grep -RInE 's[0-9]{12}r|Marking task as SUCCESS|Command exited with return code 0|tenant WD1-PROD\\.(hss|kla|mawc)' /hadoop/disk*/workday-bds-airflow/logs/dag_tenant_data_dr_pull_* 2>/dev/null | tail -200"),
+          "",
+          this.sudoWrapperExampleFor(this.basDrSlaSourceHost, `/data/pharos_script_exporter/scripts/bas_dr_sla_miss_metric.rb ${this.basDrSlaCluster} 2>/dev/null | awk '/^bds_bas_dr_sla_miss/ { tenant=$0; sub(/^.*tenant_name="/, "", tenant); sub(/".*$/, "", tenant); val=$NF+0; total++; if (val > 3600) {miss++; missed[tenant]=val} if (val > max) {max=val; maxtenant=tenant} rows[tenant]=val } END { print "total_metrics", total+0; print "sla_missed_gt_3600", miss+0; print "max_lag_seconds", max+0; print "max_lag_tenant", maxtenant; print "__MISSED__"; for (t in missed) print missed[t], t; print "__TOP10__"; for (t in rows) print rows[t], t | "sort -nr | head -10" }'`)
+        ].join("\n"),
+        expectedEvidence: `new DR task pods on healthy nodes, no affected task pods remaining on the bad worker, newer DR snapshot ids such as sYYYYMMDDHHMMr, task success markers, and BAS metric output from ${this.basDrSlaSourceHost} showing sla_missed_gt_3600 and max lag below the 3600 second alert threshold`
+      }),
+      new ReadOnlyStep({
+        id: "12-s3-vpce-connectivity",
+        purpose: "For BDS migration S3 copy failures, compare the failing S3 VPCE hostname with any alternate VPCE from the same worker/source IP before rerunning the copy.",
+        command: [
+          this.sshCommand(this.s3VpceConnectivityScript()),
+          "",
+          "# If sudo/root is needed for a worker-host shell, reuse the stdin sudo wrapper and keep the same read-only script body."
+        ].join("\n"),
+        expectedEvidence: "endpoint DNS answers, source IP, route gateway/device, curl time_connect/http_code/errormsg for failing and alternate S3 VPCE paths"
+      }),
+      new ReadOnlyStep({
+        id: "13-snapshot-cleanup-schedule-and-lock-evidence",
+        purpose: "For snapshot-cleanup or PRISM CreateDataFrame latency investigations, confirm the Airflow cleanup schedule and correlate Airflow delete durations with bounded NameNode lock/audit evidence.",
+        command: [
+          "rg -n \"SCHEDULE_ASH_CUST|LOW_RISK_TIME_WINDOW|dag_bulk_snapshot_cleaner|snapshot_cleaner_timeout\" /Users/boqiang.liang/500-work/cookbook_bitbucket/bds_kube/recipes/helm_deploy.rb",
+          "",
+          this.sshCommand("for f in /hadoop/disk*/workday-bds-airflow/logs/dag_bulk_snapshot_cleaner/scheduled__<run_time>/run_bulk_snapshot_cleaner/*.log; do echo __FILE__ $f; awk '/Deleting snapshot/ {last=$0} /Successfully deleted, duration\\(ms\\):/ {dur=$NF+0; if (dur>=3000) print dur \" | \" last \" | \" $0}' \"$f\" | sort -nr | head -40; done"),
+          "",
+          this.sshCommand("for f in /var/log/hadoop-hdfs/hadoop-hdfs-namenode-<active_nn>.log.<yyyy-mm-dd>.*.gz /var/log/hadoop-hdfs/hadoop-hdfs-namenode-<active_nn>.log; do [ -e \"$f\" ] && zgrep -n -A25 -B8 -E '<lock_window_timestamp>|Longest write-lock held' \"$f\" | head -260; done"),
+          "",
+          this.sshCommand("zgrep -h 'cmd=deleteSnapshot' /var/log/hadoop-hdfs/hdfs-audit.log.<yyyy-mm-dd>/hdfs-audit.log.<yyyy-mm-dd>-<hour>.*.gz 2>/dev/null | grep -E '<tenant>|<snapshot>|allowed=true' | head -120"),
+          "",
+          this.sshCommand("zgrep -h -E 'cmd=(getfileinfo|getFileInfo|listStatus|open|getBlockLocations)|_delta_log|WPA_|prism_table_wid' /var/log/hadoop-hdfs/hdfs-audit.log.<yyyy-mm-dd>/hdfs-audit.log.<yyyy-mm-dd>-<hour>.*.gz 2>/dev/null | head -160")
+        ].join("\n"),
+        expectedEvidence: "configured cleanup cron/timeout, top Airflow deleteSnapshot durations by run, repeated tenant snapshot backlog if present, NameNode deleteSnapshot long write-lock stack, hourly deleteSnapshot audit paths, and fast-case metadata audit RPCs that require read lock"
+      })
+    ];
+  }
+
+  toJson() {
+    return {
+      environmentName: this.environmentName,
+      configJsonPath: this.configJsonPath,
+      namespace: this.namespace,
+      controlPlaneHosts: this.controlPlaneHosts,
+      workerHosts: this.workerHosts,
+      etcdHosts: this.etcdHosts,
+      dagId: this.dagId,
+      runId: this.runId,
+      s3VpceEndpointUrls: this.s3VpceEndpointUrls,
+      alternateS3VpceEndpointUrls: this.alternateS3VpceEndpointUrls,
+      boundaryAddr: this.boundaryAddr,
+      boundaryHost: this.boundaryHost(),
+      boundaryProxyJumpHost: this.boundaryProxyJumpHost,
+      dcLogin: this.dcProfile.toJson(),
+      sudoEnvironment: {
+        tool: "mcp__devops-tool__workday_private_cloud_sudo_readonly_command",
+        credentialBoundary: "Only the dedicated sudo tool may read LDAP PROD password."
+      },
+      basDrSlaSourceHost: this.basDrSlaSourceHost,
+      basDrSlaCluster: this.basDrSlaCluster,
+      safetyRules: SAFETY_RULES,
+      commands: this.commands().map((command) => command.toJson()),
+      mcpSafeCommandInputs: this.mcpSafeCommandInputs()
+    };
+  }
+
+  toMarkdown() {
+    return [
+      "# BDS Airflow Job Status Read-Only Command Pack",
+      "",
+      this.environmentName ? `Environment: ${this.environmentName}` : "",
+      this.configJsonPath ? `Environment JSON: ${this.configJsonPath}` : "",
+      `Namespace: ${this.namespace}`,
+      `Control-plane hosts: ${this.controlPlaneHosts.length ? this.controlPlaneHosts.join(", ") : "<read kubernetes.kube_control_planes from env JSON>"}`,
+      "",
+      "## Safety Boundary",
+      "",
+      ...SAFETY_RULES.map((rule) => `- ${rule}`),
+      "",
+      "## Root Kubeconfig Note",
+      "",
+      "If `kubectl get pod` as the normal SSH user returns `The connection to the server localhost:8080 was refused`, treat that as missing user kubeconfig. Become root on the control-plane host for read-only cluster checks, then always query `-n airflow`.",
+      "",
+      "Use the explicit private-cloud host unchanged after `bauth init`. If direct SSH routes through a stale bastion or times out during banner exchange, retry with the explicit Boundary proxy jump host `proxy-jump.atl.bnd`; do not invent `.bnd` aliases for service hosts.",
+      "",
+      "```bash",
+      this.sudoWrapperExample(`kubectl get pod -n ${this.namespace} -o wide`),
+      "```",
+      "",
+      "## Read-Only Commands",
+      "",
+      ...this.commands().map((command) => command.toMarkdown()),
+      "",
+      "## MCP-Safe Remote Execution",
+      "",
+      "Run one concrete JSON input at a time. If sudo without a password is unavailable, use `workday_private_cloud_sudo_readonly_command`; do not read Keychain directly.",
+      "",
+      "```json",
+      JSON.stringify(this.mcpSafeCommandInputs(), null, 2),
+      "```",
+      "",
+      "Procedure, interpretation, and response shape live in the `bds-airflow-job-status-investigation` skill and runbook."
+    ].join("\n");
+  }
+}
+
+const inputSchema = {
+  environmentName: envNameSchema,
+  configJsonPath: pathSchema.describe("Optional local environment repo JSON file containing kubernetes.kube_control_planes."),
+  controlPlaneHosts: z.array(hostSchema).max(20).default([]),
+  workerHosts: z.array(hostSchema).max(100).default([]),
+  etcdHosts: z.array(hostSchema).max(20).default([]),
+  namespace: namespaceSchema,
+  dagId: dagSchema,
+  runId: runSchema,
+  s3VpceEndpointUrls: z.array(s3EndpointUrlSchema).max(8).default([]),
+  alternateS3VpceEndpointUrls: z.array(s3EndpointUrlSchema).max(8).default([]),
+  basDrSlaSourceHost: hostSchema.default(DEFAULT_BAS_DR_SLA_SOURCE_HOST),
+  basDrSlaCluster: envNameSchema.default(DEFAULT_BAS_DR_SLA_CLUSTER),
+  boundaryAddr: urlSchema,
+  boundaryBinaryDir: localPathSchema.default(DEFAULT_BOUNDARY_BINARY_DIR),
+  boundaryProxyJumpHost: hostSchema.default(DEFAULT_BOUNDARY_PROXY_JUMP_HOST),
+  useBoundaryBnd: z.boolean().default(false),
+  outputFormat: z.enum(["markdown", "json"]).default("markdown")
+};
+
+const inputObjectSchema = z.object(inputSchema);
+
+export async function bdsAirflowJobStatusCommandPack(rawInput) {
+  const input = inputObjectSchema.parse(rawInput ?? {});
+  const discovered = await loadEnvironmentJson(input.configJsonPath);
+  const commandPack = new BdsAirflowJobStatusCommandPack({
+    ...input,
+    controlPlaneHosts: input.controlPlaneHosts.length ? input.controlPlaneHosts : discovered.controlPlaneHosts,
+    workerHosts: input.workerHosts.length ? input.workerHosts : discovered.workerHosts,
+    etcdHosts: input.etcdHosts.length ? input.etcdHosts : discovered.etcdHosts
+  });
+  return input.outputFormat === "json" ? JSON.stringify(commandPack.toJson(), null, 2) : commandPack.toMarkdown();
+}
+
+export const __test = {
+  BdsAirflowJobStatusCommandPack,
+  bdsAirflowJobStatusCommandPack,
+  discoverKubernetesHosts
+};
+
+const sharedTool = defineReadOnlyTool({
+  name: "bds_airflow_job_status_command_pack",
+  description: "Generate a atomic-helper, read-only BDS Airflow command pack from environment Kubernetes control-plane JSON, airflow namespace checks, and S3 VPCE probes; does not SSH, run kubectl, or inspect Airflow.",
+  meta: {
+    permissions: { rule: "ask" },
+    secret_provider: "keychain"
+  },
+  inputSchema,
+  async handler(input) {
+    try {
+      return ok(await bdsAirflowJobStatusCommandPack(input));
+    } catch (error) {
+      return fail(error);
+    }
+  }
+});
+
+export default {
+  name: sharedTool.name,
+  description,
+  inputSchema: sharedTool.inputSchema,
+  meta: sharedTool.meta,
+  annotations: sharedTool.annotations,
+  invoke: sharedTool.invoke,
+  handler: sharedTool.handler
+};

package/tools/bds_bas_log_read.mjs

@@ -0,0 +1,23 @@
+export const description = "Read BAS logs from /data/workday-bas/logs on a private-cloud BAS/admin host through the approved Boundary sudo read tool.";
+
+import { createPrivateCloudLogReadTool, fixedPrivateLogSpec } from "../src/bds-atomic-log-tools.mjs";
+
+const spec = fixedPrivateLogSpec({
+  toolName: "bds_bas_log_read",
+  description: "Read BAS logs from /data/workday-bas/logs on a private-cloud BAS/admin host through the approved Boundary sudo read tool.",
+  path: "/data/workday-bas/logs/bas.log"
+});
+
+export const __test = spec;
+
+const sharedTool = createPrivateCloudLogReadTool(spec);
+
+export default {
+  name: sharedTool.name,
+  description,
+  inputSchema: sharedTool.inputSchema,
+  meta: sharedTool.meta,
+  annotations: sharedTool.annotations,
+  invoke: sharedTool.invoke,
+  handler: sharedTool.handler
+};

package/tools/bds_bdm_httpd_log_read.mjs

@@ -0,0 +1,24 @@
+export const description = "Read BDM Apache httpd access or error logs from a private-cloud BDM host through the approved Boundary sudo read tool.";
+
+import { BDM_HTTPD_EXTRA_SCHEMA, bdmHttpdCommand, createPrivateCloudLogReadTool } from "../src/bds-atomic-log-tools.mjs";
+
+export const __test = {
+  buildCommand: bdmHttpdCommand
+};
+
+const sharedTool = createPrivateCloudLogReadTool({
+  toolName: "bds_bdm_httpd_log_read",
+  description: "Read BDM Apache httpd access or error logs from a private-cloud BDM host through the approved Boundary sudo read tool.",
+  extraSchema: BDM_HTTPD_EXTRA_SCHEMA,
+  buildCommand: bdmHttpdCommand
+});
+
+export default {
+  name: sharedTool.name,
+  description,
+  inputSchema: sharedTool.inputSchema,
+  meta: sharedTool.meta,
+  annotations: sharedTool.annotations,
+  invoke: sharedTool.invoke,
+  handler: sharedTool.handler
+};