@nuvlore/extension-bds 0.1.1

package/tools/bds_scylla_tenant_bucket_command_pack.mjs

@@ -0,0 +1,323 @@
+export const description = "Generate a atomic-helper, read-only Scylla tenant bucket source/Kubernetes/S3A command pack; does not call Bitbucket, Kubernetes, or hadoop fs.";
+
+import { z } from "zod";
+import { defineReadOnlyTool, fail, ok } from "@nuvlore/extension-read-only-toolkit/devops-diagnostics";
+import { ReadOnlyStep } from "@nuvlore/extension-read-only-toolkit/read-only-command-pack";
+import {
+  DEFAULT_SCYLLA_BDM_POD,
+  DEFAULT_SCYLLA_NAMESPACE,
+  ScyllaAccessProfile
+} from "@nuvlore/extension-workday-infra-access/src/workday-access-profiles.mjs";
+
+const DEFAULT_CLUSTER = "s0130";
+const DEFAULT_NAMESPACE = DEFAULT_SCYLLA_NAMESPACE;
+const DEFAULT_BDM_POD = DEFAULT_SCYLLA_BDM_POD;
+const DEFAULT_BDM_REPO_URL = "https://bitbucket.workday.com/projects/BDS/repos/sirius/browse";
+const S0130_BDS_TENANT_BUCKET = "scylla-250478721519-wd-scylla-s0130-bds-tenant";
+
+const SAFETY_RULES = [
+  "source-code-first: read BDM tenant-health code before concluding what unhealthy means",
+  "read-only only: kcsb get, ConfigMap reads, and hadoop fs list/read commands from a BDM pod",
+  "Bitbucket reads are read-only; local clone or pull is optional human-reviewed local cache work only",
+  "prefer BDM pods for S3A tenant bucket inspection because they carry the BDS Hadoop runtime",
+  "build and verify the SPC/Kubernetes connection first, then use either kcsb or an explicit kubeconfig-backed kubectl equivalent",
+  "never use hadoop fs -get for listing; it downloads data to the local filesystem",
+  "never use hadoop fs -put, -rm, -mkdir, -touchz, -mv, -cp, -chmod, -chown, or any write-oriented filesystem operation",
+  "do not use aws s3 inside BDM or bds-service-proxy pods as the observed path lacks AWS CLI credentials",
+  "do not assume hdfs exists in new BDM or bds-service-proxy images; use hadoop fs",
+  "never print tenant key material or decrypted file contents unless explicitly approved; prefer status-only decrypt/read probes",
+  "treat unauthenticated SIR curl 401 as reachability/auth-boundary evidence only, not as evidence for the real BDS authenticated key-fetch flow",
+  "do not use curl -k for SIR checks unless the URL is HTTPS; the observed in-cluster SIR endpoint uses HTTP"
+];
+
+const clusterSchema = z
+  .string()
+  .min(1)
+  .max(64)
+  .regex(/^[A-Za-z0-9_.:-]+$/);
+
+const kubernetesNameSchema = z
+  .string()
+  .min(1)
+  .max(253)
+  .regex(/^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/);
+
+const bucketSchema = z
+  .string()
+  .min(3)
+  .max(255)
+  .regex(/^[a-z0-9][a-z0-9.-]*[a-z0-9]$/)
+  .optional();
+
+const pathNameSchema = z
+  .string()
+  .min(1)
+  .max(160)
+  .regex(/^[A-Za-z0-9_.-]+$/)
+  .optional();
+
+const urlSchema = z
+  .string()
+  .trim()
+  .url()
+  .max(2048)
+  .default(DEFAULT_BDM_REPO_URL);
+
+class BdsScyllaTenantBucketCommandPack {
+  constructor(input) {
+    this.cluster = input.cluster ?? DEFAULT_CLUSTER;
+    this.namespace = input.namespace ?? DEFAULT_NAMESPACE;
+    this.bdmPod = input.bdmPod ?? DEFAULT_BDM_POD;
+    this.environmentName = input.environmentName;
+    this.tenantName = input.tenantName;
+    this.bdmRepoUrl = input.bdmRepoUrl ?? DEFAULT_BDM_REPO_URL;
+    this.bucketName = input.bucketName ?? (this.cluster === DEFAULT_CLUSTER ? S0130_BDS_TENANT_BUCKET : undefined);
+    this.scyllaAccess = new ScyllaAccessProfile({
+      cluster: this.cluster,
+      namespace: this.namespace,
+      bdmPod: this.bdmPod
+    });
+  }
+
+  bucketRef() {
+    return this.bucketName ?? "<bds-tenant.bucket.name from bucketdetails-config>";
+  }
+
+  dataPath() {
+    return `s3a://${this.bucketRef()}/data/`;
+  }
+
+  tenantPath() {
+    if (!this.tenantName) {
+      return `s3a://${this.bucketRef()}/data/tenants/`;
+    }
+    const tenantSegment = this.environmentName
+      ? `${this.environmentName}.${this.tenantName}`
+      : `<ENV_NAME>.${this.tenantName}`;
+    return `s3a://${this.bucketRef()}/data/tenants/${tenantSegment}/`;
+  }
+
+  brokenDsInfoPath() {
+    return `${this.tenantPath()}ted/opa/consistency_check/brokenDsInfo.json`;
+  }
+
+  tenantFolderRule() {
+    return this.tenantName
+      ? `data/tenants/${this.environmentName ? `${this.environmentName}.` : "<ENV_NAME>."}${this.tenantName}`
+      : "data/tenants/<ENV_NAME>.<tenantName>";
+  }
+
+  commands() {
+    return [
+      new ReadOnlyStep({
+        id: "00-source-code-map",
+        purpose: "Read BDM source before interpreting tenant health so the RCA uses the real unhealthy criteria.",
+        command: [
+          "Use bitbucket_api_read on /rest/api/latest/projects/BDS/repos/sirius, or perform an optional HITL local cache clone:",
+          "git clone --filter=blob:none https://bitbucket.workday.com/scm/BDS/sirius.git <local-cache>/bds-sirius",
+          "Then search for tenant health code with read-only grep patterns: tenant health, TenantHealth, tenant_health, unhealthy, brokenDsInfo, consistency_check, operationsLog"
+        ].join("\n"),
+        expectedEvidence: "source files and exact predicates that translate tenant data state into healthy/unhealthy status"
+      }),
+      new ReadOnlyStep({
+        id: "01-current-pods",
+        purpose: "Confirm BDM and related BDS pods in the connected Scylla customer namespace.",
+        command: "kcsb get pod",
+        expectedEvidence: "BDM pod names and readiness, especially bdm-0/bdm-1"
+      }),
+      new ReadOnlyStep({
+        id: "02-bucket-config",
+        purpose: "Read the source of truth for the tenant bucket name before using an S3A path.",
+        command: this.scyllaAccess.bucketConfigCommand(),
+        expectedEvidence: "data.bds-tenant.bucket.name and endpoint"
+      }),
+      new ReadOnlyStep({
+        id: "03-active-tenant-census",
+        purpose: "Separate active BDS tenants from stale or disabled tenant folders before judging blast radius.",
+        command: [
+          this.scyllaAccess.podExec(`curl -fsS --max-time 5 http://127.0.0.1:8083/metrics | awk -F\\" '/bds_stats_tenant_/ && /isActiveTenant=\\"true\\"/ {print $2}' | sort -u`),
+          this.scyllaAccess.podExec(`hadoop fs -ls s3a://${this.bucketRef()}/data/tenants/ | awk '/\\.<tenant-or-env-pattern>/ {print $NF}'`)
+        ].join("\n"),
+        expectedEvidence: "active tenant names from BDM metrics and bucket tenant folders counted separately"
+      }),
+      new ReadOnlyStep({
+        id: "04-enter-bdm",
+        purpose: "Enter a BDM pod that has the BDS Hadoop runtime and hadoop fs command.",
+        command: `kcsb exec -it ${this.bdmPod} -- bash`,
+        expectedEvidence: "interactive shell prompt in the BDM pod"
+      }),
+      new ReadOnlyStep({
+        id: "05-list-data-root",
+        purpose: "List the tenant bucket data root with hadoop fs, not aws s3 or hdfs.",
+        command: `hadoop fs -ls ${this.dataPath()}`,
+        expectedEvidence: "data root listing, normally including data/tenants"
+      }),
+      new ReadOnlyStep({
+        id: "06-list-tenant-path",
+        purpose: "List the tenant-specific path when environment and tenant are known.",
+        command: `hadoop fs -ls ${this.tenantPath()}`,
+        expectedEvidence: "tenant path existence using data/tenants/<ENV_NAME>.<tenantName>, ownership, permissions, and normal children such as operationsLog.csv, ted, and tmp"
+      }),
+      new ReadOnlyStep({
+        id: "07-read-consistency-file",
+        purpose: "Read only the tenant consistency-check file when investigating unhealthy tenant status.",
+        command: `hadoop fs -cat ${this.brokenDsInfoPath()}`,
+        expectedEvidence: "broken dataset information or a clear file-not-found/access error"
+      }),
+      new ReadOnlyStep({
+        id: "08-key-fetch-and-decrypt-errors",
+        purpose: "Correlate BDM read failures with bds-service-proxy/SIR key-fetch and decrypt errors without exposing keys.",
+        command: [
+          this.scyllaAccess.podLogs({ pattern: `Exception while reading predefined file|Existing files are not readable|Failed attempt to read file|${this.tenantName ?? "<tenantName>"}` }),
+          "kcsb logs deploy/bds-service-proxy -c bds-service-proxy --since=4h | egrep 'Service unavailable|Response status: 503|Response code 503|BdsCipherClient|KMSClientException|Error in decrypt|<tenantName>'"
+        ].join("\n"),
+        expectedEvidence: "specific timestamped BDM read error, SIR 503 key endpoint, BdsCipherClient decrypt error, and downstream 500/KMSClientException"
+      }),
+      new ReadOnlyStep({
+        id: "09-healthy-tenant-comparison",
+        purpose: "Compare with other active tenants to distinguish tenant-specific failure from cluster-wide BDM, bucket, or service-proxy failure.",
+        command: [
+          this.scyllaAccess.podLogs({ pattern: "Successfully read predefined file|Tenant \\[.*\\] is healthy" }),
+          "Optionally run status-only hadoop fs -cat >/dev/null checks for other active tenants; do not print key material or file contents."
+        ].join("\n"),
+        expectedEvidence: "healthy active tenants completing the same predefined-file read/decrypt path"
+      }),
+      new ReadOnlyStep({
+        id: "10-sir-front-door-reachability",
+        purpose: "When the failure is a SIR key-fetch 503, distinguish service front-door reachability from the authenticated tenant-specific key-fetch result.",
+        command: [
+          "kcsb exec deploy/bds-service-proxy -c bds-service-proxy -- sh -lc 'for h in sir-<env>.sir sir-<env>.sir.svc sir-<env>.sir.svc.cluster.local; do getent hosts \"$h\" || nslookup \"$h\" || true; curl -sS -o /dev/null -w \"root_code=%{http_code} connect=%{time_connect} total=%{time_total}\\n\" --connect-timeout 3 --max-time 8 \"http://${h}:11710/\"; done'",
+          "kcsb exec deploy/bds-service-proxy -c bds-service-proxy -- sh -lc 'for t in <failingTenant> <healthyTenant1> <healthyTenant2>; do curl -sS -o /dev/null -w \"${t}_unauth_code=%{http_code} connect=%{time_connect} total=%{time_total}\\n\" --connect-timeout 3 --max-time 8 \"http://sir-<env>.sir:11710/internalapi/key/v1/${t}/bdstenant/encoded?kid=kid\"; done'"
+        ].join("\n"),
+        expectedEvidence: "DNS resolution and quick 404/401 responses prove the SIR front door is reachable from bds-service-proxy; 401 from manual curl is unauthenticated probe evidence only"
+      }),
+      new ReadOnlyStep({
+        id: "11-cross-namespace-rbac-boundary",
+        purpose: "Before saying SIR, Galahad, OMS, or ORS pods/logs are missing or unavailable, verify whether the operator can read those namespaces.",
+        command: [
+          this.scyllaAccess.rbacBoundaryCommand(),
+          "If RBAC says no for sir/galahad/ors/oms, report that direct backend pod/log verification is blocked and route the tenant-specific 503 to the owning channel."
+        ].join("\n"),
+        expectedEvidence: "RBAC yes/no for backend namespaces, separating lack of access from service availability claims"
+      })
+    ];
+  }
+
+  antiPatterns() {
+    return [
+      {
+        command: `aws s3 ls s3://${this.bucketRef()}/data/`,
+        reason: "Observed inside BDM and bds-service-proxy pods as NoCredentials."
+      },
+      {
+        command: `hdfs dfs -ls ${this.dataPath()}`,
+        reason: "New BDM and bds-service-proxy images may not include hdfs; KDC pods may lack the S3A class."
+      },
+      {
+        command: `hadoop fs -get ${this.dataPath()}`,
+        reason: "This downloads to local disk and can fail with local mkdir errors; it is not a list operation."
+      }
+    ];
+  }
+
+  toMarkdown() {
+    return [
+      "# BDS Scylla Tenant Bucket Read-Only Command Pack",
+      "",
+      `Cluster: ${this.cluster}`,
+      `Namespace: ${this.namespace}`,
+      `BDM pod: ${this.bdmPod}`,
+      this.bucketName ? `Bucket: ${this.bucketName}` : "Bucket: read from `bucketdetails-config` first",
+      `BDM repo: ${this.bdmRepoUrl}`,
+      this.environmentName ? `Environment: ${this.environmentName}` : "",
+      this.tenantName ? `Tenant: ${this.tenantName}` : "",
+      "",
+      "## Tenant Folder Rule",
+      "",
+      `Tenant folder: \`${this.tenantFolderRule()}\``,
+      "",
+      "A healthy discovered tenant folder commonly lists `operationsLog.csv`, `ted`, and `tmp` under that path.",
+      "Do not treat tenant folder count as active tenant count; use BDM metrics or EMS-derived BDM behavior for active BDS tenants.",
+      "For SIR key-fetch 503 cases, prove BDS-side reachability from `bds-service-proxy` before asking Galahad/OMS/ORS to inspect the backend. A manual unauthenticated HTTP 401 is only a reachability/auth-boundary signal; the real business-flow evidence comes from bds-service-proxy authenticated request logs.",
+      "Use `curl -k` only for HTTPS probes. The observed in-cluster SIR key endpoint is HTTP, so `-k` does not change that check.",
+      "",
+      "## Safety Boundary",
+      "",
+      ...SAFETY_RULES.map((rule) => `- ${rule}`),
+      "",
+      "## Read-Only Commands",
+      "",
+      ...this.commands().map((command) => command.toMarkdown()),
+      "",
+      "## Do Not Use",
+      "",
+      ...this.antiPatterns().map((entry) => `- \`${entry.command}\`: ${entry.reason}`),
+      "",
+      "Procedure, interpretation, and RCA shape live in the `bds-scylla-tenant-bucket-investigation` skill and runbook."
+    ].filter(Boolean).join("\n");
+  }
+
+  toJson() {
+    return {
+      cluster: this.cluster,
+      namespace: this.namespace,
+      bdmPod: this.bdmPod,
+      bdmRepoUrl: this.bdmRepoUrl,
+      bucketName: this.bucketName,
+      environmentName: this.environmentName,
+      tenantName: this.tenantName,
+      scyllaAccess: this.scyllaAccess.toJson(),
+      tenantFolderRule: this.tenantFolderRule(),
+      expectedTenantChildren: ["operationsLog.csv", "ted", "tmp"],
+      sourceSearchPatterns: ["tenant health", "TenantHealth", "tenant_health", "unhealthy", "brokenDsInfo", "consistency_check", "operationsLog"],
+      activeTenantSignals: ["bds_stats_tenant_* isActiveTenant=\"true\"", "BDM getBDSTenants(Some(\"Active\")) health-check behavior"],
+      keyDecryptErrorSignals: ["SIR 503 Service Unavailable", "BdsCipherClient Error in decrypt", "KMSClientException Status code : 500"],
+      sirReachabilitySignals: ["bds-service-proxy DNS resolves SIR service", "manual unauthenticated SIR HTTP probe returns 404 or 401 quickly", "curl -k is irrelevant for HTTP endpoints"],
+      backendRbacBoundary: ["kubectl auth can-i get pods/services/endpoints/deployments.apps in sir", "galahad", "ors", "oms"],
+      safetyBoundary: SAFETY_RULES,
+      commands: this.commands().map((command) => command.toJson()),
+      antiPatterns: this.antiPatterns()
+    };
+  }
+}
+
+function buildResponse(input) {
+  const commandPack = new BdsScyllaTenantBucketCommandPack(input);
+  if (input.format === "json") {
+    return JSON.stringify(commandPack.toJson(), null, 2);
+  }
+  return commandPack.toMarkdown();
+}
+
+const sharedTool = defineReadOnlyTool({
+  name: "bds_scylla_tenant_bucket_command_pack",
+  description: "Generate a atomic-helper, read-only Scylla tenant bucket source/Kubernetes/S3A command pack; does not call Bitbucket, Kubernetes, or hadoop fs.",
+  inputSchema: {
+    cluster: clusterSchema.default(DEFAULT_CLUSTER),
+    namespace: kubernetesNameSchema.default(DEFAULT_NAMESPACE),
+    bdmPod: kubernetesNameSchema.default(DEFAULT_BDM_POD),
+    bdmRepoUrl: urlSchema.describe("BDM source repository URL used to inspect tenant-health logic."),
+    environmentName: pathNameSchema.describe("Optional tenant environment name, for example WD504-IMPL."),
+    tenantName: pathNameSchema.describe("Optional tenant name, for example schellman1."),
+    bucketName: bucketSchema.describe("Optional bucket name. If omitted for s0130, the observed bds-tenant bucket is used; otherwise read bucketdetails-config first."),
+    format: z.enum(["markdown", "json"]).default("markdown")
+  },
+  annotations: { openWorldHint: true },
+  async handler(input) {
+    try {
+      return ok(buildResponse(input));
+    } catch (error) {
+      return fail(error);
+    }
+  }
+});
+
+export default {
+  name: sharedTool.name,
+  description,
+  inputSchema: sharedTool.inputSchema,
+  meta: sharedTool.meta,
+  annotations: sharedTool.annotations,
+  invoke: sharedTool.invoke,
+  handler: sharedTool.handler
+};

package/tools/bds_service_proxy_log_read.mjs

@@ -0,0 +1,23 @@
+export const description = "Read private-cloud BDS service proxy HAProxy logs through the approved Boundary sudo read tool.";
+
+import { createPrivateCloudLogReadTool, fixedPrivateLogSpec } from "../src/bds-atomic-log-tools.mjs";
+
+const spec = fixedPrivateLogSpec({
+  toolName: "bds_service_proxy_log_read",
+  description: "Read private-cloud BDS service proxy HAProxy logs through the approved Boundary sudo read tool.",
+  path: "/var/log/haproxy.log"
+});
+
+export const __test = spec;
+
+const sharedTool = createPrivateCloudLogReadTool(spec);
+
+export default {
+  name: sharedTool.name,
+  description,
+  inputSchema: sharedTool.inputSchema,
+  meta: sharedTool.meta,
+  annotations: sharedTool.annotations,
+  invoke: sharedTool.invoke,
+  handler: sharedTool.handler
+};

package/tools/bds_spark_historyserver_log_read.mjs

@@ -0,0 +1,32 @@
+export const description = "Read private-cloud Spark History Server Kubernetes logs from the bds-k8s-services namespace through the approved Boundary sudo read tool.";
+
+import { createPrivateCloudLogReadTool, PRIVATE_K8S_LOG_EXTRA_SCHEMA, privateK8sLogCommand } from "../src/bds-atomic-log-tools.mjs";
+
+function buildCommand(input) {
+  return privateK8sLogCommand(input, {
+    namespace: "bds-k8s-services",
+    deployment: "bds-shs-shs-1",
+    container: "bds-shs"
+  });
+}
+
+export const __test = {
+  buildCommand
+};
+
+const sharedTool = createPrivateCloudLogReadTool({
+  toolName: "bds_spark_historyserver_log_read",
+  description: "Read private-cloud Spark History Server Kubernetes logs from the bds-k8s-services namespace through the approved Boundary sudo read tool.",
+  extraSchema: PRIVATE_K8S_LOG_EXTRA_SCHEMA,
+  buildCommand
+});
+
+export default {
+  name: sharedTool.name,
+  description,
+  inputSchema: sharedTool.inputSchema,
+  meta: sharedTool.meta,
+  annotations: sharedTool.annotations,
+  invoke: sharedTool.invoke,
+  handler: sharedTool.handler
+};

package/tools/bds_spas_mt_log_read.mjs

@@ -0,0 +1,23 @@
+export const description = "Read SPAS-MT logs from a private-cloud SPAS-MT endpoint host through the approved Boundary sudo read tool.";
+
+import { createPrivateCloudLogReadTool, fixedPrivateLogSpec } from "../src/bds-atomic-log-tools.mjs";
+
+const spec = fixedPrivateLogSpec({
+  toolName: "bds_spas_mt_log_read",
+  description: "Read SPAS-MT logs from a private-cloud SPAS-MT endpoint host through the approved Boundary sudo read tool.",
+  path: "/data/workday-spas-mt/1.0.0/log/spas.log"
+});
+
+export const __test = spec;
+
+const sharedTool = createPrivateCloudLogReadTool(spec);
+
+export default {
+  name: sharedTool.name,
+  description,
+  inputSchema: sharedTool.inputSchema,
+  meta: sharedTool.meta,
+  annotations: sharedTool.annotations,
+  invoke: sharedTool.invoke,
+  handler: sharedTool.handler
+};