@compforge/harness-toolbox 0.4.1 → 0.4.3

package/README.md

@@ -18,45 +18,69 @@ not load database or Kubernetes drivers.
 
 ## File sources
 
-Use `LocalFileDataSource` for a local file or `PodFileDataSource` for a file in an explicit
-namespace/Pod/container. Both extend Common's `FileDataSource`, so a Service can store them directly
-as DataSources and consumers can recognize files with `instanceof FileDataSource`. No additional
-`id` or `kind` wrapper is required; consumer-owned selection names remain separate from client identity.
+A FileDataSource is a caller-declared acquisition group. Use `LocalFileDataSource` for local paths
+or `PodFileDataSource` for paths in one explicit namespace/Pod/container. Both extend Common's
+`FileDataSource`; no additional id/kind wrapper is required.
 
 ```ts
-import { ClientManager, FileDataSource } from "@compforge/harness-common";
+import { ClientManager } from "@compforge/harness-common";
 import { LocalFileDataSource, PodFileDataSource } from "@compforge/harness-toolbox/file";
 
-const source = new LocalFileDataSource("./config.yaml", { concurrency: 2 });
+const source = new LocalFileDataSource(["./config.yaml", "./secret/config.yaml"], { concurrency: 2 });
 const clients = new ClientManager();
 try {
-  if (source instanceof FileDataSource) {
-    const file = await clients.get(source);
-    const bytes = await file.read({ maxBytes: 1024 * 1024, timeoutMs: 15_000 });
-    console.log(source.path, new TextDecoder("utf-8", { fatal: true }).decode(bytes));
+  const file = await clients.get(source);
+  const results = await file.read({ maxBytes: 1024 * 1024, maxTotalBytes: 2 * 1024 * 1024, timeoutMs: 15_000 });
+  for (const result of results) {
+    if (result.status === "error") console.error(result.path, result.error.message);
+    else console.log(result.path, new TextDecoder("utf-8", { fatal: true }).decode(result.bytes));
   }
-} finally {
-  await clients.dispose();
-}
+} finally { await clients.dispose(); }
 
-// `environment` is a KubernetesEnvironment (or an equivalent ClientProvider).
-// Resolve Workloads first and create one source per chosen Pod/container.
+// Resolve Workloads first. This entire group is fetched through one exec session.
 const podSource = new PodFileDataSource(environment,
   { namespace: "app", pod: "worker-abc", container: "worker" },
-  "/etc/app/config.yaml", { concurrency: 2 });
+  ["/etc/app/config.yaml", "/etc/app/secret/config.yaml"], { concurrency: 2 });
 ```
 
-Each read reopens the file; the manager shares clients, not previous file contents. The byte limit
-rejects oversized reads, including a zero-byte limit on a nonempty file. Timeouts include queue time;
-caller cancellation and disposal cancel active reads. Keep the source path and Pod target with any
-persisted evidence. A successful read describes bytes at that location, not the application's parsed
-or effective configuration and not an atomic filesystem snapshot.
-
-Local files may be symlinks to regular files. Pod reads require `pods/exec` access and `head -c` in the
-selected container; paths are passed as argv without a shell. A Pod name is an access location, not
-a UID fence; callers needing incarnation guarantees must verify discovery around collection. Pod
-sources borrow Kubernetes access from the root manager and never close it themselves. Unsupported
-commands, missing paths and denied reads fail explicitly; they are not empty file results.
+Every read reopens all declared paths and returns one ordered result per path. Empty files succeed;
+missing, denied, nonregular and oversized files return individual errors with no partial bytes.
+An error in one file does not discard sibling results. Per-file limits reset for each path; the total
+budget bounds retained decoded bytes, including incomplete files. Queueing and the whole group
+share one deadline. Cancellation, total-budget exhaustion, deadline and transport/protocol failure
+reject the operation. Sources share clients, never previous file contents.
+
+Pod groups use one exec and require `sh`, `head -c`, `base64` and `tr`. Paths remain positional
+arguments even when they contain shell syntax or newlines. Indexed base64 records carry raw binary
+bytes plus each file's exit status; truncated or malformed records reject without replay. No remote
+temporary files are written. Local reads allow symlinks to regular files and reject nonregular files.
+Keep the path and Pod target on evidence; grouping guarantees neither an atomic filesystem snapshot
+nor application configuration precedence. A Pod name is not a UID fence; callers needing incarnation
+guarantees must check discovery around collection. File clients borrow root Kubernetes access and
+never close it themselves.
+
+Version 0.4.2 requires Common 0.5.4. Replace single path arguments with arrays, supply `maxTotalBytes`,
+and handle per-file results. Publish Common before Toolbox; this change has no single-file legacy adapter.
+
+## Redis sources
+
+`RedisClient` owns resolved configuration and reusable node connections. Use `clusterType` to
+select `single`, `cluster` or `sentinel`; `endpoints` contains that mode's seed addresses. Sentinel
+configuration additionally provides `sentinelMasterName` and optional Sentinel credentials,
+separate from data-node credentials. Cluster connections require database zero.
+
+Supply ordered TCP transports through `ConnectionSource`. Only mapping or handshake network
+failures advance to the next transport; authentication and TLS verification failures stop immediately.
+`connectTimeoutMs` bounds each route (mapping and handshake); it defaults to the command
+`timeoutMs`. `acquireTimeoutMs` bounds all candidates together and defaults to the per-route budget
+times the number of routes. A spent total budget does not start another route.
+`onRoute` reports the selected route and the preceding failure code without credentials.
+
+`client.topology()` refreshes topology using the same connections and route policy as
+`client.access.connection(endpoint, database)`. A root signal cancels connection preparation and
+active commands. Commands and pipelines are sent once, with no offline queue or automatic replay;
+a subsequent acquisition replaces an unhealthy connection. The caller owns `dispose()` and transport
+cleanup, and diagnostics own sampling policy and collected observations.
 
 ## Kubernetes resources
 

package/dist/{chunk-c6z919qb.js → chunk-5tbzf9wd.js}

@@ -26,7 +26,7 @@ function sentinelRows(value) {
   });
 }
 async function discoverSentinel(access, config) {
-  const sentinels = config.sentinelHosts.length ? config.sentinelHosts : config.endpoints;
+  const sentinels = config.endpoints;
   let lastError;
   for (const sentinel of sentinels) {
     try {

package/dist/chunk-grpygk6f.js

@@ -0,0 +1,157 @@
+import {
+  BoundedFileClient,
+  fileClientOptions,
+  filePaths
+} from "./chunk-kkts16w9.js";
+import {
+  FileError
+} from "./chunk-20kv4wc3.js";
+
+// src/file/pod.ts
+import { posix } from "node:path";
+import { clientKey, FileDataSource } from "@compforge/harness-common";
+var READ_FILES = String.raw`limit=$1
+shift
+index=0
+for path do
+  printf '%s ' "$index"
+  { if [ -f "$path" ]; then head -c "$limit" "$path" 2>/dev/null; status=$?; else status=1; fi
+    printf '\000%03d' "$status"
+  } | base64 | tr -d '\r\n'
+  printf '\n'
+  index=$((index + 1))
+done`;
+function fileRecords(paths, maxBytes, target, sink) {
+  let index = 0, header = "", started = false, carry = "", encodedSize = 0, padded = false;
+  let trailer = Buffer.alloc(0);
+  const maximumEncoded = Math.ceil((maxBytes + 5) / 3) * 4;
+  const invalid = () => new FileError("Invalid Pod file group response", { kind: "invalid_response" });
+  const body = (part) => {
+    if (padded && part)
+      throw invalid();
+    encodedSize += part.length;
+    if (encodedSize > maximumEncoded)
+      throw invalid();
+    const text = carry + part, end = text.length - text.length % 4;
+    carry = text.slice(end);
+    if (!end)
+      return;
+    const encoded = text.slice(0, end), decoded = Buffer.from(encoded, "base64");
+    if (decoded.toString("base64") !== encoded)
+      throw invalid();
+    padded = encoded.endsWith("=");
+    if (padded && carry)
+      throw invalid();
+    const bytes = Buffer.concat([trailer, decoded]);
+    if (bytes.length > 4)
+      sink.receive(index, bytes.subarray(0, -4));
+    trailer = Buffer.from(bytes.subarray(Math.max(0, bytes.length - 4)));
+  };
+  return {
+    receive(bytes) {
+      if (bytes.some((byte) => byte > 127))
+        throw invalid();
+      const text = Buffer.from(bytes).toString("ascii");
+      for (let offset = 0;offset < text.length; ) {
+        if (!started) {
+          const char = text[offset++];
+          if (char !== " ") {
+            if (!/\d/.test(char) || header.length >= String(paths.length).length)
+              throw invalid();
+            header += char;
+            continue;
+          }
+          if (index >= paths.length || header !== String(index))
+            throw invalid();
+          started = true;
+        }
+        const newline = text.indexOf(`
+`, offset);
+        body(text.slice(offset, newline < 0 ? text.length : newline));
+        if (newline < 0)
+          return;
+        if (carry || trailer.length !== 4 || trailer[0] !== 0)
+          throw invalid();
+        const status = trailer.subarray(1).toString("ascii");
+        if (!/^\d{3}$/.test(status) || Number(status) > 255)
+          throw invalid();
+        sink.complete(index, Number(status) === 0 ? undefined : new FileError(`Cannot read Pod file: ${target.namespace}/${target.pod}/${target.container} ${paths[index]}`, { kind: "operation_failed", code: Number(status) }));
+        index++;
+        header = "";
+        started = false;
+        encodedSize = 0;
+        padded = false;
+        trailer = Buffer.alloc(0);
+        offset = newline + 1;
+      }
+    },
+    finish() {
+      if (started || header || index !== paths.length)
+        throw invalid();
+    }
+  };
+}
+
+class PodFileClient extends BoundedFileClient {
+  target;
+  acquire;
+  #kubernetes;
+  constructor(paths, target, options, signal, acquire) {
+    super(paths, options, signal);
+    this.target = target;
+    this.acquire = acquire;
+  }
+  async initializeAccess() {
+    this.#kubernetes = await this.acquire();
+  }
+  async capture(signal, limits, sink) {
+    const abort = new AbortController;
+    const combined = AbortSignal.any([signal, abort.signal]);
+    const records = fileRecords(this.paths, limits.maxBytes, this.target, sink);
+    const result = await this.#kubernetes.exec(this.target.namespace, this.target, ["sh", "-c", READ_FILES, "file-source", String(limits.maxBytes + 1), ...this.paths], {
+      signal: combined,
+      timeoutMs: limits.timeoutMs,
+      collectStdout: false,
+      onStdoutBytes: (bytes) => {
+        if (combined.aborted)
+          return;
+        try {
+          records.receive(bytes);
+          combined.throwIfAborted();
+        } catch (error) {
+          abort.abort(error);
+        }
+      }
+    });
+    combined.throwIfAborted();
+    if (!result.ok)
+      throw new FileError(`Cannot acquire Pod file group: ${this.target.namespace}/${this.target.pod}/${this.target.container}`, { kind: result.timedOut ? "timeout" : "operation_failed", code: result.exitCode ?? undefined });
+    records.finish();
+  }
+}
+
+class PodFileDataSource extends FileDataSource {
+  kubernetes;
+  paths;
+  target;
+  clientKey;
+  #options;
+  constructor(kubernetes, target, paths, options) {
+    super();
+    this.kubernetes = kubernetes;
+    this.paths = filePaths(paths);
+    if (this.paths.some((path) => !posix.isAbsolute(path)))
+      throw new FileError("Pod file paths must be absolute", { kind: "invalid_argument" });
+    if ([target.namespace, target.pod, target.container].some((value) => !value.trim())) {
+      throw new FileError("Pod file namespace, Pod and Container are required", { kind: "invalid_argument" });
+    }
+    this.target = Object.freeze({ namespace: target.namespace, pod: target.pod, container: target.container });
+    this.#options = fileClientOptions(options);
+    this.clientKey = clientKey("pod-file", { access: kubernetes.clientKey, target: this.target, paths: this.paths, options: this.#options });
+  }
+  createClient(clients, signal) {
+    return new PodFileClient(this.paths, this.target, this.#options, signal, () => clients.get(this.kubernetes));
+  }
+}
+
+export { PodFileClient, PodFileDataSource };

package/dist/chunk-kkts16w9.js

@@ -0,0 +1,131 @@
+import {
+  FileError,
+  ToolboxError
+} from "./chunk-20kv4wc3.js";
+import {
+  ConcurrencyPool
+} from "./chunk-v3hxvnrc.js";
+
+// src/file/client.ts
+function fileClientOptions(options) {
+  if (!Number.isSafeInteger(options.concurrency) || options.concurrency < 1) {
+    throw new FileError("File concurrency must be a positive integer", { kind: "invalid_argument" });
+  }
+  return Object.freeze({ concurrency: options.concurrency });
+}
+function filePaths(paths) {
+  if (!paths.length || paths.some((path) => typeof path !== "string" || !path || path.includes("\x00")) || new Set(paths).size !== paths.length) {
+    throw new FileError("File paths must be nonempty, distinct and contain no NUL", { kind: "invalid_argument" });
+  }
+  return Object.freeze([...paths]);
+}
+
+class BoundedFileClient {
+  paths;
+  #controller = new AbortController;
+  #signal;
+  #pool;
+  #operations = new Set;
+  #ready = false;
+  #disposal;
+  constructor(paths, options, signal) {
+    this.paths = filePaths(paths);
+    this.#pool = new ConcurrencyPool(fileClientOptions(options).concurrency);
+    this.#signal = AbortSignal.any([signal, this.#controller.signal]);
+  }
+  async initialize() {
+    this.#signal.throwIfAborted();
+    await this.initializeAccess();
+    this.#signal.throwIfAborted();
+    this.#ready = true;
+  }
+  async initializeAccess() {}
+  async read(options) {
+    this.#signal.throwIfAborted();
+    if (!this.#ready)
+      throw new FileError("File client is not initialized", { kind: "operation_failed" });
+    if (!Number.isSafeInteger(options.maxBytes) || options.maxBytes < 0 || options.maxBytes >= Number.MAX_SAFE_INTEGER - 5 || !Number.isSafeInteger(options.maxTotalBytes) || options.maxTotalBytes < 0 || !Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0 || options.timeoutMs > 2147483647) {
+      throw new FileError("Invalid file group read limits", { kind: "invalid_argument" });
+    }
+    const controller = new AbortController;
+    const signal = AbortSignal.any([this.#signal, controller.signal, ...options.signal ? [options.signal] : []]);
+    const timer = setTimeout(() => controller.abort(new FileError("File group read timed out", { kind: "timeout" })), options.timeoutMs);
+    const operation = this.#pool.run(async () => {
+      const files = this.paths.map((path) => ({ path, chunks: [], size: 0, done: false, error: undefined }));
+      let retained = 0;
+      const discard = (file, error) => {
+        retained -= file.size;
+        file.size = 0;
+        file.chunks = [];
+        file.error ??= error;
+      };
+      const entry = (index) => {
+        const file = files[index];
+        if (!file || file.done)
+          throw new FileError("Invalid file group response", { kind: "invalid_response" });
+        return file;
+      };
+      const sink = {
+        receive: (index, bytes) => {
+          if (signal.aborted)
+            return;
+          const file = entry(index);
+          if (file.error)
+            return;
+          if (bytes.byteLength > options.maxBytes - file.size) {
+            discard(file, new FileError(`File exceeds ${options.maxBytes} bytes: ${file.path}`, { kind: "limit_exceeded" }));
+          } else if (bytes.byteLength > options.maxTotalBytes - retained) {
+            controller.abort(new FileError(`File group exceeds ${options.maxTotalBytes} bytes`, { kind: "limit_exceeded" }));
+          } else {
+            file.chunks.push(Uint8Array.from(bytes));
+            file.size += bytes.byteLength;
+            retained += bytes.byteLength;
+          }
+        },
+        complete: (index, error) => {
+          if (signal.aborted)
+            return;
+          const file = entry(index);
+          if (error)
+            discard(file, error);
+          file.done = true;
+        }
+      };
+      await this.capture(signal, options, sink);
+      signal.throwIfAborted();
+      if (files.some((file) => !file.done))
+        throw new FileError("Incomplete file group response", { kind: "invalid_response" });
+      return files.map((file) => {
+        if (file.error)
+          return { path: file.path, status: "error", error: file.error };
+        const bytes = new Uint8Array(file.size);
+        let offset = 0;
+        for (const chunk of file.chunks) {
+          bytes.set(chunk, offset);
+          offset += chunk.byteLength;
+        }
+        return { path: file.path, status: "ok", bytes };
+      });
+    }, signal);
+    this.#operations.add(operation);
+    try {
+      return await operation;
+    } catch (cause) {
+      signal.throwIfAborted();
+      if (cause instanceof ToolboxError)
+        throw cause;
+      throw new FileError("Cannot acquire file group", { kind: "operation_failed", cause });
+    } finally {
+      clearTimeout(timer);
+      this.#operations.delete(operation);
+    }
+  }
+  dispose() {
+    return this.#disposal ??= (async () => {
+      this.#controller.abort(new Error("File client disposed"));
+      await Promise.allSettled(this.#operations);
+    })();
+  }
+}
+
+export { fileClientOptions, filePaths, BoundedFileClient };

package/dist/chunk-sqbew04a.js

@@ -0,0 +1,65 @@
+import {
+  BoundedFileClient,
+  fileClientOptions,
+  filePaths
+} from "./chunk-kkts16w9.js";
+import {
+  FileError
+} from "./chunk-20kv4wc3.js";
+
+// src/file/local.ts
+import { constants } from "node:fs";
+import { open } from "node:fs/promises";
+import { resolve } from "node:path";
+import { clientKey, FileDataSource } from "@compforge/harness-common";
+class LocalFileClient extends BoundedFileClient {
+  async capture(signal, limits, sink) {
+    for (const [index, path] of this.paths.entries()) {
+      try {
+        signal.throwIfAborted();
+        const file = await open(path, constants.O_RDONLY | constants.O_NONBLOCK);
+        try {
+          signal.throwIfAborted();
+          if (!(await file.stat()).isFile())
+            throw new FileError(`Not a regular file: ${path}`, { kind: "unsupported_operation" });
+          const stream = file.createReadStream({
+            autoClose: false,
+            signal,
+            end: limits.maxBytes,
+            highWaterMark: Math.min(limits.maxBytes + 1, 64 * 1024)
+          });
+          for await (const bytes of stream)
+            sink.receive(index, bytes);
+        } finally {
+          await file.close();
+        }
+        sink.complete(index);
+      } catch (cause) {
+        signal.throwIfAborted();
+        const code = cause?.code;
+        sink.complete(index, cause instanceof FileError ? cause : new FileError(`Cannot read local file: ${path}`, {
+          kind: code === "ENOENT" || code === "ENOTDIR" ? "resource_not_found" : code === "EACCES" || code === "EPERM" ? "permission_denied" : "operation_failed",
+          code,
+          cause
+        }));
+      }
+    }
+  }
+}
+
+class LocalFileDataSource extends FileDataSource {
+  paths;
+  clientKey;
+  #options;
+  constructor(paths, options) {
+    super();
+    this.paths = filePaths(filePaths(paths).map((path) => resolve(path)));
+    this.#options = fileClientOptions(options);
+    this.clientKey = clientKey("local-file", { paths: this.paths, options: this.#options });
+  }
+  createClient(_clients, signal) {
+    return new LocalFileClient(this.paths, this.#options, signal);
+  }
+}
+
+export { LocalFileClient, LocalFileDataSource };