@nimbus-dev/connectors 0.2.1 → 0.2.3

package/shared/run-cli-json.ts

@@ -2,23 +2,36 @@ import { nimbusSpawn } from "./nimbus-spawn.ts";
 
 export type RunCliJsonResult = { ok: true; data: unknown } | { ok: false; message: string };
 
-export async function runCliOk(
+/**
+ * Spawn `command`. An empty command, or a non-zero exit, becomes the one failure message every
+ * caller reports: `<binary> exited <code>: <first 500 chars of stderr>`.
+ */
+async function spawnChecked(
   command: readonly string[],
   env: Record<string, string | undefined>,
-): Promise<{ ok: true } | { ok: false; message: string }> {
+): Promise<{ ok: true; stdout: string } | { ok: false; message: string }> {
   if (command.length === 0) {
     return { ok: false, message: "empty command" };
   }
-  const { code, stderr: err } = await nimbusSpawn(command, env);
+  const { code, stdout, stderr } = await nimbusSpawn(command, env);
   if (code !== 0) {
     return {
       ok: false,
-      message: `${command[0] ?? "cli"} exited ${String(code)}: ${err.slice(0, 500)}`,
+      message: `${command[0] ?? "cli"} exited ${String(code)}: ${stderr.slice(0, 500)}`,
     };
   }
-  return { ok: true };
+  return { ok: true, stdout };
+}
+
+export async function runCliOk(
+  command: readonly string[],
+  env: Record<string, string | undefined>,
+): Promise<{ ok: true } | { ok: false; message: string }> {
+  const r = await spawnChecked(command, env);
+  return r.ok ? { ok: true } : r;
 }
 
+/** {@link runCliOk}, throwing its failure message instead of returning it. */
 export async function runCliOkThrowing(
   command: readonly string[],
   env: Record<string, string | undefined>,
@@ -33,23 +46,32 @@ export async function runCliJson(
   command: readonly string[],
   env: Record<string, string | undefined>,
 ): Promise<RunCliJsonResult> {
-  if (command.length === 0) {
-    return { ok: false, message: "empty command" };
-  }
-  const { code, stdout: out, stderr: err } = await nimbusSpawn(command, env);
-  if (code !== 0) {
-    return {
-      ok: false,
-      message: `${command[0] ?? "cli"} exited ${String(code)}: ${err.slice(0, 500)}`,
-    };
+  const r = await spawnChecked(command, env);
+  if (!r.ok) {
+    return r;
   }
-  const trimmed = out.trim();
+  const trimmed = r.stdout.trim();
   if (trimmed === "") {
     return { ok: true, data: null };
   }
   try {
     return { ok: true, data: JSON.parse(trimmed) as unknown };
   } catch {
-    return { ok: false, message: `invalid JSON from CLI: ${out.slice(0, 200)}` };
+    return { ok: false, message: `invalid JSON from CLI: ${r.stdout.slice(0, 200)}` };
+  }
+}
+
+/**
+ * {@link runCliJson}, throwing its failure message instead of returning it. Resolves to the parsed
+ * JSON, or `null` when the command printed nothing.
+ */
+export async function runCliJsonThrowing(
+  command: readonly string[],
+  env: Record<string, string | undefined>,
+): Promise<unknown> {
+  const r = await runCliJson(command, env);
+  if (!r.ok) {
+    throw new Error(r.message);
   }
+  return r.data;
 }

package/shared/safe-cli-arg.ts

@@ -1,40 +1,70 @@
 /**
  * Guard a tool-supplied value before it is passed as an argument to a spawned CLI
- * (`gcloud` / `aws` / `bq` / …).
+ * (`gcloud` / `aws` / `az` / `kubectl` / …).
  *
  * Spawns use an argv array (no shell), so there is no classic shell injection — but
  * a value that begins with `-` would be parsed by the CLI as a FLAG rather than a
  * positional / flag-value. This "argv flag smuggling" lets tool input inject
- * arbitrary CLI flags (e.g. a `sinkName` of `--project=attacker` or `-h`). Reject
+ * arbitrary CLI flags (e.g. a `sinkName` of `--project=attacker`, or a pod name of
+ * `--kubeconfig=<path>`, whose exec credential plugin runs a command). Reject
  * any value that is empty, over-long, starts with `-`, or contains control
  * characters.
  *
  * Resource-id charsets vary per service (sink ids are `[A-Za-z0-9_.-]`, CloudWatch
  * log-group names contain `/`, BigQuery refs contain `.`), so this is the UNIVERSAL
  * guard against flag smuggling; callers may layer a stricter per-resource regex on
- * top. Returns the value unchanged so it can be used inline:
+ * top. Two CLIs need one: `aws` and `az` read some argument values from a file
+ * instead of taking them as written — see {@link awsCliArgProblem} and
+ * {@link azCliArgProblem}. A third rule is not per value at all: an argument to a CLI
+ * that Windows runs as a batch file is checked at the spawn itself, in
+ * `windows-batch-args.ts`.
+ *
+ * Returns the value unchanged so it can be used inline:
  * `cli(["describe", assertSafeCliArg(p.id, "id")])`.
  *
  * @throws {Error} when the value is unsafe to pass as a CLI argument.
  */
 export function assertSafeCliArg(value: string, label = "argument"): string {
+  const problem = cliArgProblem(value);
+  if (problem !== undefined) {
+    throw new Error(`Invalid ${label}: ${problem}`);
+  }
+  return value;
+}
+
+/** The longest value accepted as one CLI argument. */
+const MAX_CLI_ARG_LENGTH = 1024;
+
+/**
+ * Why `value` must not be passed to a spawned CLI as one argument, or `undefined` when it may.
+ *
+ * The reason is the whole of the message a refusal carries, so it says what to change. It is the
+ * rule {@link assertSafeCliArg} throws on and {@link isSafeCliArg} tests, written once so that the
+ * two can never disagree.
+ */
+export function cliArgProblem(value: unknown): string | undefined {
   if (typeof value !== "string" || value.length === 0) {
-    throw new Error(`Invalid ${label}: must be a non-empty string`);
+    return "must be a non-empty string";
   }
-  if (value.length > 1024) {
-    throw new Error(`Invalid ${label}: exceeds 1024 characters`);
+  if (value.length > MAX_CLI_ARG_LENGTH) {
+    return `exceeds ${String(MAX_CLI_ARG_LENGTH)} characters`;
   }
   if (value.startsWith("-")) {
-    throw new Error(
-      `Invalid ${label}: must not start with "-" (argv flag smuggling is not allowed)`,
-    );
+    return 'must not start with "-" (argv flag smuggling is not allowed)';
   }
+  if (hasControlCharacter(value)) {
+    return "must not contain control characters";
+  }
+  return undefined;
+}
+
+function hasControlCharacter(value: string): boolean {
   for (let i = 0; i < value.length; i += 1) {
     if ((value.codePointAt(i) ?? 0) < 0x20) {
-      throw new Error(`Invalid ${label}: must not contain control characters`);
+      return true;
     }
   }
-  return value;
+  return false;
 }
 
 /**
@@ -43,13 +73,105 @@ export function assertSafeCliArg(value: string, label = "argument"): string {
  * schema boundary before the handler ever shells out.
  */
 export function isSafeCliArg(value: unknown): value is string {
+  return cliArgProblem(value) === undefined;
+}
+
+// ---------------------------------------------------------------------------
+// aws
+// ---------------------------------------------------------------------------
+
+/**
+ * The prefixes with which the AWS CLI REPLACES a parameter's value by what it reads from
+ * somewhere else. Matched here case-insensitively and after leading whitespace, though the CLI
+ * matches them exactly: refusing a value the CLI would have taken as written costs nothing.
+ *
+ * `file://` and `fileb://` read a local file — for nearly every parameter, not only the ones
+ * documented as taking a document; the CLI exempts a short list whose value is itself a URL — and
+ * v1 of the CLI fetches an `http://` or `https://` value too, unless `cli_follow_urlparam` is
+ * turned off. The CLI then sends what it read to AWS as the value, and an
+ * error can quote it back — AWS's own validation errors name the value they reject — so a tool
+ * argument of `file://<path>` could hand the model the contents of a file on the user's machine.
+ */
+export const AWS_CLI_LOADING_PREFIXES = ["file://", "fileb://", "http://", "https://"] as const;
+
+function awsPrefixProblem(value: string): string | undefined {
+  const head = value.trimStart().toLowerCase();
+  const prefix = AWS_CLI_LOADING_PREFIXES.find((p) => head.startsWith(p));
+  return prefix === undefined
+    ? undefined
+    : `must not start with "${prefix}" (the aws CLI would read the value from that location)`;
+}
+
+/**
+ * `@=` is the AWS CLI v2 shorthand-syntax operator that loads a file into one key of a structure
+ * parameter, as in `TemplateBody@=file://<path>`. No tool passes a structure parameter today, so
+ * this is refused for the next tool that does rather than for any value that reaches the CLI now.
+ */
+function awsShorthandProblem(value: string): string | undefined {
+  return value.includes("@=")
+    ? 'must not contain "@=" (aws shorthand syntax would read a file into the value)'
+    : undefined;
+}
+
+/**
+ * Why `value` must not be passed to the AWS CLI as one argument, or `undefined` when it may: the
+ * universal rule, then a loading prefix ({@link AWS_CLI_LOADING_PREFIXES}), then `@=`.
+ */
+export function awsCliArgProblem(value: unknown): string | undefined {
   if (typeof value !== "string") {
-    return false;
+    return cliArgProblem(value);
+  }
+  return cliArgProblem(value) ?? awsPrefixProblem(value) ?? awsShorthandProblem(value);
+}
+
+/**
+ * Why `value` must not be passed to the AWS CLI as a DOCUMENT — a CloudFormation template body —
+ * or `undefined` when it may.
+ *
+ * A document legitimately spans lines and runs past the 1024 characters an argument is held to,
+ * so of the argument rules only the two about where the CLI looks remain: a leading `-`, which the
+ * CLI would parse as an option, and a loading prefix, which would make it read the "document" from
+ * a file. `@=` is not among them: shorthand syntax applies to structure parameters, and a template
+ * body is a string.
+ */
+export function awsCliDocumentProblem(value: unknown): string | undefined {
+  if (typeof value !== "string" || value.length === 0) {
+    return "must be a non-empty string";
+  }
+  if (value.startsWith("-")) {
+    return 'must not start with "-" (argv flag smuggling is not allowed)';
+  }
+  return awsPrefixProblem(value);
+}
+
+// ---------------------------------------------------------------------------
+// az
+// ---------------------------------------------------------------------------
+
+function azFileProblem(value: string): string | undefined {
+  if (value.startsWith("@")) {
+    return 'must not start with "@" (az would read the value from a file)';
+  }
+  if (value.includes("=@")) {
+    return 'must not contain "=@" (az would read what follows from a file)';
   }
-  try {
-    assertSafeCliArg(value);
-    return true;
-  } catch {
-    return false;
+  return undefined;
+}
+
+/**
+ * Why `value` must not be passed to the Azure CLI as one argument, or `undefined` when it may: the
+ * universal rule, then a leading `@`, then `=@`.
+ *
+ * Before it parses anything, `az` replaces every argument that starts with `@` by the contents of
+ * the file named after it — with `~` expanded, and `@-` read from stdin — and does the same to
+ * whatever follows the first `=` in an argument. A resource group of
+ * `@~/.azure/msal_token_cache.json` would be sent to Azure as that cache, which a "could not be
+ * found" error can quote back. An `@` anywhere else, as in an email address, is left alone by `az`
+ * and is accepted here.
+ */
+export function azCliArgProblem(value: unknown): string | undefined {
+  if (typeof value !== "string") {
+    return cliArgProblem(value);
   }
+  return cliArgProblem(value) ?? azFileProblem(value);
 }

package/shared/search-filter.ts

@@ -43,21 +43,35 @@ export function tagText(row: Record<string, unknown>): string {
 }
 
 /**
- * Extract tag names from an array of `{name: string}` tag objects (e.g. Airflow, DependencyTrack).
- * Returns "" when `tags` is absent, not an array, or contains no object entries with a string `name`.
+ * The non-empty string entries of the array at `row[key]`, space-joined; `""` when it is absent or
+ * not an array (Airflow `owners`, Prefect `tags`). Unlike {@link tagText}, an empty-string entry is
+ * skipped rather than joined — which shows in the haystack's spacing, so it is a separate rule.
  */
-export function tagNamesFromObjects(row: Record<string, unknown>): string {
-  const tags = row["tags"];
-  if (!Array.isArray(tags)) {
+export function nonEmptyStringsText(row: Record<string, unknown>, key: string): string {
+  const list = row[key];
+  if (!Array.isArray(list)) {
+    return "";
+  }
+  return list.filter((v): v is string => typeof v === "string" && v !== "").join(" ");
+}
+
+/**
+ * The non-empty string `name` of each objectish entry of the array at `row[key]`, space-joined
+ * (Wiz `projects`). Returns "" when `row[key]` is absent, not an array, or holds no object entry
+ * with a non-empty string `name`.
+ */
+export function objectNamesText(row: Record<string, unknown>, key: string): string {
+  const list = row[key];
+  if (!Array.isArray(list)) {
     return "";
   }
   const names: string[] = [];
-  for (const t of tags) {
-    const tag = asObjectish(t);
-    if (tag === undefined) {
+  for (const entry of list) {
+    const obj = asObjectish(entry);
+    if (obj === undefined) {
       continue;
     }
-    const name = tag["name"];
+    const name = obj["name"];
     if (typeof name === "string" && name !== "") {
       names.push(name);
     }
@@ -65,6 +79,14 @@ export function tagNamesFromObjects(row: Record<string, unknown>): string {
   return names.join(" ");
 }
 
+/**
+ * Extract tag names from an array of `{name: string}` tag objects (e.g. Airflow, DependencyTrack).
+ * Returns "" when `tags` is absent, not an array, or contains no object entries with a string `name`.
+ */
+export function tagNamesFromObjects(row: Record<string, unknown>): string {
+  return objectNamesText(row, "tags");
+}
+
 export function filterByQuery<T>(items: readonly T[], options: FilterByQueryOptions<T>): T[] {
   const needle = options.query.toLowerCase();
   const cap = options.limit ?? 50;
@@ -88,28 +110,46 @@ export function filterByQuery<T>(items: readonly T[], options: FilterByQueryOpti
 
 export type FieldExtractor = (item: unknown) => readonly (string | null | undefined)[] | null;
 
-/**
- * Build a {@link FieldExtractor} that reads a fixed list of string keys off each
- * objectish row, optionally appending the standard `tags` text. Collapses the
- * boilerplate `fieldsOf` body shared by the simpler connectors.
- */
-export function fieldsFromKeys(
+/** The one body behind {@link fieldsFromKeys} and {@link recordFieldsFromKeys}. */
+function keyedFields(
+  toRow: (item: unknown) => Record<string, unknown> | undefined,
   keys: readonly string[],
-  opts?: { readonly tags?: boolean },
+  tags: boolean,
 ): FieldExtractor {
   return (item: unknown) => {
-    const row = asObjectish(item);
+    const row = toRow(item);
     if (row === undefined) {
       return null;
     }
     const parts = keys.map((key) => stringField(row, key));
-    if (opts?.tags === true) {
+    if (tags) {
       parts.push(tagText(row));
     }
     return parts;
   };
 }
 
+/**
+ * Build a {@link FieldExtractor} that reads a fixed list of string keys off each
+ * objectish row, optionally appending the standard `tags` text. Collapses the
+ * boilerplate `fieldsOf` body shared by the simpler connectors.
+ */
+export function fieldsFromKeys(
+  keys: readonly string[],
+  opts?: { readonly tags?: boolean },
+): FieldExtractor {
+  return keyedFields(asObjectish, keys, opts?.tags === true);
+}
+
+/**
+ * {@link fieldsFromKeys} for rows that must be plain objects: an ARRAY row is skipped
+ * ({@link asRecord}) instead of being read as an object that has none of the keys — which a
+ * query made only of spaces would still match. The BI connectors' filters use this rule.
+ */
+export function recordFieldsFromKeys(keys: readonly string[]): FieldExtractor {
+  return keyedFields(asRecord, keys, false);
+}
+
 /**
  * Read a nested string field by key path off an objectish row, returning `""`
  * when any path segment is missing or the leaf is not a string. Shared by the

package/shared/walk-files.ts

@@ -0,0 +1,118 @@
+/**
+ * walk-files — the bounded recursive directory walk the local-filesystem
+ * connectors share, and the path-traversal guard that goes with it.
+ *
+ * `great-expectations`, `localdb` and `dataprofile` each scan a configured
+ * directory tree for the files they care about (`*.json`, `*.sql`, and a set of
+ * tabular extensions), and each had written out the same walk: recurse, stop at
+ * a depth limit, stop at a file-count limit, swallow an unreadable directory
+ * rather than aborting the scan.
+ *
+ * Those two limits are why this is worth sharing rather than tolerating. They
+ * are the guard against a connector being pointed — by configuration or by a
+ * symlink — at a tree deep or wide enough to exhaust the process, and a guard
+ * maintained in three places is a guard that can be tightened in one and left
+ * alone in the other two. They were already inconsistent: 1000 files for
+ * great-expectations against 2000 for the others. That difference is real (a
+ * GX run directory is small) so it stays a per-caller argument rather than
+ * being averaged away — but it is now visibly an argument rather than three
+ * independent constants that happen to differ.
+ *
+ * The same three connectors each carried a copy of {@link assertWithinDir} too,
+ * identical but for the directory named in the error — a second security
+ * control in three hand-written copies, shared for the same reason.
+ */
+
+import type { Dirent } from "node:fs";
+import { readdir } from "node:fs/promises";
+import { isAbsolute, join, relative } from "node:path";
+
+export interface WalkFilesOptions<T> {
+  /** Hard cap on results. The walk stops as soon as it is reached. */
+  readonly maxFiles: number;
+  /** Hard cap on recursion depth, counted from `root` at 0. */
+  readonly maxDepth: number;
+  /**
+   * What to keep for one file entry, or `undefined` to skip it. Receives the
+   * directory entry and its full path, so a caller can select on the name, the
+   * extension, or anything else it can decide without reading the file.
+   */
+  readonly select: (entry: Dirent, fullPath: string) => T | undefined;
+}
+
+/**
+ * Collect what `select` keeps, walking `root` depth-first within the limits.
+ *
+ * An unreadable directory is SKIPPED rather than fatal: a scan of a
+ * user-configured tree meets permission-denied subdirectories routinely, and
+ * failing the whole tool call for one of them would make the connector useless
+ * on a real machine. A caller that needs to know a directory was unreadable
+ * should stat it itself.
+ *
+ * Symlinks are not followed — `readdir` reports a link as neither a file nor a
+ * directory here, so a cycle cannot make the walk diverge.
+ */
+export async function walkFiles<T>(root: string, options: WalkFilesOptions<T>): Promise<T[]> {
+  const found: T[] = [];
+
+  async function walk(dir: string, depth: number): Promise<void> {
+    if (depth > options.maxDepth || found.length >= options.maxFiles) {
+      return;
+    }
+    let entries: Dirent[];
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      if (found.length >= options.maxFiles) {
+        return;
+      }
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        await walk(full, depth + 1); // NOSONAR S9382: depth-first under one shared maxFiles cap — a subtree must finish before the next starts or the cap stops at a nondeterministic point, and concurrent readdirs over a wide tree are the exhaustion this walk exists to bound.
+      } else if (entry.isFile()) {
+        const kept = options.select(entry, full);
+        if (kept !== undefined) {
+          found.push(kept);
+        }
+      }
+    }
+  }
+
+  await walk(root, 0);
+  return found;
+}
+
+/**
+ * A {@link WalkFilesOptions.select} that keeps the full path of every file with
+ * one of `extensions`, compared case-insensitively.
+ */
+export function byExtension(
+  ...extensions: readonly string[]
+): (e: Dirent, full: string) => string | undefined {
+  const wanted = extensions.map((x) => x.toLowerCase());
+  return (entry: Dirent, full: string): string | undefined =>
+    wanted.some((x) => entry.name.toLowerCase().endsWith(x)) ? full : undefined;
+}
+
+/**
+ * The path-traversal guard: throws `"path escapes <what>"` unless `candidate` (an
+ * absolute, resolved path) is `root` itself or a descendant of it. `what` names the
+ * directory for the message, e.g. `"the configured local DB scripts dir"`.
+ *
+ * A `relative()` that starts with `..` climbed out of `root` (this also covers
+ * `..<sep>`); one that is absolute could not be expressed relative to it at all — a
+ * different drive on Windows. It fails closed: a child whose own name begins with
+ * `..` (such as `..cache`) is refused too.
+ */
+export function assertWithinDir(candidate: string, root: string, what: string): void {
+  const rel = relative(root, candidate);
+  if (rel === "") {
+    return;
+  }
+  if (rel.startsWith("..") || isAbsolute(rel)) {
+    throw new Error(`path escapes ${what}`);
+  }
+}