@nimbus-dev/connectors 0.2.0 → 0.2.2

package/shared/rest-tool-kit.ts

@@ -1,18 +1,88 @@
 /**
- * rest-tool-kit — re-export of the shared REST fetch wrapper for Bearer-auth
- * connectors, now owned by `@nimbus-dev/sdk/connector-kit`.
+ * rest-tool-kit — the REST tool plumbing for token-auth connectors.
  *
- * These helpers originated here and were upstreamed to the SDK in 1.11.0. The
- * SDK is now the single owner; this module stays as the connector-facing import
- * path so the ~99 connectors keep their existing relative imports.
+ * `makeRestFetcher` and `makeRestToolRegistrar` originated here and were upstreamed
+ * to the SDK in 1.11.0. The SDK is now their single owner; this module re-exports
+ * them so the ~99 connectors keep their existing relative imports. Named
+ * re-exports, deliberately not `export *` — see the note in `mcp-tool-kit.ts`.
  *
- * Named re-exports, deliberately not `export *` — see the note in
- * `mcp-tool-kit.ts`.
+ * The two helpers defined BELOW are this repository's own and stay here:
+ * {@link makeRestWriteToolRegistrar} registers through the consent kit, which the
+ * SDK does not have, and {@link toRestFetchResult} is the tail of the connector
+ * fetches whose credential is not a Bearer token, which the SDK's fetch cannot send.
  */
 
+import {
+  type HttpJsonBodyResponse,
+  mcpJsonResultIfOk,
+  type RestFetchResult,
+  requireProcessEnv,
+  type ZodObjectSchema,
+} from "@nimbus-dev/sdk/connector-kit";
+import type { WriteToolConfig, WriteToolRegistrar } from "./consent-kit.ts";
+
 export type {
   RestFetcherConfig,
   RestFetchResult,
   RestToolRegistrar,
 } from "@nimbus-dev/sdk/connector-kit";
 export { makeRestFetcher, makeRestToolRegistrar } from "@nimbus-dev/sdk/connector-kit";
+
+/** The per-connector registrar {@link makeRestWriteToolRegistrar} builds. */
+export type RestWriteToolRegistrar = <T>(
+  name: string,
+  cfg: WriteToolConfig<T>,
+  description: string,
+  schema: ZodObjectSchema<T>,
+  buildPath: (args: T) => string,
+  buildInit?: (args: T) => RequestInit,
+) => void;
+
+/**
+ * The write twin of `makeRestToolRegistrar`: the same standard tool body —
+ * `token = requireProcessEnv(<env>)`, `res = await <fetch>(token, buildPath(args)
+ * [, buildInit(args)])`, `return mcpJsonResultIfOk(<label>, res, <snippetMax>)` —
+ * registered through the connector's consent-kit write registrar instead of its
+ * read-only one, so every mutation it registers is gated.
+ *
+ * The connector passes its OWN `registerWriteTool` in rather than this kit building
+ * one: the standalone launcher decides whether a connector's mutations are gated by
+ * reading that connector's own `server.ts`/`tools.ts`, and a registrar constructed in
+ * here would be invisible to it.
+ */
+export function makeRestWriteToolRegistrar(cfg: {
+  readonly registerWriteTool: WriteToolRegistrar;
+  readonly tokenEnv: string;
+  readonly serviceLabel: string;
+  readonly fetch: (
+    token: string,
+    pathOrUrl: string,
+    init?: RequestInit,
+  ) => Promise<HttpJsonBodyResponse>;
+  /** Body-snippet length for the `<label> <status>: <snippet>` error (mcpJsonResultIfOk default 300). */
+  readonly snippetMax?: number;
+}): RestWriteToolRegistrar {
+  return (name, toolCfg, description, schema, buildPath, buildInit) => {
+    cfg.registerWriteTool(name, toolCfg, description, schema, async (args) => {
+      const token = requireProcessEnv(cfg.tokenEnv);
+      const res = await cfg.fetch(token, buildPath(args), buildInit?.(args));
+      return mcpJsonResultIfOk(cfg.serviceLabel, res, cfg.snippetMax);
+    });
+  };
+}
+
+/**
+ * Read a response into the {@link RestFetchResult} shape a REST tool's fetch
+ * returns. `json` is the parsed body, or `null` when the body is not JSON (an empty
+ * 204, an HTML error page): what that means is the tool's decision, not this one's.
+ */
+export async function toRestFetchResult(res: Response): Promise<RestFetchResult> {
+  const text = await res.text();
+  let json: unknown;
+  try {
+    json = JSON.parse(text) as unknown;
+  } catch {
+    json = null;
+  }
+  return { ok: res.ok, status: res.status, json, text };
+}

package/shared/run-cli-json.ts

@@ -2,23 +2,36 @@ import { nimbusSpawn } from "./nimbus-spawn.ts";
 
 export type RunCliJsonResult = { ok: true; data: unknown } | { ok: false; message: string };
 
-export async function runCliOk(
+/**
+ * Spawn `command`. An empty command, or a non-zero exit, becomes the one failure message every
+ * caller reports: `<binary> exited <code>: <first 500 chars of stderr>`.
+ */
+async function spawnChecked(
   command: readonly string[],
   env: Record<string, string | undefined>,
-): Promise<{ ok: true } | { ok: false; message: string }> {
+): Promise<{ ok: true; stdout: string } | { ok: false; message: string }> {
   if (command.length === 0) {
     return { ok: false, message: "empty command" };
   }
-  const { code, stderr: err } = await nimbusSpawn(command, env);
+  const { code, stdout, stderr } = await nimbusSpawn(command, env);
   if (code !== 0) {
     return {
       ok: false,
-      message: `${command[0] ?? "cli"} exited ${String(code)}: ${err.slice(0, 500)}`,
+      message: `${command[0] ?? "cli"} exited ${String(code)}: ${stderr.slice(0, 500)}`,
     };
   }
-  return { ok: true };
+  return { ok: true, stdout };
+}
+
+export async function runCliOk(
+  command: readonly string[],
+  env: Record<string, string | undefined>,
+): Promise<{ ok: true } | { ok: false; message: string }> {
+  const r = await spawnChecked(command, env);
+  return r.ok ? { ok: true } : r;
 }
 
+/** {@link runCliOk}, throwing its failure message instead of returning it. */
 export async function runCliOkThrowing(
   command: readonly string[],
   env: Record<string, string | undefined>,
@@ -33,23 +46,32 @@ export async function runCliJson(
   command: readonly string[],
   env: Record<string, string | undefined>,
 ): Promise<RunCliJsonResult> {
-  if (command.length === 0) {
-    return { ok: false, message: "empty command" };
-  }
-  const { code, stdout: out, stderr: err } = await nimbusSpawn(command, env);
-  if (code !== 0) {
-    return {
-      ok: false,
-      message: `${command[0] ?? "cli"} exited ${String(code)}: ${err.slice(0, 500)}`,
-    };
+  const r = await spawnChecked(command, env);
+  if (!r.ok) {
+    return r;
   }
-  const trimmed = out.trim();
+  const trimmed = r.stdout.trim();
   if (trimmed === "") {
     return { ok: true, data: null };
   }
   try {
     return { ok: true, data: JSON.parse(trimmed) as unknown };
   } catch {
-    return { ok: false, message: `invalid JSON from CLI: ${out.slice(0, 200)}` };
+    return { ok: false, message: `invalid JSON from CLI: ${r.stdout.slice(0, 200)}` };
+  }
+}
+
+/**
+ * {@link runCliJson}, throwing its failure message instead of returning it. Resolves to the parsed
+ * JSON, or `null` when the command printed nothing.
+ */
+export async function runCliJsonThrowing(
+  command: readonly string[],
+  env: Record<string, string | undefined>,
+): Promise<unknown> {
+  const r = await runCliJson(command, env);
+  if (!r.ok) {
+    throw new Error(r.message);
   }
+  return r.data;
 }

package/shared/search-filter.ts

@@ -43,21 +43,35 @@ export function tagText(row: Record<string, unknown>): string {
 }
 
 /**
- * Extract tag names from an array of `{name: string}` tag objects (e.g. Airflow, DependencyTrack).
- * Returns "" when `tags` is absent, not an array, or contains no object entries with a string `name`.
+ * The non-empty string entries of the array at `row[key]`, space-joined; `""` when it is absent or
+ * not an array (Airflow `owners`, Prefect `tags`). Unlike {@link tagText}, an empty-string entry is
+ * skipped rather than joined — which shows in the haystack's spacing, so it is a separate rule.
  */
-export function tagNamesFromObjects(row: Record<string, unknown>): string {
-  const tags = row["tags"];
-  if (!Array.isArray(tags)) {
+export function nonEmptyStringsText(row: Record<string, unknown>, key: string): string {
+  const list = row[key];
+  if (!Array.isArray(list)) {
+    return "";
+  }
+  return list.filter((v): v is string => typeof v === "string" && v !== "").join(" ");
+}
+
+/**
+ * The non-empty string `name` of each objectish entry of the array at `row[key]`, space-joined
+ * (Wiz `projects`). Returns "" when `row[key]` is absent, not an array, or holds no object entry
+ * with a non-empty string `name`.
+ */
+export function objectNamesText(row: Record<string, unknown>, key: string): string {
+  const list = row[key];
+  if (!Array.isArray(list)) {
     return "";
   }
   const names: string[] = [];
-  for (const t of tags) {
-    const tag = asObjectish(t);
-    if (tag === undefined) {
+  for (const entry of list) {
+    const obj = asObjectish(entry);
+    if (obj === undefined) {
       continue;
     }
-    const name = tag["name"];
+    const name = obj["name"];
     if (typeof name === "string" && name !== "") {
       names.push(name);
     }
@@ -65,6 +79,14 @@ export function tagNamesFromObjects(row: Record<string, unknown>): string {
   return names.join(" ");
 }
 
+/**
+ * Extract tag names from an array of `{name: string}` tag objects (e.g. Airflow, DependencyTrack).
+ * Returns "" when `tags` is absent, not an array, or contains no object entries with a string `name`.
+ */
+export function tagNamesFromObjects(row: Record<string, unknown>): string {
+  return objectNamesText(row, "tags");
+}
+
 export function filterByQuery<T>(items: readonly T[], options: FilterByQueryOptions<T>): T[] {
   const needle = options.query.toLowerCase();
   const cap = options.limit ?? 50;
@@ -88,28 +110,46 @@ export function filterByQuery<T>(items: readonly T[], options: FilterByQueryOpti
 
 export type FieldExtractor = (item: unknown) => readonly (string | null | undefined)[] | null;
 
-/**
- * Build a {@link FieldExtractor} that reads a fixed list of string keys off each
- * objectish row, optionally appending the standard `tags` text. Collapses the
- * boilerplate `fieldsOf` body shared by the simpler connectors.
- */
-export function fieldsFromKeys(
+/** The one body behind {@link fieldsFromKeys} and {@link recordFieldsFromKeys}. */
+function keyedFields(
+  toRow: (item: unknown) => Record<string, unknown> | undefined,
   keys: readonly string[],
-  opts?: { readonly tags?: boolean },
+  tags: boolean,
 ): FieldExtractor {
   return (item: unknown) => {
-    const row = asObjectish(item);
+    const row = toRow(item);
     if (row === undefined) {
       return null;
     }
     const parts = keys.map((key) => stringField(row, key));
-    if (opts?.tags === true) {
+    if (tags) {
       parts.push(tagText(row));
     }
     return parts;
   };
 }
 
+/**
+ * Build a {@link FieldExtractor} that reads a fixed list of string keys off each
+ * objectish row, optionally appending the standard `tags` text. Collapses the
+ * boilerplate `fieldsOf` body shared by the simpler connectors.
+ */
+export function fieldsFromKeys(
+  keys: readonly string[],
+  opts?: { readonly tags?: boolean },
+): FieldExtractor {
+  return keyedFields(asObjectish, keys, opts?.tags === true);
+}
+
+/**
+ * {@link fieldsFromKeys} for rows that must be plain objects: an ARRAY row is skipped
+ * ({@link asRecord}) instead of being read as an object that has none of the keys — which a
+ * query made only of spaces would still match. The BI connectors' filters use this rule.
+ */
+export function recordFieldsFromKeys(keys: readonly string[]): FieldExtractor {
+  return keyedFields(asRecord, keys, false);
+}
+
 /**
  * Read a nested string field by key path off an objectish row, returning `""`
  * when any path segment is missing or the leaf is not a string. Shared by the

package/shared/walk-files.ts

@@ -0,0 +1,118 @@
+/**
+ * walk-files — the bounded recursive directory walk the local-filesystem
+ * connectors share, and the path-traversal guard that goes with it.
+ *
+ * `great-expectations`, `localdb` and `dataprofile` each scan a configured
+ * directory tree for the files they care about (`*.json`, `*.sql`, and a set of
+ * tabular extensions), and each had written out the same walk: recurse, stop at
+ * a depth limit, stop at a file-count limit, swallow an unreadable directory
+ * rather than aborting the scan.
+ *
+ * Those two limits are why this is worth sharing rather than tolerating. They
+ * are the guard against a connector being pointed — by configuration or by a
+ * symlink — at a tree deep or wide enough to exhaust the process, and a guard
+ * maintained in three places is a guard that can be tightened in one and left
+ * alone in the other two. They were already inconsistent: 1000 files for
+ * great-expectations against 2000 for the others. That difference is real (a
+ * GX run directory is small) so it stays a per-caller argument rather than
+ * being averaged away — but it is now visibly an argument rather than three
+ * independent constants that happen to differ.
+ *
+ * The same three connectors each carried a copy of {@link assertWithinDir} too,
+ * identical but for the directory named in the error — a second security
+ * control in three hand-written copies, shared for the same reason.
+ */
+
+import type { Dirent } from "node:fs";
+import { readdir } from "node:fs/promises";
+import { isAbsolute, join, relative } from "node:path";
+
+export interface WalkFilesOptions<T> {
+  /** Hard cap on results. The walk stops as soon as it is reached. */
+  readonly maxFiles: number;
+  /** Hard cap on recursion depth, counted from `root` at 0. */
+  readonly maxDepth: number;
+  /**
+   * What to keep for one file entry, or `undefined` to skip it. Receives the
+   * directory entry and its full path, so a caller can select on the name, the
+   * extension, or anything else it can decide without reading the file.
+   */
+  readonly select: (entry: Dirent, fullPath: string) => T | undefined;
+}
+
+/**
+ * Collect what `select` keeps, walking `root` depth-first within the limits.
+ *
+ * An unreadable directory is SKIPPED rather than fatal: a scan of a
+ * user-configured tree meets permission-denied subdirectories routinely, and
+ * failing the whole tool call for one of them would make the connector useless
+ * on a real machine. A caller that needs to know a directory was unreadable
+ * should stat it itself.
+ *
+ * Symlinks are not followed — `readdir` reports a link as neither a file nor a
+ * directory here, so a cycle cannot make the walk diverge.
+ */
+export async function walkFiles<T>(root: string, options: WalkFilesOptions<T>): Promise<T[]> {
+  const found: T[] = [];
+
+  async function walk(dir: string, depth: number): Promise<void> {
+    if (depth > options.maxDepth || found.length >= options.maxFiles) {
+      return;
+    }
+    let entries: Dirent[];
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      if (found.length >= options.maxFiles) {
+        return;
+      }
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        await walk(full, depth + 1); // NOSONAR S9382: depth-first under one shared maxFiles cap — a subtree must finish before the next starts or the cap stops at a nondeterministic point, and concurrent readdirs over a wide tree are the exhaustion this walk exists to bound.
+      } else if (entry.isFile()) {
+        const kept = options.select(entry, full);
+        if (kept !== undefined) {
+          found.push(kept);
+        }
+      }
+    }
+  }
+
+  await walk(root, 0);
+  return found;
+}
+
+/**
+ * A {@link WalkFilesOptions.select} that keeps the full path of every file with
+ * one of `extensions`, compared case-insensitively.
+ */
+export function byExtension(
+  ...extensions: readonly string[]
+): (e: Dirent, full: string) => string | undefined {
+  const wanted = extensions.map((x) => x.toLowerCase());
+  return (entry: Dirent, full: string): string | undefined =>
+    wanted.some((x) => entry.name.toLowerCase().endsWith(x)) ? full : undefined;
+}
+
+/**
+ * The path-traversal guard: throws `"path escapes <what>"` unless `candidate` (an
+ * absolute, resolved path) is `root` itself or a descendant of it. `what` names the
+ * directory for the message, e.g. `"the configured local DB scripts dir"`.
+ *
+ * A `relative()` that starts with `..` climbed out of `root` (this also covers
+ * `..<sep>`); one that is absolute could not be expressed relative to it at all — a
+ * different drive on Windows. It fails closed: a child whose own name begins with
+ * `..` (such as `..cache`) is refused too.
+ */
+export function assertWithinDir(candidate: string, root: string, what: string): void {
+  const rel = relative(root, candidate);
+  if (rel === "") {
+    return;
+  }
+  if (rel.startsWith("..") || isAbsolute(rel)) {
+    throw new Error(`path escapes ${what}`);
+  }
+}

package/standalone/src/launcher.ts

@@ -10,7 +10,10 @@ const ID_RE = /^[a-z0-9-]+$/;
  *
  * A registration CALL, or the registrar handed to a shared kit — not a bare substring, which the
  * registrar's own `const registerWriteTool = ...` would satisfy even with nothing registered.
- * Kept in step with the twin in check-connector-consent.ts.
+ *
+ * The ONE copy. `audit:connector-consent` (scripts/check-connector-consent.ts) imports it rather
+ * than keeping a twin, so the launcher's runtime verdict and the audit's static one cannot drift.
+ * They had: the audit's twin learned to tolerate a carriage return (below) and this copy never did.
  *
  * Deliberately NOT a regular expression. The previous pattern was
  * `^\s*register[A-Za-z]*WriteTool\(` under `/m`, and it drew two rounds of ReDoS reports. The
@@ -25,9 +28,15 @@ const ID_RE = /^[a-z0-9-]+$/;
  * bounded work, no star sits next to an overlapping literal, and the accepted language is
  * unchanged — including the trailing-comma form, which still requires the comma to end the line.
  */
-function registersWriteTool(src: string): boolean {
+export function registersWriteTool(src: string): boolean {
   for (const line of src.split("\n")) {
-    const t = line.trimStart();
+    // trim(), not trimStart(): the equality below is exact, and a CRLF checkout leaves a trailing
+    // carriage return that breaks it. Observed in the consent audit on the first run of this repo
+    // on its own, before .gitattributes existed: it reported imap and protonmail as declaring
+    // ungated writes while both register through the kit, on a line reading `registerWriteTool,`
+    // plus a CR. That failed SAFE (a false finding there, a false refusal here — never a false
+    // green), but a verdict that depends on the checkout's line endings is waiting to be wrong.
+    const t = line.trim();
     if (t === "registerWriteTool,") return true;
     if (!t.startsWith("register")) continue;
     const at = t.indexOf("WriteTool(");
@@ -57,13 +66,13 @@ function connectorsDir(): string {
  * The id is validated against a strict allow-list BEFORE being joined into a path. A separator or
  * `..` would otherwise let the id escape the connectors directory and import an arbitrary module.
  */
-export function resolveConnectorEntry(id: string): string {
+export function resolveConnectorEntry(id: string, root: string = connectorsDir()): string {
   if (!ID_RE.test(id)) {
     throw new Error(
       `invalid connector id ${JSON.stringify(id)}: expected only lowercase letters, digits and hyphens`,
     );
   }
-  return join(connectorsDir(), id, "src", "server.ts");
+  return join(root, id, "src", "server.ts");
 }
 
 export type Eligibility =
@@ -154,6 +163,9 @@ export type ConnectorImporter = (entry: string) => Promise<{
 export async function runStandalone(
   argv: readonly string[],
   importConnector: ConnectorImporter = (entry) => import(entry),
+  // Injectable for the same reason as standaloneEligibility's root: every real connector is
+  // migrated, so only a fixture can show the refusal path end to end.
+  root: string = connectorsDir(),
 ): Promise<number> {
   const id = argv[0];
   if (id === undefined) {
@@ -162,7 +174,7 @@ export async function runStandalone(
   }
   let entry: string;
   try {
-    entry = resolveConnectorEntry(id);
+    entry = resolveConnectorEntry(id, root);
   } catch (e) {
     process.stderr.write(`${e instanceof Error ? e.message : String(e)}\n`);
     return 2;
@@ -172,7 +184,7 @@ export async function runStandalone(
     return 2;
   }
 
-  const verdict = standaloneEligibility(id);
+  const verdict = standaloneEligibility(id, root);
   if (!verdict.eligible) {
     process.stderr.write(`${verdict.reason}\n`);
     return 3;