@nimbus-dev/connectors 0.2.1 → 0.2.3

package/shared/cli-json-kit.ts

@@ -0,0 +1,191 @@
+/**
+ * cli-json-kit — the shared pieces of the CLI-backed connectors.
+ *
+ * Five connectors reach their service by spawning a cloud CLI and parsing its
+ * JSON (athena, cloudwatch and sagemaker through `aws`; cloud-logging and
+ * vertex-ai through `gcloud`), and each had written out the same three things:
+ *
+ *   - the `cliArg` Zod refinement that rejects an argument starting with `-`
+ *     (argv flag smuggling) or carrying control characters;
+ *   - a `runX` wrapper: spawn, trim, throw `"<label> <verb> failed: <stderr>"`
+ *     on a non-zero exit, otherwise `JSON.parse` the stdout;
+ *   - `isRecord` / `strField` / `asArray`, the untyped-payload readers used to
+ *     walk the parsed result.
+ *
+ * The argument guard is the reason this is worth sharing rather than tolerating:
+ * it is a security control, and a security control that exists in five
+ * hand-written copies is one that can be strengthened in four of them. Every
+ * caller-supplied value that reaches a CLI's argv, in all eleven connectors
+ * that spawn one, now passes one of the argument schemas here — including in
+ * the connectors that spawn through `run-cli-json.ts` rather than
+ * {@link createCliJsonRunner}; bigquery's values never reach argv at all. And
+ * `scripts/cli-argument-guards.test.ts` fails a tool whose schema lets a value
+ * the CLI would misread through, whether or not something later refuses it.
+ *
+ * The `gcloud` connectors (bigquery, cloud-logging, gcp, vertex-ai) also share
+ * how they hand gcloud its project and credentials: see {@link gcloudEnv} and
+ * {@link gcloudProjectArgs}.
+ */
+
+import { z } from "zod";
+import { optionalEnv } from "./env-json-api.ts";
+import {
+  awsCliArgProblem,
+  awsCliDocumentProblem,
+  azCliArgProblem,
+  cliArgProblem,
+} from "./safe-cli-arg.ts";
+
+/** Body-snippet length in the thrown error. The value every connector used. */
+export const DEFAULT_STDERR_SNIPPET = 400;
+
+/**
+ * A Zod string that refuses every value `problem` names a reason against, with that reason as the
+ * message — so a refused call says which rule it broke rather than listing all of them.
+ */
+function cliArgSchema(problem: (value: unknown) => string | undefined): z.ZodString {
+  return z
+    .string()
+    .min(1)
+    .refine((value) => problem(value) === undefined, {
+      error: (issue) => problem(issue.input) ?? "is not a safe CLI argument",
+    });
+}
+
+/**
+ * A value passed to a CLI as an argument.
+ *
+ * Rejected at the SCHEMA boundary, before the value can reach argv: a leading
+ * `-` would be read by the CLI as a flag rather than as the name it is meant to
+ * be, and control characters have no legitimate place in one.
+ */
+export const cliArg = cliArgSchema(cliArgProblem);
+
+/**
+ * A value passed to the `aws` CLI as an argument: {@link cliArg}, and not a value the CLI would
+ * replace by reading a file or fetching a URL (`file://`, `fileb://`, `http://`, `https://`), nor
+ * one carrying the shorthand file-load operator `@=`.
+ */
+export const awsCliArg = cliArgSchema(awsCliArgProblem);
+
+/**
+ * A DOCUMENT passed to the `aws` CLI as one argument — a CloudFormation template body. Unlike
+ * {@link awsCliArg} it may span lines and run past 1024 characters; it still may not start with `-`
+ * or with a prefix that makes the CLI read it from somewhere else.
+ */
+export const awsCliDocument = cliArgSchema(awsCliDocumentProblem);
+
+/**
+ * A value passed to the `az` CLI as an argument: {@link cliArg}, and not one `az` would replace by
+ * a file's contents — a leading `@`, or `=@` anywhere.
+ */
+export const azCliArg = cliArgSchema(azCliArgProblem);
+
+/** The spawn seam. Production passes `nimbusSpawn`; tests pass a fake. */
+export type SpawnFn = (
+  command: readonly string[],
+  env: Record<string, string | undefined>,
+) => Promise<{ code: number; stdout: string; stderr: string }>;
+
+export interface CliJsonRunnerConfig {
+  /**
+   * Builds the full argv for one call. Given the connector's own arguments, it
+   * returns the complete command — binary, subcommand, arguments and whatever
+   * output flag that CLI needs (`--output json` for aws, `--format json` for
+   * gcloud).
+   */
+  readonly argv: (args: readonly string[]) => readonly string[];
+  /** Prefix for the thrown error, e.g. `"aws athena"`. */
+  readonly label: string;
+  /**
+   * What empty stdout means. The `aws` connectors treat it as `{}` and the
+   * `gcloud` ones as `[]`, because that is the shape each CLI omits when there
+   * is nothing to report.
+   */
+  readonly emptyResult: unknown;
+  /** Extra environment for the spawn. Defaults to none — the CLI reads its own. */
+  readonly env?: () => Record<string, string | undefined>;
+  readonly snippetMax?: number;
+}
+
+/**
+ * A `(args) => Promise<unknown>` runner for one CLI.
+ *
+ * Throws `"<label> <first arg> failed: <stderr snippet>"` on a non-zero exit,
+ * so a caller sees which subcommand failed and why.
+ */
+export function createCliJsonRunner(
+  config: CliJsonRunnerConfig,
+  spawn: SpawnFn,
+): (args: readonly string[]) => Promise<unknown> {
+  const snippetMax = config.snippetMax ?? DEFAULT_STDERR_SNIPPET;
+  return async (args: readonly string[]): Promise<unknown> => {
+    const { code, stdout, stderr } = await spawn(config.argv(args), config.env?.() ?? {});
+    const out = stdout.trim();
+    if (code !== 0) {
+      throw new Error(
+        `${config.label} ${args[0] ?? ""} failed: ${stderr.trim().slice(0, snippetMax)}`,
+      );
+    }
+    return out === "" ? config.emptyResult : (JSON.parse(out) as unknown);
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Untyped-payload readers
+//
+// External CLI output is `unknown` at the boundary (Non-Negotiable #5), and
+// these three are how every one of these connectors narrows it.
+// ---------------------------------------------------------------------------
+
+/** A non-null, non-array object. */
+export function isRecord(v: unknown): v is Record<string, unknown> {
+  return v !== null && typeof v === "object" && !Array.isArray(v);
+}
+
+/** `r[key]` when it is a string, else `""` — never `undefined`, never a throw. */
+export function strField(r: Record<string, unknown>, key: string): string {
+  const v = r[key];
+  return typeof v === "string" ? v : "";
+}
+
+/** `parsed[key]` when `parsed` is a record and the value is an array, else `[]`. */
+export function asArray(parsed: unknown, key: string): unknown[] {
+  if (!isRecord(parsed)) {
+    return [];
+  }
+  const arr = parsed[key];
+  return Array.isArray(arr) ? arr : [];
+}
+
+/** Case-insensitive substring match on one string field of an untyped entry. */
+export function fieldMatches(entry: unknown, key: string, query: string): boolean {
+  return isRecord(entry) && strField(entry, key).toLowerCase().includes(query.toLowerCase());
+}
+
+// ---------------------------------------------------------------------------
+// gcloud
+// ---------------------------------------------------------------------------
+
+/**
+ * `["--project", <id>]` when `GOOGLE_CLOUD_PROJECT` is set at spawn, else `[]`, which leaves
+ * gcloud on the project it is configured with.
+ */
+export function gcloudProjectArgs(): string[] {
+  const project = optionalEnv("GOOGLE_CLOUD_PROJECT", "");
+  return project === "" ? [] : ["--project", project];
+}
+
+/**
+ * The environment for a `gcloud` spawn: the process environment, with
+ * `GOOGLE_APPLICATION_CREDENTIALS` — the key file gcloud reads natively — re-set to its trimmed
+ * value when it is set.
+ */
+export function gcloudEnv(): Record<string, string | undefined> {
+  const env = { ...process.env } as Record<string, string | undefined>;
+  const credentials = process.env["GOOGLE_APPLICATION_CREDENTIALS"]?.trim();
+  if (credentials !== undefined && credentials !== "") {
+    env["GOOGLE_APPLICATION_CREDENTIALS"] = credentials;
+  }
+  return env;
+}

package/shared/collection-tool-kit.ts

@@ -0,0 +1,143 @@
+/**
+ * collection-tool-kit — the `<prefix>_list` / `<prefix>_get` / `<prefix>_search`
+ * triple that most read-only REST connectors expose over a single collection.
+ *
+ * Sonar reports the tail of this shape as duplicated across readwise, mercury,
+ * zoom, lever and intercom, but the flagged blocks understate it: the triple is
+ * written out by hand in roughly thirty connectors and is the same three tool
+ * bodies every time —
+ *
+ *   list   → `jsonResult(await get(PATH))`
+ *   get    → `jsonResult(await get(ITEM_PATH(encodeURIComponent(id))))`
+ *   search → `matchesResult(rows(await get(PATH)), filter, args)`
+ *
+ * — with only the paths, the envelope key and the descriptions differing. Those
+ * are the inputs here.
+ *
+ * What is NOT hidden: the descriptions. Each connector still writes its own,
+ * because they document that API's envelope shape and pagination and are the
+ * only thing the model sees when choosing a tool. A generated description would
+ * be a real regression in tool quality to save a line.
+ *
+ * Connectors whose tools do anything else — extra query parameters, a second
+ * request, a non-standard error, a POST — keep registering those directly on
+ * `reg`. This kit is for the plain triple, not a framework to force the rest
+ * into.
+ */
+
+import { z } from "zod";
+import { matchesResult, type SearchFilter, searchToolInputSchema } from "./mcp-search-tool.ts";
+import { mcpJsonResult } from "./mcp-tool-kit.ts";
+import type { ZodToolRegistrar } from "./run-read-only-mcp-connector.ts";
+
+/** The per-connector search cap every connector in this family uses. */
+export const DEFAULT_SEARCH_MAX = 100;
+
+/** A `(path) => Promise<unknown>` client, e.g. from `createJsonGetter`. */
+export type JsonGetter = (path: string) => Promise<unknown>;
+
+/**
+ * Read the row array out of a list response.
+ *
+ * Most APIs wrap the rows in a named key (`{ results }`, `{ data }`,
+ * `{ accounts }`, `{ conversations }`); a few return a bare array, which is
+ * {@link identityRows}.
+ */
+export type RowsAccessor = (root: unknown) => unknown;
+
+/** Rows under a single envelope key, e.g. `envelopeRows("results")`. */
+export function envelopeRows(key: string): RowsAccessor {
+  return (root: unknown): unknown => (root as Record<string, unknown> | null)?.[key];
+}
+
+/** The response IS the row array. */
+export const identityRows: RowsAccessor = (root: unknown): unknown => root;
+
+export interface CollectionListSpec {
+  /** Path appended to the getter's base. */
+  readonly path: string;
+  readonly description: string;
+  /** Tool name suffix. Defaults to `"list"`, giving `<prefix>_list`. */
+  readonly tool?: string;
+}
+
+export interface CollectionItemSpec {
+  /** Builds the path for one id. The id arrives already percent-encoded. */
+  readonly path: (encodedId: string) => string;
+  readonly description: string;
+  /** Tool name suffix. Defaults to `"get"`. */
+  readonly tool?: string;
+}
+
+export interface CollectionSearchSpec {
+  /** Path to search over. Defaults to the list path. */
+  readonly path?: string;
+  readonly description: string;
+  readonly rows: RowsAccessor;
+  readonly filter: SearchFilter;
+  /** Per-connector `limit` cap. Defaults to {@link DEFAULT_SEARCH_MAX}. */
+  readonly maxLimit?: number;
+  /** Tool name suffix. Defaults to `"search"`. */
+  readonly tool?: string;
+}
+
+export interface CollectionToolsConfig {
+  /** Tool-name prefix, e.g. `"mercury"` for `mercury_list`. */
+  readonly prefix: string;
+  readonly get: JsonGetter;
+  readonly list: CollectionListSpec;
+  readonly item?: CollectionItemSpec;
+  readonly search?: CollectionSearchSpec;
+}
+
+/** The names {@link registerCollectionTools} would register for `config`. */
+export function collectionToolNames(config: CollectionToolsConfig): string[] {
+  const names = [`${config.prefix}_${config.list.tool ?? "list"}`];
+  if (config.item !== undefined) {
+    names.push(`${config.prefix}_${config.item.tool ?? "get"}`);
+  }
+  if (config.search !== undefined) {
+    names.push(`${config.prefix}_${config.search.tool ?? "search"}`);
+  }
+  return names;
+}
+
+/**
+ * Register the list/get/search triple described by `config`.
+ *
+ * `item` and `search` are optional: a few collections are list-only, and a few
+ * have no id to fetch by.
+ */
+export function registerCollectionTools(
+  reg: ZodToolRegistrar,
+  config: CollectionToolsConfig,
+): void {
+  const { prefix, get, list, item, search } = config;
+
+  reg(`${prefix}_${list.tool ?? "list"}`, list.description, z.object({}), async () => {
+    return mcpJsonResult(await get(list.path));
+  });
+
+  if (item !== undefined) {
+    reg(
+      `${prefix}_${item.tool ?? "get"}`,
+      item.description,
+      z.object({ id: z.string().min(1) }),
+      async (p) => {
+        return mcpJsonResult(await get(item.path(encodeURIComponent(p.id))));
+      },
+    );
+  }
+
+  if (search !== undefined) {
+    reg(
+      `${prefix}_${search.tool ?? "search"}`,
+      search.description,
+      searchToolInputSchema(search.maxLimit ?? DEFAULT_SEARCH_MAX),
+      async (p) => {
+        const root = await get(search.path ?? list.path);
+        return matchesResult(search.rows(root), search.filter, p);
+      },
+    );
+  }
+}

package/shared/connector-mode.ts

@@ -45,8 +45,9 @@ export function getConnectorMode(): ConnectorMode {
 
 /**
  * TEST-ONLY: clear the lock between cases. Never called from production code — the
- * `audit:connector-consent` gate permits `setConnectorMode` only in the two sanctioned entrypoints,
- * and this function exists so tests never need to reach for it.
+ * `audit:connector-consent` gate lets no production file here but this one name `setConnectorMode`
+ * (its one production caller is the gateway's `run-bundled-connector.ts`), and this function
+ * exists so tests never need to reach for it.
  *
  * This matters more than it looks. `bun test` runs MANY TEST FILES IN ONE PROCESS — verified: two
  * files sharing a module observed the same pid, and state set by the first was visible to the

package/shared/consent-kit.ts

@@ -2,11 +2,9 @@ import type { McpListResult, ZodObjectSchema } from "@nimbus-dev/sdk/connector-k
 
 import { type AuditOutcome, appendAuditEntry } from "./audit-chain.ts";
 import { getConnectorMode } from "./connector-mode.ts";
+import { createWriteBudget, parseWriteBudget, WRITE_BUDGET_ENV } from "./write-budget.ts";
 import { parseWriteScope, scopeAllows } from "./write-scope.ts";
 
-/** Mutations allowed per process lifetime when NIMBUS_MCP_WRITE_BUDGET is unset. */
-const DEFAULT_BUDGET = 10;
-
 /** The subset of the SDK's `RegisteredTool` this kit needs. */
 export type ToolHandle = { disable(): void };
 
@@ -84,6 +82,11 @@ function refused(why: string): McpListResult {
   return { content: [{ type: "text", text: JSON.stringify({ ok: false, error: why }) }] };
 }
 
+/** The message of whatever was thrown, which need not be an `Error`. */
+function messageOf(e: unknown): string {
+  return e instanceof Error ? e.message : String(e);
+}
+
 /**
  * Ask the human, through the client, and report whether they said yes.
  *
@@ -172,19 +175,25 @@ export function createWriteToolRegistrar(
 
   const scope = parseWriteScope(process.env[cfg.scopeEnv], cfg.scopeKinds);
   const auditLog = process.env["NIMBUS_MCP_AUDIT_LOG"];
-  let remaining = Number(process.env["NIMBUS_MCP_WRITE_BUDGET"] ?? DEFAULT_BUDGET);
+  // Throws on a value that is not plain digits, an empty one included, as the scope parse above
+  // does on a malformed term: read with `Number()`, `abc` became NaN, which `remaining <= 0` never
+  // refused.
+  const budget = createWriteBudget(parseWriteBudget(process.env[WRITE_BUDGET_ENV]));
 
   async function record(
     tool: string,
     outcome: AuditOutcome,
     detail: Record<string, unknown>,
   ): Promise<void> {
-    // Client-visible channel: any MCP client can display or persist this.
+    // Meant as a client-visible channel, but it reaches no client today: the SDK sends
+    // `notifications/message` only for a server that declared the MCP `logging` capability, and
+    // no connector declares it. The durable log below is the only record.
     await server.sendLoggingMessage({
       level: outcome === "executed" ? "info" : "warning",
       data: { connector: cfg.connector, tool, outcome },
     });
-    // Durable channel: only when the operator configured a path.
+    // Durable channel: only when the operator configured a path. An append that fails rejects, and
+    // every record before the mutation is awaited, so a write whose entry did not land never runs.
     if (auditLog !== undefined && auditLog !== "") {
       await appendAuditEntry(auditLog, {
         ts: new Date().toISOString(),
@@ -196,6 +205,38 @@ export function createWriteToolRegistrar(
     }
   }
 
+  /**
+   * Take one unit of budget if any is left. Spending the last unit disables every write tool and
+   * tells the client.
+   *
+   * Nothing may be awaited between the check and the decrement, which is why both are the one
+   * synchronous call `budget.take()`. Calls run concurrently (the SDK starts each request's
+   * handler as it arrives), and with an await between the two every call in flight passed the
+   * check before any of them spent: three approved calls all mutated under a budget of one.
+   */
+  function spendOne(): boolean {
+    if (!budget.take()) return false;
+    if (!budget.hasLeft()) {
+      for (const h of handles) h.disable();
+      server.sendToolListChanged();
+    }
+    return true;
+  }
+
+  /**
+   * The refusal for a spent budget, worded the same for the client at both places it is checked.
+   * The audit reason tells them apart: `budget exhausted after approval` follows an `accepted`
+   * entry, because the budget ran out while that write's prompt was open.
+   */
+  async function budgetExhausted(
+    tool: string,
+    target: { kind: string; value: string },
+    reason: "budget exhausted" | "budget exhausted after approval",
+  ): Promise<McpListResult> {
+    await record(tool, "refused", { reason, target });
+    return refused("write budget exhausted for this session");
+  }
+
   /**
    * Decide the write surface, once, at the first moment client capabilities are knowable.
    *
@@ -269,11 +310,9 @@ export function createWriteToolRegistrar(
         return refused(`out of scope: ${target.kind}:${target.value} is not in ${cfg.scopeEnv}`);
       }
 
-      // 2. BUDGET — same reasoning: a refusal that consent cannot lift comes before consent.
-      if (remaining <= 0) {
-        await record(name, "refused", { reason: "budget exhausted", target });
-        return refused("write budget exhausted for this session");
-      }
+      // 2. BUDGET — same reasoning: a refusal that consent cannot lift comes before consent. Not
+      //    the last word on the budget, though: see step 4.
+      if (!budget.hasLeft()) return budgetExhausted(name, target, "budget exhausted");
 
       // 3. CONSENT.
       await record(name, "requested", { target, params: args });
@@ -283,7 +322,13 @@ export function createWriteToolRegistrar(
       }
       await record(name, "accepted", { target });
 
-      // 4. PRE-STATE — after approval, before the mutation, so an unrecoverable action leaves a
+      // 4. SPEND — the budget checked AGAIN, and decremented in the same synchronous step: every
+      //    call in flight passed step 2 before any of them got this far, so step 2 alone cannot
+      //    hold the cap (see `spendOne`). Spent BEFORE the mutation so a throwing one still
+      //    consumes budget — otherwise a failing destructive tool could be retried without limit.
+      if (!spendOne()) return budgetExhausted(name, target, "budget exhausted after approval");
+
+      // 5. PRE-STATE — after approval, before the mutation, so an unrecoverable action leaves a
       //    record of what it destroyed. Capture failure is NOT fatal: refusing here would turn a
       //    transient read error into a blocked action the owner already approved.
       let preState: Record<string, unknown> = {};
@@ -291,29 +336,39 @@ export function createWriteToolRegistrar(
         try {
           preState = await toolCfg.capturePreState(args);
         } catch (e) {
-          preState = { captureFailed: e instanceof Error ? e.message : String(e) };
+          preState = { captureFailed: messageOf(e) };
         }
       }
 
-      // 5. MUTATE. Decrement BEFORE the call so a throwing mutation still consumes budget —
-      //    otherwise a failing destructive tool could be retried without limit.
-      remaining -= 1;
-      if (remaining <= 0) {
-        for (const h of handles) h.disable();
-        server.sendToolListChanged();
+      // 6. MUTATE.
+      let result: McpListResult;
+      try {
+        result = await handler(args);
+      } catch (e) {
+        // A tool that threw may still have changed something first. If recording the failure fails
+        // too, the tool's own error still leads what the call reports: an audit error alone reads
+        // as a write that never started, and so as one that is safe to try again.
+        try {
+          await record(name, "failed", { target, preState, error: messageOf(e) });
+        } catch (error_) {
+          throw new Error(
+            `${name} failed: ${messageOf(e)}. Recording that failure also failed: ${messageOf(error_)}`,
+            { cause: e },
+          );
+        }
+        throw e;
       }
+      // Outside the `try`: the mutation has happened, so a failure to RECORD it must not be
+      // recorded as the mutation failing, which it was when both shared one `catch`. The call still
+      // rejects, saying the write ran, so it is not taken for one that is safe to try again.
       try {
-        const result = await handler(args);
         await record(name, "executed", { target, preState });
-        return result;
       } catch (e) {
-        await record(name, "failed", {
-          target,
-          preState,
-          error: e instanceof Error ? e.message : String(e),
+        throw new Error(`${name} ran, but recording that it ran failed: ${messageOf(e)}`, {
+          cause: e,
         });
-        throw e;
       }
+      return result;
     };
 
     pending.push(() => {

package/shared/cursor-list-tool.ts

@@ -0,0 +1,29 @@
+/**
+ * cursor-list-tool — the input contract of a paginated `<prefix>_list` tool.
+ *
+ * Six connectors (bigeye, looker, monte-carlo, powerbi, snowflake, tableau) page through a
+ * collection the same way: the caller passes back an opaque `cursor` — the previous page's
+ * `nextCursor`, or nothing / `null` for the first page — plus a `limit`, and receives
+ * `{ items, nextCursor }`. Each declared that input schema separately. What the cursor encodes (an
+ * offset, a page number, a relay token) and the default page size stay each connector's own.
+ */
+
+import { z } from "zod";
+import type { ZodObjectSchema } from "./mcp-tool-kit.ts";
+
+/** What a paginated `_list` tool is called with. */
+export interface CursorListOptions {
+  readonly cursor?: string | null | undefined;
+  readonly limit?: number | undefined;
+}
+
+/**
+ * The paginated `_list` input schema: an optional, nullable opaque `cursor` and an optional
+ * integer `limit` from 1 to `maxLimit`.
+ */
+export function cursorListInputSchema(maxLimit = 500): ZodObjectSchema<CursorListOptions> {
+  return z.object({
+    cursor: z.string().nullable().optional(),
+    limit: z.number().int().min(1).max(maxLimit).optional(),
+  });
+}