@cohortapp/agent-sdk 2.3.0

package/lib/voice/tool-bridge.mjs

@@ -0,0 +1,370 @@
+/**
+ * Maestro — Voice tool bridge
+ *
+ * Converts maestro's existing action handlers (defined in
+ * lib/tool-definitions.js + executed via lib/action-executor.js) into
+ * the function-tool format OpenAI Realtime expects, and wires the
+ * provider's tool.call events back through the same action executor and
+ * action-classification policy used by the Slack / Gmail surfaces.
+ *
+ * Why this matters:
+ *   - The voice channel must NOT bypass the approval policy that protects
+ *     the Slack / Gmail surfaces. A user saying "wire fifty thousand to
+ *     this account" should require the same verbal-confirmation gate that
+ *     a Slack-typed equivalent would, not silently fire the transfer.
+ *   - We surface that gate by tagging high-risk actions in the Realtime
+ *     tool description and (optionally) by inserting a pre-execution
+ *     verbal-confirm step that asks the model to read back the action
+ *     before invoking it.
+ *
+ * Public API:
+ *   actionsToRealtimeTools(opts) → array of Realtime function tool defs
+ *   registerToolBridge(provider, opts) → attach handler to provider events
+ */
+
+import { readFileSync, existsSync } from "node:fs";
+import { createRequire } from "node:module";
+import { redactSensitiveUrl, isSensitiveParamName, REDACTED } from "../security/redact.mjs";
+
+// js-yaml is a declared maestro dependency, but action policies are usually
+// authored as JSON, so we keep YAML loading optional. createRequire gives us a
+// working `require` inside this ESM module — a bare `require("js-yaml")` here is
+// a ReferenceError under "type":"module", which previously got swallowed and
+// silently disabled the verbal-confirm gate for YAML policies.
+const require = createRequire(import.meta.url);
+import { join, resolve } from "node:path";
+
+// We dynamically import the action-executor only on first invocation
+// to avoid circular import order issues during module load.
+let _toolDefs = null;
+let _executor = null;
+
+async function loadMaestroBindings() {
+  if (_toolDefs && _executor) return { _toolDefs, _executor };
+  try {
+    const td = await import("../tool-definitions.js");
+    const ae = await import("../action-executor.js");
+    _toolDefs = td;
+    _executor = ae;
+    return { _toolDefs, _executor };
+  } catch (err) {
+    throw new Error(`tool-bridge: failed to load maestro bindings: ${err.message}`);
+  }
+}
+
+/**
+ * Load the action-classification policy (YAML or JSON) so we know which
+ * actions are low/medium/high risk and how each should be gated.
+ *
+ * Returns a normalised map: { actionName: { risk, approval, ... } }
+ */
+export function loadActionClassification(agentRoot, opts = {}) {
+  const root = resolve(
+    agentRoot || opts.agentRoot || process.env.AGENT_ROOT || process.env.AGENT_DIR || process.cwd()
+  );
+  // Try JSON first (no js-yaml dependency), then YAML.
+  const candidates = [
+    join(root, "policies/action-classification.json"),
+    join(root, "policies/action-classification.yaml"),
+  ];
+  for (const path of candidates) {
+    if (!existsSync(path)) continue;
+    const raw = readFileSync(path, "utf-8");
+    if (path.endsWith(".json")) return normaliseClassification(JSON.parse(raw));
+    // YAML policy file. createRequire makes this resolve correctly when
+    // js-yaml is installed; if it genuinely is not, we fall back to {} rather
+    // than crashing — but we log it so a misconfigured policy is visible
+    // instead of silently disabling the gate.
+    try {
+      const yaml = require("js-yaml");
+      return normaliseClassification(yaml.load(raw));
+    } catch (err) {
+      console.error(
+        `[tool-bridge] Could not load YAML action policy at ${path} ` +
+        `(${err.message}); action-classification gate is INACTIVE for this session.`
+      );
+      return {};
+    }
+  }
+  return {};
+}
+
+function normaliseClassification(raw) {
+  if (!raw || typeof raw !== "object") return {};
+  const out = {};
+  // Two shapes we accept:
+  //   { actions: { send_slack_message: { risk: "high", ... } } }
+  //   { send_slack_message: { risk: "high", ... } }
+  const actions = raw.actions || raw;
+  for (const [name, def] of Object.entries(actions)) {
+    if (!def || typeof def !== "object") continue;
+    out[name] = {
+      risk: def.risk || def.level || "low",
+      approval: def.approval || def.gate || "none",
+      verbal_confirm: def.verbal_confirm ?? (def.risk === "high"),
+      notes: def.notes || "",
+    };
+  }
+  return out;
+}
+
+/**
+ * Build the array of function tools for the Realtime session. By default
+ * exposes every "communication"-tier maestro tool (send-slack, send-email,
+ * send-whatsapp, etc.) plus the "low-risk" research tools.
+ *
+ * @param {object} opts
+ * @param {string[]} [opts.include]  Specific tool names to include
+ * @param {string[]} [opts.exclude]  Names to exclude
+ * @param {string}   [opts.accessLevel] "voice" (default), "full"
+ * @param {object}   [opts.classification] action-classification map
+ * @returns {Promise<Array<RealtimeTool>>}
+ */
+export async function actionsToRealtimeTools(opts = {}) {
+  const { _toolDefs } = await loadMaestroBindings();
+  const classification = opts.classification || loadActionClassification(opts.agentRoot);
+  const accessLevel = opts.accessLevel || "voice";
+  let sourceTools;
+  if (typeof _toolDefs.getToolsForAccessLevel === "function") {
+    // The framework already exposes this — use it so the voice surface
+    // automatically picks up new tools added to the registry.
+    sourceTools = _toolDefs.getToolsForAccessLevel(accessLevel);
+  } else if (Array.isArray(_toolDefs.ALL_TOOLS)) {
+    sourceTools = _toolDefs.ALL_TOOLS;
+  } else {
+    sourceTools = [];
+  }
+
+  const include = new Set(opts.include || []);
+  const exclude = new Set(opts.exclude || []);
+
+  const out = [];
+  for (const t of sourceTools) {
+    const name = t.name || t.toolName;
+    if (!name) continue;
+    if (include.size && !include.has(name)) continue;
+    if (exclude.has(name)) continue;
+    const risk = classification[name]?.risk || "low";
+    const verbalConfirm = classification[name]?.verbal_confirm ?? (risk === "high");
+    const baseDesc = t.description || `Maestro action: ${name}`;
+    const description = verbalConfirm
+      ? `${baseDesc}\n\n⚠ HIGH-RISK action — before calling this tool you MUST verbally read back the parameters (recipient, amount, key fields) and obtain explicit user confirmation in the SAME turn. Do not call this tool on the first turn it is mentioned.`
+      : baseDesc;
+
+    out.push({
+      type: "function",
+      name,
+      description,
+      parameters: toolParameters(t),
+    });
+  }
+  return out;
+}
+
+function toolParameters(t) {
+  // We accept either: t.parameters (already JSON Schema) OR t.input_schema
+  // (Anthropic-style). Both pass through unchanged; the OpenAI Realtime
+  // function format wants JSON Schema in `parameters`.
+  if (t.parameters && typeof t.parameters === "object") return t.parameters;
+  if (t.input_schema && typeof t.input_schema === "object") return t.input_schema;
+  if (t.inputSchema && typeof t.inputSchema === "object") return t.inputSchema;
+  return { type: "object", properties: {}, additionalProperties: true };
+}
+
+// ── Audit redaction (L11) ─────────────────────────────────────────────────
+// The audit hook receives raw tool arguments and results, which on the voice
+// surface routinely contain PII and message bodies (recipients, amounts, email
+// text, etc.). Callers that ship audit entries to a shared sink can opt in to
+// redaction so secrets/PII don't leak into logs. Default behaviour is unchanged
+// (no redaction) — you must pass opts.redact, opts.auditAllowlist, or
+// opts.redactAudit to enable any of this.
+
+const DEFAULT_TRUNCATE_LEN = 256;
+
+/**
+ * Deep-redact a JSON-ish value for audit logging:
+ *   - values of object keys whose NAME looks sensitive (token/secret/password/
+ *     auth/… per lib/security/redact.mjs) are replaced with REDACTED ("***")
+ *   - string values are passed through redactSensitiveUrl (strips embedded
+ *     credentials + masks sensitive query params) and truncated to maxLen
+ *   - arrays/objects are walked recursively (bounded depth, cycle-safe)
+ *
+ * Pure; never throws.
+ *
+ * @param {*} value
+ * @param {{ maxLen?: number, allowlist?: Set<string>|null, depth?: number,
+ *           seen?: WeakSet<object> }} [opts]
+ * @returns {*}
+ */
+export function redactAuditValue(value, opts = {}) {
+  const maxLen = opts.maxLen ?? DEFAULT_TRUNCATE_LEN;
+  const allowlist = opts.allowlist || null;
+  const depth = opts.depth ?? 0;
+  const seen = opts.seen || new WeakSet();
+
+  if (value == null) return value;
+  if (typeof value === "string") {
+    let out = redactSensitiveUrl(value);
+    if (maxLen > 0 && out.length > maxLen) out = out.slice(0, maxLen) + "…";
+    return out;
+  }
+  if (typeof value === "number" || typeof value === "boolean") return value;
+  if (typeof value !== "object") return REDACTED; // function/symbol/bigint → mask
+  if (depth >= 6) return REDACTED; // bound recursion
+  if (seen.has(value)) return REDACTED; // cycle guard
+  seen.add(value);
+
+  if (Array.isArray(value)) {
+    return value.map((v) =>
+      redactAuditValue(v, { maxLen, allowlist, depth: depth + 1, seen })
+    );
+  }
+
+  const out = {};
+  for (const [k, v] of Object.entries(value)) {
+    // Field allowlist: if provided, only top-level keys (depth 0) on the list
+    // survive; the rest are dropped entirely.
+    if (allowlist && depth === 0 && !allowlist.has(k)) continue;
+    if (isSensitiveParamName(k)) {
+      out[k] = REDACTED;
+      continue;
+    }
+    out[k] = redactAuditValue(v, { maxLen, allowlist, depth: depth + 1, seen });
+  }
+  return out;
+}
+
+/**
+ * Build the redactor applied to each audit entry's args/result, from
+ * registerToolBridge opts. Returns null when no redaction was requested (so the
+ * default no-op path stays allocation-free).
+ *
+ * @param {object} opts
+ * @returns {((entry:object)=>object)|null}
+ */
+function buildAuditRedactor(opts) {
+  // Fully-custom redactor wins.
+  if (typeof opts.redact === "function") return opts.redact;
+
+  const wantAllowlist = Array.isArray(opts.auditAllowlist);
+  const wantRedact = opts.redactAudit === true || wantAllowlist;
+  if (!wantRedact) return null;
+
+  const allowlist = wantAllowlist ? new Set(opts.auditAllowlist) : null;
+  const maxLen = typeof opts.auditMaxLen === "number" ? opts.auditMaxLen : DEFAULT_TRUNCATE_LEN;
+  return (entry) => {
+    const next = { ...entry };
+    if ("args" in next) next.args = redactAuditValue(next.args, { allowlist, maxLen });
+    if ("result" in next) next.result = redactAuditValue(next.result, { allowlist, maxLen });
+    return next;
+  };
+}
+
+/**
+ * Attach a tool-call handler to a voice provider. When the model emits a
+ * tool.call event, we look up the action in maestro's executor, run it,
+ * and feed the result back into the conversation via
+ * provider.sendToolResult().
+ *
+ * Returns a teardown fn that removes the listeners.
+ *
+ * @param {VoiceProvider} provider
+ * @param {object} opts
+ * @param {(name:string,args:object)=>Promise<object>} [opts.executor]
+ *        Override the default executor (useful for tests).
+ * @param {Array<string>} [opts.allow]   Allow-list of tool names; others
+ *        are refused without invocation.
+ * @param {object} [opts.classification] action-classification map
+ * @param {(entry:object)=>void} [opts.audit]  Hook called for each tool
+ *        call with {name, args, result, error, started_at, finished_at}.
+ * @param {(entry:object)=>object} [opts.redact]  Optional redactor applied to
+ *        the audit entry BEFORE it reaches opts.audit. Receives the full entry,
+ *        returns the (redacted) entry to log. Use this for full control.
+ * @param {boolean} [opts.redactAudit]  Convenience flag: when true, apply the
+ *        built-in deep redactor to entry.args/entry.result — masks sensitive
+ *        field values, strips URL credentials, and truncates long strings.
+ * @param {string[]} [opts.auditAllowlist]  Convenience: keep only these
+ *        top-level arg/result fields in the audit entry (implies redactAudit).
+ * @param {number} [opts.auditMaxLen]  Max length for redacted strings
+ *        (default 256). Only used by the built-in redactor.
+ *
+ * Default behaviour is unchanged: with none of the redact options set, raw
+ * args/results reach the audit sink exactly as before.
+ */
+export function registerToolBridge(provider, opts = {}) {
+  const allow = opts.allow ? new Set(opts.allow) : null;
+  const classification = opts.classification || {};
+  const rawAudit = typeof opts.audit === "function" ? opts.audit : () => {};
+  const redactor = buildAuditRedactor(opts);
+  // Single choke point so every audit site goes through the redactor (if any).
+  // Redaction must never break auditing, so failures fall back to the raw entry.
+  const audit = redactor
+    ? (entry) => {
+        let safe = entry;
+        try {
+          safe = redactor(entry);
+        } catch {
+          safe = entry;
+        }
+        rawAudit(safe);
+      }
+    : rawAudit;
+
+  const handler = async (evt) => {
+    const { call_id, name, arguments: args } = evt;
+    const startedAt = new Date().toISOString();
+
+    if (!call_id || !name) {
+      audit({ name, args, error: "missing_call_id_or_name", started_at: startedAt, finished_at: new Date().toISOString() });
+      return;
+    }
+    if (allow && !allow.has(name)) {
+      provider.sendToolResult(call_id, {
+        ok: false,
+        error: "tool_not_allowed",
+        message: `Tool "${name}" is not in the voice-session allow-list. The agent must invoke a different tool or escalate to a human.`,
+      });
+      audit({ name, args, error: "tool_not_allowed", started_at: startedAt, finished_at: new Date().toISOString() });
+      return;
+    }
+
+    let result;
+    let error = null;
+    try {
+      if (opts.executor) {
+        result = await opts.executor(name, args || {});
+      } else {
+        const { _executor } = await loadMaestroBindings();
+        const fn = _executor.executeAction || _executor.default;
+        if (typeof fn !== "function") {
+          throw new Error("maestro action-executor is missing executeAction()");
+        }
+        result = await fn(name, args || {});
+      }
+    } catch (err) {
+      error = err.message || String(err);
+      result = { ok: false, error };
+    }
+
+    provider.sendToolResult(call_id, result);
+    audit({
+      name,
+      args,
+      result,
+      error,
+      risk: classification[name]?.risk || "low",
+      started_at: startedAt,
+      finished_at: new Date().toISOString(),
+    });
+  };
+
+  provider.on("tool.call", handler);
+  return () => provider.off("tool.call", handler);
+}
+
+export default {
+  actionsToRealtimeTools,
+  registerToolBridge,
+  loadActionClassification,
+  redactAuditValue,
+};

package/lib/voice/tts.mjs

@@ -0,0 +1,104 @@
+/**
+ * Maestro — Text-to-speech helper for outbound voice messages.
+ *
+ * Wraps OpenAI's `/v1/audio/speech` endpoint. Used by the voice channel
+ * adapter's send() path when an agent wants to deliver a voice note
+ * over WhatsApp, Slack, or a one-shot Twilio <Play> phone call.
+ *
+ * Why a separate helper:
+ *   - One-shot TTS is much cheaper than spinning up a full Realtime
+ *     session ($0.015/M chars for gpt-4o-mini-tts vs. $32-$64/M tokens
+ *     for gpt-realtime-2).
+ *   - WhatsApp / Slack / SMS can't take live audio anyway — they want
+ *     a file. TTS is the right primitive for asynchronous delivery.
+ *   - The Realtime API is reserved for true live conversations.
+ *
+ * Public API:
+ *   synthesizeSpeech({ text, voice, format, model }) → Promise<Buffer>
+ *   ttsCostEstimate(text, model) → number (USD)
+ *
+ * Voices (May 2026): alloy, ash, ballad, coral, echo, sage, shimmer,
+ * verse + the GA-only Realtime voices cedar / marin (only available
+ * via gpt-realtime-2; tts-1 / gpt-4o-mini-tts use the original eight).
+ *
+ * Formats: mp3 (default; ~16 kbps for tts-1), opus (best for WhatsApp
+ * PTT), aac, flac, wav, pcm.
+ */
+
+const OPENAI_TTS_URL = "https://api.openai.com/v1/audio/speech";
+
+const DEFAULT_MODEL = "gpt-4o-mini-tts"; // best quality/cost for prose
+const DEFAULT_VOICE = "alloy";
+const DEFAULT_FORMAT = "mp3";
+
+/**
+ * @param {object} opts
+ * @param {string} opts.text     The text to synthesize (max ~4096 chars per call).
+ * @param {string} [opts.voice]  Voice name (default: alloy).
+ * @param {string} [opts.format] Audio format (default: mp3). Use "opus" for WhatsApp PTT.
+ * @param {string} [opts.model]  TTS model id (default: gpt-4o-mini-tts).
+ * @param {number} [opts.speed]  Playback speed multiplier 0.25–4.0 (default: 1.0).
+ * @param {string} [opts.apiKey] Override OPENAI_API_KEY.
+ * @returns {Promise<Buffer>}    Audio bytes.
+ */
+export async function synthesizeSpeech(opts = {}) {
+  const text = String(opts.text || "").trim();
+  if (!text) throw new Error("synthesizeSpeech: text required");
+  if (text.length > 4096) {
+    throw new Error(`synthesizeSpeech: text > 4096 chars (got ${text.length}); chunk it first`);
+  }
+  const apiKey = opts.apiKey || process.env.OPENAI_API_KEY;
+  if (!apiKey) throw new Error("OPENAI_API_KEY not set");
+
+  const body = {
+    model: opts.model || DEFAULT_MODEL,
+    input: text,
+    voice: opts.voice || DEFAULT_VOICE,
+    response_format: opts.format || DEFAULT_FORMAT,
+  };
+  if (typeof opts.speed === "number") body.speed = opts.speed;
+
+  const r = await fetch(OPENAI_TTS_URL, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${apiKey}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(body),
+  });
+  if (!r.ok) {
+    const errText = await r.text();
+    throw new Error(`tts failed: HTTP ${r.status} ${errText.slice(0, 300)}`);
+  }
+  const arr = await r.arrayBuffer();
+  return Buffer.from(arr);
+}
+
+/**
+ * Rough USD cost estimate for a TTS call.
+ *   gpt-4o-mini-tts: $0.015 / 1k characters
+ *   tts-1 / tts-1-hd: $0.015 / 1k characters (legacy)
+ */
+export function ttsCostEstimate(text, model = DEFAULT_MODEL) {
+  const chars = String(text || "").length;
+  return Number(((chars / 1000) * 0.015).toFixed(6));
+}
+
+/**
+ * Choose a sensible default voice for an agent based on its config.
+ * Falls back to "alloy" so any agent gets a working default without
+ * needing to declare one. Operators can pin a specific voice in
+ * config/voice.yaml under `voice:`.
+ */
+export function pickDefaultVoice(agentVoiceConfig = {}) {
+  const v = (agentVoiceConfig.voice || "").toLowerCase();
+  // tts-1 / gpt-4o-mini-tts voices (not the Realtime-only cedar/marin)
+  const valid = ["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"];
+  if (valid.includes(v)) return v;
+  // Realtime-only voices fall back gracefully.
+  if (v === "cedar") return "ash";
+  if (v === "marin") return "shimmer";
+  return "alloy";
+}
+
+export default { synthesizeSpeech, ttsCostEstimate, pickDefaultVoice };