@cohortapp/agent-sdk 2.3.0

package/lib/org/verify.test.mjs

@@ -0,0 +1,194 @@
+/**
+ * verify.test.mjs — detective layer: pinned-signer snapshot verification +
+ * event-chain re-derivation + quarantine (no-throw) semantics.
+ * Run: node --test lib/org/verify.test.mjs
+ */
+"use strict";
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+import { createPublicKey } from "node:crypto";
+
+import { verifySnapshot, verifyEvents, quarantine, sshEd25519ToSpkiPem } from "./verify.mjs";
+import { canonicalize, computeRowHash } from "./protocol.mjs";
+import { generateDeviceKey, signWith, verifySignature } from "./keys.mjs";
+
+/** Encode an Ed25519 SPKI PEM into the OpenSSH `ssh-ed25519 <base64>` blob. */
+function spkiToSshEd25519(spkiPem) {
+  const der = createPublicKey(spkiPem).export({ type: "spki", format: "der" });
+  const raw = der.subarray(12); // 12-byte Ed25519 SPKI prefix, then 32 raw bytes
+  const sshStr = (buf) => {
+    const len = Buffer.alloc(4); len.writeUInt32BE(buf.length);
+    return Buffer.concat([len, buf]);
+  };
+  const blob = Buffer.concat([sshStr(Buffer.from("ssh-ed25519")), sshStr(raw)]);
+  return blob.toString("base64");
+}
+
+// --- helpers ---------------------------------------------------------------
+
+/** Sign a snapshot body the SAME way verify expects (canonical body w/o sig). */
+function signSnapshot(privateKeyPem, body) {
+  const material = Buffer.from(canonicalize(body));
+  return { ...body, sig: signWith(privateKeyPem, material) };
+}
+
+/** Build a valid chain of N events from genesis. */
+function buildChain(n) {
+  const events = [];
+  let prev = "";
+  for (let i = 0; i < n; i++) {
+    const row = {
+      seq: i + 1,
+      family: "board",
+      kind: "create",
+      entity_id: `item-${i}`,
+      actor: "agent-a",
+      at: `2026-06-10T00:00:0${i}.000Z`,
+      payload: { i },
+    };
+    const row_hash = computeRowHash(prev, row);
+    events.push({ ...row, prev_hash: prev, row_hash });
+    prev = row_hash;
+  }
+  return events;
+}
+
+// --- verifySnapshot --------------------------------------------------------
+
+test("verifySnapshot: accepts a snapshot signed by a pinned signer", () => {
+  const server = generateDeviceKey();
+  const snap = signSnapshot(server.privateKeyPem, { orgId: "acme", agents: [{ id: "a" }], seq: 7 });
+  const v = verifySnapshot(snap, [server.publicKeyPem]);
+  assert.equal(v.ok, true);
+  assert.equal(v.signer, server.publicKeyPem);
+});
+
+test("verifySnapshot: rejects when signer NOT in pinned list (forgery)", () => {
+  const evil = generateDeviceKey();
+  const pinned = generateDeviceKey();
+  const snap = signSnapshot(evil.privateKeyPem, { orgId: "acme", agents: [] });
+  const v = verifySnapshot(snap, [pinned.publicKeyPem]);
+  assert.equal(v.ok, false);
+  assert.match(v.reason, /pinned signer/);
+});
+
+test("verifySnapshot: rejects a tampered body (sig no longer matches)", () => {
+  const server = generateDeviceKey();
+  const snap = signSnapshot(server.privateKeyPem, { orgId: "acme", agents: [{ id: "a" }] });
+  snap.agents.push({ id: "injected" }); // mutate after signing
+  const v = verifySnapshot(snap, [server.publicKeyPem]);
+  assert.equal(v.ok, false);
+});
+
+test("verifySnapshot: default-deny when no pinned signers", () => {
+  const server = generateDeviceKey();
+  const snap = signSnapshot(server.privateKeyPem, { orgId: "acme" });
+  assert.equal(verifySnapshot(snap, []).ok, false);
+  assert.equal(verifySnapshot(snap, null).ok, false);
+});
+
+test("verifySnapshot: rejects unsigned / malformed snapshot", () => {
+  const server = generateDeviceKey();
+  assert.equal(verifySnapshot({ orgId: "x" }, [server.publicKeyPem]).ok, false);
+  assert.equal(verifySnapshot(null, [server.publicKeyPem]).ok, false);
+});
+
+test("verifySnapshot: tries multiple pinned signers (key rotation window)", () => {
+  const oldKey = generateDeviceKey();
+  const newKey = generateDeviceKey();
+  const snap = signSnapshot(newKey.privateKeyPem, { orgId: "acme" });
+  const v = verifySnapshot(snap, [oldKey.publicKeyPem, newKey.publicKeyPem]);
+  assert.equal(v.ok, true);
+});
+
+// --- verifyEvents ----------------------------------------------------------
+
+test("verifyEvents: accepts a valid chain from genesis", () => {
+  const events = buildChain(5);
+  assert.deepEqual(verifyEvents(events, ""), { ok: true });
+});
+
+test("verifyEvents: catches a rewritten payload at the right index", () => {
+  const events = buildChain(4);
+  events[2].payload = { i: 999 }; // rewrite without re-hashing
+  const v = verifyEvents(events, "");
+  assert.equal(v.ok, false);
+  assert.equal(v.index, 2);
+  assert.match(v.reason, /row_hash/);
+});
+
+test("verifyEvents: catches a reordered/broken prev link", () => {
+  const events = buildChain(3);
+  events[1].prev_hash = "deadbeef";
+  const v = verifyEvents(events, "");
+  assert.equal(v.ok, false);
+  assert.equal(v.index, 1);
+  assert.match(v.reason, /prev_hash/);
+});
+
+test("verifyEvents: honours a non-genesis startPrev", () => {
+  const all = buildChain(5);
+  const slice = all.slice(2); // events 3..5
+  const startPrev = all[1].row_hash;
+  assert.deepEqual(verifyEvents(slice, startPrev), { ok: true });
+  // wrong startPrev → first link breaks
+  assert.equal(verifyEvents(slice, "wrong").ok, false);
+});
+
+test("verifyEvents: non-array input → quarantine verdict, no throw", () => {
+  const v = verifyEvents(null, "");
+  assert.equal(v.ok, false);
+  assert.match(v.reason, /array/);
+});
+
+// --- quarantine ------------------------------------------------------------
+
+test("quarantine: passes through a verdict", () => {
+  assert.deepEqual(quarantine(() => ({ ok: true })), { ok: true });
+  assert.deepEqual(quarantine(() => ({ ok: false, reason: "x" })), { ok: false, reason: "x" });
+});
+
+test("quarantine: converts a throw into {ok:false}", () => {
+  const v = quarantine(() => { throw new Error("hostile payload"); });
+  assert.equal(v.ok, false);
+  assert.match(v.reason, /hostile payload/);
+});
+
+test("quarantine: rejects a non-verdict return", () => {
+  assert.equal(quarantine(() => 42).ok, false);
+});
+
+// --- sshEd25519ToSpkiPem (the SPEC-format trust-root loader) ----------------
+
+test("sshEd25519ToSpkiPem: round-trips an SSH one-liner back to the exact SPKI PEM", () => {
+  const key = generateDeviceKey();
+  const sshBlob = spkiToSshEd25519(key.publicKeyPem);
+  // bare base64 blob
+  const fromBlob = sshEd25519ToSpkiPem(sshBlob);
+  assert.equal(createPublicKey(fromBlob).export({ type: "spki", format: "pem" }).toString(),
+    createPublicKey(key.publicKeyPem).export({ type: "spki", format: "pem" }).toString(),
+    "the reconstructed SPKI PEM matches the original key");
+  // a full allowed-signers / authorized line
+  const fromLine = sshEd25519ToSpkiPem(`alice@example.com namespaces="org" ssh-ed25519 ${sshBlob} comment`);
+  assert.ok(fromLine, "parses a full allowed-signers line");
+  // a signature made with the private key verifies against the reconstructed PEM.
+  const msg = Buffer.from("policy-bundle-bytes");
+  const sigB64 = signWith(key.privateKeyPem, msg);
+  assert.equal(verifySignature(fromBlob, msg, sigB64), true, "signature verifies against the converted PEM");
+  assert.equal(verifySignature(fromLine, msg, sigB64), true);
+});
+
+test("sshEd25519ToSpkiPem: returns null on malformed / non-ed25519 input", () => {
+  assert.equal(sshEd25519ToSpkiPem(""), null);
+  assert.equal(sshEd25519ToSpkiPem(null), null);
+  assert.equal(sshEd25519ToSpkiPem("not-base64-!!!"), null);
+  // a valid base64 but wrong key type (ssh-rsa) blob → null
+  const sshStr = (buf) => { const l = Buffer.alloc(4); l.writeUInt32BE(buf.length); return Buffer.concat([l, buf]); };
+  const rsaBlob = Buffer.concat([sshStr(Buffer.from("ssh-rsa")), sshStr(Buffer.alloc(32))]).toString("base64");
+  assert.equal(sshEd25519ToSpkiPem(rsaBlob), null);
+  // an ed25519 tag with the wrong raw-key length → null
+  const shortBlob = Buffer.concat([sshStr(Buffer.from("ssh-ed25519")), sshStr(Buffer.alloc(16))]).toString("base64");
+  assert.equal(sshEd25519ToSpkiPem(shortBlob), null);
+});

package/lib/rag/embed.mjs

@@ -0,0 +1,188 @@
+/**
+ * Maestro — Embeddings helper (OpenAI text-embedding-3-small by default)
+ *
+ * Wraps OpenAI's `/v1/embeddings` for use by the RAG indexer and search.
+ * Embeddings are L2-normalized at write time so search-time cosine
+ * similarity is a plain dot product (fast and cache-friendly).
+ *
+ * Cost (May 2026):
+ *   - text-embedding-3-small: $0.02 / 1M tokens, 1536-dim float32
+ *   - text-embedding-3-large: $0.13 / 1M tokens, 3072-dim float32
+ *
+ * For a typical agent corpus (~2k chunks × 500 tokens), -small costs
+ * ~$0.02 to index once. -large is ~7x more but only marginally better
+ * on the queries that BM25 already handles well, so -small is the
+ * sensible default.
+ *
+ * Public API:
+ *   embedTexts(texts, opts?)              → { vectors, model, usage }
+ *   embedQuery(text, opts?)               → Float32Array (normalized)
+ *   cosineSimilarity(aBytes, bBytes)      → number
+ *   serializeVector(floatArr)             → Buffer
+ *   deserializeVector(buf)                → Float32Array
+ */
+
+const DEFAULT_MODEL = "text-embedding-3-small"; // 1536 dim
+const DEFAULT_BATCH_SIZE = 100;                  // OpenAI accepts up to 2048 inputs/req
+const MAX_RETRIES = 3;
+
+const OPENAI_EMBED_URL = "https://api.openai.com/v1/embeddings";
+
+/**
+ * Embed a list of texts. Batches automatically; retries with
+ * exponential backoff on transient failures.
+ *
+ * @param {string[]} texts
+ * @param {object} [opts]
+ * @param {string} [opts.model]      Model id (default: text-embedding-3-small)
+ * @param {string} [opts.apiKey]     Override OPENAI_API_KEY
+ * @param {number} [opts.batchSize]  Inputs per request (default 100)
+ * @param {boolean} [opts.normalize] L2-normalize each vector (default true)
+ * @returns {Promise<{ vectors: Float32Array[], model: string, usage: object }>}
+ */
+export async function embedTexts(texts, opts = {}) {
+  if (!Array.isArray(texts) || texts.length === 0) {
+    return { vectors: [], model: opts.model || DEFAULT_MODEL, usage: { total_tokens: 0 } };
+  }
+  const apiKey = opts.apiKey || process.env.OPENAI_API_KEY;
+  if (!apiKey) throw new Error("OPENAI_API_KEY not set");
+  const model = opts.model || DEFAULT_MODEL;
+  const batchSize = opts.batchSize || DEFAULT_BATCH_SIZE;
+  const normalize = opts.normalize !== false;
+
+  const vectors = new Array(texts.length);
+  let totalTokens = 0;
+
+  for (let i = 0; i < texts.length; i += batchSize) {
+    const batch = texts.slice(i, i + batchSize);
+    // OpenAI rejects empty strings — substitute a single space so the
+    // index of every chunk stays stable across the response.
+    const inputs = batch.map((t) => (t && t.trim() ? t : " "));
+    const data = await callEmbeddings({ apiKey, model, inputs });
+    totalTokens += data.usage?.total_tokens || 0;
+    for (let j = 0; j < data.data.length; j++) {
+      const emb = data.data[j].embedding; // number[]
+      const f32 = new Float32Array(emb);
+      vectors[i + j] = normalize ? l2Normalize(f32) : f32;
+    }
+  }
+  return { vectors, model, usage: { total_tokens: totalTokens } };
+}
+
+/**
+ * Embed a single query string. Returns a normalized Float32Array.
+ */
+export async function embedQuery(text, opts = {}) {
+  const r = await embedTexts([text || ""], opts);
+  return r.vectors[0];
+}
+
+async function callEmbeddings({ apiKey, model, inputs }) {
+  let lastErr;
+  for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      const r = await fetch(OPENAI_EMBED_URL, {
+        method: "POST",
+        headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+        body: JSON.stringify({ model, input: inputs }),
+      });
+      if (r.ok) return r.json();
+      // Retry on 429 (rate limit) and 5xx; fail fast on 4xx else.
+      const text = await r.text();
+      if (r.status === 429 || r.status >= 500) {
+        lastErr = new Error(`embeddings HTTP ${r.status}: ${text.slice(0, 200)}`);
+        await sleep(Math.min(2 ** attempt * 500, 8000));
+        continue;
+      }
+      throw new Error(`embeddings HTTP ${r.status}: ${text.slice(0, 200)}`);
+    } catch (err) {
+      lastErr = err;
+      if (attempt < MAX_RETRIES) await sleep(Math.min(2 ** attempt * 500, 8000));
+    }
+  }
+  throw lastErr;
+}
+
+function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); }
+
+// ---------------------------------------------------------------------------
+// Vector ops
+// ---------------------------------------------------------------------------
+
+/**
+ * L2-normalize a Float32Array in place + return it. Used at write time
+ * so search-time similarity reduces to a plain dot product.
+ */
+export function l2Normalize(v) {
+  let sumSq = 0;
+  for (let i = 0; i < v.length; i++) sumSq += v[i] * v[i];
+  const norm = Math.sqrt(sumSq);
+  if (norm === 0) return v;
+  for (let i = 0; i < v.length; i++) v[i] /= norm;
+  return v;
+}
+
+/**
+ * Cosine similarity of two NORMALIZED vectors. Returns 1.0 for identical
+ * direction, 0 for orthogonal, -1 for opposite. For un-normalized input
+ * use cosineSimilarityRaw().
+ */
+export function cosineSimilarity(a, b) {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot;
+}
+
+export function cosineSimilarityRaw(a, b) {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+// ---------------------------------------------------------------------------
+// (De)serialization — Float32Array ↔ Buffer for sqlite BLOB storage
+// ---------------------------------------------------------------------------
+
+export function serializeVector(f32) {
+  if (!(f32 instanceof Float32Array)) {
+    throw new Error("serializeVector expects Float32Array");
+  }
+  return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
+}
+
+export function deserializeVector(buf) {
+  if (!buf) return null;
+  // Make sure we get a Float32Array view over the underlying bytes,
+  // independent of the Buffer's parent ArrayBuffer slicing.
+  const ab = new ArrayBuffer(buf.length);
+  Buffer.from(ab).set(buf);
+  return new Float32Array(ab);
+}
+
+/**
+ * Rough USD cost estimate. text-embedding-3-small = $0.02 / 1M tokens.
+ * Tokens ≈ chars/4 for English.
+ */
+export function embedCostEstimate(totalChars, model = DEFAULT_MODEL) {
+  const tokens = Math.ceil(totalChars / 4);
+  const ratePerM = model.includes("large") ? 0.13 : 0.02;
+  return Number(((tokens / 1e6) * ratePerM).toFixed(6));
+}
+
+export default {
+  embedTexts,
+  embedQuery,
+  cosineSimilarity,
+  cosineSimilarityRaw,
+  serializeVector,
+  deserializeVector,
+  l2Normalize,
+  embedCostEstimate,
+};