@kontourai/flow-agents 0.3.0 → 1.0.0

package/kits/knowledge/adapters/shared/codec.js

@@ -0,0 +1,325 @@
+/**
+ * Knowledge Kit — Shared Codec
+ *
+ * Utility functions shared between Knowledge Kit store adapters:
+ *   - Error helpers (MISSING_EVIDENCE, NOT_FOUND)
+ *   - YAML frontmatter codec (zero-dep subset)
+ *   - Wikilink parser / indexer
+ *   - Graph index helpers
+ *   - Validation constants and helpers
+ *
+ * @module adapters/shared/codec
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+// ---------------------------------------------------------------------------
+// Error helpers
+// ---------------------------------------------------------------------------
+
+export function missingEvidenceError(message) {
+  const err = new Error(message);
+  err.code = "MISSING_EVIDENCE";
+  return err;
+}
+
+export function notFoundError(id) {
+  const err = new Error(`Record not found: ${id}`);
+  err.code = "NOT_FOUND";
+  return err;
+}
+
+// ---------------------------------------------------------------------------
+// YAML frontmatter codec  (no external deps — handles the subset we need)
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a markdown file that begins with a YAML frontmatter block.
+ * Returns { meta, body }.
+ */
+export function parseMarkdown(text) {
+  if (!text.startsWith("---\n")) {
+    return { meta: {}, body: text };
+  }
+  const end = text.indexOf("\n---\n", 4);
+  if (end === -1) {
+    return { meta: {}, body: text };
+  }
+  const yaml = text.slice(4, end);
+  const body = text.slice(end + 5).replace(/^\n+/, "");
+  return { meta: parseYaml(yaml), body };
+}
+
+/**
+ * Minimal YAML parser: handles the scalar/list/nested-object subset
+ * emitted by serializeYaml below. Not a general YAML parser.
+ */
+export function parseYaml(yaml) {
+  const lines = yaml.split("\n");
+  return parseYamlLines(lines, 0, 0).value;
+}
+
+export function parseYamlLines(lines, start, baseIndent) {
+  const obj = {};
+  let i = start;
+  while (i < lines.length) {
+    const line = lines[i];
+    if (line.trim() === "" || line.trim().startsWith("#")) { i++; continue; }
+    const indent = line.search(/\S/);
+    if (indent < baseIndent) break;
+    if (indent > baseIndent) { i++; continue; }
+
+    // key: value  OR  key:
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) { i++; continue; }
+    const key = line.slice(indent, colonIdx).trim();
+    const rest = line.slice(colonIdx + 1).trim();
+
+    if (rest.startsWith("[")) {
+      // Inline array: [a, b, c]
+      const inner = rest.slice(1, rest.lastIndexOf("]"));
+      obj[key] = inner ? inner.split(",").map((s) => unquote(s.trim())).filter(Boolean) : [];
+      i++;
+    } else if (rest === "") {
+      // Block: peek ahead
+      i++;
+      if (i < lines.length) {
+        const nextLine = lines[i];
+        const nextIndent = nextLine.search(/\S/);
+        if (nextIndent > baseIndent && nextLine.trimStart().startsWith("- ")) {
+          // Block sequence
+          const arr = [];
+          while (i < lines.length) {
+            const l = lines[i];
+            if (l.trim() === "") { i++; continue; }
+            const ind = l.search(/\S/);
+            if (ind < nextIndent) break;
+            if (l.trimStart().startsWith("- ")) {
+              const itemText = l.trimStart().slice(2).trim();
+              if (itemText.includes(": ") || (i + 1 < lines.length && lines[i + 1].search(/\S/) > ind + 1)) {
+                // Object item
+                const childLines = [" ".repeat(ind + 2) + itemText];
+                i++;
+                while (i < lines.length) {
+                  const cl = lines[i];
+                  const ci = cl.search(/\S/);
+                  if (cl.trim() === "" || ci <= ind) break;
+                  childLines.push(cl);
+                  i++;
+                }
+                arr.push(parseYamlLines(childLines, 0, ind + 2).value);
+              } else {
+                arr.push(unquote(itemText));
+                i++;
+              }
+            } else {
+              break;
+            }
+          }
+          obj[key] = arr;
+        } else if (nextIndent > baseIndent) {
+          // Nested mapping
+          const result = parseYamlLines(lines, i, nextIndent);
+          obj[key] = result.value;
+          i = result.next;
+        } else {
+          obj[key] = null;
+        }
+      } else {
+        obj[key] = null;
+      }
+    } else {
+      obj[key] = unquote(rest);
+      i++;
+    }
+  }
+  return { value: obj, next: i };
+}
+
+export function unquote(s) {
+  if (s.startsWith('"') && s.endsWith('"')) {
+    return s.slice(1, -1).replace(/\\(\\|n|r|")/g, (_, c) => {
+      if (c === "\\") return "\\";
+      if (c === "n") return "\n";
+      if (c === "r") return "\r";
+      if (c === '"') return '"';
+      return c;
+    });
+  }
+  if (s.startsWith("'") && s.endsWith("'")) {
+    return s.slice(1, -1);
+  }
+  return s;
+}
+
+/**
+ * Serialize an object to YAML-ish text suitable for frontmatter.
+ * Only handles strings, numbers, arrays of primitives, and shallow objects.
+ */
+export function serializeYaml(obj, indent = 0) {
+  const pad = " ".repeat(indent);
+  const lines = [];
+  for (const [key, value] of Object.entries(obj)) {
+    if (value === undefined || value === null) continue;
+    if (Array.isArray(value)) {
+      if (value.length === 0) {
+        lines.push(`${pad}${key}: []`);
+      } else if (value.every((v) => typeof v !== "object")) {
+        lines.push(`${pad}${key}: [${value.map(yamlScalar).join(", ")}]`);
+      } else {
+        lines.push(`${pad}${key}:`);
+        for (const item of value) {
+          if (typeof item === "object" && item !== null) {
+            const entries = Object.entries(item).filter(([, v]) => v !== undefined && v !== null);
+            if (entries.length === 0) { lines.push(`${pad}  - {}`); continue; }
+            const [firstKey, firstVal] = entries[0];
+            if (typeof firstVal === "object" && firstVal !== null && !Array.isArray(firstVal)) {
+              lines.push(`${pad}  - ${firstKey}:`);
+              lines.push(serializeYaml(firstVal, indent + 6));
+            } else {
+              lines.push(`${pad}  - ${firstKey}: ${yamlScalar(firstVal)}`);
+            }
+            for (const [k, v] of entries.slice(1)) {
+              if (typeof v === "object" && v !== null && !Array.isArray(v)) {
+                lines.push(`${pad}    ${k}:`);
+                lines.push(serializeYaml(v, indent + 6));
+              } else {
+                lines.push(`${pad}    ${k}: ${yamlScalar(v)}`);
+              }
+            }
+          } else {
+            lines.push(`${pad}  - ${yamlScalar(item)}`);
+          }
+        }
+      }
+    } else if (typeof value === "object") {
+      lines.push(`${pad}${key}:`);
+      lines.push(serializeYaml(value, indent + 2));
+    } else {
+      lines.push(`${pad}${key}: ${yamlScalar(value)}`);
+    }
+  }
+  return lines.join("\n");
+}
+
+export function yamlScalar(v) {
+  if (typeof v === "string") {
+    // Quote if it contains special chars or actual newlines/carriage returns
+    if (/[:#\[\]{},&*?|<>=!%@`"'\n\r]/.test(v) || v.trim() !== v || v === "") {
+      return `"${v.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n").replace(/\r/g, "\\r")}"`;
+    }
+    return v;
+  }
+  return String(v);
+}
+
+export function serializeMarkdown(meta, body) {
+  return `---\n${serializeYaml(meta)}\n---\n\n${body}`;
+}
+
+// ---------------------------------------------------------------------------
+// Wikilink parser / indexer
+// ---------------------------------------------------------------------------
+
+const WIKILINK_RE = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g;
+
+/**
+ * Extract all [[target_id]] and [[target_id|label]] links from body text.
+ * Returns Link objects.
+ */
+export function extractWikilinks(body) {
+  const links = [];
+  for (const match of body.matchAll(WIKILINK_RE)) {
+    links.push({ target_id: match[1].trim(), kind: "related", label: match[2]?.trim() });
+  }
+  return links;
+}
+
+/**
+ * Merge explicit links array with wikilink-derived links.
+ * De-duplicates by (target_id, kind); explicit links win on conflict.
+ */
+export function mergeLinks(explicit, wikilinks) {
+  const key = (l) => `${l.target_id}::${l.kind}`;
+  const seen = new Set(explicit.map(key));
+  const merged = [...explicit];
+  for (const wl of wikilinks) {
+    if (!seen.has(key(wl))) {
+      merged.push(wl);
+      seen.add(key(wl));
+    }
+  }
+  return merged;
+}
+
+// ---------------------------------------------------------------------------
+// Graph index
+// ---------------------------------------------------------------------------
+
+export const GRAPH_SCHEMA_VERSION = "1.0";
+
+export function emptyGraph() {
+  return { schema_version: GRAPH_SCHEMA_VERSION, forward: {}, reverse: {} };
+}
+
+export function loadGraph(graphPath) {
+  if (!fs.existsSync(graphPath)) return emptyGraph();
+  try {
+    return JSON.parse(fs.readFileSync(graphPath, "utf8"));
+  } catch {
+    return emptyGraph();
+  }
+}
+
+export function saveGraph(graphPath, graph) {
+  fs.writeFileSync(graphPath, JSON.stringify(graph, null, 2) + "\n", "utf8");
+}
+
+export function addLinksToGraph(graph, sourceId, links) {
+  if (!graph.forward[sourceId]) graph.forward[sourceId] = [];
+  for (const link of links) {
+    const { target_id, kind, label } = link;
+    // Idempotent: skip if already present
+    const exists = graph.forward[sourceId].some(
+      (l) => l.target_id === target_id && l.kind === kind
+    );
+    if (!exists) {
+      const entry = { target_id, kind };
+      if (label) entry.label = label;
+      graph.forward[sourceId].push(entry);
+      if (!graph.reverse[target_id]) graph.reverse[target_id] = [];
+      graph.reverse[target_id].push({ source_id: sourceId, kind });
+    }
+  }
+}
+
+export function removeLinksFromGraph(graph, sourceId) {
+  const oldForward = graph.forward[sourceId] || [];
+  for (const link of oldForward) {
+    const rev = graph.reverse[link.target_id] || [];
+    graph.reverse[link.target_id] = rev.filter((r) => r.source_id !== sourceId);
+    if (graph.reverse[link.target_id].length === 0) delete graph.reverse[link.target_id];
+  }
+  delete graph.forward[sourceId];
+}
+
+// ---------------------------------------------------------------------------
+// Validation helpers
+// ---------------------------------------------------------------------------
+
+export const VALID_TYPES = new Set(["raw", "compiled", "concept", "snapshot", "person"]);
+export const VALID_STATUSES = new Set(["active", "implemented", "retired"]);
+const CATEGORY_SEGMENT_RE = /^[a-z0-9_-]+$/;
+
+// Status transition table: from → allowed targets
+export const VALID_STATUS_TRANSITIONS = {
+  active:      new Set(["implemented", "retired"]),
+  implemented: new Set(["retired"]),
+  retired:     new Set(),  // terminal — no further transitions
+};
+
+export function validateCategory(cat) {
+  if (!cat || typeof cat !== "string") return false;
+  return cat.split(".").every((seg) => CATEGORY_SEGMENT_RE.test(seg));
+}

package/kits/knowledge/adapters/similarity-vector/index.js

@@ -0,0 +1,284 @@
+/**
+ * Knowledge Kit — Vector Similarity Adapter
+ *
+ * Provides a drop-in SimilarityDetector implementation backed by dense vector
+ * embeddings (cosine similarity) instead of the default category-prefix /
+ * link-overlap heuristic.
+ *
+ * SimilarityDetector interface (from adapters/flow-runner/index.js):
+ *   async (concept: Record, candidates: Record[], store: KnowledgeStoreAdapter) => string[]
+ *
+ * Usage:
+ *   import { createVectorSimilarityDetector } from './adapters/similarity-vector/index.js';
+ *
+ *   // Ollama (default):
+ *   const detector = createVectorSimilarityDetector();
+ *
+ *   // Ollama with non-default model/host:
+ *   const detector = createVectorSimilarityDetector({
+ *     host: 'http://localhost:11434',
+ *     model: 'nomic-embed-text',
+ *     threshold: 0.60,
+ *   });
+ *
+ *   // Injectable embed fn (for tests / custom providers):
+ *   const detector = createVectorSimilarityDetector({
+ *     embed: async (texts) => texts.map(() => [0.1, 0.9, 0.0]),
+ *     threshold: 0.60,
+ *   });
+ *
+ *   // Pass to synthesize:
+ *   await runner.synthesize(conceptId, {
+ *     proposedBody: '...',
+ *     rationale: '...',
+ *     similarityDetector: detector,
+ *   });
+ *
+ * Zero npm dependencies — uses Node.js built-in fetch (Node >= 18).
+ *
+ * Fail-closed policy:
+ *   If the embedding call fails (network error, non-200, malformed response),
+ *   the detector throws an Error with code="EMBED_FAILURE". This is intentional:
+ *   silently returning [] would look identical to "no similar records found" and
+ *   mask infrastructure failures as legitimate empty clusters, blocking synthesis
+ *   with a misleading MISSING_EVIDENCE rather than a clear infrastructure error.
+ *
+ * @module adapters/similarity-vector
+ */
+
+// ---------------------------------------------------------------------------
+// Pure cosine similarity (exported for tests)
+// ---------------------------------------------------------------------------
+
+/**
+ * Compute the cosine similarity between two equal-length numeric vectors.
+ *
+ * Returns a value in [-1, 1]:
+ *   1.0  — identical direction
+ *   0.0  — orthogonal
+ *  -1.0  — opposite direction
+ *
+ * Edge cases:
+ *   - Zero-magnitude vector(s): returns 0 (no similarity signal).
+ *   - Empty or unequal-length vectors: returns 0.
+ *
+ * @param {number[]} a
+ * @param {number[]} b
+ * @returns {number}
+ */
+export function cosineSimilarity(a, b) {
+  if (!Array.isArray(a) || !Array.isArray(b)) return 0;
+  if (a.length !== b.length || a.length === 0) return 0;
+
+  let dot = 0;
+  let magA = 0;
+  let magB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+
+  const denom = Math.sqrt(magA) * Math.sqrt(magB);
+  if (denom === 0) return 0;
+  return dot / denom;
+}
+
+// ---------------------------------------------------------------------------
+// Ollama embed call
+// ---------------------------------------------------------------------------
+
+/**
+ * Call ollama's /api/embed endpoint.
+ *
+ * Throws an Error with code="EMBED_FAILURE" on any failure.
+ *
+ * @param {string} host
+ * @param {string} model
+ * @param {string[]} texts
+ * @returns {Promise<number[][]>}
+ */
+async function ollamaEmbed(host, model, texts) {
+  const url = `${host}/api/embed`;
+  let response;
+  try {
+    response = await fetch(url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ model, input: texts }),
+    });
+  } catch (cause) {
+    const err = new Error(
+      `EMBED_FAILURE: embedding call to ${url} failed — ${cause.message}`
+    );
+    err.code = "EMBED_FAILURE";
+    err.cause = cause;
+    throw err;
+  }
+
+  if (!response.ok) {
+    const body = await response.text().catch(() => "(unreadable)");
+    const err = new Error(
+      `EMBED_FAILURE: embedding call to ${url} returned HTTP ${response.status}: ${body}`
+    );
+    err.code = "EMBED_FAILURE";
+    throw err;
+  }
+
+  let data;
+  try {
+    data = await response.json();
+  } catch (cause) {
+    const err = new Error(
+      `EMBED_FAILURE: embedding response from ${url} was not valid JSON — ${cause.message}`
+    );
+    err.code = "EMBED_FAILURE";
+    err.cause = cause;
+    throw err;
+  }
+
+  // ollama /api/embed returns { embeddings: number[][] }
+  if (!data.embeddings || !Array.isArray(data.embeddings)) {
+    const err = new Error(
+      `EMBED_FAILURE: embedding response missing .embeddings array (got: ${JSON.stringify(Object.keys(data || {}))})`
+    );
+    err.code = "EMBED_FAILURE";
+    throw err;
+  }
+
+  if (data.embeddings.length !== texts.length) {
+    const err = new Error(
+      `EMBED_FAILURE: expected ${texts.length} embedding(s), got ${data.embeddings.length}`
+    );
+    err.code = "EMBED_FAILURE";
+    throw err;
+  }
+
+  return data.embeddings;
+}
+
+// ---------------------------------------------------------------------------
+// createVectorSimilarityDetector
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a SimilarityDetector backed by dense vector embeddings.
+ *
+ * The returned detector satisfies the SimilarityDetector interface:
+ *   async (concept, candidates, store) => string[]
+ *
+ * @param {object} [options]
+ * @param {((texts: string[]) => Promise<number[][]>)} [options.embed]
+ *   Injectable embedding function. When provided, `host` and `model` are
+ *   ignored. Signature: async (texts: string[]) => number[][]
+ *   Must return one vector per input text.
+ * @param {string} [options.host="http://localhost:11434"]
+ *   Ollama server base URL. Used when `embed` is not provided.
+ * @param {string} [options.model="nomic-embed-text"]
+ *   Embedding model name passed to ollama. Used when `embed` is not provided.
+ * @param {number} [options.threshold=0.60]
+ *   Minimum cosine similarity score for a candidate to be included.
+ *   Range: [-1, 1]. Default 0.60 is calibrated for nomic-embed-text where
+ *   semantically related texts from the same domain typically score ≥ 0.70
+ *   and unrelated texts score < 0.50.
+ * @param {((record: object) => string)} [options.text]
+ *   Extractor that converts a store record to the text to embed.
+ *   Default: `record.title + "\n" + record.body`.
+ * @returns {(concept: object, candidates: object[], store: object) => Promise<string[]>}
+ */
+export function createVectorSimilarityDetector(options = {}) {
+  const {
+    embed: injectEmbed = null,
+    host = "http://localhost:11434",
+    model = "nomic-embed-text",
+    threshold = 0.60,
+    text: extractText = defaultTextExtractor,
+  } = options;
+
+  // Resolve the actual embed function once (avoid re-resolving on each call)
+  const embedFn = injectEmbed
+    ? injectEmbed
+    : (texts) => ollamaEmbed(host, model, texts);
+
+  /**
+   * SimilarityDetector: returns candidate IDs whose cosine similarity to the
+   * concept embedding meets or exceeds `threshold`.
+   *
+   * Fail-closed: any embedding failure throws EMBED_FAILURE rather than
+   * silently returning [].
+   *
+   * @param {object} concept
+   * @param {object[]} candidates
+   * @param {object} _store  (not used by vector detector; kept for interface compat)
+   * @returns {Promise<string[]>}
+   */
+  async function vectorSimilarityDetector(concept, candidates, _store) {
+    if (!candidates || candidates.length === 0) {
+      return [];
+    }
+
+    // Exclude retired records from the working set (Addendum B — R3)
+    const activeCandidates = candidates.filter(
+      (c) => (c.status || "active") !== "retired"
+    );
+
+    if (activeCandidates.length === 0) {
+      return [];
+    }
+
+    const conceptText = extractText(concept);
+
+    // Build the batch: concept first, then all active candidates.
+    // One round-trip minimises latency and keeps the batch API simple.
+    const allTexts = [conceptText, ...activeCandidates.map(extractText)];
+
+    // Embedding call — throws EMBED_FAILURE on any infrastructure error.
+    const embeddings = await embedFn(allTexts);
+
+    // Validate count: the embed fn must return one vector per input text.
+    // A count mismatch would produce silent wrong results (undefined vectors
+    // scoring 0 and being excluded) — throw EMBED_FAILURE instead.
+    if (!Array.isArray(embeddings) || embeddings.length !== allTexts.length) {
+      const err = new Error(
+        `EMBED_FAILURE: embed function returned ${Array.isArray(embeddings) ? embeddings.length : typeof embeddings} vector(s) but expected ${allTexts.length} (1 concept + ${activeCandidates.length} active candidates)`
+      );
+      err.code = 'EMBED_FAILURE';
+      throw err;
+    }
+
+    const conceptVec = embeddings[0];
+    const similar = [];
+
+    for (let i = 0; i < activeCandidates.length; i++) {
+      const candidateVec = embeddings[i + 1];
+      const score = cosineSimilarity(conceptVec, candidateVec);
+      if (score >= threshold) {
+        similar.push(activeCandidates[i].id);
+      }
+    }
+
+    return similar;
+  }
+
+  return vectorSimilarityDetector;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Default text extractor: title + newline + body.
+ * Gracefully handles missing fields.
+ *
+ * @param {object} record
+ * @returns {string}
+ */
+function defaultTextExtractor(record) {
+  const title = record?.title || "";
+  const body = record?.body || "";
+  return `${title}\n${body}`;
+}
+
+export default createVectorSimilarityDetector;