@remnic/core 1.1.12 → 1.1.13

package/src/taxonomy/default-taxonomy.ts

@@ -0,0 +1,87 @@
+/**
+ * Default MECE taxonomy that maps every existing MemoryCategory value
+ * to exactly one taxonomy category, ordered by priority.
+ */
+
+import type { Taxonomy } from "./types.js";
+
+export const DEFAULT_TAXONOMY: Taxonomy = {
+  version: 1,
+  categories: [
+    {
+      id: "corrections",
+      name: "Corrections",
+      description: "Corrections to previously stored information",
+      filingRules: ["Any update that supersedes a prior fact"],
+      priority: 10,
+      memoryCategories: ["correction"],
+    },
+    {
+      id: "principles",
+      name: "Principles",
+      description: "Rules, guidelines, and recurring patterns",
+      filingRules: ["A guiding principle, rule, or skill"],
+      priority: 20,
+      memoryCategories: ["principle", "rule", "skill"],
+    },
+    {
+      id: "procedures",
+      name: "Procedures",
+      description: "Ordered multi-step workflows the user repeats",
+      filingRules: ["A repeatable sequence of steps or commands for a task"],
+      priority: 25,
+      memoryCategories: ["procedure"],
+    },
+    {
+      id: "entities",
+      name: "Entities",
+      description: "People, organizations, places, projects",
+      filingRules: ["Named entity with attributes"],
+      priority: 30,
+      memoryCategories: ["entity", "relationship"],
+    },
+    {
+      id: "decisions",
+      name: "Decisions",
+      description: "Choices made and their rationale",
+      filingRules: ["A decision or commitment with reasoning"],
+      priority: 35,
+      memoryCategories: ["decision", "commitment"],
+    },
+    {
+      id: "preferences",
+      name: "Preferences",
+      description: "User likes, dislikes, and style choices",
+      filingRules: ["Anything expressing a preference or taste"],
+      priority: 40,
+      memoryCategories: ["preference"],
+    },
+    {
+      id: "facts",
+      name: "Facts",
+      description: "Objective statements about the world",
+      filingRules: ["Any factual claim or piece of information"],
+      priority: 50,
+      memoryCategories: ["fact"],
+    },
+    {
+      id: "moments",
+      name: "Moments",
+      description: "Significant events or experiences",
+      filingRules: ["A specific event worth remembering"],
+      priority: 60,
+      memoryCategories: ["moment"],
+    },
+    {
+      id: "reasoning-traces",
+      name: "Reasoning Traces",
+      description:
+        "Stored intermediate reasoning / solution chains for a problem the agent previously solved",
+      filingRules: [
+        "A multi-step reasoning chain or solution walkthrough the agent can replay for a similar problem",
+      ],
+      priority: 55,
+      memoryCategories: ["reasoning_trace"],
+    },
+  ],
+};

package/src/taxonomy/index.ts

@@ -0,0 +1,26 @@
+/**
+ * MECE Taxonomy — Knowledge directory with resolver decision tree.
+ *
+ * Re-exports all public types and functions from the taxonomy subsystem.
+ */
+
+export type {
+  Taxonomy,
+  TaxonomyCategory,
+  ResolverDecision,
+} from "./types.js";
+
+export { DEFAULT_TAXONOMY } from "./default-taxonomy.js";
+
+export { resolveCategory } from "./resolver.js";
+
+export { generateResolverDocument } from "./resolver-doc-generator.js";
+
+export {
+  loadTaxonomy,
+  saveTaxonomy,
+  validateSlug,
+  validateTaxonomy,
+  getTaxonomyDir,
+  getTaxonomyFilePath,
+} from "./taxonomy-loader.js";

package/src/taxonomy/resolver-doc-generator.ts

@@ -0,0 +1,57 @@
+/**
+ * Generates a markdown decision-tree document (RESOLVER.md) from a
+ * taxonomy definition.
+ *
+ * The document walks a user through filing a new piece of knowledge
+ * by checking each category in priority order (lowest number first).
+ */
+
+import type { Taxonomy } from "./types.js";
+
+/**
+ * Produce a markdown decision tree for the given taxonomy.
+ *
+ * Categories are listed in priority order (lowest number = checked first).
+ * Each step asks whether the knowledge fits the category and, if so,
+ * instructs the reader to file it there.
+ */
+export function generateResolverDocument(taxonomy: Taxonomy): string {
+  const sorted = [...taxonomy.categories].sort((a, b) => {
+    if (a.priority !== b.priority) return a.priority - b.priority;
+    return a.id.localeCompare(b.id);
+  });
+
+  const lines: string[] = [
+    "# Memory Filing Resolver",
+    "",
+    "Given a new piece of knowledge, follow this tree to determine where it belongs.",
+    "",
+  ];
+
+  let step = 1;
+  for (const cat of sorted) {
+    lines.push(`## Step ${step}: ${cat.description}?`);
+    lines.push("");
+    for (const rule of cat.filingRules) {
+      lines.push(`- ${rule}`);
+    }
+    lines.push("");
+    lines.push(
+      `> YES: File under **${cat.id}/** (priority ${cat.priority})`,
+    );
+    lines.push("");
+    step++;
+  }
+
+  lines.push("## Tie-breaking");
+  lines.push("");
+  lines.push(
+    "If a fact could go in multiple categories, file under the one with the **lowest priority number**.",
+  );
+  lines.push("");
+  lines.push(`---`);
+  lines.push(`*Generated from taxonomy v${taxonomy.version}*`);
+  lines.push("");
+
+  return lines.join("\n");
+}

package/src/taxonomy/resolver.ts

@@ -0,0 +1,184 @@
+/**
+ * Resolver decision tree for the MECE taxonomy.
+ *
+ * Given extracted content and its MemoryCategory, determines which
+ * taxonomy category the knowledge should be filed under.
+ */
+
+import type { MemoryCategory } from "../types.js";
+import type { ResolverDecision, Taxonomy, TaxonomyCategory } from "./types.js";
+
+const DEFAULT_CATEGORY_ID = "facts";
+
+/**
+ * Resolve a piece of content to a taxonomy category.
+ *
+ * Algorithm:
+ * 1. Find all taxonomy categories whose `memoryCategories` include
+ *    the given `memoryCategory`.
+ * 2. If exactly one match, return it with confidence 1.0.
+ * 3. If multiple matches, prefer the strongest exact-token keyword
+ *    overlap from filing rules. Use priority only as a tie-breaker.
+ *    If no filing rules match, choose the explicit/default fallback
+ *    category instead of letting low-priority generic terms win.
+ * 4. If no match, fall back to the "facts" category (or first
+ *    category if "facts" is absent) with low confidence.
+ * 5. Always populate `alternatives` with other plausible categories.
+ */
+export function resolveCategory(
+  content: string,
+  memoryCategory: MemoryCategory,
+  taxonomy: Taxonomy,
+): ResolverDecision {
+  const contentTokens = tokenizeKeywordText(content);
+
+  // Step 1: find matching categories
+  const matches = taxonomy.categories.filter((cat) =>
+    cat.memoryCategories.includes(memoryCategory),
+  );
+
+  if (matches.length === 0) {
+    // No taxonomy category accepts this MemoryCategory — fall back
+    const fallback =
+      taxonomy.categories.find((c) => c.id === DEFAULT_CATEGORY_ID) ??
+      taxonomy.categories[0];
+    if (!fallback) {
+      return {
+        categoryId: DEFAULT_CATEGORY_ID,
+        confidence: 0,
+        reason: "Taxonomy is empty; using default category",
+        alternatives: [],
+      };
+    }
+    const alternatives = taxonomy.categories
+      .filter((c) => c.id !== fallback.id)
+      .map((c) => ({
+        categoryId: c.id,
+        reason: c.description,
+      }));
+    return {
+      categoryId: fallback.id,
+      confidence: 0.3,
+      reason: `No taxonomy category maps to MemoryCategory "${memoryCategory}"; falling back to "${fallback.name}"`,
+      alternatives,
+    };
+  }
+
+  if (matches.length === 1) {
+    const match = matches[0]!;
+    const alternatives = taxonomy.categories
+      .filter((c) => c.id !== match.id)
+      .map((c) => ({
+        categoryId: c.id,
+        reason: c.description,
+      }));
+    return {
+      categoryId: match.id,
+      confidence: 1.0,
+      reason: `Unique match: MemoryCategory "${memoryCategory}" maps to "${match.name}"`,
+      alternatives,
+    };
+  }
+
+  // Multiple matches — use filing rule keyword heuristics + priority
+  const scored = matches.map((cat) => ({
+    cat,
+    keywordScore: computeKeywordScoreForTokens(contentTokens, cat),
+  }));
+
+  // Sort by keyword score descending, then priority ascending (lower wins)
+  scored.sort((a, b) => {
+    if (b.keywordScore !== a.keywordScore) return b.keywordScore - a.keywordScore;
+    return a.cat.priority - b.cat.priority;
+  });
+
+  const topScored = scored[0]!;
+  const best = topScored.keywordScore > 0
+    ? topScored
+    : {
+        cat: selectFallbackCategory(matches) ?? topScored.cat,
+        keywordScore: 0,
+      };
+  const runnerUp = scored[1];
+
+  // Confidence is higher when keyword match clearly differentiates
+  const confidence =
+    best.keywordScore > 0 && (!runnerUp || best.keywordScore > runnerUp.keywordScore)
+      ? 0.9
+      : 0.7;
+
+  const alternatives = taxonomy.categories
+    .filter((c) => c.id !== best.cat.id)
+    .map((c) => ({
+      categoryId: c.id,
+      reason: c.description,
+    }));
+
+  const reason =
+    best.keywordScore > 0
+      ? `Filing rules for "${best.cat.name}" matched content keywords (priority ${best.cat.priority})`
+      : `No filing rules matched content keywords; using fallback category "${best.cat.name}"`;
+
+  return {
+    categoryId: best.cat.id,
+    confidence,
+    reason,
+    alternatives,
+  };
+}
+
+/**
+ * Compute a simple keyword overlap score between content and
+ * a category's filing rules + description.
+ */
+function computeKeywordScoreForTokens(contentTokens: Set<string>, cat: TaxonomyCategory): number {
+  let score = 0;
+  const ruleText = [...cat.filingRules, cat.description]
+    .join(" ")
+    .toLowerCase();
+
+  const keywords = tokenizeKeywordText(ruleText);
+
+  for (const kw of keywords) {
+    if (contentTokens.has(kw)) {
+      score += 1;
+    }
+  }
+  return score;
+}
+
+function tokenizeKeywordText(value: string): Set<string> {
+  return new Set(
+    value
+      .toLowerCase()
+      .split(/[^a-z0-9]+/)
+      .map((word) => word.trim())
+      .filter((word) => word.length >= 3 && !TAXONOMY_KEYWORD_STOPWORDS.has(word)),
+  );
+}
+
+function selectFallbackCategory(
+  categories: readonly TaxonomyCategory[],
+): TaxonomyCategory | undefined {
+  return categories.find((cat) => cat.id === DEFAULT_CATEGORY_ID) ??
+    categories.find((cat) => {
+      const text = `${cat.id} ${cat.name} ${cat.description} ${cat.filingRules.join(" ")}`.toLowerCase();
+      return /\b(general|fallback)\b/.test(text);
+    });
+}
+
+const TAXONOMY_KEYWORD_STOPWORDS = new Set([
+  "about",
+  "all",
+  "and",
+  "are",
+  "for",
+  "from",
+  "has",
+  "into",
+  "not",
+  "the",
+  "this",
+  "was",
+  "with",
+]);

package/src/taxonomy/taxonomy-loader.ts

@@ -0,0 +1,186 @@
+/**
+ * Loads and validates a user-customized taxonomy from disk, merging
+ * with the built-in defaults.
+ *
+ * User taxonomies are stored at `<memoryDir>/.taxonomy/taxonomy.json`.
+ */
+
+import { readFile, mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { Taxonomy, TaxonomyCategory } from "./types.js";
+import { DEFAULT_TAXONOMY } from "./default-taxonomy.js";
+
+const TAXONOMY_DIR = ".taxonomy";
+const TAXONOMY_FILE = "taxonomy.json";
+
+/** Maximum allowed slug length */
+const MAX_SLUG_LENGTH = 32;
+
+/** Regex for valid slug: lowercase letters, digits, hyphens */
+const SLUG_RE = /^[a-z][a-z0-9-]*$/;
+
+/**
+ * Validate a taxonomy category slug.
+ * Throws if the slug is invalid.
+ */
+export function validateSlug(slug: string): void {
+  if (slug.length === 0) {
+    throw new Error("Taxonomy category ID must not be empty");
+  }
+  if (slug.length > MAX_SLUG_LENGTH) {
+    throw new Error(
+      `Taxonomy category ID "${slug}" exceeds ${MAX_SLUG_LENGTH} characters`,
+    );
+  }
+  if (!SLUG_RE.test(slug)) {
+    throw new Error(
+      `Taxonomy category ID "${slug}" is invalid: must be lowercase letters, digits, and hyphens, starting with a letter`,
+    );
+  }
+}
+
+/**
+ * Validate an entire taxonomy for structural correctness.
+ * Throws on the first error found.
+ */
+export function validateTaxonomy(taxonomy: Taxonomy): void {
+  if (typeof taxonomy.version !== "number" || taxonomy.version < 1) {
+    throw new Error("Taxonomy version must be a positive integer");
+  }
+  if (!Array.isArray(taxonomy.categories)) {
+    throw new Error("Taxonomy categories must be an array");
+  }
+
+  const seenIds = new Set<string>();
+  for (const cat of taxonomy.categories) {
+    validateSlug(cat.id);
+    if (seenIds.has(cat.id)) {
+      throw new Error(`Duplicate taxonomy category ID: "${cat.id}"`);
+    }
+    seenIds.add(cat.id);
+
+    if (typeof cat.name !== "string" || cat.name.trim().length === 0) {
+      throw new Error(`Taxonomy category "${cat.id}" must have a non-empty name`);
+    }
+    if (typeof cat.description !== "string" || cat.description.trim().length === 0) {
+      throw new Error(`Taxonomy category "${cat.id}" must have a non-empty description`);
+    }
+    if (!Array.isArray(cat.filingRules)) {
+      throw new Error(`Taxonomy category "${cat.id}" filingRules must be an array`);
+    }
+    if (typeof cat.priority !== "number" || !Number.isFinite(cat.priority)) {
+      throw new Error(`Taxonomy category "${cat.id}" must have a finite numeric priority`);
+    }
+    if (!Array.isArray(cat.memoryCategories)) {
+      throw new Error(`Taxonomy category "${cat.id}" memoryCategories must be an array`);
+    }
+    if (cat.parentId !== undefined) {
+      if (typeof cat.parentId !== "string") {
+        throw new Error(`Taxonomy category "${cat.id}" parentId must be a string if set`);
+      }
+    }
+  }
+
+  // Validate parentId references
+  for (const cat of taxonomy.categories) {
+    if (cat.parentId !== undefined && !seenIds.has(cat.parentId)) {
+      throw new Error(
+        `Taxonomy category "${cat.id}" references unknown parentId "${cat.parentId}"`,
+      );
+    }
+  }
+}
+
+/**
+ * Load a taxonomy from the user's memory directory.
+ *
+ * If `<memoryDir>/.taxonomy/taxonomy.json` exists, loads it, validates
+ * it, and merges with the defaults (user categories override defaults
+ * by ID). If the file does not exist, returns the defaults.
+ */
+export async function loadTaxonomy(memoryDir: string): Promise<Taxonomy> {
+  const taxonomyPath = path.join(memoryDir, TAXONOMY_DIR, TAXONOMY_FILE);
+  let raw: string;
+  try {
+    raw = await readFile(taxonomyPath, "utf-8");
+  } catch (err: unknown) {
+    // Only fall back to defaults for missing file; rethrow permission / I/O errors
+    if (err instanceof Error && (err as NodeJS.ErrnoException).code === "ENOENT") {
+      return structuredClone(DEFAULT_TAXONOMY);
+    }
+    throw err;
+  }
+
+  const parsed: unknown = JSON.parse(raw);
+  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+    throw new Error("taxonomy.json must be a JSON object");
+  }
+
+  const obj = parsed as Record<string, unknown>;
+  const userVersion = typeof obj.version === "number" ? obj.version : DEFAULT_TAXONOMY.version;
+  const userCategories = Array.isArray(obj.categories)
+    ? (obj.categories as TaxonomyCategory[])
+    : [];
+
+  // Validate: reject duplicate IDs in user categories before merging.
+  // Without this check, duplicates are silently collapsed with last-write-wins
+  // semantics when inserted into the Map.
+  const userIdCounts = new Map<string, number>();
+  for (const cat of userCategories) {
+    const id = typeof cat.id === "string" ? cat.id : String(cat.id);
+    userIdCounts.set(id, (userIdCounts.get(id) ?? 0) + 1);
+  }
+  const duplicateIds = [...userIdCounts.entries()]
+    .filter(([, count]) => count > 1)
+    .map(([id]) => id);
+  if (duplicateIds.length > 0) {
+    throw new Error(
+      `Duplicate category IDs in taxonomy.json: ${duplicateIds.map((id) => `"${id}"`).join(", ")}`,
+    );
+  }
+
+  // Merge: user categories override defaults by ID
+  const mergedMap = new Map<string, TaxonomyCategory>();
+  for (const cat of DEFAULT_TAXONOMY.categories) {
+    mergedMap.set(cat.id, { ...cat });
+  }
+  for (const cat of userCategories) {
+    mergedMap.set(cat.id, cat);
+  }
+
+  const merged: Taxonomy = {
+    version: userVersion,
+    categories: [...mergedMap.values()],
+  };
+
+  validateTaxonomy(merged);
+  return merged;
+}
+
+/**
+ * Save a taxonomy to the user's memory directory.
+ */
+export async function saveTaxonomy(
+  memoryDir: string,
+  taxonomy: Taxonomy,
+): Promise<void> {
+  validateTaxonomy(taxonomy);
+  const dir = path.join(memoryDir, TAXONOMY_DIR);
+  await mkdir(dir, { recursive: true });
+  const filePath = path.join(dir, TAXONOMY_FILE);
+  await writeFile(filePath, JSON.stringify(taxonomy, null, 2) + "\n", "utf-8");
+}
+
+/**
+ * Get the taxonomy directory path for a given memory directory.
+ */
+export function getTaxonomyDir(memoryDir: string): string {
+  return path.join(memoryDir, TAXONOMY_DIR);
+}
+
+/**
+ * Get the taxonomy file path for a given memory directory.
+ */
+export function getTaxonomyFilePath(memoryDir: string): string {
+  return path.join(memoryDir, TAXONOMY_DIR, TAXONOMY_FILE);
+}

package/src/taxonomy/types.ts

@@ -0,0 +1,48 @@
+/**
+ * Taxonomy types for the MECE knowledge directory.
+ *
+ * A taxonomy defines a set of categories that partition the knowledge
+ * space so that every memory maps to exactly one category (Mutually
+ * Exclusive, Collectively Exhaustive).
+ */
+
+/**
+ * A single category in the taxonomy.
+ *
+ * @property id          - Slug: lowercase letters, digits, hyphens; max 32 chars.
+ * @property name        - Human-readable display name.
+ * @property description - What belongs in this category.
+ * @property filingRules - Prose rules used by the resolver decision tree.
+ * @property parentId    - Optional parent category for nesting.
+ * @property priority    - Tie-breaker: lower number wins when a memory
+ *                         could belong to multiple categories.
+ * @property memoryCategories - Which MemoryCategory values map here.
+ */
+export interface TaxonomyCategory {
+  id: string;
+  name: string;
+  description: string;
+  filingRules: string[];
+  parentId?: string;
+  priority: number;
+  memoryCategories: string[];
+}
+
+/**
+ * A versioned taxonomy comprising an ordered list of categories.
+ */
+export interface Taxonomy {
+  version: number;
+  categories: TaxonomyCategory[];
+}
+
+/**
+ * The output of the resolver: which category a piece of knowledge
+ * belongs to, with confidence and alternatives.
+ */
+export interface ResolverDecision {
+  categoryId: string;
+  confidence: number;
+  reason: string;
+  alternatives: Array<{ categoryId: string; reason: string }>;
+}

package/src/telemetry-transcript.ts

@@ -0,0 +1,70 @@
+const TELEMETRY_TURN_MARKER = /^\[(?:user|assistant|system)\]\s+\[(?:action|observation|state|reward|step|turn|frame)\s+\d+\]:/i;
+const BARE_TELEMETRY_MARKER = /^\[(?:action|observation|state|reward|step|turn|frame)\s+\d+\]:/i;
+const DURABLE_MEMORY_CUE_PHRASES = [
+  "remember",
+  "don't forget",
+  "prefer",
+  "preference",
+  "decided",
+  "decision",
+  "deadline",
+  "commitment",
+  "promise",
+  "my name",
+  "i am",
+  "i work",
+  "we use",
+  "correction",
+  "actually",
+  "always",
+  "never",
+  "when you",
+];
+
+export function stripRolePrefix(line: string): string {
+  return line.replace(/^\[(?:user|assistant|system)\]\s+/i, "").trim();
+}
+
+export function isTelemetryMarkerLine(line: string): boolean {
+  return TELEMETRY_TURN_MARKER.test(line) ||
+    BARE_TELEMETRY_MARKER.test(stripRolePrefix(line));
+}
+
+export function isMechanicalTelemetryLine(line: string): boolean {
+  const stripped = stripRolePrefix(line);
+  return (
+    isTelemetryMarkerLine(line) ||
+    /^(?:left|right|up|down|wait|noop|stop|start|forward|backward)$/i.test(stripped) ||
+    /^(?:active rules|objects on (?:the )?map|inventory|score|reward|position|location):?$/i.test(stripped) ||
+    /^rule `[^`]+`/i.test(stripped) ||
+    /^(?:[a-z][\w-]*|\w+ `[^`]+`)\s+\d+\s+steps?\s+(?:to\s+the\s+)?(?:left|right|up|down)\b/i.test(stripped)
+  );
+}
+
+export function hasDurableMemoryCue(conversation: string): boolean {
+  const lower = conversation.toLowerCase();
+  if (DURABLE_MEMORY_CUE_PHRASES.some((phrase) => lower.includes(phrase))) {
+    return true;
+  }
+  const ifIndex = lower.indexOf("if ");
+  return ifIndex >= 0 && lower.indexOf(" then", ifIndex + 3) >= 0;
+}
+
+export function looksLikeMechanicalTelemetryTranscript(
+  conversation: string,
+): boolean {
+  if (hasDurableMemoryCue(conversation)) return false;
+
+  const lines = conversation
+    .split(/\n+/)
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0);
+  if (lines.length < 8) return false;
+
+  const markerLines = lines.filter((line) => isTelemetryMarkerLine(line)).length;
+  if (markerLines < 4) return false;
+
+  const mechanicalLines = lines.filter((line) => isMechanicalTelemetryLine(line)).length;
+  const mechanicalRatio = mechanicalLines / lines.length;
+  return mechanicalRatio >= 0.45;
+}