flint-agent 1.14.0

package/src/memory/store.js

@@ -0,0 +1,208 @@
+import { readFileSync, writeFileSync, renameSync, existsSync, mkdirSync, chmodSync } from "node:fs";
+import { createHmac, randomBytes, timingSafeEqual } from "node:crypto";
+import path from "node:path";
+import { config } from "../config.js";
+
+const MEMORY_DIR = path.join(config.projectRoot, "memory");
+const MEMORIES_FILE = path.join(MEMORY_DIR, "memories.jsonl");
+const HMAC_FILE = path.join(MEMORY_DIR, "memories.hmac");
+const KEY_FILE = path.join(MEMORY_DIR, ".hmac-key");
+
+const MAX_CONTENT_LENGTH = 10000;
+const MAX_IMPORTANCE = 3;
+const MIN_IMPORTANCE = 1;
+
+// ── HMAC Integrity ──────────────────────────────────────────
+
+function getHmacKey() {
+  // Key from env takes priority (for CI/deployment)
+  if (process.env.AGENT_MEMORY_HMAC_KEY) {
+    return process.env.AGENT_MEMORY_HMAC_KEY;
+  }
+  // Otherwise generate and persist a key per project
+  ensureDir();
+  if (existsSync(KEY_FILE)) {
+    return readFileSync(KEY_FILE, "utf-8").trim();
+  }
+  const key = randomBytes(32).toString("hex");
+  writeFileSync(KEY_FILE, key, { encoding: "utf-8", mode: 0o600 });
+  return key;
+}
+
+function computeHmac(data) {
+  return createHmac("sha256", getHmacKey()).update(data, "utf-8").digest("hex");
+}
+
+function verifyHmac(data) {
+  if (!existsSync(HMAC_FILE)) {
+    // No HMAC file: only trust if data file is also empty/missing (true first run)
+    // If data exists without HMAC → possible tampering (attacker deleted .hmac)
+    if (existsSync(MEMORIES_FILE) && data.trim().length > 0) {
+      console.error("[SECURITY] Memory data exists without HMAC file — possible tampering (HMAC deleted)");
+      // Auto-heal: create HMAC for existing data (migration from pre-HMAC version)
+      // This is a one-time migration — after this, HMAC is required
+      saveHmac(data);
+      return true;
+    }
+    return true; // genuinely empty first run
+  }
+  const stored = readFileSync(HMAC_FILE, "utf-8").trim();
+  const computed = computeHmac(data);
+  // Timing-safe comparison to prevent timing attacks
+  try {
+    const storedBuf = Buffer.from(stored, "hex");
+    const computedBuf = Buffer.from(computed, "hex");
+    if (storedBuf.length !== computedBuf.length || !timingSafeEqual(storedBuf, computedBuf)) {
+      console.error("[SECURITY] Memory file integrity check FAILED — possible tampering detected");
+      return false;
+    }
+  } catch {
+    console.error("[SECURITY] Memory HMAC comparison error — rejecting data");
+    return false;
+  }
+  return true;
+}
+
+function saveHmac(data) {
+  writeFileSync(HMAC_FILE, computeHmac(data), "utf-8");
+}
+
+// ── Core ─────────────────────────────────────────────────────
+
+// PERF-01: In-process cache — load file once, invalidate on write
+let _cache = null;
+
+/** Invalidate in-memory cache (call after external file modification) */
+export function invalidateCache() { _cache = null; }
+
+function ensureDir() {
+  if (!existsSync(MEMORY_DIR)) mkdirSync(MEMORY_DIR, { recursive: true });
+}
+
+export function loadAll() {
+  if (_cache !== null) return _cache;
+  ensureDir();
+  if (!existsSync(MEMORIES_FILE)) { _cache = []; return _cache; }
+  const raw = readFileSync(MEMORIES_FILE, "utf-8");
+
+  // Verify integrity before parsing
+  if (!verifyHmac(raw)) {
+    // Tampering detected — return empty to prevent poisoned memory from loading
+    _cache = [];
+    return _cache;
+  }
+
+  const lines = raw.split("\n").filter(Boolean);
+  _cache = lines.map((line) => {
+    try { return JSON.parse(line); } catch { return null; }
+  }).filter(Boolean);
+  return _cache;
+}
+
+function atomicWrite(data) {
+  ensureDir();
+  const tmp = MEMORIES_FILE + ".tmp";
+  writeFileSync(tmp, data, "utf-8");
+  renameSync(tmp, MEMORIES_FILE);
+  // Update HMAC after successful write
+  saveHmac(data);
+}
+
+function saveAll(memories) {
+  const data = memories.map((m) => JSON.stringify(m)).join("\n") + (memories.length ? "\n" : "");
+  atomicWrite(data);
+  _cache = memories; // update cache after write
+}
+
+function nextId(memories) {
+  if (!memories.length) return 1;
+  let max = 0;
+  for (const m of memories) {
+    if (m.id > max) max = m.id;
+  }
+  return max + 1;
+}
+
+function clampImportance(val) {
+  const n = typeof val === "number" ? val : MIN_IMPORTANCE;
+  return Math.max(MIN_IMPORTANCE, Math.min(MAX_IMPORTANCE, Math.round(n)));
+}
+
+export function insertMemory({ content, category = "general", importance = 1, sessionId = null }) {
+  if (!content || typeof content !== "string") {
+    throw new Error("Memory content must be a non-empty string");
+  }
+  const trimmed = content.slice(0, MAX_CONTENT_LENGTH);
+  const memories = loadAll();
+  const entry = {
+    id: nextId(memories),
+    content: trimmed,
+    category: String(category || "general"),
+    importance: clampImportance(importance),
+    session_id: sessionId,
+    created_at: new Date().toISOString(),
+  };
+  memories.push(entry);
+  saveAll(memories);
+  return entry;
+}
+
+export function searchMemories(query, limit = 10) {
+  if (!query || typeof query !== "string") return [];
+  const terms = query.toLowerCase().split(/\s+/).filter(Boolean);
+  if (!terms.length) return [];
+
+  const memories = loadAll();
+  const scored = [];
+
+  for (const m of memories) {
+    const text = `${m.content} ${m.category}`.toLowerCase();
+    let score = 0;
+    for (const term of terms) {
+      if (text.includes(term)) score++;
+    }
+    if (score === 0) continue;
+    score += (m.importance - 1) * 0.5;
+    scored.push({ ...m, score });
+  }
+
+  scored.sort((a, b) => b.score - a.score);
+  return scored.slice(0, limit);
+}
+
+export function getMemory(id) {
+  const memories = loadAll();
+  return memories.find((m) => m.id === id) || null;
+}
+
+export function listRecentMemories(limit = 10) {
+  const memories = loadAll();
+  return memories.slice(-limit).reverse();
+}
+
+export function deleteMemory(id) {
+  const memories = loadAll();
+  const idx = memories.findIndex((m) => m.id === id);
+  if (idx === -1) return false;
+  memories.splice(idx, 1);
+  saveAll(memories);
+  return true;
+}
+
+export function getMemoryStats() {
+  const memories = loadAll();
+  const byCategory = {};
+  for (const m of memories) {
+    byCategory[m.category] = (byCategory[m.category] || 0) + 1;
+  }
+  return {
+    total: memories.length,
+    byCategory,
+    oldest: memories[0]?.created_at || null,
+    newest: memories[memories.length - 1]?.created_at || null,
+  };
+}
+
+export function clearAllMemories() {
+  saveAll([]);
+}

package/src/memory/tools.js

@@ -0,0 +1,196 @@
+import { insertMemory, searchMemories, getMemory, listRecentMemories, deleteMemory } from "./store.js";
+import { updateMemoryMd } from "./markdown.js";
+import { createSkill, updateSkill, removeSkill, listSkillSummaries } from "./skills.js";
+import { searchFts, getSkill as getSkillById } from "./sqlite-store.js";
+
+export const tools = [
+  {
+    type: "function",
+    function: {
+      name: "memory_write",
+      description: "Save a fact, decision, or observation to persistent memory. Use this when you learn something important that should be remembered across sessions.",
+      parameters: {
+        type: "object",
+        properties: {
+          content: { type: "string", description: "The information to remember" },
+          category: { type: "string", description: "Category: facts, decisions, preferences, bugs, patterns", default: "general" },
+          importance: { type: "integer", description: "1-3, where 3 is critical", default: 1 },
+        },
+        required: ["content"],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "memory_search",
+      description: "Search persistent memory for relevant information. Use this to recall previously saved knowledge.",
+      parameters: {
+        type: "object",
+        properties: {
+          query: { type: "string", description: "Search query" },
+          limit: { type: "integer", description: "Max results", default: 10 },
+        },
+        required: ["query"],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "memory_get",
+      description: "Get a specific memory by ID or list recent memories.",
+      parameters: {
+        type: "object",
+        properties: {
+          id: { type: "integer", description: "Memory ID to retrieve" },
+          last: { type: "integer", description: "Get last N memories instead" },
+        },
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "memory_delete",
+      description: "Delete a memory by ID.",
+      parameters: {
+        type: "object",
+        properties: {
+          id: { type: "integer", description: "Memory ID to delete" },
+        },
+        required: ["id"],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "skill_add",
+      description: "Create a reusable skill — a named procedure the agent remembers permanently. Use when you learn a multi-step workflow the user might need again (e.g. 'how we deploy', 'how to run tests for this project').",
+      parameters: {
+        type: "object",
+        properties: {
+          name: { type: "string", description: "Short name for the skill, e.g. 'Deploy to staging'" },
+          content: { type: "string", description: "Full procedure in markdown. Include steps, commands, and caveats." },
+          tags: { type: "string", description: "Comma-separated tags, e.g. 'deploy, aws, staging'" },
+        },
+        required: ["name", "content"],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "skill_update",
+      description: "Update the content of an existing skill.",
+      parameters: {
+        type: "object",
+        properties: {
+          id: { type: "string", description: "Skill ID (from skill_list)" },
+          content: { type: "string", description: "New content replacing the old" },
+        },
+        required: ["id", "content"],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "skill_remove",
+      description: "Delete a skill permanently.",
+      parameters: {
+        type: "object",
+        properties: {
+          id: { type: "string", description: "Skill ID to delete" },
+        },
+        required: ["id"],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "memory_expand",
+      description: "Retrieve full content of a skill or memory entry by ID. Use when the system prompt shows a skill index (§ heading) and you need the full procedure. Alias: load_skill.",
+      parameters: {
+        type: "object",
+        properties: {
+          id: { type: "string", description: "Entry ID (skill-xxx or memory ID)" },
+          layer: { type: "string", description: "Layer: skills, reflections, patterns, facts, user_model", default: "skills" },
+        },
+        required: ["id"],
+      },
+    },
+  },
+];
+
+export const handlers = {
+  memory_write(args) {
+    const entry = insertMemory({
+      content: args.content,
+      category: args.category ?? "general",
+      importance: args.importance ?? 1,
+    });
+    updateMemoryMd();
+    return `Saved memory #${entry.id} [${entry.category}]: ${entry.content}`;
+  },
+
+  memory_search(args) {
+    const results = searchMemories(args.query, args.limit ?? 10);
+    if (!results.length) return "No memories found matching your query.";
+    return results.map((m) => {
+      const date = m.created_at?.slice(0, 10) || "";
+      return `[#${m.id}] (${m.category}) ${date}: ${m.content} [score: ${m.score.toFixed(1)}]`;
+    }).join("\n");
+  },
+
+  memory_get(args) {
+    if (args.id != null) {
+      const m = getMemory(args.id);
+      if (!m) return `Memory #${args.id} not found.`;
+      return `[#${m.id}] (${m.category}) ${m.created_at}: ${m.content} [importance: ${m.importance}]`;
+    }
+    const recent = listRecentMemories(args.last ?? 5);
+    if (!recent.length) return "No memories stored yet.";
+    return recent.map((m) => `[#${m.id}] (${m.category}) ${m.created_at?.slice(0, 10)}: ${m.content}`).join("\n");
+  },
+
+  memory_delete(args) {
+    const ok = deleteMemory(args.id);
+    if (ok) {
+      updateMemoryMd();
+      return `Deleted memory #${args.id}.`;
+    }
+    return `Memory #${args.id} not found.`;
+  },
+
+  skill_add(args) {
+    const tags = args.tags ? args.tags.split(",").map(t => t.trim()).filter(Boolean) : [];
+    const id = createSkill(args.name, args.content, tags);
+    return `Created skill "${args.name}" (${id}). Stored as markdown in ~/.flint/memory/skills/ and indexed for search.`;
+  },
+
+  skill_update(args) {
+    const ok = updateSkill(args.id, args.content);
+    return ok ? `Updated skill ${args.id}.` : `Skill ${args.id} not found.`;
+  },
+
+  skill_remove(args) {
+    const ok = removeSkill(args.id);
+    return ok ? `Removed skill ${args.id} (file deleted).` : `Skill ${args.id} not found.`;
+  },
+
+  memory_expand(args) {
+    const layer = args.layer || "skills";
+    if (layer === "skills") {
+      const skill = getSkillById(args.id);
+      if (!skill) return `Skill ${args.id} not found.`;
+      return `# ${skill.name}\n\n${skill.content}`;
+    }
+    // For other layers — FTS search by id
+    const results = searchFts(args.id, { limit: 1 });
+    if (results.length) return results[0].text;
+    return `Entry ${args.id} not found in layer ${layer}.`;
+  },
+};

package/src/memory/user-model.js

@@ -0,0 +1,86 @@
+// User Model — Layer 5 of memory architecture (dialectic).
+//
+// Builds a profile of the user over time by observing conversations.
+// Traits: communication style, expertise level, preferences, recurring topics.
+// Not explicit "user said X" (that's facts L4). This is "agent OBSERVED X".
+//
+// Inspired by Honcho (Plastic Labs) but implemented in own SQLite, no dependency.
+// Internal task reference removed.
+
+import { upsertTrait, getAllTraits, indexEntry } from "./sqlite-store.js";
+
+/**
+ * Observe a user message and update traits.
+ * Called on each user turn. Lightweight language-neutral observations only.
+ *
+ * - Communication style (terse / verbose) from word count
+ * - Language (ru / en / mixed) from unicode block presence
+ *
+ * Expertise level and role hint used to be guessed from hardcoded RU/EN keyword
+ * regex (git rebase|API|CSS|...). Removed 2026-04-19 — biased to English,
+ * misses Ukrainian/German/Chinese users, and is the LLM's job anyway. Those
+ * traits are inferred by the LLM-based fact extractor at session end
+ * (see src/memory/extract-facts.js, category "preference" / "person").
+ *
+ * @param {string} userMessage
+ * @param {object} context — { toolsUsed, responseLanguage, sessionLength }
+ */
+export function observeUser(userMessage, context = {}) {
+  if (!userMessage || userMessage.length < 5) return;
+
+  // Communication style — word count is language-neutral
+  const words = userMessage.split(/\s+/).length;
+  if (words <= 5) {
+    updateTrait("communication_style", "terse", `message: ${words} words`, 0.6);
+  } else if (words > 50) {
+    updateTrait("communication_style", "verbose", `message: ${words} words`, 0.6);
+  }
+
+  // Language — unicode block observation (ru / en / mixed).
+  // This is observation of the user's script, not interpretation of intent.
+  const hasCyrillic = /\p{Script=Cyrillic}/u.test(userMessage);
+  const hasLatin = /\p{Script=Latin}/u.test(userMessage);
+  const hasCjk = /\p{Script=Han}|\p{Script=Hiragana}|\p{Script=Katakana}|\p{Script=Hangul}/u.test(userMessage);
+  const hasArabic = /\p{Script=Arabic}/u.test(userMessage);
+  let lang = null;
+  if (hasCjk) lang = hasLatin ? "cjk+latin" : "cjk";
+  else if (hasArabic) lang = hasLatin ? "arabic+latin" : "arabic";
+  else if (hasCyrillic && !hasLatin) lang = "cyrillic";
+  else if (hasLatin && !hasCyrillic) lang = "latin";
+  else if (hasCyrillic && hasLatin) lang = "cyrillic+latin";
+  if (lang) updateTrait("language", lang, "script observation", 0.7);
+}
+
+function updateTrait(trait, value, evidence, confidence) {
+  upsertTrait(trait, value, evidence, confidence);
+  indexEntry("user_model", `trait-${trait}`, `${trait}: ${value}`);
+}
+
+/**
+ * Get current user profile as structured object.
+ * @returns {Record<string, {value, confidence, evidence}>}
+ */
+export function getProfile() {
+  const traits = getAllTraits();
+  const profile = {};
+  for (const t of traits) {
+    profile[t.trait] = { value: t.value, confidence: t.confidence, evidence: t.evidence };
+  }
+  return profile;
+}
+
+/**
+ * Format user model as system-prompt-ready block.
+ * Only includes traits with confidence >= threshold.
+ * @param {number} minConfidence
+ * @returns {string}
+ */
+export function formatForPrompt(minConfidence = 0.5) {
+  const traits = getAllTraits().filter(t => t.confidence >= minConfidence);
+  if (!traits.length) return "";
+  const lines = ["User profile (observed, adapt your behavior):"];
+  for (const t of traits) {
+    lines.push(`  § ${t.trait}: ${t.value} (${Math.round(t.confidence * 100)}%)`);
+  }
+  return lines.join("\n");
+}