flint-agent 1.14.0

package/src/memory/rules.js

@@ -0,0 +1,105 @@
+// Rules memory — Layer 3 of memory architecture.
+//
+// Aggregates `better` bullets from ALL reflections (Layer 1) into a
+// stable ruleset: groups semantically-similar lessons, counts frequency,
+// surfaces top-N canonical rules.
+//
+// Purpose: Layer 1 injects last-5 reflections — random recent slice.
+// Layer 3 injects the most-repeated lessons across the full history —
+// load-bearing advice that shows up session after session. Answers the
+// question "what has the agent learned most often to do differently?"
+
+import { loadAll as loadAllReflections } from "./reflections.js";
+import { tokenize } from "./patterns.js";
+
+function jaccard(setA, setB) {
+  let inter = 0;
+  for (const x of setA) if (setB.has(x)) inter++;
+  const union = setA.size + setB.size - inter;
+  return union === 0 ? 0 : inter / union;
+}
+
+/**
+ * Cluster bullets by Jaccard token similarity (greedy agglomerative).
+ * Two bullets merge if overlap ≥ threshold with any cluster member.
+ */
+function clusterByJaccard(bullets, threshold = 0.4) {
+  const items = bullets.map(b => ({ text: b.text, tags: b.tags, tokens: new Set(tokenize(b.text)) }));
+  const clusters = [];
+  for (const item of items) {
+    if (item.tokens.size === 0) continue;
+    let matched = null;
+    for (const c of clusters) {
+      for (const m of c.members) {
+        if (jaccard(item.tokens, m.tokens) >= threshold) {
+          matched = c;
+          break;
+        }
+      }
+      if (matched) break;
+    }
+    if (matched) {
+      matched.members.push(item);
+      for (const t of item.tags) matched.tags.add(t);
+    } else {
+      clusters.push({ members: [item], tags: new Set(item.tags) });
+    }
+  }
+  return clusters;
+}
+
+/**
+ * Compile top-N stable rules from all reflections.
+ * @param {object} opts
+ * @param {number} opts.topN — max rules to return (default 8)
+ * @param {number} opts.minCount — min occurrences to keep a cluster (default 2)
+ * @param {number} opts.similarity — Jaccard threshold for clustering (default 0.4)
+ * @returns {Array<{text:string, count:number, tags:string[]}>}
+ */
+export function compileRules(opts = {}) {
+  const topN = opts.topN || 8;
+  const minCount = opts.minCount || 2;
+  const similarity = opts.similarity || 0.4;
+
+  const reflections = loadAllReflections();
+  if (!reflections.length) return [];
+
+  const bullets = [];
+  for (const r of reflections) {
+    const tags = Array.isArray(r.tags) ? r.tags : [];
+    for (const b of (r.better || [])) {
+      if (typeof b !== "string" || b.length < 15) continue;
+      bullets.push({ text: b, tags });
+    }
+  }
+  if (!bullets.length) return [];
+
+  const clusters = clusterByJaccard(bullets, similarity);
+
+  const result = [];
+  for (const c of clusters) {
+    if (c.members.length < minCount) continue;
+    // Canonical = shortest member ≥ 30 chars, fallback to first
+    let canonical = c.members[0].text;
+    for (const m of c.members) {
+      if (m.text.length >= 30 && m.text.length < canonical.length) canonical = m.text;
+    }
+    result.push({ text: canonical, count: c.members.length, tags: [...c.tags].slice(0, 3) });
+  }
+  result.sort((a, b) => b.count - a.count);
+  return result.slice(0, topN);
+}
+
+/**
+ * Format rules as a prompt block.
+ * @param {Array<{text,count,tags}>} rules
+ */
+export function formatForPrompt(rules) {
+  if (!rules?.length) return "";
+  const lines = ["Stable rules distilled from past sessions (most-repeated lessons):"];
+  for (const r of rules) {
+    const tags = r.tags?.length ? ` [${r.tags.slice(0, 3).join(",")}]` : "";
+    lines.push(`  - (${r.count}×)${tags} ${r.text}`);
+  }
+  return lines.join("\n");
+}

package/src/memory/session-facts.js

@@ -0,0 +1,125 @@
+// Per-session fact storage — each session has its own facts file
+// Stored in sessions/{sessionId}.facts.jsonl
+// HMAC integrity protection against tampering
+
+import { readFileSync, writeFileSync, appendFileSync, existsSync, mkdirSync } from "node:fs";
+import { createHmac, randomBytes, timingSafeEqual } from "node:crypto";
+import path from "node:path";
+import { config } from "../config.js";
+
+function factsPath(sessionId) {
+  return path.join(config.sessionsDir, `${sessionId}.facts.jsonl`);
+}
+
+function hmacPath(sessionId) {
+  return path.join(config.sessionsDir, `${sessionId}.facts.hmac`);
+}
+
+function getFactsKey() {
+  if (process.env.AGENT_MEMORY_HMAC_KEY) return process.env.AGENT_MEMORY_HMAC_KEY;
+  const keyFile = path.join(config.sessionsDir, ".hmac-key");
+  if (existsSync(keyFile)) return readFileSync(keyFile, "utf-8").trim();
+  mkdirSync(config.sessionsDir, { recursive: true });
+  const key = randomBytes(32).toString("hex");
+  writeFileSync(keyFile, key, { encoding: "utf-8", mode: 0o600 });
+  return key;
+}
+
+function computeFactsHmac(data) {
+  return createHmac("sha256", getFactsKey()).update(data, "utf-8").digest("hex");
+}
+
+function verifyFactsHmac(sessionId, data) {
+  const hp = hmacPath(sessionId);
+  if (!existsSync(hp)) {
+    // If facts file has data but no HMAC → auto-heal (migration)
+    if (data.trim().length > 0) {
+      updateFactsHmac(sessionId);
+    }
+    return true;
+  }
+  const stored = readFileSync(hp, "utf-8").trim();
+  const computed = computeFactsHmac(data);
+  try {
+    const storedBuf = Buffer.from(stored, "hex");
+    const computedBuf = Buffer.from(computed, "hex");
+    if (storedBuf.length !== computedBuf.length || !timingSafeEqual(storedBuf, computedBuf)) {
+      console.error(`[SECURITY] Session facts integrity FAILED for ${sessionId} — possible tampering`);
+      return false;
+    }
+  } catch {
+    console.error(`[SECURITY] Session facts HMAC error for ${sessionId} — rejecting`);
+    return false;
+  }
+  return true;
+}
+
+function updateFactsHmac(sessionId) {
+  const fp = factsPath(sessionId);
+  if (!existsSync(fp)) return;
+  const data = readFileSync(fp, "utf-8");
+  writeFileSync(hmacPath(sessionId), computeFactsHmac(data), "utf-8");
+}
+
+export function saveSessionFact(sessionId, fact) {
+  if (!sessionId || !fact?.content) return;
+  mkdirSync(config.sessionsDir, { recursive: true });
+  const entry = {
+    content: fact.content,
+    category: fact.category || "auto",
+    source: fact.source || "compression",
+    created_at: new Date().toISOString(),
+  };
+  appendFileSync(factsPath(sessionId), JSON.stringify(entry) + "\n", "utf-8");
+  updateFactsHmac(sessionId);
+  return entry;
+}
+
+export function saveSessionFacts(sessionId, facts) {
+  if (!sessionId || !facts?.length) return;
+  for (const fact of facts) {
+    saveSessionFact(sessionId, fact);
+  }
+}
+
+export function loadSessionFacts(sessionId) {
+  if (!sessionId) return [];
+  const fp = factsPath(sessionId);
+  if (!existsSync(fp)) return [];
+  const raw = readFileSync(fp, "utf-8");
+
+  if (!verifyFactsHmac(sessionId, raw)) {
+    return []; // tampering detected — return empty
+  }
+
+  const lines = raw.split("\n").filter(Boolean);
+  return lines.map((line) => {
+    try { return JSON.parse(line); } catch { return null; }
+  }).filter(Boolean);
+}
+
+export function getSessionFactsSummary(sessionId, maxLines = 30) {
+  const facts = loadSessionFacts(sessionId);
+  if (!facts.length) return "";
+
+  const grouped = {};
+  for (const f of facts) {
+    const cat = f.category || "auto";
+    if (!grouped[cat]) grouped[cat] = [];
+    grouped[cat].push(f);
+  }
+
+  let summary = `Session facts (${facts.length}):\n`;
+  for (const [cat, items] of Object.entries(grouped)) {
+    summary += `[${cat}]\n`;
+    for (const f of items.slice(-maxLines)) {
+      summary += `- ${f.content}\n`;
+    }
+  }
+  return summary;
+}
+
+export function clearSessionFacts(sessionId) {
+  const fp = factsPath(sessionId);
+  if (existsSync(fp)) writeFileSync(fp, "", "utf-8");
+}

package/src/memory/skills.js

@@ -0,0 +1,191 @@
+// Skills memory — Layer 3 of memory architecture.
+//
+// Reusable procedures stored as markdown files in ~/.flint/memory/skills/.
+// Synced bidirectionally with SQLite skills table.
+// Human-editable: user can write/edit .md files directly.
+// Agent-editable: agent can create/update via tool actions.
+//
+// Internal task reference removed.
+
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync, unlinkSync } from "node:fs";
+import { join, basename, resolve } from "node:path";
+import { homedir } from "node:os";
+import { insertSkill, getAllSkills, getSkill, deleteSkill, indexEntry } from "./sqlite-store.js";
+
+const SKILLS_DIR = process.env.FLINT_DATA_DIR
+  ? join(resolve(process.env.FLINT_DATA_DIR), "memory", "skills")
+  : join(homedir(), ".flint", "memory", "skills");
+const MAX_SKILLS = 50;
+
+function ensureDir() {
+  if (!existsSync(SKILLS_DIR)) mkdirSync(SKILLS_DIR, { recursive: true });
+}
+
+/**
+ * Parse a markdown skill file. Optional YAML-like frontmatter:
+ *   ---
+ *   tags: deploy, aws
+ *   ---
+ *   # Skill title
+ *   Content...
+ */
+function parseSkillFile(filePath) {
+  const raw = readFileSync(filePath, "utf-8");
+  const fileName = basename(filePath, ".md");
+  let tags = [];
+  let content = raw;
+  let name = fileName;
+
+  // Extract frontmatter
+  const fmMatch = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+  if (fmMatch) {
+    const fm = fmMatch[1];
+    content = fmMatch[2].trim();
+    const tagsMatch = fm.match(/tags:\s*(.+)/);
+    if (tagsMatch) tags = tagsMatch[1].split(",").map(t => t.trim()).filter(Boolean);
+  }
+
+  // Extract title from first # heading
+  const titleMatch = content.match(/^#\s+(.+)/m);
+  if (titleMatch) name = titleMatch[1].trim();
+
+  return { id: `skill-${fileName}`, name, content, tags, source_file: filePath };
+}
+
+/**
+ * Sync markdown files → SQLite table.
+ * Called at startup. Adds new files, updates changed, removes deleted.
+ */
+export function syncFromDisk() {
+  ensureDir();
+  const files = readdirSync(SKILLS_DIR).filter(f => f.endsWith(".md"));
+  const dbSkills = getAllSkills();
+  const dbMap = new Map(dbSkills.map(s => [s.id, s]));
+  const fileIds = new Set();
+
+  for (const f of files) {
+    const filePath = join(SKILLS_DIR, f);
+    try {
+      const parsed = parseSkillFile(filePath);
+      fileIds.add(parsed.id);
+      const existing = dbMap.get(parsed.id);
+      // Insert or update if content changed
+      if (!existing || existing.content !== parsed.content) {
+        insertSkill(parsed);
+        indexEntry("skills", parsed.id, [parsed.name, parsed.content].join(" "));
+      }
+    } catch {}
+  }
+
+  // Remove skills from DB that no longer have files
+  for (const s of dbSkills) {
+    if (s.source_file && !fileIds.has(s.id)) {
+      deleteSkill(s.id);
+    }
+  }
+
+  return { synced: fileIds.size, total: files.length };
+}
+
+/**
+ * Create a new skill from agent (writes both DB + file).
+ * @param {string} name — human-readable name
+ * @param {string} content — markdown content
+ * @param {string[]} tags — optional tags
+ * @returns {string} skill id
+ */
+export function createSkill(name, content, tags = []) {
+  ensureDir();
+  // Enforce cap
+  const existing = getAllSkills();
+  if (existing.length >= MAX_SKILLS) {
+    // Evict oldest
+    const oldest = existing[existing.length - 1];
+    if (oldest) {
+      deleteSkill(oldest.id);
+      if (oldest.source_file && existsSync(oldest.source_file)) {
+        try { unlinkSync(oldest.source_file); } catch {}
+      }
+    }
+  }
+
+  const slug = name.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "-").slice(0, 60);
+  const id = `skill-${slug}`;
+  const filePath = join(SKILLS_DIR, `${slug}.md`);
+
+  // Build markdown with frontmatter
+  let md = "";
+  if (tags.length) {
+    md += `---\ntags: ${tags.join(", ")}\n---\n`;
+  }
+  md += `# ${name}\n\n${content}\n`;
+
+  writeFileSync(filePath, md, "utf-8");
+  insertSkill({ id, name, content, source_file: filePath, tags });
+  indexEntry("skills", id, [name, content].join(" "));
+  return id;
+}
+
+/**
+ * Update an existing skill content.
+ */
+export function updateSkill(id, content) {
+  const skill = getSkill(id);
+  if (!skill) return false;
+  skill.content = content;
+  insertSkill(skill);
+  indexEntry("skills", id, [skill.name, content].join(" "));
+  // Update file
+  if (skill.source_file && existsSync(skill.source_file)) {
+    let md = "";
+    if (skill.tags?.length) {
+      md += `---\ntags: ${skill.tags.join(", ")}\n---\n`;
+    }
+    md += `# ${skill.name}\n\n${content}\n`;
+    writeFileSync(skill.source_file, md, "utf-8");
+  }
+  return true;
+}
+
+/**
+ * Remove a skill (DB + file).
+ */
+export function removeSkill(id) {
+  const skill = getSkill(id);
+  if (!skill) return false;
+  deleteSkill(id);
+  if (skill.source_file && existsSync(skill.source_file)) {
+    try { unlinkSync(skill.source_file); } catch {}
+  }
+  return true;
+}
+
+/**
+ * List all skills as summaries (for prompt injection).
+ * @returns {Array<{id, name, tags}>}
+ */
+export function listSkillSummaries() {
+  return getAllSkills().map(s => ({ id: s.id, name: s.name, tags: s.tags }));
+}
+
+/**
+ * Format skills as a system-prompt-ready block.
+ * Progressive disclosure: only names + tags, agent uses memory_expand to get full content.
+ * @returns {string}
+ */
+export function formatForPrompt() {
+  const skills = getAllSkills();
+  if (!skills.length) return "";
+  const lines = ["Available skills (use load_skill or memory_expand to see full content):"];
+  for (const s of skills) {
+    const tagStr = s.tags?.length ? ` [${s.tags.join(", ")}]` : "";
+    // First non-empty line of content as description (~60 chars)
+    const desc = s.content
+      ? s.content.split("\n").find(l => l.trim() && !l.startsWith("#"))?.trim().slice(0, 60) || ""
+      : "";
+    lines.push(`  § ${s.name}${tagStr}${desc ? " — " + desc : ""}`);
+  }
+  return lines.join("\n");
+}
+
+export const __internal = { SKILLS_DIR, MAX_SKILLS };