@caoscompanybr/merlin 3.5.0

package/.merlin-core/core/alkimia/memory/thoughts-indexer.js

@@ -0,0 +1,477 @@
+/**
+ * Thoughts Indexer -- Scans thoughts/shared/ and builds searchable index.
+ *
+ * Generates INDEX.json (machine-readable) and INDEX.md (human-readable)
+ * for the ThoughtsProvider to use during bracket-aware memory retrieval.
+ *
+ * @module core/alkimia/memory/thoughts-indexer
+ * @version 3.0.0
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { estimateTokens } = require("../utils/tokens");
+
+const SUBDIRS = [
+  "plans",
+  "research",
+  "handoffs",
+  "decisions",
+  "reminders",
+  "objectives",
+  "learnings",
+  "processes",
+  "feedback",
+];
+const TYPE_MAP = {
+  plans: "plan",
+  research: "research",
+  handoffs: "handoff",
+  decisions: "decision",
+  reminders: "reminder",
+  objectives: "objective",
+  learnings: "learning",
+  processes: "process",
+  feedback: "feedback",
+};
+
+/** TTL for learnings: 30-day tactical lessons expire; permanent ones never do. */
+const LESSON_TTL_DAYS = 30;
+
+const TERMINAL_STATUSES = [
+  "concluded",
+  "achieved",
+  "abandoned",
+  "completed",
+  "complete",
+  "expired",
+  "done",
+  "milestone_complete",
+  "implemented",
+  "superseded",
+  "shipped",
+  "deployed",
+  "deprecated",
+];
+const ACTIVE_STATUSES = [
+  "active",
+  "pending",
+  "draft",
+  "acknowledged",
+  "in_progress",
+];
+const STALE_THRESHOLD_DAYS = 14;
+
+/**
+ * Classify thought lifecycle based on status, type, and age.
+ *
+ * @param {string|null} status - Frontmatter status field
+ * @param {string} type - Thought type (plan, handoff, etc.)
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {Object} [frontmatter] - Full frontmatter (for type-specific fields like ttl)
+ * @returns {"active"|"terminal"|"stale"}
+ */
+function classifyLifecycle(status, type, dateStr, frontmatter = {}) {
+  if (status && TERMINAL_STATUSES.includes(status.toLowerCase())) {
+    return "terminal";
+  }
+
+  // Learnings are permanent knowledge — never stale, UNLESS tagged with a
+  // 30-day TTL (tactical lessons) and now past it → terminal (expired).
+  if (type === "learning") {
+    if (frontmatter.ttl === "30d" && daysSinceDate(dateStr) > LESSON_TTL_DAYS) {
+      return "terminal";
+    }
+    return "active";
+  }
+
+  // Feedback is granular operational memory — active until FIFO-pruned (not aged out).
+  if (type === "feedback") return "active";
+
+  // Processes are permanent operating procedures — never stale. A process leaves the
+  // active set only via the terminal `deprecated` status (handled by the check above).
+  if (type === "process") return "active";
+
+  // No status or explicit active status — check staleness
+  if (!status || ACTIVE_STATUSES.includes(status.toLowerCase())) {
+    const daysSince = daysSinceDate(dateStr);
+    if (daysSince > STALE_THRESHOLD_DAYS) return "stale";
+    return "active";
+  }
+
+  return "active";
+}
+
+/**
+ * Calculate days since a date string.
+ */
+function daysSinceDate(dateStr) {
+  if (!dateStr || dateStr === "unknown") return 30;
+  try {
+    const then = new Date(dateStr);
+    const now = new Date();
+    return Math.max(0, Math.floor((now - then) / 86_400_000));
+  } catch {
+    return 30;
+  }
+}
+
+/**
+ * Parse YAML frontmatter from markdown content.
+ * Lightweight parser -- no external dependencies.
+ */
+function parseFrontmatter(content) {
+  if (!content.startsWith("---")) return { frontmatter: {}, body: content };
+
+  const end = content.indexOf("\n---", 3);
+  if (end === -1) return { frontmatter: {}, body: content };
+
+  const yamlBlock = content.slice(4, end).trim();
+  const body = content.slice(end + 4).trim();
+  const frontmatter = {};
+
+  for (const line of yamlBlock.split("\n")) {
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) continue;
+    const key = line.slice(0, colonIdx).trim();
+    let val = line.slice(colonIdx + 1).trim();
+    // Strip surrounding quotes
+    if (
+      (val.startsWith('"') && val.endsWith('"')) ||
+      (val.startsWith("'") && val.endsWith("'"))
+    ) {
+      val = val.slice(1, -1);
+    }
+    // Parse arrays: [tag1, tag2]
+    if (val.startsWith("[") && val.endsWith("]")) {
+      val = val
+        .slice(1, -1)
+        .split(",")
+        .map((s) => s.trim().replace(/^["']|["']$/g, ""));
+    }
+    frontmatter[key] = val;
+  }
+
+  return { frontmatter, body };
+}
+
+/**
+ * Extract keywords from document content.
+ * Sources: filename tokens, frontmatter tags, markdown headers.
+ */
+function extractKeywords(filename, frontmatter, body) {
+  const keywords = new Set();
+
+  // 1. Filename tokens (e.g. "2026-02-19-payments-firestore-cleanup.md" → payments, firestore, cleanup)
+  const nameTokens = path
+    .basename(filename, ".md")
+    .replace(/^\d{4}-\d{2}-\d{2}-/, "") // strip date prefix
+    .split(/[-_]/)
+    .filter((t) => t.length > 2);
+  for (const t of nameTokens) keywords.add(t.toLowerCase());
+
+  // 2. Frontmatter tags/keywords
+  const fmTags = frontmatter.tags || frontmatter.keywords || [];
+  const tagArray = Array.isArray(fmTags) ? fmTags : String(fmTags).split(",");
+  for (const t of tagArray) {
+    const trimmed = t.trim().toLowerCase();
+    if (trimmed) keywords.add(trimmed);
+  }
+
+  // 3. Frontmatter project/topic
+  if (frontmatter.project)
+    keywords.add(String(frontmatter.project).toLowerCase());
+  if (frontmatter.topic) {
+    for (const word of String(frontmatter.topic).toLowerCase().split(/\s+/)) {
+      if (word.length > 3) keywords.add(word);
+    }
+  }
+
+  // 4. Learning-specific: domain, category, applies_to (area-of-action matching)
+  if (frontmatter.domain)
+    keywords.add(String(frontmatter.domain).toLowerCase());
+  if (frontmatter.category)
+    keywords.add(String(frontmatter.category).toLowerCase());
+  const appliesTo = frontmatter.applies_to || [];
+  const appliesArray = Array.isArray(appliesTo)
+    ? appliesTo
+    : String(appliesTo).split(",");
+  for (const entry of appliesArray) {
+    const tokens = String(entry)
+      .trim()
+      .toLowerCase()
+      .replace(/[/\\]/g, " ")
+      .split(/[\s-_]+/)
+      .filter((t) => t.length > 2);
+    for (const t of tokens) keywords.add(t);
+  }
+
+  // 5. Markdown headers (## and ###)
+  const headerRegex = /^#{2,3}\s+(.+)$/gm;
+  let match;
+  while ((match = headerRegex.exec(body)) !== null) {
+    const headerWords = match[1]
+      .toLowerCase()
+      .replace(/[^a-z0-9\s-]/g, "")
+      .split(/\s+/)
+      .filter((w) => w.length > 3);
+    for (const w of headerWords) keywords.add(w);
+  }
+
+  return Array.from(keywords);
+}
+
+/**
+ * Extract section titles (## level) from body.
+ */
+function extractSections(body) {
+  const sections = [];
+  const regex = /^##\s+(.+)$/gm;
+  let match;
+  while ((match = regex.exec(body)) !== null) {
+    sections.push(match[1].trim());
+  }
+  return sections;
+}
+
+/**
+ * Extract title from document (first # heading or first line).
+ */
+function extractTitle(body, frontmatter) {
+  if (frontmatter.title) return frontmatter.title;
+
+  const h1Match = body.match(/^#\s+(.+)$/m);
+  if (h1Match) return h1Match[1].trim();
+
+  const firstLine = body.split("\n").find((l) => l.trim().length > 0);
+  return firstLine ? firstLine.trim().slice(0, 100) : "Untitled";
+}
+
+/**
+ * Extract summary (first 3 non-empty, non-heading lines after the title).
+ */
+function extractSummary(body) {
+  const lines = body.split("\n");
+  const summaryLines = [];
+  let pastTitle = false;
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!pastTitle && trimmed.startsWith("#")) {
+      pastTitle = true;
+      continue;
+    }
+    if (!pastTitle) continue;
+    if (!trimmed) continue;
+    if (trimmed.startsWith("#")) continue;
+    if (trimmed.startsWith("---")) continue;
+    if (trimmed.startsWith("|")) continue; // skip tables
+    summaryLines.push(trimmed);
+    if (summaryLines.length >= 3) break;
+  }
+
+  return summaryLines.join(" ").slice(0, 300);
+}
+
+/**
+ * Build index entry for a single markdown file.
+ */
+function indexFile(filePath, type) {
+  const content = fs.readFileSync(filePath, "utf8");
+  const { frontmatter, body } = parseFrontmatter(content);
+  const relativePath =
+    path.basename(path.dirname(filePath)) + "/" + path.basename(filePath);
+  const lineCount = content.split("\n").length;
+
+  // Extract date from filename (YYYY-MM-DD prefix)
+  const dateMatch = path.basename(filePath).match(/^(\d{4}-\d{2}-\d{2})/);
+  const date = dateMatch ? dateMatch[1] : frontmatter.date || "unknown";
+
+  // Generate deterministic citation ID: type:date:slug
+  const slug = path
+    .basename(filePath, ".md")
+    .replace(/^\d{4}-\d{2}-\d{2}-/, "")
+    .substring(0, 40);
+  const citationId = `${type}:${date}:${slug}`;
+
+  const status = frontmatter.status || null;
+  const lifecycle = classifyLifecycle(status, type, date, frontmatter);
+
+  return {
+    path: relativePath,
+    type,
+    citationId,
+    title: extractTitle(body, frontmatter),
+    date,
+    status,
+    lifecycle,
+    summary: extractSummary(body),
+    keywords: extractKeywords(path.basename(filePath), frontmatter, body),
+    sections: extractSections(body),
+    lineCount,
+    estimatedTokens: estimateTokens(content),
+    frontmatter,
+  };
+}
+
+/**
+ * Scan thoughts/shared/ and build the full index.
+ *
+ * @param {string} thoughtsPath - Path to .merlin-core/thoughts
+ * @returns {{ entries: object[], generated: string }}
+ */
+function buildIndex(thoughtsPath) {
+  const sharedPath = path.join(thoughtsPath, "shared");
+  const entries = [];
+
+  for (const subdir of SUBDIRS) {
+    const dirPath = path.join(sharedPath, subdir);
+    if (!fs.existsSync(dirPath)) continue;
+
+    const files = fs
+      .readdirSync(dirPath)
+      .filter((f) => f.endsWith(".md") && f !== ".gitkeep" && f !== "INDEX.md");
+
+    for (const file of files) {
+      try {
+        const entry = indexFile(path.join(dirPath, file), TYPE_MAP[subdir]);
+        entries.push(entry);
+      } catch (err) {
+        console.warn(
+          `[thoughts-indexer] Failed to index ${subdir}/${file}: ${err.message}`,
+        );
+      }
+    }
+  }
+
+  // Cross-reference: infer plan status from handoff references
+  const handoffPlanRefs = new Set();
+  for (const entry of entries) {
+    if (entry.type === "handoff" && entry.frontmatter?.plan) {
+      handoffPlanRefs.add(String(entry.frontmatter.plan).trim());
+    }
+  }
+  if (handoffPlanRefs.size > 0) {
+    for (const entry of entries) {
+      if (
+        entry.type === "plan" &&
+        !TERMINAL_STATUSES.includes((entry.status || "").toLowerCase())
+      ) {
+        const filename = path.basename(entry.path);
+        if (
+          handoffPlanRefs.has(filename) ||
+          handoffPlanRefs.has(entry.citationId)
+        ) {
+          entry.status = "concluded";
+          entry.lifecycle = "terminal";
+          entry.statusInferred = true;
+        }
+      }
+    }
+  }
+
+  // Sort by date descending (newest first)
+  entries.sort((a, b) => (b.date || "").localeCompare(a.date || ""));
+
+  return {
+    entries,
+    generated: new Date().toISOString(),
+    version: "3.0.0",
+  };
+}
+
+/**
+ * Write INDEX.json and INDEX.md to thoughts/shared/.
+ */
+function writeIndex(thoughtsPath) {
+  const sharedPath = path.join(thoughtsPath, "shared");
+  const index = buildIndex(thoughtsPath);
+
+  // Write JSON
+  const jsonPath = path.join(sharedPath, "INDEX.json");
+  fs.writeFileSync(jsonPath, JSON.stringify(index, null, 2));
+
+  // Write human-readable MD
+  const mdPath = path.join(sharedPath, "INDEX.md");
+
+  // Lifecycle summary counts
+  const lifecycleCounts = { active: 0, stale: 0, terminal: 0 };
+  for (const entry of index.entries) {
+    const lc = entry.lifecycle || "active";
+    if (lifecycleCounts[lc] !== undefined) lifecycleCounts[lc]++;
+  }
+
+  const LIFECYCLE_ICON = { active: "●", stale: "◐", terminal: "○" };
+
+  const lines = [
+    "# Thoughts Index",
+    "",
+    `> Auto-generated: ${index.generated} | ${index.entries.length} documents`,
+    "",
+    "## Lifecycle Summary",
+    "",
+    `- **● Active:** ${lifecycleCounts.active} items`,
+    `- **◐ Stale:** ${lifecycleCounts.stale} items (>${STALE_THRESHOLD_DAYS} days, no terminal status)`,
+    `- **○ Terminal:** ${lifecycleCounts.terminal} items (concluded/achieved/abandoned/completed/expired/done)`,
+    "",
+  ];
+
+  const byType = {};
+  for (const entry of index.entries) {
+    if (!byType[entry.type]) byType[entry.type] = [];
+    byType[entry.type].push(entry);
+  }
+
+  for (const [type, docs] of Object.entries(byType)) {
+    // Pluralize: words ending in 's' take 'es' (process → Processes), else 's'.
+    const singular = type.charAt(0).toUpperCase() + type.slice(1);
+    const plural = singular.endsWith("s") ? `${singular}es` : `${singular}s`;
+    lines.push(`## ${plural} (${docs.length})`);
+    lines.push("");
+    for (const doc of docs) {
+      const icon = LIFECYCLE_ICON[doc.lifecycle] || "●";
+      const inferred = doc.statusInferred ? " [inferred]" : "";
+      lines.push(
+        `- ${icon} **${doc.title}** (${doc.date}) — ${doc.lineCount} lines, ~${doc.estimatedTokens} tokens${inferred}`,
+      );
+      lines.push(
+        `  \`${doc.path}\` | Keywords: ${doc.keywords.slice(0, 8).join(", ")}`,
+      );
+    }
+    lines.push("");
+  }
+
+  fs.writeFileSync(mdPath, lines.join("\n"));
+
+  return index;
+}
+
+/**
+ * Compute keyword overlap (Jaccard similarity) between two keyword arrays.
+ * Used for contradiction detection between learnings.
+ *
+ * @param {string[]} kw1
+ * @param {string[]} kw2
+ * @returns {number} 0.0 (no overlap) to 1.0 (identical)
+ */
+function computeKeywordOverlap(kw1, kw2) {
+  const set1 = new Set(kw1);
+  const set2 = new Set(kw2);
+  const intersection = [...set1].filter((k) => set2.has(k));
+  const union = new Set([...set1, ...set2]);
+  return union.size > 0 ? intersection.length / union.size : 0;
+}
+
+module.exports = {
+  buildIndex,
+  writeIndex,
+  indexFile,
+  parseFrontmatter,
+  extractKeywords,
+  computeKeywordOverlap,
+  classifyLifecycle,
+  TERMINAL_STATUSES,
+  ACTIVE_STATUSES,
+};