@mseep/core 3.0.0

package/lib/engine/auto-linker.js

@@ -0,0 +1,177 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+  listNodes, loadNode, parseNodeFrontmatter,
+  appendEdge, loadEdges,
+} from './graph.js';
+
+/**
+ * Auto-links a newly created knowledge node to:
+ * 1. Matching project components (by keyword in title/content)
+ * 2. Related existing knowledge nodes (by semantic similarity via search)
+ *
+ * @param {object} opts
+ * @param {string} opts.nodeId - the new node's ID
+ * @param {string} opts.title - node title
+ * @param {string} [opts.content] - node content
+ * @param {string[]} [opts.tags] - node tags
+ * @param {string} [opts.nodesDir] - knowledge nodes directory
+ * @param {string} [opts.graphFile] - graph.jsonl path
+ * @returns {Promise<Array<{from: string, to: string, type: string, reason: string}>>}
+ */
+export async function autoLink({ nodeId, title, content = '', tags = [], nodesDir, graphFile }) {
+  const links = [];
+  const existingEdges = loadEdges({ graphFile });
+  const allNodeIds = listNodes({ nodesDir });
+  const today = new Date().toISOString().split('T')[0];
+
+  // Step 1: Component matching — find components whose name appears in title/content
+  const searchText = `${title} ${content} ${tags.join(' ')}`.toLowerCase();
+
+  for (const id of allNodeIds) {
+    if (id === nodeId) continue;
+    const raw = loadNode(id, { nodesDir });
+    if (!raw) continue;
+    const parsed = parseNodeFrontmatter(raw);
+
+    // Only match against components
+    if (parsed.type !== 'component') continue;
+
+    const componentName = (parsed.title ?? '').toLowerCase();
+    if (!componentName) continue;
+
+    // Check if component name (or significant words from it) appears in the new node's text
+    const nameWords = componentName.split(/[\s\-_]+/).filter(w => w.length > 2);
+    const matches = nameWords.some(word => searchText.includes(word));
+
+    if (matches && !edgeExists(existingEdges, nodeId, id)) {
+      const edge = { from: nodeId, to: id, type: 'applies-to', weight: 1.0, created: today };
+      appendEdge(edge, { graphFile });
+      links.push({ from: nodeId, to: id, type: 'applies-to', reason: `component name match: "${parsed.title}"` });
+    }
+  }
+
+  // Step 2: Knowledge matching — find related existing notes by title similarity
+  // Simple approach: check if significant words from the new title appear in existing node titles
+  const titleWords = title.toLowerCase().split(/[\s\-_]+/).filter(w => w.length > 3);
+
+  for (const id of allNodeIds) {
+    if (id === nodeId) continue;
+    if (links.length >= 3) break; // cap at 3 auto-links
+
+    const raw = loadNode(id, { nodesDir });
+    if (!raw) continue;
+    const parsed = parseNodeFrontmatter(raw);
+
+    // Skip components (already handled above)
+    if (parsed.type === 'component') continue;
+
+    const existingTitle = (parsed.title ?? '').toLowerCase();
+    if (!existingTitle) continue;
+
+    // Check for word overlap between titles
+    const existingWords = existingTitle.split(/[\s\-_]+/).filter(w => w.length > 3);
+    const overlap = titleWords.filter(w => existingWords.includes(w));
+
+    if (overlap.length >= 1 && !edgeExists(existingEdges, nodeId, id)) {
+      const edge = { from: nodeId, to: id, type: 'see-also', weight: 1.0, created: today };
+      appendEdge(edge, { graphFile });
+      links.push({ from: nodeId, to: id, type: 'see-also', reason: `related knowledge: "${parsed.title}"` });
+    }
+  }
+
+  return links;
+}
+
+/**
+ * Reverse auto-link: when a new component is created, find existing notes
+ * that mention the component name and link them.
+ */
+export async function autoLinkReverse({ componentId, componentTitle, nodesDir, graphFile }) {
+  const links = [];
+  const existingEdges = loadEdges({ graphFile });
+  const allNodeIds = listNodes({ nodesDir });
+  const today = new Date().toISOString().split('T')[0];
+  const nameWords = componentTitle.toLowerCase().split(/[\s\-_]+/).filter(w => w.length > 2);
+
+  for (const id of allNodeIds) {
+    if (id === componentId) continue;
+    const raw = loadNode(id, { nodesDir });
+    if (!raw) continue;
+    const parsed = parseNodeFrontmatter(raw);
+    if (parsed.type === 'component') continue;
+
+    const nodeText = `${parsed.title ?? ''} ${parsed.body ?? ''}`.toLowerCase();
+    const matches = nameWords.some(word => nodeText.includes(word));
+
+    if (matches && !edgeExists(existingEdges, id, componentId)) {
+      const edge = { from: id, to: componentId, type: 'applies-to', weight: 1.0, created: today };
+      appendEdge(edge, { graphFile });
+      links.push({ from: id, to: componentId, type: 'applies-to', reason: `mentions component: "${componentTitle}"` });
+    }
+  }
+
+  return links;
+}
+
+/**
+ * Creates see-also edges between knowledge nodes and skills when their tags overlap.
+ * Called during `booklib index` after indexing knowledge nodes.
+ *
+ * @param {object} opts
+ * @param {Array<{id: string, title: string, tags: string[]}>} opts.knowledgeNodes
+ * @param {Array<{name: string, tags: string[]}>} opts.skillTags
+ * @param {string} [opts.graphFile]
+ * @returns {Promise<Array<{from: string, to: string, type: string, reason: string}>>}
+ */
+export async function autoLinkSkills({ knowledgeNodes, skillTags, graphFile }) {
+  const links = [];
+  const existingEdges = loadEdges({ graphFile });
+  const today = new Date().toISOString().split('T')[0];
+
+  // For each knowledge node, check tag overlap with each skill
+  for (const node of knowledgeNodes) {
+    const nodeTagSet = new Set(node.tags || []);
+
+    // Extract significant words (>3 chars) from title
+    const titleWords = node.title.toLowerCase().split(/[\s\-_]+/).filter(w => w.length > 3);
+    for (const word of titleWords) {
+      nodeTagSet.add(word);
+    }
+
+    // Check overlap with each skill
+    for (const skill of skillTags) {
+      const skillTagSet = new Set(skill.tags || []);
+
+      // Count overlapping tags
+      const overlap = Array.from(nodeTagSet).filter(tag => skillTagSet.has(tag));
+
+      // Create edge if ≥2 tags overlap and edge doesn't already exist
+      if (overlap.length >= 2 && !edgeExists(existingEdges, node.id, skill.name)) {
+        const edge = {
+          from: node.id,
+          to: skill.name,
+          type: 'see-also',
+          weight: 1.0,
+          created: today,
+        };
+        appendEdge(edge, { graphFile });
+        links.push({
+          from: node.id,
+          to: skill.name,
+          type: 'see-also',
+          reason: `tag overlap: [${overlap.join(', ')}]`,
+        });
+      }
+    }
+  }
+
+  return links;
+}
+
+function edgeExists(edges, from, to) {
+  return edges.some(e =>
+    (e.from === from && e.to === to) ||
+    (e.from === to && e.to === from)
+  );
+}

package/lib/engine/bm25-index.js

@@ -0,0 +1,178 @@
+import fs from 'node:fs';
+
+const K1 = 1.5;
+const B = 0.75;
+const MIN_TOKEN_LENGTH = 2;
+
+/**
+ * Tokenizes text into lowercase alphanumeric terms of at least 2 characters.
+ * @param {string} text
+ * @returns {string[]}
+ */
+function tokenize(text) {
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(token => token.length >= MIN_TOKEN_LENGTH);
+}
+
+/**
+ * Counts term frequencies in a token array.
+ * @param {string[]} tokens
+ * @returns {Map<string, number>}
+ */
+function countTermFrequencies(tokens) {
+  const freq = new Map();
+  for (const token of tokens) {
+    freq.set(token, (freq.get(token) ?? 0) + 1);
+  }
+  return freq;
+}
+
+/**
+ * Computes BM25 IDF for a term.
+ * Formula: log((N - df + 0.5) / (df + 0.5) + 1)
+ * @param {number} docCount - Total number of documents
+ * @param {number} docFrequency - Number of documents containing the term
+ * @returns {number}
+ */
+function computeIdf(docCount, docFrequency) {
+  return Math.log((docCount - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
+}
+
+/**
+ * Scores a single document against a set of query terms using BM25.
+ * @param {Object} doc - Document with freq map and len
+ * @param {string[]} queryTerms
+ * @param {Map<string, number>} df - Document frequency map
+ * @param {number} docCount
+ * @param {number} avgDocLen
+ * @returns {number}
+ */
+function scoreBm25(doc, queryTerms, df, docCount, avgDocLen) {
+  let score = 0;
+  for (const term of queryTerms) {
+    const termFreq = doc.freq[term] ?? 0;
+    if (termFreq === 0) continue;
+
+    const docFrequency = df[term] ?? 0;
+    const idf = computeIdf(docCount, docFrequency);
+    const normalizedTf =
+      (termFreq * (K1 + 1)) /
+      (termFreq + K1 * (1 - B + B * (doc.len / avgDocLen)));
+
+    score += idf * normalizedTf;
+  }
+  return score;
+}
+
+/**
+ * BM25 full-text search index.
+ *
+ * Supports incremental document addition, JSON persistence, and
+ * Robertson BM25 ranking.
+ */
+export class BM25Index {
+  constructor() {
+    this._docs = [];
+    this._df = {};
+    this._avgLen = 0;
+    this._totalLen = 0;
+  }
+
+  /**
+   * Builds the index from scratch from an array of chunks.
+   * @param {{ text: string, metadata: object }[]} chunks
+   */
+  build(chunks) {
+    this._docs = [];
+    this._df = {};
+    this._avgLen = 0;
+    this._totalLen = 0;
+
+    for (const chunk of chunks) {
+      this._appendDoc(chunk);
+    }
+  }
+
+  /**
+   * Incrementally adds one document to the index, updating avgLen and df.
+   * @param {{ text: string, metadata: object }} chunk
+   */
+  add(chunk) {
+    this._appendDoc(chunk);
+  }
+
+  /**
+   * Searches the index and returns top-K results sorted by score descending.
+   * @param {string} query
+   * @param {number} topK
+   * @returns {{ score: number, text: string, metadata: object }[]}
+   */
+  search(query, topK = 20) {
+    if (this._docs.length === 0) return [];
+
+    const queryTerms = tokenize(query);
+    if (queryTerms.length === 0) return [];
+
+    const scoredDocs = this._docs.map(doc => ({
+      score: scoreBm25(doc, queryTerms, this._df, this._docs.length, this._avgLen),
+      text: doc.text,
+      metadata: doc.metadata,
+    }));
+
+    const matchingDocs = scoredDocs.filter(r => r.score > 0);
+    return matchingDocs
+      .sort((a, b) => b.score - a.score)
+      .slice(0, topK);
+  }
+
+  /**
+   * Persists the index to a JSON file.
+   * @param {string} filePath
+   */
+  save(filePath) {
+    const serialized = JSON.stringify({
+      docs: this._docs,
+      df: this._df,
+      avgLen: this._avgLen,
+    });
+    fs.writeFileSync(filePath, serialized, 'utf8');
+  }
+
+  /**
+   * Restores a BM25Index instance from a JSON file.
+   * @param {string} filePath
+   * @returns {BM25Index}
+   */
+  static load(filePath) {
+    const raw = fs.readFileSync(filePath, 'utf8');
+    const { docs, df, avgLen } = JSON.parse(raw);
+    const idx = new BM25Index();
+    idx._docs = docs;
+    idx._df = df;
+    idx._avgLen = avgLen;
+    idx._totalLen = docs.reduce((sum, d) => sum + d.len, 0);
+    return idx;
+  }
+
+  /**
+   * Internal: adds one document and updates df and avgLen.
+   * @param {{ text: string, metadata: object }} chunk
+   */
+  _appendDoc(chunk) {
+    const tokens = tokenize(chunk.text);
+    const freqMap = countTermFrequencies(tokens);
+    const freqObj = Object.fromEntries(freqMap);
+
+    for (const term of freqMap.keys()) {
+      this._df[term] = (this._df[term] ?? 0) + 1;
+    }
+
+    const len = tokens.length;
+    this._docs.push({ text: chunk.text, metadata: chunk.metadata, freq: freqObj, len });
+
+    this._totalLen += len;
+    this._avgLen = this._totalLen / this._docs.length;
+  }
+}

package/lib/engine/capture.js

@@ -0,0 +1,120 @@
+// lib/engine/capture.js
+import { spawnSync } from 'node:child_process';
+import { writeFileSync, readFileSync, unlinkSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+// ── AI prompt builders (exported for testing) ─────────────────────────────────
+
+/** Returns the prompt used to ask Claude to structure raw dictation into a clean note. */
+export function buildDictatePrompt(rawText) {
+  return `You are a knowledge management assistant. Structure the following raw notes into a clean markdown knowledge note.
+
+Raw input:
+${rawText}
+
+Return ONLY valid YAML frontmatter + markdown. The frontmatter must include:
+- title: (concise, descriptive title extracted or inferred from the content)
+- tags: (2-5 relevant tags as a YAML list)
+- type: note
+
+Fix grammar and typos. Preserve all meaning. Do not add information not present in the input.
+Return nothing except the markdown document starting with ---.`;
+}
+
+/** Returns the prompt used to summarize a conversation transcript into a structured note. */
+export function buildSummarizePrompt(transcript, title = '') {
+  const titleLine = title ? `Title: ${title}\n\n` : '';
+  return `You are a knowledge management assistant. Summarize this AI conversation into a structured knowledge note.
+
+${titleLine}Conversation:
+${transcript}
+
+Return ONLY a markdown document starting with ---. Include this frontmatter:
+- title: (descriptive title${title ? ` — use "${title}" as basis` : ''})
+- tags: (2-5 relevant tags)
+- type: note
+
+And these sections in the body:
+## Key Decisions
+(bullet list of decisions made)
+
+## Findings
+(bullet list of key findings or conclusions)
+
+## Context
+(1-2 sentences of background)
+
+Then append the full transcript inside a details element:
+<details>
+<summary>Full conversation transcript</summary>
+
+${transcript}
+
+</details>`;
+}
+
+// ── AI call (requires ANTHROPIC_API_KEY env var) ──────────────────────────────
+
+/** Calls claude-haiku via the Anthropic Messages API. Requires ANTHROPIC_API_KEY. */
+export async function callAnthropicAPI(prompt) {
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) {
+    throw new Error('ANTHROPIC_API_KEY not set. Export it or add to .env.local');
+  }
+  const response = await fetch('https://api.anthropic.com/v1/messages', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'x-api-key': apiKey,
+      'anthropic-version': '2023-06-01',
+    },
+    body: JSON.stringify({
+      model: 'claude-haiku-4-5-20251001',
+      max_tokens: 2048,
+      messages: [{ role: 'user', content: prompt }],
+    }),
+  });
+  if (!response.ok) {
+    const body = await response.text();
+    throw new Error(`Anthropic API ${response.status}: ${body}`);
+  }
+  const data = await response.json();
+  return data.content[0].text;
+}
+
+// ── Input helpers ─────────────────────────────────────────────────────────────
+
+/** Opens $EDITOR (or vi) with optional initial content. Returns edited content. */
+export function openEditor(initialContent = '') {
+  const tmpFile = join(tmpdir(), `booklib-edit-${Date.now()}.md`);
+  writeFileSync(tmpFile, initialContent, 'utf8');
+  const editor = process.env.EDITOR ?? 'vi';
+  spawnSync(editor, [tmpFile], { stdio: 'inherit' });
+  const content = readFileSync(tmpFile, 'utf8');
+  unlinkSync(tmpFile);
+  return content.trim();
+}
+
+/** Reads all of stdin when piped. Returns empty string when stdin is a TTY. */
+export async function readStdin() {
+  if (process.stdin.isTTY) return '';
+  return new Promise(resolve => {
+    let data = '';
+    process.stdin.setEncoding('utf8');
+    process.stdin.on('data', chunk => { data += chunk; });
+    process.stdin.on('end', () => resolve(data.trim()));
+  });
+}
+
+/** Prompts user to type input interactively until Ctrl+D. */
+export async function readInteractive(prompt = 'Type your note (Ctrl+D when done):') {
+  process.stdout.write(prompt + '\n');
+  return new Promise(resolve => {
+    let data = '';
+    process.stdin.setEncoding('utf8');
+    process.stdin.resume();
+    process.stdin.on('data', chunk => { data += chunk; });
+    process.stdin.on('end', () => resolve(data.trim()));
+  });
+}