@syndash/research-vault-mcp 1.1.2 → 1.1.3

package/src/vault.ts

@@ -5,32 +5,49 @@
 import { readFileSync, readdirSync, existsSync, statSync } from 'fs'
 import { join, basename } from 'path'
 import { homedir } from 'os'
+import { getVaultEntry } from './vault_get.ts'
+import type { DecayScore, VaultEntry, VaultGetInput } from './types.ts'
+import { flagGuidance, passGuidance } from './guidance.ts'
+import { okEnvelope } from './response.ts'
+import {
+  coverageMetadata,
+  itemFreshness,
+  matchedFields,
+  queueFreshness,
+  releaseMetadata,
+  snippetFromContent,
+  staleVerdict,
+  whyMatched,
+} from './evidence_metadata.ts'
+
+const DEFAULT_VAULT_ROOT = `${homedir()}/Documents/Evensong/research-vault`
+
+function getVaultRoot(): string {
+  return process.env.VAULT_ROOT ?? DEFAULT_VAULT_ROOT
+}
+
+function getKnowledgeDir(): string {
+  return join(getVaultRoot(), 'knowledge')
+}
+
+function getRawDir(): string {
+  return join(getVaultRoot(), 'raw')
+}
 
-const VAULT_ROOT = process.env.VAULT_ROOT ?? `${homedir()}/Documents/Evensong/research-vault`
-const KNOWLEDGE_DIR = join(VAULT_ROOT, 'knowledge')
-const RAW_DIR = join(VAULT_ROOT, 'raw')
-const DECAY_PATH = join(VAULT_ROOT, '.meta', 'decay-scores.json')
-const TAXONOMY_PATH = join(VAULT_ROOT, 'knowledge', '_taxonomy.md')
-
-// ─── Types ───────────────────────────────────────────────────────────────────
-
-interface VaultEntry {
-  id: string
-  title: string
-  category: string
-  path: string
-  modified: string
-  size: number
+function getDecayPath(): string {
+  return join(getVaultRoot(), '.meta', 'decay-scores.json')
 }
 
-interface DecayScore {
-  itemId: string
-  score: number
-  lastAccess: string
-  accessCount: number
-  summaryLevel: 'deep' | 'shallow' | 'none'
-  nextReviewAt: string
-  difficulty: number
+function getTaxonomyPath(): string {
+  return join(getVaultRoot(), 'knowledge', '_taxonomy.md')
+}
+
+function getPackageJson(): { name?: string; version?: string } {
+  try {
+    return JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf-8'))
+  } catch {
+    return {}
+  }
 }
 
 // ─── Helpers ─────────────────────────────────────────────────────────────────
@@ -42,9 +59,16 @@ function normalizeId(raw: string): string {
     .replace(/\.md$/, '')
 }
 
+export function normalizeDecayScoresStore(parsed: unknown): DecayScore[] {
+  if (Array.isArray(parsed)) return parsed as DecayScore[]
+  if (parsed && typeof parsed === 'object') return Object.values(parsed) as DecayScore[]
+  return []
+}
+
 function loadDecayScores(): DecayScore[] {
   try {
-    return JSON.parse(readFileSync(DECAY_PATH, 'utf-8'))
+    const parsed = JSON.parse(readFileSync(getDecayPath(), 'utf-8'))
+    return normalizeDecayScoresStore(parsed)
   } catch {
     return []
   }
@@ -52,7 +76,7 @@ function loadDecayScores(): DecayScore[] {
 
 function loadTaxonomy(): string {
   try {
-    return readFileSync(TAXONOMY_PATH, 'utf-8')
+    return readFileSync(getTaxonomyPath(), 'utf-8')
   } catch {
     return ''
   }
@@ -80,12 +104,13 @@ function loadFileMeta(filePath: string): { title: string; modified: string; size
 
 function scanKnowledge(): VaultEntry[] {
   const entries: VaultEntry[] = []
-  if (!existsSync(KNOWLEDGE_DIR)) return entries
+  const knowledgeDir = getKnowledgeDir()
+  if (!existsSync(knowledgeDir)) return entries
 
-  const categories = readdirSync(KNOWLEDGE_DIR)
+  const categories = readdirSync(knowledgeDir)
   for (const cat of categories) {
     if (cat.startsWith('_')) continue
-    const catPath = join(KNOWLEDGE_DIR, cat)
+    const catPath = join(knowledgeDir, cat)
     if (!existsSync(catPath) || !statSync(catPath).isDirectory()) continue
 
     const subEntries = readdirSync(catPath)
@@ -125,18 +150,19 @@ function scanKnowledge(): VaultEntry[] {
 
 function scanRaw(): string[] {
   const pending: string[] = []
-  if (!existsSync(RAW_DIR)) return pending
+  const rawDir = getRawDir()
+  if (!existsSync(rawDir)) return pending
 
   try {
-    const entries = readdirSync(RAW_DIR)
+    const entries = readdirSync(rawDir)
     for (const entry of entries) {
       if (entry === '_inbox') {
-        const inbox = join(RAW_DIR, entry)
+        const inbox = join(rawDir, entry)
         if (existsSync(inbox)) {
           pending.push(...readdirSync(inbox).filter(f => /\.(md|pdf|txt)$/.test(f)))
         }
       } else if (/^\d{4}-\d{2}$/.test(entry)) {
-        const monthDir = join(RAW_DIR, entry)
+        const monthDir = join(rawDir, entry)
         if (existsSync(monthDir)) {
           pending.push(
             ...readdirSync(monthDir)
@@ -151,24 +177,121 @@ function scanRaw(): string[] {
   return pending
 }
 
+function scanRawQueueItems() {
+  const rawDir = getRawDir()
+  return scanRaw().map(item => {
+    let filePath = join(rawDir, item)
+    if (!existsSync(filePath)) filePath = join(rawDir, '_inbox', item)
+    let sourceMtime: string | null = null
+    try {
+      sourceMtime = statSync(filePath).mtime.toISOString()
+    } catch {}
+
+    const title = normalizeId(basename(item)).replace(/\.[^.]+$/, '')
+    return {
+      id: normalizeId(basename(item)).replace(/\.[^.]+$/, ''),
+      title,
+      source_mtime: sourceMtime,
+    }
+  })
+}
+
+function entryScoreAliases(entry: VaultEntry): Set<string> {
+  const stem = basename(entry.path).replace(/\.md$/, '')
+  const aliases = new Set([
+    entry.id,
+    stem,
+    normalizeId(entry.id),
+    normalizeId(stem),
+    entry.id.replace(/--/g, '-'),
+    stem.replace(/--/g, '-'),
+  ])
+
+  const datePrefixed = stem.match(/^\d{8}--(.+)$/)
+  if (datePrefixed) aliases.add(datePrefixed[1])
+
+  return aliases
+}
+
+function scoreAliases(score: DecayScore): Set<string> {
+  return new Set([
+    score.itemId,
+    normalizeId(score.itemId),
+    score.itemId.replace(/--/g, '-'),
+  ])
+}
+
+function scoreMatchesEntry(score: DecayScore, entry: VaultEntry): boolean {
+  const entryAliases = entryScoreAliases(entry)
+  return [...scoreAliases(score)].some(alias => entryAliases.has(alias))
+}
+
+function matchedScoresForEntries<T extends DecayScore>(scores: T[], entries: VaultEntry[]): T[] {
+  return scores.filter(score => entries.some(entry => scoreMatchesEntry(score, entry)))
+}
+
+function pendingRawQueueItems(entries: VaultEntry[]) {
+  const analyzedIds = new Set(entries.flatMap(entry => [...entryScoreAliases(entry)]))
+  return scanRawQueueItems().filter(item => !analyzedIds.has(normalizeId(item.id)))
+}
+
+function sourceRef(entry: VaultEntry): string {
+  return `vault://knowledge/${entry.category}/${entry.id}`
+}
+
+function readEntryContent(entry: VaultEntry): string {
+  try {
+    return readFileSync(entry.path, 'utf-8')
+  } catch {
+    return ''
+  }
+}
+
+function scoreForEntry(scoreMap: Map<string, DecayScore & { lastAnalyzedAt?: string }>, item: VaultEntry) {
+  return [...entryScoreAliases(item)]
+    .map(alias => scoreMap.get(alias))
+    .find(Boolean)
+}
+
 // ─── MCP Tools ───────────────────────────────────────────────────────────────
 
 const vaultTools = [
+  {
+    name: 'vault_get',
+    description: 'Read a bounded public-safe excerpt or capped content from a Research Vault note by id.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        id: { type: 'string', description: 'Research Vault note id returned by vault_search' },
+        include_content: { type: 'boolean', description: 'Return capped content instead of the default excerpt' },
+        max_chars: { type: 'number', description: 'Maximum returned content characters, capped at 12000' }
+      },
+      required: ['id']
+    },
+    call: async (args: VaultGetInput) => {
+      const result = getVaultEntry(args, scanKnowledge())
+      return {
+        content: [{ type: 'text', text: JSON.stringify(result.envelope, null, 2) }],
+        ...(result.isError ? { isError: true } : {})
+      }
+    }
+  },
+
   {
     name: 'vault_search',
     description: 'Search the Research Vault knowledge base. Returns analyzed papers with retention scores.',
     inputSchema: {
       type: 'object',
       properties: {
-        query: { type: 'string', description: 'Search query (matches title, category)' },
+        query: { type: 'string', description: 'Search query (matches title, content, id, and category)' },
         category: { type: 'string', description: 'Filter by category (e.g., "ai-agents/benchmarking")' },
         limit: { type: 'number', description: 'Max results (default 10)' }
       }
     },
     call: async ({ query, category, limit = 10 }: { query?: string; category?: string; limit?: number }) => {
       let items = scanKnowledge()
-      const scores = loadDecayScores()
-      const scoreMap = new Map(scores.map(s => [normalizeId(s.itemId), s]))
+      const scores = loadDecayScores() as Array<DecayScore & { lastAnalyzedAt?: string }>
+      const scoreMap = new Map(scores.flatMap(s => [...scoreAliases(s)].map(alias => [alias, s] as const)))
 
       if (category) {
         items = items.filter(item =>
@@ -177,17 +300,17 @@ const vaultTools = [
       }
 
       if (query) {
-        const q = query.toLowerCase()
-        items = items.filter(item =>
-          item.title.toLowerCase().includes(q) ||
-          item.id.toLowerCase().includes(q) ||
-          item.category.toLowerCase().includes(q)
-        )
+        items = items.filter(item => {
+          const content = readEntryContent(item)
+          return matchedFields({ ...item, content }, query).length > 0
+        })
       }
 
       const results = items.slice(0, limit).map(item => {
-        const sid = item.id.replace(/--/g, '-')
-        const score = scoreMap.get(item.id) || scoreMap.get(sid)
+        const content = readEntryContent(item)
+        const score = scoreForEntry(scoreMap, item)
+        const fields = matchedFields({ ...item, content }, query)
+        const freshness = itemFreshness({ ...item, score })
         return {
           id: item.id,
           title: item.title,
@@ -196,14 +319,41 @@ const vaultTools = [
           summaryLevel: score?.summaryLevel ?? null,
           nextReview: score?.nextReviewAt ?? null,
           accessCount: score?.accessCount ?? 0,
-          modified: item.modified
+          modified: item.modified,
+          matched_fields: fields,
+          why_matched: whyMatched({ ...item, content }, query, fields),
+          snippet: snippetFromContent(content, query, 280),
+          source_ref: sourceRef(item),
+          section_anchor: undefined,
+          canonical_group: undefined,
+          ...freshness,
         }
       })
 
+      const hasStale = results.some(result => result.freshness_verdict === 'FLAG')
+      const envelope = okEnvelope(
+        { query, category, results, total: results.length },
+        hasStale
+          ? flagGuidance(
+            'Search completed, but one or more results lack fresh analysis metadata.',
+            'Use source_ref for readonly follow-up and refresh analysis metadata in the operator lane if needed.',
+            'vault_get',
+          )
+          : passGuidance(
+            'Search completed with provenance and freshness metadata.',
+            'Use source_ref or vault_get for bounded follow-up evidence.',
+            'vault_get',
+          ),
+        {
+          freshness: hasStale ? 'Some search results are missing or stale analysis timestamps.' : 'Search result analysis metadata is fresh.',
+          provenance: 'vault_search result source_ref values are vault:// references without local paths.',
+        },
+      )
+
       return {
         content: [{
           type: 'text',
-          text: JSON.stringify({ query, category, results, total: results.length }, null, 2)
+          text: JSON.stringify(envelope, null, 2)
         }]
       }
     }
@@ -214,12 +364,21 @@ const vaultTools = [
     description: 'Get Research Vault health — item counts by decay level, top/bottom retention.',
     inputSchema: { type: 'object', properties: {} },
     call: async () => {
-      const scores = loadDecayScores()
+      const scores = loadDecayScores() as Array<DecayScore & { lastAnalyzedAt?: string }>
       const entries = scanKnowledge()
-      const deep = scores.filter(s => s.summaryLevel === 'deep')
-      const shallow = scores.filter(s => s.summaryLevel === 'shallow')
-      const none = scores.filter(s => s.summaryLevel === 'none')
-      const sorted = [...scores].sort((a, b) => b.score - a.score)
+      const matchedScores = matchedScoresForEntries(scores, entries)
+      const deep = matchedScores.filter(s => s.summaryLevel === 'deep')
+      const shallow = matchedScores.filter(s => s.summaryLevel === 'shallow')
+      const none = matchedScores.filter(s => s.summaryLevel === 'none')
+      const sorted = [...matchedScores].sort((a, b) => b.score - a.score)
+      const queueItems = pendingRawQueueItems(entries)
+      const coverage = coverageMetadata({
+        total: entries.length,
+        analyzed: matchedScores.length,
+        scores: matchedScores,
+        queueItems,
+      })
+      const release = releaseMetadata(process.env, getPackageJson())
 
       const top5 = sorted.slice(0, 5).map(s => {
         const sid = s.itemId.replace(/--/g, '-')
@@ -232,20 +391,46 @@ const vaultTools = [
         return { itemId: s.itemId, score: s.score, lastAccess: s.lastAccess.slice(0, 10), title: entry?.title || s.itemId }
       })
 
-      const pending = scanRaw()
+      const statusData = {
+        total: entries.length,
+        analyzed: matchedScores.length,
+        deep: deep.length,
+        shallow: shallow.length,
+        dormant: none.length,
+        pending_raw: queueItems.length,
+        top5,
+        bottom5,
+        ...coverage,
+        release,
+      }
+      const releaseFlag = release.freshness_verdict === 'FLAG'
+      const analysisFlag = staleVerdict(coverage.last_analyzed_at).verdict === 'FLAG'
+      const envelope = okEnvelope(
+        statusData,
+        releaseFlag || analysisFlag
+          ? flagGuidance(
+            releaseFlag
+              ? 'Vault status is available, but release freshness was not fully provided by the runtime environment.'
+              : 'Vault status is available, but analysis freshness is stale or missing.',
+            releaseFlag
+              ? 'Set RESEARCH_VAULT_NPM_LATEST_VERSION, RESEARCH_VAULT_NPM_MODIFIED_AT, and RESEARCH_VAULT_PUBLIC_REPO_URL in the runtime environment.'
+              : 'Run the operator analysis lane before relying on freshness-sensitive claims.',
+          )
+          : passGuidance(
+            'Vault status includes coverage, queue freshness, and release metadata.',
+            'Use pending_raw and oldest_pending_age to decide whether operator analysis is needed.',
+          ),
+        {
+          as_of: coverage.as_of,
+          freshness: analysisFlag ? 'Analysis freshness is stale or missing.' : 'Analysis freshness is within the accepted window.',
+          release: release.freshness_verdict,
+        },
+      )
+
       return {
         content: [{
           type: 'text',
-          text: JSON.stringify({
-            total: entries.length,
-            analyzed: scores.length,
-            deep: deep.length,
-            shallow: shallow.length,
-            dormant: none.length,
-            pending_raw: pending.length,
-            top5,
-            bottom5
-          }, null, 2)
+          text: JSON.stringify(envelope, null, 2)
         }]
       }
     }
@@ -261,27 +446,56 @@ const vaultTools = [
       }
     },
     call: async ({ count = 5 }: { count?: number } = {}) => {
-      const pending = scanRaw()
       const entries = scanKnowledge()
-      const analyzedIds = new Set(entries.map(e => normalizeId(e.id)))
-      const unanalyzed = pending.filter(p => {
-        const id = normalizeId(p)
-        return !analyzedIds.has(id)
-      })
+      const unanalyzed = pendingRawQueueItems(entries)
+      const freshness = queueFreshness(unanalyzed)
 
       if (unanalyzed.length === 0) {
-        return { content: [{ type: 'text', text: JSON.stringify({ message: 'Queue empty — all papers analyzed', analyzed: entries.length }) }] }
+        const envelope = okEnvelope(
+          {
+            message: 'Queue empty; all visible raw papers are analyzed.',
+            analyzed: entries.length,
+            pending: 0,
+            preview: [],
+            oldest_pending_age: null,
+            next_action: 'none',
+          },
+          passGuidance(
+            'Batch analysis queue is empty.',
+            'No operator batch analysis action is needed.',
+          ),
+        )
+        return { content: [{ type: 'text', text: JSON.stringify(envelope, null, 2) }] }
       }
 
+      const envelope = okEnvelope(
+        {
+          message: `${unanalyzed.length} papers pending analysis`,
+          pending: unanalyzed.length,
+          preview: unanalyzed.slice(0, count).map(item => ({
+            id: item.id,
+            title: item.title,
+          })),
+          oldest_pending_age: freshness.oldest_pending_age,
+          oldest_pending_at: freshness.oldest_pending_at,
+          next_action: 'operator_run_batch_analysis',
+        },
+        flagGuidance(
+          'Raw queue has pending items that require operator-side batch analysis.',
+          'Run the private operator batch analysis lane; this public response intentionally omits shell commands and local paths.',
+        ),
+        {
+          freshness: freshness.oldest_pending_age === null
+            ? 'Pending queue age could not be calculated.'
+            : `Oldest pending item is ${freshness.oldest_pending_age} days old.`,
+          provenance: 'Queue preview exposes ids and titles only.',
+        },
+      )
+
       return {
         content: [{
           type: 'text',
-          text: JSON.stringify({
-            message: `${unanalyzed.length} papers pending analysis`,
-            pending: unanalyzed.length,
-            preview: unanalyzed.slice(0, count),
-            hint: 'cd ~/Desktop/research-vault && bun run scripts/batch-analyze.ts --count N'
-          }, null, 2)
+          text: JSON.stringify(envelope, null, 2)
         }]
       }
     }
@@ -296,11 +510,22 @@ const vaultTools = [
       const entries = scanKnowledge()
       const catCounts: Record<string, number> = {}
       for (const e of entries) catCounts[e.category] = (catCounts[e.category] || 0) + 1
+      const envelope = okEnvelope(
+        { taxonomy, categories: catCounts },
+        passGuidance(
+          'Taxonomy loaded with public-safe evidence metadata.',
+          'Use category counts for readonly routing and vault_search for bounded follow-up evidence.',
+          'vault_search',
+        ),
+        {
+          provenance: 'Taxonomy response is sanitized through the public-safe envelope before serialization.',
+        },
+      )
 
       return {
         content: [{
           type: 'text',
-          text: JSON.stringify({ taxonomy, categories: catCounts }, null, 2)
+          text: JSON.stringify(envelope, null, 2)
         }]
       }
     }

package/src/vault_get.ts

@@ -0,0 +1,109 @@
+import { readFileSync, statSync } from 'fs'
+import { basename } from 'path'
+import { passGuidance } from './guidance.ts'
+import { errorEnvelope, okEnvelope } from './response.ts'
+import type { VaultEntry, VaultGetInput } from './types.ts'
+
+const DEFAULT_EXCERPT_CHARS = 1200
+const HARD_MAX_CHARS = 12000
+
+function fileStem(entry: VaultEntry): string {
+  return basename(entry.path).replace(/\.md$/, '')
+}
+
+function sourceRef(entry: VaultEntry): string {
+  return `vault://knowledge/${entry.category}/${entry.id}`
+}
+
+function requestedLimit(input: VaultGetInput): number {
+  const fallback = input.include_content ? HARD_MAX_CHARS : DEFAULT_EXCERPT_CHARS
+  const ceiling = input.include_content ? HARD_MAX_CHARS : DEFAULT_EXCERPT_CHARS
+  const requested = input.max_chars === undefined
+    ? fallback
+    : Number.isFinite(input.max_chars)
+      ? Math.max(0, Math.floor(input.max_chars))
+      : fallback
+  return Math.min(requested, ceiling)
+}
+
+function resolveEntry(
+  id: string,
+  entries: VaultEntry[],
+): { entry?: VaultEntry; reason?: string } {
+  const exact = entries.filter(entry => entry.id === id)
+  if (exact.length === 1) return { entry: exact[0] }
+  if (exact.length > 1) return { reason: 'Multiple Research Vault notes matched the requested id.' }
+
+  const stemMatches = entries.filter(entry => fileStem(entry) === id)
+  if (stemMatches.length === 1) return { entry: stemMatches[0] }
+  if (stemMatches.length > 1) return { reason: 'Multiple Research Vault notes matched the requested file stem.' }
+
+  return { reason: 'No Research Vault note matched the requested id.' }
+}
+
+export function getVaultEntry(
+  input: VaultGetInput,
+  entries: VaultEntry[],
+) {
+  const id = input.id?.trim()
+  if (!id) {
+    return {
+      envelope: errorEnvelope(
+        'vault_get requires a non-empty id.',
+        'Call vault_search first, then retry vault_get with an exact id from the search results.',
+        {},
+      ),
+      isError: true,
+    }
+  }
+
+  const resolved = resolveEntry(id, entries)
+  if (!resolved.entry) {
+    return {
+      envelope: errorEnvelope(
+        resolved.reason ?? 'No Research Vault note matched the requested id.',
+        'Call vault_search first, then retry vault_get with an exact id from the search results.',
+        {},
+      ),
+      isError: true,
+    }
+  }
+
+  const entry = resolved.entry
+  const content = readFileSync(entry.path, 'utf-8')
+  const stat = statSync(entry.path)
+  const limit = requestedLimit(input)
+  const body = content.slice(0, limit)
+  const truncated = content.length > body.length
+  const contentKind = input.include_content ? 'full' : 'excerpt'
+
+  return {
+    envelope: okEnvelope(
+      {
+        id: entry.id,
+        title: entry.title,
+        category: entry.category,
+        source_ref: sourceRef(entry),
+        content: body,
+        content_kind: contentKind,
+        truncated,
+        chars_returned: body.length,
+        total_chars: content.length,
+        max_chars: limit,
+        modified: stat.mtime.toISOString(),
+        size: stat.size,
+      },
+      passGuidance(
+        input.include_content
+          ? 'Returned bounded note content from the Research Vault read surface.'
+          : 'Returned a bounded excerpt from the Research Vault read surface.',
+        truncated
+          ? 'Use include_content with a larger max_chars only if this excerpt is insufficient.'
+          : 'Use the returned public-safe note reference for follow-up.',
+        truncated ? 'vault_get' : undefined,
+      ),
+      {},
+    ),
+    isError: false,
+  }
+}