dsh-plugin-wiki-tools 0.8.0 → 0.9.0

package/index.js

@@ -257,6 +257,37 @@ export function createTools(vault, options = {}) {
     presentCall: args => ({ card: 'generic', title: `Scaffold wiki vault (${args.mode})`, kind: 'other', rawInput: { mode: args.mode } }),
   })
 
+  const wikiArchive = defineTool({
+    name: 'wiki_archive',
+    description:
+      'Archive one cold raw source: move it from .raw/ to .archive/ (same subpath, file kept on disk) '
+      + 'and drop its manifest entry so future ingests treat it as new. Use when a source is no longer '
+      + 'active but should not be deleted. Pages derived from it are untouched.',
+    parameters: {
+      source_path: {
+        type: 'string',
+        required: true,
+        description: 'Vault-relative .raw/ source path, e.g. .raw/articles/note.md.',
+      },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: true,
+      },
+      render: (_args, value) => [{
+        type: 'text',
+        text: typeof value === 'object' && value !== null && 'archivedTo' in value
+          ? 'wiki_archive: ' + value.archivedFrom + ' → ' + value.archivedTo + ' (manifest entry dropped)'
+          : 'wiki_archive: failed',
+      }],
+    },
+    async execute(args) {
+      return await vault.archiveSource({ sourcePath: args.source_path })
+    },
+    presentCall: args => ({ card: 'generic', title: 'Archive source: ' + args.source_path, kind: 'other', rawInput: { source: args.source_path } }),
+  })
+
   const wikiLint = defineTool({
     name: 'wiki_lint',
     description:
@@ -280,7 +311,7 @@ export function createTools(vault, options = {}) {
     presentCall: () => ({ card: 'generic', title: 'Lint wiki vault', kind: 'other' }),
   })
 
-  return [wikiQuery, wikiWrite, wikiRename, wikiScaffold, wikiLint]
+  return [wikiQuery, wikiWrite, wikiRename, wikiScaffold, wikiArchive, wikiLint]
 }
 
 /**

package/lib/bm25.js

@@ -0,0 +1,107 @@
+/**
+ * BM25 ranking over wiki pages, the retrieval core the wiki-retrieve skill
+ * describes. Latin text tokenizes on word boundaries; CJK runs tokenize into
+ * character bigrams (plus unigrams) so Chinese queries rank without a
+ * segmenter. Exact-phrase matching stays in search.js as a complementary
+ * signal: BM25 ranks term overlap, not contiguous text.
+ *
+ * @module dsh-plugin-wiki-tools/lib/bm25
+ */
+
+const K1 = 1.2
+const B = 0.75
+
+/**
+ * Tokenize text into ranking terms: latin words lowercased, CJK unigrams plus
+ * bigrams.
+ * @param {string} text - any text.
+ * @returns {string[]} terms in order.
+ */
+export function tokenize(text) {
+  const source = typeof text === 'string' ? text : ''
+  const terms = []
+  let buffer = ''
+  const flush = () => {
+    if (buffer.length > 0) {
+      terms.push(buffer.toLowerCase())
+      buffer = ''
+    }
+  }
+  for (let index = 0; index < source.length; index += 1) {
+    const code = source.charCodeAt(index)
+    const isLatin = (code >= 0x41 && code <= 0x5a) || (code >= 0x61 && code <= 0x7a) || (code >= 0x30 && code <= 0x39)
+    const isCjk = code >= 0x4e00 && code <= 0x9fff
+    if (isLatin) {
+      buffer += source[index]
+    } else if (isCjk) {
+      flush()
+      terms.push(source[index])
+      const previous = source[index - 1]
+      if (previous !== undefined && previous.charCodeAt(0) >= 0x4e00 && previous.charCodeAt(0) <= 0x9fff) {
+        terms.push(previous + source[index])
+      }
+    } else {
+      flush()
+    }
+  }
+  flush()
+  return terms
+}
+
+/**
+ * Build a BM25 index over documents.
+ * @param {Map<string, string>} documents - docKey → searchable text.
+ * @returns {Bm25Index} the built index.
+ */
+export function buildIndex(documents) {
+  /** @type {Map<string, Map<string, number>>} */
+  const termFreqs = new Map()
+  /** @type {Map<string, number>} */
+  const lengths = new Map()
+  /** @type {Map<string, number>} */
+  const docFreq = new Map()
+  for (const [key, text] of documents) {
+    const counts = new Map()
+    for (const term of tokenize(text)) counts.set(term, (counts.get(term) ?? 0) + 1)
+    termFreqs.set(key, counts)
+    lengths.set(key, [...counts.values()].reduce((sum, n) => sum + n, 0))
+    for (const term of counts.keys()) docFreq.set(term, (docFreq.get(term) ?? 0) + 1)
+  }
+  const avgLength = lengths.size === 0 ? 0 : [...lengths.values()].reduce((sum, n) => sum + n, 0) / lengths.size
+  return { termFreqs, lengths, docFreq, avgLength, total: lengths.size }
+}
+
+/**
+ * @typedef {object} Bm25Index
+ * @property {Map<string, Map<string, number>>} termFreqs
+ * @property {Map<string, number>} lengths
+ * @property {Map<string, number>} docFreq
+ * @property {number} avgLength
+ * @property {number} total
+ */
+
+/**
+ * Score one query against every indexed document.
+ * @param {Bm25Index} index - built index.
+ * @param {string} query - raw query text.
+ * @returns {Map<string, number>} docKey → BM25 score; only nonzero scores included.
+ */
+export function rank(index, query) {
+  const terms = tokenize(query)
+  const scores = new Map()
+  if (terms.length === 0 || index.total === 0) return scores
+  for (const [key, counts] of index.termFreqs) {
+    let score = 0
+    const length = index.lengths.get(key) ?? 0
+    const normalizer = K1 * (1 - B + B * (length / (index.avgLength || 1)))
+    for (const term of terms) {
+      const freq = counts.get(term)
+      if (freq === undefined) continue
+      const df = index.docFreq.get(term) ?? 0
+      const idf = Math.log(1 + (index.total - df + 0.5) / (df + 0.5))
+      score += idf * (freq * (K1 + 1)) / (freq + normalizer)
+    }
+    if (score > 0) scores.set(key, score)
+  }
+  return scores
+}

package/lib/search.js

@@ -9,6 +9,7 @@
 import { readFile } from 'node:fs/promises'
 import { join } from 'node:path'
 import { buildAliasMap, collectMarkdown, isMachineryPage, resolveLinkTarget } from './vault.js'
+import { buildIndex, rank } from './bm25.js'
 
 /**
  * Answer the quick mode: the hot cache and master index verbatim. The caller
@@ -54,25 +55,30 @@ export async function searchVault(root, { query, limit = 10 }) {
       if (resolved !== undefined) inbound.get(resolved)?.push(page.name)
     }
   }
+  // BM25 ranks term overlap (latin words; CJK unigram+bigram); the substring
+  // bonuses below keep exact title/alias/tag phrases and partial words ahead,
+  // and a substring-only hit still surfaces when tokenization misses it.
+  const documents = new Map(searchable.map(page => [
+    page.name,
+    `${page.name} ${(page.aliases ?? []).join(' ')} ${tagsOf(page)} ${page.content}`,
+  ]))
+  const bm25Scores = rank(buildIndex(documents), query)
+
   const results = []
   let totalMatches = 0
   for (const page of searchable) {
-    const aliasText = (page.aliases ?? []).join(' ').toLowerCase()
-    const titleHits = (countOccurrences(page.name.toLowerCase(), needle)
-      + (aliasText.includes(needle) ? 1 : 0)) * 5
-    const tags = Array.isArray(page.fields?.tags) ? page.fields.tags.join(' ').toLowerCase() : ''
-    const tagHits = tags.includes(needle) ? 4 : 0
-    const headings = page.content.split('\n').filter(line => line.startsWith('#')).join('\n').toLowerCase()
-    const headingHits = countOccurrences(headings, needle) * 2
-    const body = page.content.toLowerCase()
-    const bodyHits = countOccurrences(body, needle)
-    const score = titleHits + tagHits + headingHits + bodyHits
-    if (score === 0) continue
+    const bm25 = bm25Scores.get(page.name) ?? 0
+    const titleHit = page.name.toLowerCase().includes(needle)
+      || (page.aliases ?? []).some(alias => alias.toLowerCase().includes(needle))
+    const tagHit = tagsOf(page).includes(needle)
+    const bodyHit = page.content.toLowerCase().includes(needle)
+    if (bm25 === 0 && !titleHit && !tagHit && !bodyHit) continue
     totalMatches += 1
+    const score = bm25 + (titleHit ? 5 : 0) + (tagHit ? 4 : 0) + (bodyHit ? 1 : 0)
     results.push({
       name: page.name,
       path: page.path,
-      score,
+      score: Math.round(score * 100) / 100,
       snippets: matchLines(page.content, needle).slice(0, 2),
       inbound: inbound.get(page.name) ?? [],
       outbound: new Set(page.links).size,
@@ -100,19 +106,12 @@ function matchLines(content, needle) {
   return lines
 }
 
+
 /**
- * Count substring occurrences.
- * @param {string} haystack - lowercased text.
- * @param {string} needle - lowercased substring.
- * @returns {number} occurrence count.
+ * A page's tags as one lowercase string for matching.
+ * @param {{ fields?: Record<string, unknown> }} page - collected page.
+ * @returns {string} space-joined tags.
  */
-function countOccurrences(haystack, needle) {
-  if (!haystack.includes(needle)) return 0
-  let count = 0
-  let index = haystack.indexOf(needle)
-  while (index >= 0) {
-    count += 1
-    index = haystack.indexOf(needle, index + needle.length)
-  }
-  return count
+function tagsOf(page) {
+  return Array.isArray(page.fields?.tags) ? page.fields.tags.join(' ').toLowerCase() : ''
 }

package/lib/vault.js

@@ -304,6 +304,41 @@ export class Vault {
     })
   }
 
+  /**
+   * Archive one raw source: move it from `.raw/` to `.archive/` (same
+   * subpath) and drop its manifest entry, the wiki skill's cold-source
+   * hygiene rule. Archived files leave the ingest set but stay on disk.
+   * @param {object} input - the archive request.
+   * @param {string} input.sourcePath - vault-relative `.raw/` path.
+   * @returns {Promise<{ archivedFrom: string, archivedTo: string }>}
+   */
+  async archiveSource({ sourcePath }) {
+    return await this.enqueue(join(this.root, '.archive'), async () => {
+      await this.assertRoot()
+      const absolute = isAbsolute(sourcePath) ? sourcePath : join(this.root, sourcePath)
+      const rel = relative(this.root, absolute).split(sep).join('/')
+      if (!rel.startsWith('.raw/')) {
+        throw new Error('wiki-tools: wiki_archive moves .raw/ sources only')
+      }
+      const raw = await readFile(absolute).catch(error => {
+        if (error.code === 'ENOENT') throw new Error(`wiki-tools: source ${sourcePath} not found under the vault`)
+        throw error
+      })
+      const destination = join(this.root, '.archive', rel.slice('.raw/'.length))
+      await mkdir(join(destination, '..'), { recursive: true })
+      await writeFile(destination, raw)
+      await rm(absolute)
+      const manifestPath = join(this.root, '.raw', '.manifest.json')
+      const manifest = await readJson(manifestPath, { sources: {} })
+      delete manifest.sources[rel]
+      await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8')
+      await this.prependLog(`## [${today()}] archive | ${rel}`, [
+        `- Archived: \`${rel}\` → \`.archive/${rel.slice('.raw/'.length)}\``,
+      ])
+      return { archivedFrom: rel, archivedTo: `.archive/${rel.slice('.raw/'.length)}` }
+    })
+  }
+
   /**
    * Remove one page's entry lines from the master index and its folder
    * `_index.md`, used when a rename replaces rather than refreshes.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-plugin-wiki-tools",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Native DeepSeek Harness tools for an Obsidian wiki vault: wiki_query, wiki_write, and wiki_lint implement the mechanical core (path routing, delta tracking, index/log bookkeeping, health checks) of the wiki skill suite.",
   "license": "MIT",
   "type": "module",