dsh-plugin-discover 0.0.0-stage → 1.0.0

package/lib/store.js

@@ -0,0 +1,184 @@
+/**
+ * Index storage: the `current` pointer, manifests, the build lock, and
+ * delayed cleanup of superseded index directories.
+ */
+import {
+  existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync,
+  statSync, unlinkSync, writeFileSync,
+} from 'node:fs'
+import { hostname } from 'node:os'
+import { join } from 'node:path'
+import {
+  buildLogPath, currentPath, isIndexDirName, lockPath, newIndexDirName, resolveIndexDir,
+} from './paths.js'
+
+/** A lock older than this is treated as abandoned even if its PID is alive. */
+export const LOCK_STALE_MS = 30 * 60 * 1000
+
+/** Directory names that look like published indexes, newest first. */
+export function listIndexDirs(indexDir) {
+  let names
+  try {
+    names = readdirSync(indexDir, { withFileTypes: true })
+      .filter(entry => entry.isDirectory() && isIndexDirName(entry.name))
+      .map(entry => entry.name)
+  } catch {
+    return []
+  }
+  return names.sort((a, b) => b.localeCompare(a))
+}
+
+/** Read the `current` pointer; null when missing or malformed. */
+export function readCurrent(indexDir) {
+  try {
+    const name = readFileSync(currentPath(indexDir), 'utf8').trim()
+    if (!isIndexDirName(name)) return null
+    const dir = resolveIndexDir(indexDir, name)
+    if (!dir || !existsSync(join(dir, 'manifest.json')) || !existsSync(join(dir, 'records.json'))) return null
+    return { name, dir, mtimeMs: statSync(currentPath(indexDir)).mtimeMs }
+  } catch {
+    return null
+  }
+}
+
+/** Read the manifest behind the current pointer, if both are readable. */
+export function readManifest(indexDir) {
+  const current = readCurrent(indexDir)
+  if (!current) return null
+  try {
+    const manifest = JSON.parse(readFileSync(join(current.dir, 'manifest.json'), 'utf8'))
+    if (!manifest || typeof manifest !== 'object') return null
+    return { ...current, manifest }
+  } catch {
+    return null
+  }
+}
+
+/** Read the current record set; null when no usable index exists. */
+export function readRecords(indexDir) {
+  const current = readCurrent(indexDir)
+  if (!current) return null
+  try {
+    const records = JSON.parse(readFileSync(join(current.dir, 'records.json'), 'utf8'))
+    if (!Array.isArray(records)) return null
+    return { ...current, records }
+  } catch {
+    return null
+  }
+}
+
+/** Atomically point `current` at a published index directory. */
+export function publishCurrent(indexDir, name) {
+  if (!isIndexDirName(name)) throw new Error(`invalid index directory name: ${name}`)
+  const target = currentPath(indexDir)
+  const temp = `${target}.tmp-${process.pid}`
+  writeFileSync(temp, `${name}\n`)
+  try {
+    renameSync(temp, target)
+  } catch (error) {
+    // Windows can refuse to replace an existing file held open by a reader.
+    if (error && (error.code === 'EPERM' || error.code === 'EEXIST')) {
+      try { unlinkSync(target) } catch {}
+      renameSync(temp, target)
+    } else {
+      try { unlinkSync(temp) } catch {}
+      throw error
+    }
+  }
+}
+
+/** Create a fresh unpublished index directory. */
+export function createIndexDir(indexDir, now = new Date()) {
+  mkdirSync(indexDir, { recursive: true })
+  const name = newIndexDirName(now)
+  const dir = join(indexDir, name)
+  mkdirSync(dir, { recursive: true })
+  return { name, dir }
+}
+
+/** Read the build lock; null when absent or unreadable. */
+export function readLock(indexDir) {
+  try {
+    const value = JSON.parse(readFileSync(lockPath(indexDir), 'utf8'))
+    if (!value || typeof value !== 'object') return null
+    return value
+  } catch {
+    return null
+  }
+}
+
+/** Whether a lock belongs to a dead or over-age builder. */
+export function isLockStale(lock, now = Date.now()) {
+  if (!lock || typeof lock !== 'object') return true
+  if (typeof lock.startedAt !== 'number' || typeof lock.pid !== 'number') return true
+  if (now - lock.startedAt > LOCK_STALE_MS) return true
+  try {
+    process.kill(lock.pid, 0)
+    return false
+  } catch (error) {
+    return error && error.code === 'ESRCH'
+  }
+}
+
+/**
+ * Acquire the cross-process build lock. A stale lock is reclaimed once.
+ * @returns `{ ok: true, release }` or `{ ok: false, lock }`.
+ */
+export function acquireLock(indexDir, { pid = process.pid, now = Date.now() } = {}) {
+  mkdirSync(indexDir, { recursive: true })
+  const file = lockPath(indexDir)
+  const payload = JSON.stringify({ pid, startedAt: now, host: hostname() })
+  for (let attempt = 0; attempt < 2; attempt += 1) {
+    try {
+      writeFileSync(file, payload, { flag: 'wx' })
+      return {
+        ok: true,
+        lock: { pid, startedAt: now, host: hostname() },
+        release() {
+          const held = readLock(indexDir)
+          if (held && held.pid === pid && held.startedAt === now) {
+            try { unlinkSync(file) } catch {}
+          }
+        },
+      }
+    } catch (error) {
+      if (!error || error.code !== 'EEXIST') throw error
+      const existing = readLock(indexDir)
+      if (existing && isLockStale(existing, now)) {
+        try { unlinkSync(file) } catch {}
+        continue
+      }
+      return { ok: false, lock: existing }
+    }
+  }
+  return { ok: false, lock: readLock(indexDir) }
+}
+
+/** Remove index directories superseded by `keepName`, keeping the newest two. */
+export function cleanupOldIndexes(indexDir, keepName, { now = Date.now(), maxAgeMs = 7 * 24 * 60 * 60 * 1000 } = {}) {
+  const names = listIndexDirs(indexDir)
+  const keep = new Set([keepName, ...names.filter(name => name !== keepName).slice(0, 1)])
+  const removed = []
+  for (const name of names) {
+    if (keep.has(name)) continue
+    const dir = join(indexDir, name)
+    let age = 0
+    try { age = now - statSync(dir).mtimeMs } catch { continue }
+    if (age < maxAgeMs) continue
+    try {
+      rmSync(dir, { recursive: true, force: true })
+      removed.push(name)
+    } catch {}
+  }
+  return removed
+}
+
+/** Read the tail of the build log, for diagnostics in tool output. */
+export function readBuildLog(indexDir, maxChars = 4000) {
+  try {
+    const text = readFileSync(buildLogPath(indexDir), 'utf8')
+    return text.length > maxChars ? text.slice(-maxChars) : text
+  } catch {
+    return ''
+  }
+}

package/lib/tokenize.js

@@ -0,0 +1,141 @@
+/**
+ * Query and document tokenization. Chinese runs use overlapping bigrams plus
+ * the last character — no dictionary, no segmenter. Stopwords and negation
+ * apply to queries only; the indexed documents keep every token.
+ */
+
+const EN_STOPWORDS = new Set([
+  'a', 'an', 'the', 'i', 'me', 'my', 'we', 'our', 'you', 'your', 'it', 'its', 'this', 'that',
+  'to', 'for', 'of', 'in', 'on', 'with', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'be',
+  'do', 'does', 'did', 'can', 'could', 'should', 'would', 'will', 'want', 'wants', 'need',
+  'needs', 'please', 'help', 'some', 'any', 'how', 'what', 'which', 'when', 'where', 'who',
+  'why', 'plugin', 'plugins', 'dsh', 'deepseek', 'harness', 'tool', 'tools', 'something',
+  'into', 'from', 'at', 'by', 'as', 'about', 'after', 'before', 'over', 'under', 'between',
+  'through', 'during', 'out', 'up', 'down', 'off', 'again', 'more', 'most', 'other', 'such',
+  'only', 'own', 'same', 'too', 'very', 'just', 'now', 'also', 'make', 'makes', 'making',
+  'get', 'gets', 'getting', 'give', 'gives', 'take', 'takes', 'use', 'uses', 'using', 'used',
+  'like', 'well', 'even', 'still', 'back', 'way', 'thing', 'things',
+])
+
+const ZH_STOPWORDS = new Set([
+  '帮我', '帮忙', '我想', '我要', '想要', '一个', '这个', '那个', '可以', '能否', '能够', '需要',
+  '希望', '请问', '有没有', '什么', '怎么', '如何', '哪些', '哪个', '插件', '工具', '功能',
+  '我', '你', '他', '她', '它', '我们', '你们', '他们', '的', '了', '吗', '呢', '吧', '啊',
+  '请', '找', '要', '做', '用', '在', '有', '和', '与', '或', '及', '就', '也', '都', '还',
+  '不', '没', '没有', '能', '会', '把', '给', '被', '让', '对', '从', '到', '上', '下',
+  '但', '但是', '不过', '然而', '而且', '并且', '或者', '还是', '如果', '因为', '所以', '然后',
+])
+
+const NEGATION_RE = /(?:不要|除了|不含|不带|无需|不想|without|except(?:ing)?|exclude)\s*([^，。；;,.!?！？\n]+)/gi
+
+function isCjk(code) {
+  return (code >= 0x3400 && code <= 0x9fff)
+    || (code >= 0xf900 && code <= 0xfaff)
+    || (code >= 0x3040 && code <= 0x30ff)
+}
+
+function splitCamel(value) {
+  return value
+    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+    .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
+}
+
+/** Tokenize one string into lowercase words and CJK bigrams. */
+export function tokenize(text) {
+  if (typeof text !== 'string' || text === '') return []
+  const value = splitCamel(text.normalize('NFKC'))
+  const tokens = []
+  let latin = ''
+  let cjk = ''
+  const flushLatin = () => {
+    if (latin.length >= 2 && !/^\d+$/.test(latin)) tokens.push(latin)
+    latin = ''
+  }
+  const flushCjk = () => {
+    if (cjk.length === 0) return
+    if (cjk.length === 1) {
+      tokens.push(cjk)
+    } else {
+      for (let index = 0; index + 1 < cjk.length; index += 1) tokens.push(cjk.slice(index, index + 2))
+      tokens.push(cjk.slice(-1))
+    }
+    cjk = ''
+  }
+  for (const character of value) {
+    const code = character.codePointAt(0)
+    if (isCjk(code)) {
+      flushLatin()
+      cjk += character
+      continue
+    }
+    flushCjk()
+    if (/[\p{L}\p{N}]/u.test(character)) latin += character.toLowerCase()
+    else flushLatin()
+  }
+  flushLatin()
+  flushCjk()
+  return tokens
+}
+
+/** Remove query stopwords, preserving order. */
+export function dropStopwords(tokens) {
+  return tokens.filter(token => !EN_STOPWORDS.has(token) && !ZH_STOPWORDS.has(token))
+}
+
+/** Pull negated clauses out of a query, returning the cleaned text and the exclusions. */
+export function extractNegation(text) {
+  if (typeof text !== 'string' || text === '') return { text: '', not: [] }
+  const not = []
+  const cleaned = text.replace(NEGATION_RE, (_match, clause) => {
+    const terms = dropStopwords(tokenize(clause))
+      .filter(term => term.length >= 2)
+      .slice(0, 3)
+    for (const term of terms) if (!not.includes(term)) not.push(term)
+    return ' '
+  })
+  return { text: cleaned, not }
+}
+
+/** Prepare one route: cleaned text, positive terms, and exclusions. */
+export function prepareQuery(text) {
+  if (typeof text !== 'string' || text.trim() === '') return { text: '', terms: [], not: [] }
+  const { text: cleaned, not } = extractNegation(text)
+  let terms = dropStopwords(tokenize(cleaned))
+  if (terms.length === 0) terms = tokenize(cleaned)
+  const excluded = new Set(not)
+  terms = [...new Set(terms)].filter(term => !excluded.has(term))
+  return { text: cleaned.trim(), terms, not }
+}
+
+/**
+ * Prepare the query group: the original plus up to three model rewrites.
+ * The original's negations apply to every route.
+ */
+export function prepareQueries(query, alts = []) {
+  const original = prepareQuery(query)
+  const rewrites = []
+  const seen = new Set()
+  for (const alt of Array.isArray(alts) ? alts : []) {
+    if (typeof alt !== 'string' || alt.trim() === '') continue
+    const packet = prepareQuery(alt)
+    if (packet.terms.length === 0) continue
+    const key = packet.terms.join(' ')
+    if (seen.has(key)) continue
+    seen.add(key)
+    rewrites.push(packet)
+    if (rewrites.length >= 3) break
+  }
+  const not = [...new Set([...original.not, ...rewrites.flatMap(packet => packet.not)])]
+  const excluded = new Set(not)
+  const clean = packet => ({ ...packet, terms: packet.terms.filter(term => !excluded.has(term)) })
+  return {
+    original: clean(original),
+    rewrites: rewrites.map(clean),
+    not,
+  }
+}
+
+/** The minimum distinct terms a document must match for one route. */
+export function minHitsFor(terms) {
+  return Math.max(1, Math.ceil(terms.length * 0.34))
+}

package/lib/vector-store.js

@@ -0,0 +1,142 @@
+/**
+ * fp16 vector and README-excerpt caches outside the immutable index
+ * directories, keyed by content hash so a rebuilt index reuses unchanged work.
+ */
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { createHash } from 'node:crypto'
+
+/** One fp16 value; inputs are normalized to [-1, 1]. */
+export function float32ToFloat16(value) {
+  const buffer = new ArrayBuffer(4)
+  const floatView = new Float32Array(buffer)
+  const intView = new Int32Array(buffer)
+  floatView[0] = value
+  const bits = intView[0]
+  const sign = (bits >> 16) & 0x8000
+  let exponent = (bits >> 23) & 0xff
+  let mantissa = bits & 0x7fffff
+  if (exponent === 0xff) return sign | 0x7c00 | (mantissa ? 0x200 : 0)
+  const halfExponent = exponent - 127 + 15
+  if (halfExponent >= 31) return sign | 0x7c00
+  if (halfExponent <= 0) {
+    if (halfExponent < -10) return sign
+    mantissa = (mantissa | 0x800000) >> (1 - halfExponent)
+    return sign | ((mantissa + 0x1000) >> 13)
+  }
+  return sign | (halfExponent << 10) | ((mantissa + 0x1000) >> 13)
+}
+
+/** Decode one fp16 bit pattern. */
+export function float16ToFloat32(bits) {
+  const sign = bits & 0x8000 ? -1 : 1
+  const exponent = (bits >> 10) & 0x1f
+  const mantissa = bits & 0x3ff
+  if (exponent === 0) return sign * mantissa * 2 ** -24
+  if (exponent === 31) return mantissa ? Number.NaN : sign * Number.POSITIVE_INFINITY
+  return sign * (1 + mantissa / 1024) * 2 ** (exponent - 15)
+}
+
+/** Serialize a 384-dimensional float vector as base64 of its fp16 bits. */
+export function encodeVector(vector) {
+  const bits = new Uint16Array(vector.length)
+  for (let index = 0; index < vector.length; index += 1) bits[index] = float32ToFloat16(vector[index])
+  return Buffer.from(bits.buffer).toString('base64')
+}
+
+/** Decode a base64 fp16 vector back to float32. */
+export function decodeVector(base64) {
+  const buffer = Buffer.from(base64, 'base64')
+  const bits = new Uint16Array(buffer.buffer, buffer.byteOffset, Math.floor(buffer.byteLength / 2))
+  const vector = new Float32Array(bits.length)
+  for (let index = 0; index < bits.length; index += 1) vector[index] = float16ToFloat32(bits[index])
+  return vector
+}
+
+/** Deterministic short hash used for excerpt and content keys. */
+export function hashText(text) {
+  return createHash('sha256').update(String(text)).digest('hex').slice(0, 24)
+}
+
+function readJsonLines(file) {
+  const map = new Map()
+  if (!existsSync(file)) return map
+  try {
+    const text = readFileSync(file, 'utf8')
+    for (const line of text.split('\n')) {
+      const trimmed = line.trim()
+      if (!trimmed) continue
+      try {
+        const value = JSON.parse(trimmed)
+        if (value && typeof value.k === 'string') map.set(value.k, value)
+      } catch {}
+    }
+  } catch {}
+  return map
+}
+
+/** One store per index directory; caches survive index rebuilds. */
+export class VectorStore {
+  constructor(indexDir) {
+    this.indexDir = indexDir
+    this.dir = join(indexDir, 'cache')
+    this.fields = null
+    this.readmes = null
+    this.excerpts = null
+    this.pending = { fields: [], readmes: [], excerpts: [] }
+  }
+
+  file(name) {
+    return join(this.dir, name)
+  }
+
+  ensureDir() {
+    mkdirSync(this.dir, { recursive: true })
+  }
+
+  getField(key) {
+    this.fields ??= readJsonLines(this.file('field-vectors.jsonl'))
+    return this.fields.get(key) ?? null
+  }
+
+  setField(key, vector, contentHash) {
+    const value = { k: key, v: encodeVector(vector), at: Date.now(), contentHash }
+    this.fields ??= readJsonLines(this.file('field-vectors.jsonl'))
+    this.fields.set(key, value)
+    this.pending.fields.push(value)
+  }
+
+  getReadme(key) {
+    this.readmes ??= readJsonLines(this.file('readme-vectors.jsonl'))
+    return this.readmes.get(key) ?? null
+  }
+
+  setReadme(key, vector, excerptHash) {
+    const value = { k: key, v: encodeVector(vector), at: Date.now(), excerptHash }
+    this.readmes ??= readJsonLines(this.file('readme-vectors.jsonl'))
+    this.readmes.set(key, value)
+    this.pending.readmes.push(value)
+  }
+
+  getExcerpt(repo) {
+    this.excerpts ??= readJsonLines(this.file('readme-excerpts.jsonl'))
+    return this.excerpts.get(repo) ?? null
+  }
+
+  setExcerpt(repo, excerpt) {
+    const value = { k: repo, ...excerpt, at: Date.now() }
+    this.excerpts ??= readJsonLines(this.file('readme-excerpts.jsonl'))
+    this.excerpts.set(repo, value)
+    this.pending.excerpts.push(value)
+  }
+
+  flush() {
+    this.ensureDir()
+    for (const [name, values] of Object.entries(this.pending)) {
+      if (values.length === 0) continue
+      const file = this.file(`${name === 'fields' ? 'field-vectors' : name === 'readmes' ? 'readme-vectors' : 'readme-excerpts'}.jsonl`)
+      appendFileSync(file, values.map(value => `${JSON.stringify(value)}\n`).join(''))
+      values.length = 0
+    }
+  }
+}