dsh-memory-dba 0.0.0-stage → 0.1.0

package/src/stream.mjs

@@ -0,0 +1,74 @@
+/**
+ * Turn-to-stream writer.
+ *
+ * The stream is a timeline, not a transcript: one line per finished turn, no
+ * detail. Anything longer belongs in a workspace note that retrieval can reach.
+ * Writing is best-effort — a failed write must never fail the turn.
+ */
+
+import { appendLayer, readLayer } from './store.mjs'
+
+const SECRET_PATTERNS = [
+  /sk-[A-Za-z0-9_-]{16,}/g,
+  /\bBearer\s+[A-Za-z0-9._-]{16,}/gi,
+  /\b[A-Fa-f0-9]{32,}\b/g,
+  /\b\d{13,19}\b/g,
+]
+
+/** Remove credential-looking runs before text reaches a committable file. */
+export function sanitize(text) {
+  let out = String(text || '')
+  for (const pattern of SECRET_PATTERNS) out = out.replace(pattern, '[redacted]')
+  return out
+}
+
+/** Collapse whitespace so one turn stays one line. */
+export function oneLine(text, maxChars = 600) {
+  const flat = String(text || '').replace(/\s+/g, ' ').trim()
+  if (flat.length <= maxChars) return flat
+  let end = Math.max(0, maxChars - 1)
+  if (end > 0 && /[\uD800-\uDBFF]/.test(flat[end - 1]) && /[\uDC00-\uDFFF]/.test(flat[end])) end -= 1
+  return `${flat.slice(0, end)}…`
+}
+
+/** Local `YYYY-MM-DD HH:mm` timestamp. */
+export function stamp(date = new Date()) {
+  const pad = n => String(n).padStart(2, '0')
+  return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ${pad(date.getHours())}:${pad(date.getMinutes())}`
+}
+
+/**
+ * Format one turn as a stream line.
+ *
+ * @param {{user?: string, assistant?: string, tag?: string, at?: Date}} turn
+ * @returns {string|null} Line, or null when the turn is not worth remembering.
+ */
+export function formatTurn({ user, assistant, tag, at }) {
+  const u = oneLine(sanitize(user), 300)
+  const a = oneLine(sanitize(assistant), 600)
+  if (!u || !a) return null
+  const label = tag ? ` | tag=${oneLine(sanitize(tag), 100)}` : ''
+  return `- ${stamp(at)}${label} | 用户：${u} | 助手：${a}`
+}
+
+/**
+ * Append a turn unless it repeats the last line.
+ *
+ * @param {string} home Memory directory.
+ * @param {{user?: string, assistant?: string, tag?: string, at?: Date}} turn
+ * @returns {{written: boolean, reason?: string, line?: string}}
+ */
+export function appendTurn(home, turn) {
+  const line = formatTurn(turn)
+  if (!line) return { written: false, reason: 'empty-turn' }
+  let last = ''
+  try {
+    const stream = readLayer(home, 'stream.md').trimEnd()
+    last = stream.slice(stream.lastIndexOf('\n') + 1)
+  } catch (error) {
+    if (error?.name !== 'MissingLayer') throw error
+  }
+  if (last === line) return { written: false, reason: 'duplicate' }
+  appendLayer(home, 'stream.md', line)
+  return { written: true, line }
+}

package/src/tokenize.mjs

@@ -0,0 +1,73 @@
+/**
+ * Tokenizer for mixed Chinese/English memory text.
+ *
+ * A trigram index cannot answer two-character Chinese words (a two-character
+ * query is shorter than one trigram), so CJK runs emit both unigrams and
+ * bigrams. Latin/digit runs stay whole words, which keeps identifiers such as
+ * `202301233055` searchable while ordinary words still match exactly.
+ */
+
+const CJK = /[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\u3040-\u30FF\uAC00-\uD7AF]/
+const WORD = /[A-Za-z0-9_@.\-]+/g
+const CJK_RUN = /[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\u3040-\u30FF\uAC00-\uD7AF]+/g
+
+/**
+ * Tokens for indexing and querying. Lowercases Latin text, keeps digits and
+ * CJK as-is, and deduplicates nothing (term frequency comes from repetition).
+ *
+ * @param {string} text Raw memory or query text.
+ * @returns {string[]} Tokens in source order.
+ */
+export function tokenize(text) {
+  const tokens = []
+  if (!text) return tokens
+  const lower = String(text).toLowerCase()
+
+  for (const match of lower.matchAll(CJK_RUN)) {
+    const run = match[0]
+    for (let i = 0; i < run.length; i += 1) {
+      tokens.push(run[i])
+      if (i + 1 < run.length) tokens.push(run.slice(i, i + 2))
+    }
+  }
+
+  const withoutCjk = lower.replace(CJK_RUN, ' ')
+  for (const match of withoutCjk.matchAll(WORD)) {
+    const word = match[0].replace(/^[-.]+|[-.]+$/g, '')
+    if (word) tokens.push(word)
+  }
+  return tokens
+}
+
+/**
+ * Query terms with per-term weight. CJK bigrams carry the most signal, single
+ * characters the least; a bare digit string is exact-match material.
+ *
+ * @param {string} query User query text.
+ * @returns {Map<string, number>} term -> boost
+ */
+export function queryTerms(query) {
+  const weights = new Map()
+  const bump = (term, weight) => {
+    if (!term) return
+    weights.set(term, Math.max(weights.get(term) || 0, weight))
+  }
+  const lower = String(query || '').toLowerCase()
+
+  for (const match of lower.matchAll(CJK_RUN)) {
+    const run = match[0]
+    if (run.length === 1) bump(run, 1)
+    for (let i = 0; i < run.length - 1; i += 1) bump(run.slice(i, i + 2), 1.6)
+  }
+  for (const match of lower.replace(CJK_RUN, ' ').matchAll(WORD)) {
+    const word = match[0].replace(/^[-.]+|[-.]+$/g, '')
+    if (!word) continue
+    bump(word, /\d{4,}/.test(word) ? 2 : 1.2)
+  }
+  return weights
+}
+
+/** True when the query contains no indexable term at all. */
+export function isBlankQuery(query) {
+  return queryTerms(query).size === 0
+}

package/src/vector-worker.mjs

@@ -0,0 +1,37 @@
+import { parentPort } from 'node:worker_threads'
+import { dirname } from 'node:path'
+import { statSync } from 'node:fs'
+import { embed } from './embed.mjs'
+import { cosineInt8, encodeVector, loadVectorCache } from './vector.mjs'
+
+/** The cache is re-parsed only when the file changes; each turn otherwise reuses it. */
+let memo
+
+function cacheFor(cachePath) {
+  const { mtimeMs } = statSync(cachePath)
+  if (!memo || memo.path !== cachePath || memo.mtimeMs !== mtimeMs) {
+    memo = { path: cachePath, mtimeMs, cache: loadVectorCache(dirname(dirname(cachePath))) }
+  }
+  return memo.cache
+}
+
+parentPort.on('message', async ({ query, cachePath, keys, topK, url, model, timeoutMs, keepAlive, sab }) => {
+  const header = new Int32Array(sab)
+  try {
+    const cache = cacheFor(cachePath)
+    if (cache.model !== model) throw new Error('model mismatch')
+    const vector = await embed(query, { url, model, timeoutMs, keepAlive })
+    if (vector.length !== cache.dim) throw new Error('dimension mismatch')
+    const encoded = encodeVector(vector)
+    const hits = keys.flatMap((key, index) => cache.entries[key] ? [{ index, score: cosineInt8(encoded, cache.entries[key]) }] : [])
+      .sort((left, right) => right.score - left.score || left.index - right.index).slice(0, Math.min(8, topK))
+    const floats = new Float32Array(sab)
+    for (let index = 0; index < hits.length; index += 1) {
+      header[2 + index * 2] = hits[index].index
+      floats[3 + index * 2] = hits[index].score
+    }
+    header[1] = hits.length
+    Atomics.store(header, 0, 1)
+  } catch { Atomics.store(header, 0, -1) }
+  Atomics.notify(header, 0)
+})

package/src/vector.mjs

@@ -0,0 +1,121 @@
+import { createHash } from 'node:crypto'
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { Worker } from 'node:worker_threads'
+
+export class VectorCorrupt extends Error {
+  constructor() {
+    super('invalid vector cache')
+    this.name = 'VectorCorrupt'
+  }
+}
+
+export class MissingVectors extends Error {
+  constructor() {
+    super('vector cache missing')
+    this.name = 'MissingVectors'
+  }
+}
+
+export function docKey(text) {
+  return createHash('sha1').update(text, 'utf8').digest('hex')
+}
+
+export function encodeVector(vector) {
+  if (!(vector instanceof Float32Array) || !vector.length || !vector.every(Number.isFinite)) throw new VectorCorrupt()
+  let maximum = 0
+  for (const value of vector) maximum = Math.max(maximum, Math.abs(value))
+  const scale = maximum / 127
+  const quantized = Int8Array.from(vector, value => scale ? Math.round(value / scale) : 0)
+  return { s: scale, q: Buffer.from(quantized.buffer).toString('base64') }
+}
+
+function quantized(entry) {
+  if (!entry || typeof entry.s !== 'number' || !Number.isFinite(entry.s) || entry.s < 0 || typeof entry.q !== 'string' || !entry.q.length) throw new VectorCorrupt()
+  const buffer = Buffer.from(entry.q, 'base64')
+  if (buffer.toString('base64') !== entry.q) throw new VectorCorrupt()
+  const values = new Int8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
+  if (values.some(value => value === -128 || (entry.s === 0 && value !== 0))) throw new VectorCorrupt()
+  return values
+}
+
+export function decodeVector(entry) {
+  return Float32Array.from(quantized(entry), value => value * entry.s)
+}
+
+export function cosineInt8(left, right) {
+  const first = quantized(left)
+  const second = quantized(right)
+  if (first.length !== second.length) throw new VectorCorrupt()
+  let dot = 0
+  let firstNorm = 0
+  let secondNorm = 0
+  for (let index = 0; index < first.length; index += 1) {
+    dot += first[index] * second[index]
+    firstNorm += first[index] ** 2
+    secondNorm += second[index] ** 2
+  }
+  return firstNorm && secondNorm ? Math.max(-1, Math.min(1, dot / Math.sqrt(firstNorm * secondNorm))) : 0
+}
+
+function validateCache(cache) {
+  if (!cache || cache.version !== 1 || typeof cache.model !== 'string' || !cache.model.trim() || !Number.isInteger(cache.dim) || cache.dim < 1 || !cache.entries || typeof cache.entries !== 'object' || Array.isArray(cache.entries)) throw new VectorCorrupt()
+  for (const [key, entry] of Object.entries(cache.entries)) {
+    if (!/^[a-f0-9]{40}$/.test(key) || quantized(entry).length !== cache.dim) throw new VectorCorrupt()
+  }
+  return cache
+}
+
+export function loadVectorCache(home) {
+  let raw
+  try { raw = readFileSync(join(home, '.vec', 'embeddings.json'), 'utf8') } catch (error) {
+    if (error?.code === 'ENOENT') throw new MissingVectors()
+    throw error
+  }
+  try { return validateCache(JSON.parse(raw)) } catch { throw new VectorCorrupt() }
+}
+
+export function saveVectorCache(home, cache) {
+  validateCache(cache)
+  const file = join(home, '.vec', 'embeddings.json')
+  mkdirSync(dirname(file), { recursive: true })
+  const temporary = `${file}.tmp-${process.pid}`
+  const clean = { version: 1, model: cache.model, dim: cache.dim, entries: Object.fromEntries(Object.entries(cache.entries).map(([key, entry]) => [key, { s: entry.s, q: entry.q }])) }
+  writeFileSync(temporary, JSON.stringify(clean), 'utf8')
+  renameSync(temporary, file)
+}
+
+let worker
+// After a failed lookup, skip semantic recall briefly: a down or cold embedder must
+// not cost the synchronous budget on every single turn.
+const coolDown = new Map()
+
+export function semanticRecall(home, query, keys, { topK = 8, url = 'http://127.0.0.1:11434', model = 'bge-m3', timeoutMs = 1500, keepAlive = '30m' } = {}) {
+  const started = performance.now()
+  try {
+    const cachePath = join(home, '.vec', 'embeddings.json')
+    if (!keys.length || !existsSync(cachePath) || !Number.isFinite(timeoutMs) || timeoutMs <= 0) return null
+    const coolKey = `${url}|${model}`
+    if ((coolDown.get(coolKey) || 0) > started) return null
+    if (!worker) {
+      const instance = new Worker(new URL('./vector-worker.mjs', import.meta.url))
+      instance.on('error', () => { if (worker === instance) worker = undefined })
+      instance.on('exit', () => { if (worker === instance) worker = undefined })
+      instance.unref()
+      worker = instance
+    }
+    const sab = new SharedArrayBuffer(8 + 8 * 8)
+    const header = new Int32Array(sab)
+    worker.postMessage({ query, cachePath, keys, topK: Math.min(8, Math.max(1, Math.floor(topK))), url, model, timeoutMs, keepAlive, sab })
+    const remaining = timeoutMs - (performance.now() - started)
+    if (remaining <= 0 || Atomics.wait(header, 0, 0, remaining) === 'timed-out' || Atomics.load(header, 0) !== 1) {
+      coolDown.set(coolKey, performance.now() + 60_000)
+      return null
+    }
+    const floats = new Float32Array(sab)
+    const result = new Map()
+    for (let index = 0; index < header[1]; index += 1) result.set(keys[header[2 + index * 2]], floats[3 + index * 2])
+    coolDown.delete(coolKey)
+    return result
+  } catch { return null }
+}