@heixui/dsh-session-suite 0.0.0-stage → 0.3.0

package/lib/step-tag-filter.js

@@ -0,0 +1,199 @@
+/**
+ * Streaming sanitizer for the four internal step tags
+ * (`<发>` / `<执行>` / `<好>` / `<现在>`, U+003C + word + U+003E).
+ *
+ * These are prompt-side stepping markers, not user-facing content. Weak or
+ * mis-switched models occasionally emit them as a long run inside a normal
+ * assistant reply; once streamed they land in the durable assistant message,
+ * the live transcript, share exports, and every later request that replays the
+ * history. The only layer that can clean all of those at once is the
+ * `llm/stream` waterfall: filter the token deltas (live transcript) and the
+ * terminal `block-end` block text (what the assembler actually persists).
+ *
+ * The filter is deliberately conservative:
+ * - only the four exact literals are removed, never other angle-bracket text;
+ * - a possible partial tag at a delta boundary is held back until it can be
+ *   classified, so `<执` + `行>` is removed across two deltas;
+ * - every other chunk passes through unchanged (same object), and a filter
+ *   failure yields the original chunk instead of breaking the stream.
+ *
+ * @module dsh-session-suite/step-tag-filter
+ */
+
+/** The four internal step tags, built from code points on purpose. */
+export const STEP_TAGS = Object.freeze([
+  '\u003c发\u003e',
+  '\u003c执行\u003e',
+  '\u003c好\u003e',
+  '\u003c现在\u003e',
+])
+
+/** Longest tag length, used to size the boundary carry. */
+const MAX_TAG_LENGTH = STEP_TAGS.reduce((max, tag) => Math.max(max, tag.length), 0)
+
+/**
+ * Remove every occurrence of the four step tags from a complete string.
+ * Other angle-bracket text (for example `\u003c我现在写\u003e`) is preserved.
+ *
+ * @param {string} text source text
+ * @returns {string} text without the four step tags
+ */
+export function stripStepTags(text) {
+  if (typeof text !== 'string' || text.length === 0) return text
+  if (!text.includes('\u003c')) return text
+  let out = ''
+  let index = 0
+  while (index < text.length) {
+    if (text.charCodeAt(index) === 0x3c) {
+      let matched = false
+      for (const tag of STEP_TAGS) {
+        if (text.startsWith(tag, index)) {
+          index += tag.length
+          matched = true
+          break
+        }
+      }
+      if (matched) continue
+    }
+    out += text[index]
+    index += 1
+  }
+  return out
+}
+
+/**
+ * Per-(block kind, index) carry state for the streaming filter.
+ * Holds back the longest suffix that is still a proper prefix of a step tag, so
+ * a tag split across two deltas is removed as one unit instead of leaking a
+ * fragment.
+ */
+class StepTagCarry {
+  /** @type {Map<string, string>} */
+  pending = new Map()
+
+  /**
+   * @param {'text'|'reasoning'} kind delta kind
+   * @param {number} index content-block index
+   * @param {string} text incoming delta
+   * @returns {string} text safe to emit now (may be empty)
+   */
+  push(kind, index, text) {
+    const key = `${kind}#${index}`
+    const buffer = (this.pending.get(key) ?? '') + text
+    const stripped = stripStepTags(buffer)
+    let hold = 0
+    const limit = Math.min(MAX_TAG_LENGTH - 1, stripped.length)
+    for (let length = limit; length > 0; length -= 1) {
+      const suffix = stripped.slice(-length)
+      let isPartialTag = false
+      for (const tag of STEP_TAGS) {
+        if (tag.length > length && tag.startsWith(suffix)) {
+          isPartialTag = true
+          break
+        }
+      }
+      if (isPartialTag) {
+        hold = length
+        break
+      }
+    }
+    const emit = hold === 0 ? stripped : stripped.slice(0, -hold)
+    this.pending.set(key, hold === 0 ? '' : stripped.slice(-hold))
+    return emit
+  }
+
+  /**
+   * Emit everything still carried for one key.
+   *
+   * @param {'text'|'reasoning'} kind delta kind
+   * @param {number} index content-block index
+   * @returns {Array<object>} zero or one synthetic delta chunk
+   */
+  flush(kind, index) {
+    const key = `${kind}#${index}`
+    const text = this.pending.get(key) ?? ''
+    this.pending.delete(key)
+    if (text.length === 0) return []
+    return [{ type: kind === 'text' ? 'text-delta' : 'reasoning-delta', index, text }]
+  }
+
+  /** @returns {Array<object>} one synthetic delta chunk per non-empty carry, in arrival order */
+  flushAll() {
+    const chunks = []
+    for (const key of [...this.pending.keys()]) {
+      const separator = key.indexOf('#')
+      const kind = key.slice(0, separator)
+      const index = Number(key.slice(separator + 1))
+      chunks.push(...this.flush(kind, index))
+    }
+    return chunks
+  }
+}
+
+/**
+ * Wrap an LLM chunk stream, removing the four step tags from text/reasoning
+ * deltas and from the terminal `block-end` block text.
+ *
+ * @param {AsyncIterable<object>} stream adapter chunk stream
+ * @returns {AsyncGenerator<object>} sanitized chunk stream
+ */
+export async function* stripStepTagsFromStream(stream) {
+  const iterator = stream[Symbol.asyncIterator]()
+  const carry = new StepTagCarry()
+  let completed = false
+  try {
+    while (true) {
+      const step = await iterator.next()
+      if (step.done === true) {
+        completed = true
+        for (const pending of carry.flushAll()) yield pending
+        return
+      }
+      const chunk = step.value
+      try {
+        if (
+          chunk !== null &&
+          typeof chunk === 'object' &&
+          (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta') &&
+          typeof chunk.text === 'string'
+        ) {
+          const kind = chunk.type === 'text-delta' ? 'text' : 'reasoning'
+          const text = carry.push(kind, chunk.index, chunk.text)
+          if (text.length > 0) yield { ...chunk, text }
+          continue
+        }
+        if (chunk !== null && typeof chunk === 'object' && chunk.type === 'block-end') {
+          for (const pending of carry.flushAll()) yield pending
+          const block = chunk.block
+          if (
+            block !== null &&
+            typeof block === 'object' &&
+            (block.type === 'text' || block.type === 'reasoning') &&
+            typeof block.text === 'string'
+          ) {
+            const text = stripStepTags(block.text)
+            if (text !== block.text) {
+              yield { ...chunk, block: { ...block, text } }
+              continue
+            }
+          }
+          yield chunk
+          continue
+        }
+        for (const pending of carry.flushAll()) yield pending
+        yield chunk
+      } catch {
+        // A filter bug must never break the model stream: pass the chunk through.
+        yield chunk
+      }
+    }
+  } finally {
+    if (!completed) {
+      try {
+        await iterator.return?.()
+      } catch {
+        // Cleanup is best-effort; the consumer already abandoned the stream.
+      }
+    }
+  }
+}

package/lib/tags.js

@@ -0,0 +1,140 @@
+/**
+ * Controlled vocabulary for memory tags.
+ *
+ * History: tags were whatever a caller passed, used verbatim. In practice that
+ * produced a library where the same domain appeared under different spellings
+ * (`dsh` and `DSH`), UI words leaked in as if they were domains (`快照`,
+ * `删除` — those are features, not subjects), and near-duplicates fragmented the
+ * list (`记忆` / `机制` / `项目经验`). The user's report was blunt: 标签打得有点乱.
+ *
+ * Two rules fix that without taking control away from the caller:
+ *
+ *   1. **Normalise before storing.** Case, surrounding space, full-width
+ *      punctuation, and the half-dozen spellings a domain actually appears in
+ *      all collapse to one canonical label.
+ *   2. **Refuse what is not a subject.** A tag must name a domain of knowledge.
+ *      Words that describe the plugin's own UI, or that are pure scaffolding,
+ *      are dropped rather than stored, because a tag nobody can filter by is
+ *      worse than no tag.
+ *
+ * The vocabulary is a suggestion list, not a jail: an unknown tag that looks
+ * like a real subject is still accepted, after normalisation. Refusing all
+ * novelty would make the feature useless for new domains, and the user has been
+ * explicit that they want the system to decide, not to be asked.
+ */
+
+/**
+ * Canonical labels, with the spellings that must fold into each.
+ *
+ * Keys are the stored form; values are recognised aliases. Chinese entries use
+ * the form the user's own notes use.
+ */
+export const TAG_ALIASES = {
+  错误教训: ['错误教训', '教训', '踩坑', '错误', 'bug', 'mistake', 'lesson'],
+  技术方案: ['技术方案', '方案', '技术', '实现', 'design', 'technique'],
+  迭代版本: ['迭代版本', '迭代', '版本', 'version', 'iteration'],
+  偏差档案: ['偏差档案', '偏差', 'deviation'],
+  调度: ['调度', 'scheduling', 'routing'],
+  配置: ['配置', '设置', 'config', 'configuration', 'settings'],
+  安全: ['安全', 'security', 'safety'],
+  检索: ['检索', '搜索', 'search', 'retrieval'],
+  记忆: ['记忆', 'memory'],
+  模型: ['模型', 'model', 'provider'],
+  工具: ['工具', 'tool', 'tools'],
+  档案: ['档案', 'archive', 'profile'],
+  事实: ['事实', 'fact', 'facts'],
+  偏好: ['偏好', 'preference', 'preferences'],
+  项目经验: ['项目经验', '经验', 'experience'],
+  工作区: ['工作区', 'workspace'],
+  dsh: ['dsh', 'DSH', 'DeepSeek Harness', '深寻'],
+}
+
+/**
+ * Words that describe the plugin's own UI or are pure scaffolding. These were
+ * observed as tags and are not subjects: a user cannot sensibly filter a memory
+ * library by `全部标签`.
+ */
+export const TAG_STOPWORDS = new Set([
+  '全部标签',
+  '快照',
+  '删除',
+  '回滚',
+  '重发',
+  '编辑',
+  '共装',
+  '降链',
+  '降级链',
+  '插件',
+  'all',
+  'all tags',
+  'snapshot',
+  'delete',
+  'rollback',
+  'resend',
+  'edit',
+  'plugin',
+  'tag',
+  'tags',
+])
+
+/** @param {string} value raw tag @returns {string} comparable form */
+function comparable(value) {
+  return String(value ?? '')
+    .replace(/[\u3000\s]+/g, ' ')
+    .trim()
+    .replace(/[，。、；：,.;:]+$/g, '')
+    .toLowerCase()
+}
+
+/** alias lookup, built once */
+const ALIAS_INDEX = (() => {
+  const index = new Map()
+  for (const [canonical, aliases] of Object.entries(TAG_ALIASES)) {
+    for (const alias of aliases) index.set(comparable(alias), canonical)
+  }
+  return index
+})()
+
+/**
+ * Canonicalise one tag.
+ *
+ * @param {unknown} raw caller-supplied tag
+ * @returns {string | null} the canonical label, or null when it is not a subject
+ */
+export function normalizeTag(raw) {
+  if (typeof raw !== 'string') return null
+  const key = comparable(raw)
+  if (!key) return null
+  if (TAG_STOPWORDS.has(key)) return null
+  const canonical = ALIAS_INDEX.get(key)
+  if (canonical !== undefined) return canonical
+
+  // Unknown but plausibly a subject: keep it, trimmed, with internal spacing
+  // collapsed. Reject one-character noise and anything absurdly long, which are
+  // the shapes auto-extraction produces when it splits a sentence badly.
+  const trimmed = String(raw).replace(/\s+/g, ' ').trim()
+  if (trimmed.length < 2 || trimmed.length > 24) return null
+  if (!/[\p{L}\p{N}]/u.test(trimmed)) return null
+  return trimmed
+}
+
+/**
+ * Canonicalise a tag list, dropping duplicates and non-subjects.
+ *
+ * @param {unknown} raw caller-supplied tags
+ * @returns {string[]} canonical tags, in first-seen order
+ */
+export function normalizeTags(raw) {
+  if (!Array.isArray(raw)) return []
+  const out = []
+  const seen = new Set()
+  for (const item of raw) {
+    const tag = normalizeTag(item)
+    if (tag === null) continue
+    const key = tag.toLowerCase()
+    if (seen.has(key)) continue
+    seen.add(key)
+    out.push(tag)
+  }
+  return out
+}

package/lib/util.js

@@ -0,0 +1,261 @@
+/**
+ * Small, dependency-free helpers used by both the snapshot engine and the
+ * HTTP surface of dsh-session-suite.
+ *
+ * Everything here is plain Node: the Host half of a DSH plugin is an ordinary
+ * ESM module running in the Electron main process, so `node:` builtins are the
+ * only imports allowed (no @deepseek-ai runtime packages are resolved for a
+ * profile-installed bundle).
+ */
+
+import { createHash } from 'node:crypto'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+
+/** Directories and files a checkpoint never records. */
+export const DEFAULT_IGNORES = [
+  'node_modules',
+  '.git',
+  '.hg',
+  '.svn',
+  '.pnpm-store',
+  '.pnpm',
+  '.yarn',
+  '.cache',
+  '.next',
+  '.nuxt',
+  '.svelte-kit',
+  '.turbo',
+  '.parcel-cache',
+  '.venv',
+  'venv',
+  '__pycache__',
+  'dist',
+  'build',
+  'out',
+  'coverage',
+  '.idea',
+  '.vscode',
+  '.DS_Store',
+  'Thumbs.db',
+  '*.log',
+  '*.tmp',
+  '*.temp',
+  '*.lock',
+  'package-lock.json',
+  'pnpm-lock.yaml',
+  'yarn.lock',
+  '.dsh-ignore',
+]
+
+const LOCKFILE_NAMES = new Set(['package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lockb'])
+
+/**
+ * Glob-ish ignore matcher: exact segment, `*` prefix/suffix patterns, and
+ * bare `*.ext` suffixes. Matching is applied to every path segment.
+ *
+ * @param {readonly string[]} patterns ignore patterns
+ * @returns {(relPath: string, name: string) => boolean} predicate
+ */
+export function createIgnoreMatcher(patterns) {
+  const segments = new Set()
+  const suffixes = []
+  for (const raw of patterns) {
+    const pattern = raw.trim()
+    if (!pattern) continue
+    if (pattern.startsWith('*.')) suffixes.push(pattern.slice(1).toLowerCase())
+    else if (pattern.startsWith('*')) suffixes.push(pattern.slice(1).toLowerCase())
+    else segments.add(pattern)
+  }
+  return (relPath, name) => {
+    if (segments.has(name)) return true
+    const lower = name.toLowerCase()
+    for (const suffix of suffixes) {
+      if (lower.endsWith(suffix)) return true
+    }
+    if (LOCKFILE_NAMES.has(name)) return true
+    // `.git` nested inside an allowed directory is still ignored.
+    return relPath.split('/').some(part => part === '.git' || part === 'node_modules')
+  }
+}
+
+/** @param {Buffer} data content */
+export function sha256(data) {
+  return createHash('sha256').update(data).digest('hex')
+}
+
+/** @param {string} text content */
+export function sha256Text(text) {
+  return createHash('sha256').update(text, 'utf8').digest('hex')
+}
+
+/**
+ * Deterministic short id, independent of `crypto.randomUUID` availability.
+ *
+ * @param {string} seed seed material
+ * @returns {string} 12-char id
+ */
+export function shortId(seed) {
+  return sha256Text(`${seed}:${process.hrtime.bigint()}`).slice(0, 12)
+}
+
+/** @param {string} dir directory to create */
+export async function ensureDir(dir) {
+  await fs.mkdir(dir, { recursive: true })
+  return dir
+}
+
+/**
+ * @param {string} file target file
+ * @returns {Promise<unknown | undefined>} parsed JSON, or undefined when absent/corrupt
+ */
+export async function readJson(file) {
+  try {
+    const text = await fs.readFile(file, 'utf8')
+    return JSON.parse(text)
+  } catch {
+    return undefined
+  }
+}
+
+/**
+ * Atomic JSON write (temp file + rename) so a crash never truncates state.
+ *
+ * @param {string} file target file
+ * @param {unknown} value JSON-serializable value
+ */
+export async function writeJson(file, value) {
+  await ensureDir(path.dirname(file))
+  const tmp = `${file}.${process.pid}.${Date.now()}.tmp`
+  await fs.writeFile(tmp, `${JSON.stringify(value, null, 2)}\n`, 'utf8')
+  await fs.rename(tmp, file)
+}
+
+/** @param {number} bytes raw size */
+export function formatBytes(bytes) {
+  if (!Number.isFinite(bytes) || bytes <= 0) return '0 B'
+  const units = ['B', 'KB', 'MB', 'GB']
+  let value = bytes
+  let index = 0
+  while (value >= 1024 && index < units.length - 1) {
+    value /= 1024
+    index += 1
+  }
+  return `${value >= 10 || index === 0 ? Math.round(value) : value.toFixed(1)} ${units[index]}`
+}
+
+/**
+ * Look like a binary blob? A NUL byte within the first 8 KiB is the classic
+ * signal, and we also refuse files the design explicitly excludes.
+ *
+ * @param {Buffer} head first bytes of the file
+ * @returns {boolean} true when the file must be skipped
+ */
+export function looksBinary(head) {
+  return head.includes(0)
+}
+
+/**
+ * Remove a directory tree without throwing when it is already gone.
+ *
+ * @param {string} dir directory
+ */
+export async function removeDir(dir) {
+  await fs.rm(dir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 })
+}
+
+/**
+ * @param {string} file file
+ * @returns {Promise<{ size: number, mtimeMs: number } | undefined>} stat or undefined
+ */
+export async function statOrUndefined(file) {
+  try {
+    return await fs.stat(file)
+  } catch {
+    return undefined
+  }
+}
+
+/** @param {number} ms delay */
+export function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+/**
+ * Bound concurrency without pulling in a dependency.
+ *
+ * @template T, R
+ * @param {readonly T[]} items inputs
+ * @param {number} limit parallelism
+ * @param {(item: T, index: number) => Promise<R>} worker per-item work
+ * @returns {Promise<R[]>} results in input order
+ */
+export async function mapLimit(items, limit, worker) {
+  const results = new Array(items.length)
+  let cursor = 0
+  const runners = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
+    while (cursor < items.length) {
+      const index = cursor
+      cursor += 1
+      results[index] = await worker(items[index], index)
+    }
+  })
+  await Promise.all(runners)
+  return results
+}
+
+/**
+ * Normalize a workspace-relative path to forward slashes.
+ *
+ * @param {string} rel relative path
+ * @returns {string} normalized
+ */
+export function toPosix(rel) {
+  return rel.split(path.sep).join('/')
+}
+
+/** @param {string} value unknown value */
+export function asString(value) {
+  return typeof value === 'string' ? value : undefined
+}
+
+/**
+ * Is this step message something the user actually typed?
+ *
+ * `role === 'user'` is not enough. The harness injects its own pipeline
+ * messages with that role — `runtime-context`, `time-context`, `skill-catalog`,
+ * `agent-message` — and compaction adds a synthetic `user/message` with
+ * `source.kind === 'compact-checkpoint'`. A plugin that reads instructions out
+ * of "user" messages will therefore read its own prompt text back, because
+ * whatever it injects is delivered inside a `runtime-context` message.
+ *
+ * That is not hypothetical: scanning runtime-context messages made this bundle
+ * parse the `【记住：…】` example in its own instruction footer as a user command,
+ * 154 times, and store a memory whose entire body was the ellipsis placeholder.
+ *
+ * @param {any} message step message
+ * @returns {boolean} whether the message is user-authored
+ */
+export function isUserAuthored(message) {
+  if (message?.role !== 'user') return false
+  const source = typeof message.source === 'object' && message.source ? message.source : undefined
+  return source?.kind === undefined || source.kind === 'user'
+}
+
+/**
+ * Is a captured value real content rather than a placeholder?
+ *
+ * A placeholder is worth rejecting explicitly because the value came out of a
+ * regex over text that may include our own examples. `【记住：…】` yields `…`,
+ * which is a non-empty string and would otherwise sail straight into the store.
+ *
+ * @param {unknown} value captured value
+ * @returns {boolean} whether it is worth storing
+ */
+export function isMeaningfulText(value) {
+  const text = String(value ?? '').trim()
+  // One character carries no recoverable meaning, and punctuation/symbol-only
+  // values are placeholders by construction.
+  if (text.length < 2) return false
+  return /[\p{L}\p{N}]/u.test(text)
+}