dsh-locator 0.5.3 → 0.6.0

package/src/config.ts

@@ -25,6 +25,16 @@ export interface Config {
   scanWorkers: number
   /** Soft Go heap limit in MiB for the engine; 0 leaves it to GOMEMLIMIT/GOGC. */
   memoryLimitMB: number
+  /** Index text-file bodies and DSH session messages into the content database. */
+  contentIndex: boolean
+  /** Maximum bytes read from one text file. */
+  contentMaxFileBytes: number
+  /** Maximum text files indexed per build; 0 means no cap. */
+  contentMaxFiles: number
+  /** Most recent session logs indexed; 0 means all. */
+  contentMaxSessions: number
+  /** Extra session directories; empty means $DSH_HOME/sessions. */
+  contentSessionDirs: string[]
   /**
    * Try the NTFS USN/MFT enumeration before the portable walk (Windows only,
    * requires running DSH as administrator). Falls back automatically.
@@ -61,6 +71,11 @@ export const Config: Schema<Config> = Schema.object({
   refreshIntervalMs: Schema.number().default(1_800_000),
   scanWorkers: Schema.number().default(0),
   memoryLimitMB: Schema.number().default(0),
+  contentIndex: Schema.boolean().default(true),
+  contentMaxFileBytes: Schema.number().default(1_000_000),
+  contentMaxFiles: Schema.number().default(50_000),
+  contentMaxSessions: Schema.number().default(50),
+  contentSessionDirs: Schema.array(Schema.string()).default([]),
   windowsUsn: Schema.boolean().default(false),
   stateDir: Schema.string().default(''),
   binaryPath: Schema.string().default(''),

package/src/content.ts

@@ -0,0 +1,253 @@
+/**
+ * Content search: text files and DSH session messages in SQLite FTS5.
+ *
+ * The file-name index stays in the Go engine. This store is built beside it:
+ * the engine's `dump` RPC supplies the already-scanned file list, Node reads
+ * text files and decompresses `session.v4.jsonl.zstd` logs, and SQLite's
+ * trigram tokenizer gives substring/CJK search with snippets.
+ * @module dsh-locator/content
+ */
+
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { basename, dirname, extname, join } from 'node:path'
+import { zstdDecompressSync } from 'node:zlib'
+import type { LocatorRuntime } from './runtime.ts'
+
+const TEXT_EXTENSIONS = new Set([
+  '.txt', '.md', '.markdown', '.rst', '.log', '.csv', '.tsv', '.json', '.jsonl',
+  '.yaml', '.yml', '.toml', '.ini', '.cfg', '.conf', '.xml', '.html', '.htm',
+  '.css', '.scss', '.less', '.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx',
+  '.go', '.rs', '.py', '.rb', '.php', '.java', '.kt', '.swift', '.c', '.h',
+  '.cc', '.cpp', '.hpp', '.cs', '.sh', '.bash', '.zsh', '.fish', '.ps1',
+  '.sql', '.graphql', '.proto', '.tex', '.org', '.adoc',
+])
+
+/** One searchable document. */
+export interface ContentHit {
+  kind: 'file' | 'session'
+  path: string
+  title: string
+  snippet: string
+  sessionId?: string
+  messageId?: string
+  score: number
+}
+
+/** Build progress shown by the panel/status. */
+export interface ContentState {
+  building: boolean
+  files: number
+  sessions: number
+  messages: number
+  error: string
+  updatedAt: number
+}
+
+/** The content store; one instance per plugin. */
+export class ContentIndex {
+  private db: import('node:sqlite').DatabaseSync | null = null
+  private readonly state: ContentState = { building: false, files: 0, sessions: 0, messages: 0, error: '', updatedAt: 0 }
+  private generation = 0
+
+  constructor(private readonly rt: LocatorRuntime, private readonly options: {
+    stateDir: string
+    sessionDirs: string[]
+    maxFileBytes: number
+    maxFiles: number
+    maxSessions: number
+  }) {}
+
+  status(): ContentState { return { ...this.state } }
+
+  /** Close the database when the plugin unmounts. */
+  close(): void {
+    this.generation++
+    this.db?.close()
+    this.db = null
+    this.state.building = false
+  }
+
+  /** Build in the background; stale generations stop at their next checkpoint. */
+  start(): void {
+    const generation = ++this.generation
+    this.state.building = true
+    this.state.error = ''
+    void this.build(generation).catch(error => {
+      if (generation !== this.generation) return
+      this.state.error = messageOf(error)
+      this.state.building = false
+    })
+  }
+
+  private async build(generation: number): Promise<void> {
+    const { DatabaseSync } = await import('node:sqlite')
+    const path = join(this.options.stateDir, 'content.db')
+    this.db?.close()
+    this.db = new DatabaseSync(path)
+    this.db.exec('PRAGMA journal_mode = WAL')
+    this.db.exec('PRAGMA synchronous = OFF')
+    this.db.exec('DROP TABLE IF EXISTS docs')
+    this.db.exec(`CREATE VIRTUAL TABLE docs USING fts5(
+      body,
+      kind UNINDEXED,
+      path UNINDEXED,
+      title UNINDEXED,
+      session_id UNINDEXED,
+      message_id UNINDEXED,
+      tokenize = 'trigram'
+    )`)
+    const insert = this.db.prepare('INSERT INTO docs (body, kind, path, title, session_id, message_id) VALUES (?, ?, ?, ?, ?, ?)')
+    const flush = (rows: Array<[string, string, string, string, string | null, string | null]>) => {
+      if (rows.length === 0) return
+      this.db!.exec('BEGIN')
+      try {
+        for (const row of rows) insert.run(...row)
+        this.db!.exec('COMMIT')
+      } catch (error) {
+        this.db!.exec('ROLLBACK')
+        throw error
+      }
+    }
+
+    await this.indexSessions(flush, generation)
+    if (generation !== this.generation) return
+    await this.indexFiles(flush, generation)
+    if (generation !== this.generation) return
+    this.state.building = false
+    this.state.updatedAt = Date.now()
+  }
+
+  private async indexSessions(flush: (rows: Array<[string, string, string, string, string | null, string | null]>) => void, generation: number): Promise<void> {
+    const files: Array<{ path: string; mtime: number }> = []
+    for (const root of this.sessionRoots()) {
+      walkSessions(root, files, 0)
+    }
+    files.sort((a, b) => b.mtime - a.mtime)
+    const limit = this.options.maxSessions > 0 ? this.options.maxSessions : files.length
+    let batch: Array<[string, string, string, string, string | null, string | null]> = []
+    for (const file of files.slice(0, limit)) {
+      if (generation !== this.generation) return
+      let text: string
+      try { text = zstdDecompressSync(readFileSync(file.path)).toString('utf8') } catch { continue }
+      const sessionId = basename(dirname(file.path))
+      let title = ''
+      let messages = 0
+      for (const line of text.split('\n')) {
+        if (!line) continue
+        let event: { type?: string; data?: Record<string, unknown> }
+        try { event = JSON.parse(line) } catch { continue }
+        const extracted = messageText(event)
+        if (extracted === undefined) continue
+        if (title === '') title = extracted.text.replace(/\s+/g, ' ').slice(0, 90)
+        batch.push([extracted.text.slice(0, 8_000), 'session', file.path, title || sessionId, sessionId, extracted.id])
+        messages++
+        if (batch.length >= 200) { flush(batch); batch = [] }
+      }
+      this.state.sessions++
+      this.state.messages += messages
+    }
+    flush(batch)
+  }
+
+  private async indexFiles(flush: (rows: Array<[string, string, string, string, string | null, string | null]>) => void, generation: number): Promise<void> {
+    let after = 0
+    let batch: Array<[string, string, string, string, string | null, string | null]> = []
+    let seen = 0
+    for (;;) {
+      if (generation !== this.generation) return
+      const page = await this.rt.dump(after, 2_000)
+      for (const entry of page.entries) {
+        if (entry.dir || entry.size <= 0 || entry.size > this.options.maxFileBytes) continue
+        if (!TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue
+        if (this.options.maxFiles > 0 && seen >= this.options.maxFiles) break
+        let body: string
+        try { body = readFileSync(entry.path, 'utf8') } catch { continue }
+        if (body.includes('\u0000')) continue
+        seen++
+        this.state.files++
+        batch.push([body.slice(0, this.options.maxFileBytes), 'file', entry.path, entry.name, null, null])
+        if (batch.length >= 200) { flush(batch); batch = [] }
+      }
+      after = page.next
+      if (page.done || page.entries.length === 0) break
+    }
+    flush(batch)
+  }
+
+  private sessionRoots(): string[] {
+    if (this.options.sessionDirs.length > 0) return this.options.sessionDirs
+    const home = process.env.DSH_HOME || join(homedir(), '.dsh')
+    return [join(home, 'sessions')]
+  }
+
+  /** Query the FTS table; short queries fall back to LIKE because trigram needs 3 chars. */
+  search(query: string, limit: number): ContentHit[] {
+    if (this.db === null) return []
+    const text = query.trim()
+    if (text === '') return []
+    const capped = Math.max(1, Math.min(100, limit))
+    try {
+      let rows: Array<Record<string, unknown>>
+      if ([...text].length < 3) {
+        rows = this.db.prepare(`SELECT kind, path, title, session_id, message_id, body, 0 AS score
+          FROM docs WHERE body LIKE '%' || ? || '%' LIMIT ?`).all(text, capped) as Array<Record<string, unknown>>
+      } else {
+        const match = text.split(/\s+/).filter(Boolean).map(term => '"' + term.replace(/"/g, '') + '"').join(' AND ')
+        rows = this.db.prepare(`SELECT kind, path, title, session_id, message_id,
+          snippet(docs, 0, char(1), char(2), '…', 16) AS snippet, bm25(docs) AS score
+          FROM docs WHERE docs MATCH ? ORDER BY score LIMIT ?`).all(match, capped) as Array<Record<string, unknown>>
+      }
+      return rows.map(row => ({
+        kind: row.kind === 'session' ? 'session' : 'file',
+        path: String(row.path ?? ''),
+        title: String(row.title ?? ''),
+        snippet: String(row.snippet ?? row.body ?? '').replace(/[\u0001\u0002]/g, ''),
+        ...(row.session_id ? { sessionId: String(row.session_id) } : {}),
+        ...(row.message_id ? { messageId: String(row.message_id) } : {}),
+        score: Number(row.score ?? 0),
+      }))
+    } catch (error) {
+      this.state.error = messageOf(error)
+      return []
+    }
+  }
+}
+
+function messageText(event: { type?: string; data?: Record<string, unknown> }): { text: string; id: string } | undefined {
+  if (event.type !== 'user/message' && event.type !== 'assistant/message') return undefined
+  const data = event.data ?? {}
+  if (event.type === 'user/message') {
+    const source = (data.source ?? {}) as Record<string, unknown>
+    if (source.kind === 'time-context') return undefined
+  }
+  const message = (data.message ?? {}) as Record<string, unknown>
+  const content = event.type === 'user/message' ? data.content : message.content
+  if (!Array.isArray(content)) return undefined
+  const text = content
+    .filter((block): block is { type: string; text: string } => typeof block === 'object' && block !== null && (block as { type?: string }).type === 'text' && typeof (block as { text?: unknown }).text === 'string')
+    .map(block => block.text)
+    .join('\n')
+    .trim()
+  if (text === '') return undefined
+  const id = event.type === 'user/message'
+    ? String(data.id ?? '')
+    : String(message.id ?? '')
+  return { text, id: id || `seq-${String(data.seq ?? '')}` }
+}
+
+function walkSessions(root: string, out: Array<{ path: string; mtime: number }>, depth: number): void {
+  if (depth > 4 || !existsSync(root)) return
+  let entries
+  try { entries = readdirSync(root, { withFileTypes: true }) } catch { return }
+  for (const entry of entries) {
+    const path = join(root, entry.name)
+    if (entry.isDirectory()) { walkSessions(path, out, depth + 1); continue }
+    if (!entry.isFile() || !entry.name.endsWith('.v4.jsonl.zstd')) continue
+    try { out.push({ path, mtime: statSync(path).mtimeMs }) } catch { /* ignore */ }
+  }
+}
+
+function messageOf(error: unknown): string {
+  return error instanceof Error ? error.message : String(error)
+}

package/src/find.ts

@@ -0,0 +1,76 @@
+/**
+ * locator_find — search file bodies and DSH session messages.
+ * @module dsh-locator/find
+ */
+
+import { defineTool } from '@deepseek-ai/dsh-tools'
+import type { ContentIndex, ContentState } from './content.ts'
+
+const HIT = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    kind: { type: 'string', enum: ['file', 'session'], required: true },
+    path: { type: 'string', required: true },
+    title: { type: 'string' },
+    snippet: { type: 'string' },
+    sessionId: { type: 'string' },
+    messageId: { type: 'string' },
+    score: { type: 'number' },
+  },
+} as const
+
+/** Build the locator_find tool. */
+export function makeFindTool(content: ContentIndex) {
+  return defineTool({
+    name: 'locator_find',
+    description: [
+      'Search the **contents** of text files and DSH session messages (dsh-locator).',
+      '',
+      'Use it when the user remembers a phrase, a sentence, a decision or a code snippet rather than a file name. `locator_search` searches names and paths; this searches bodies and returns snippets plus `sessionId`/`messageId` for session hits, so the result can be opened at its source.',
+      '',
+      'Content is built asynchronously after start: results may be partial while `state.building` is true. Text files are capped by `contentMaxFileBytes`; session logs are decompressed from `$DSH_HOME/sessions`.',
+    ].join('\n'),
+    parameters: {
+      query: { type: 'string', required: true, description: 'Words or a phrase to find in file/session content.' },
+      limit: { type: 'integer', description: 'Maximum hits (default 30, at most 100).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          ok: { type: 'boolean', required: true },
+          query: { type: 'string' },
+          hits: { type: 'array', items: HIT },
+          state: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+              building: { type: 'boolean' },
+              files: { type: 'integer' },
+              sessions: { type: 'integer' },
+              messages: { type: 'integer' },
+              updatedAt: { type: 'integer' },
+              error: { type: 'string' },
+            },
+          },
+          error: { type: 'string' },
+        },
+      },
+      render: (_args, value) => {
+        const v = value as { ok: boolean; hits?: Array<{ kind: string; title?: string; snippet?: string; path?: string }>; error?: string; state?: ContentState }
+        if (!v.ok) return [{ type: 'text', text: `locator_find failed: ${v.error ?? 'unknown'}` }]
+        const lines = (v.hits ?? []).map((hit, index) => `${String(index + 1)}. [${hit.kind}] ${hit.title ?? ''} — ${hit.snippet ?? ''}\n   ${hit.path ?? ''}`)
+        return [{ type: 'text', text: `${String(v.hits?.length ?? 0)} content hit(s)${v.state?.building ? ' (index still building)' : ''}\n${lines.join('\n')}` }]
+      },
+    },
+    async execute(args) {
+      const query = args.query.trim()
+      if (query === '') return { ok: false as const, error: 'query is empty' }
+      const limit = Math.max(1, Math.min(100, args.limit ?? 30))
+      const hits = content.search(query, limit)
+      return { ok: true as const, query, hits, state: content.status() }
+    },
+  })
+}

package/src/http.ts

@@ -15,6 +15,7 @@
 import type { Context } from '@deepseek-ai/cordis'
 import type { IncomingMessage, ServerResponse } from 'node:http'
 import type { LocatorRuntime, SearchArgs } from './runtime.ts'
+import type { ContentIndex } from './content.ts'
 import { openWithDefaultApp, revealInFileManager, withinRoots } from './reveal.ts'
 import { statSync } from 'node:fs'
 
@@ -46,7 +47,7 @@ interface SearchBody {
 const BODY_LIMIT = 64 * 1024
 
 /** Mount the panel's routes when this profile runs a Web server. */
-export function registerHttp(ctx: Context, rt: LocatorRuntime): void {
+export function registerHttp(ctx: Context, rt: LocatorRuntime, content?: ContentIndex): void {
   const inject = (ctx as unknown as {
     inject?: (deps: string[], callback: (scoped: Context) => void) => void
   }).inject
@@ -69,6 +70,17 @@ export function registerHttp(ctx: Context, rt: LocatorRuntime): void {
       path: '/dsh-locator/reveal',
       handler: handler(scoped, async body => await reveal(rt, body)),
     }))
+    if (content !== undefined) {
+      scoped.effect(() => server.register({
+        kind: 'exact',
+        path: '/dsh-locator/content',
+        handler: handler(scoped, async body => {
+          const query = typeof body.query === 'string' ? body.query : ''
+          const limit = typeof body.limit === 'number' ? body.limit : 30
+          return { ok: true, query, hits: content.search(query, limit), state: content.status() }
+        }),
+      }))
+    }
     scoped.effect(() => server.register({
       kind: 'exact',
       path: '/dsh-locator/open',

package/src/index.ts

@@ -7,7 +7,9 @@
 import type { Context } from '@deepseek-ai/cordis'
 import { Config as ConfigSchema } from './config.ts'
 import type { Config } from './config.ts'
+import { ContentIndex } from './content.ts'
 import { makeExportTool } from './export.ts'
+import { makeFindTool } from './find.ts'
 import { registerHttp } from './http.ts'
 import { resolveBinary, resolveRoots, resolveStateDir } from './paths.ts'
 import { LocatorRuntime } from './runtime.ts'
@@ -34,15 +36,26 @@ export function apply(ctx: Context, config: Config): void {
   const binary = resolveBinary(config, cwd)
   const logger = ctx.logger('dsh-locator')
   const rt = new LocatorRuntime(config, roots, stateDir, binary, cwd)
+  const content = config.contentIndex
+    ? new ContentIndex(rt, {
+        stateDir,
+        sessionDirs: config.contentSessionDirs,
+        maxFileBytes: config.contentMaxFileBytes,
+        maxFiles: config.contentMaxFiles,
+        maxSessions: config.contentMaxSessions,
+      })
+    : undefined
 
   ctx.effect(() => {
     // Let the host finish starting before a cold scan starts reading the disk.
     const timer = setTimeout(() => {
       rt.start()
       logger.info(`index roots: ${roots.join(', ')} (state ${stateDir})`)
+      if (content !== undefined) void startContent(rt, content, logger)
     }, 1500)
     return () => {
       clearTimeout(timer)
+      content?.close()
       void rt.stop()
     }
   })
@@ -51,8 +64,24 @@ export function apply(ctx: Context, config: Config): void {
   ctx.tools.register(makeStatusTool(rt))
   ctx.tools.register(makeRebuildTool(rt))
   ctx.tools.register(makeExportTool(rt))
+  if (content !== undefined) ctx.tools.register(makeFindTool(content))
 
   // The Web GUI panel is optional: profiles without a Web server keep the
   // three tools and never mount an HTTP route.
-  registerHttp(ctx, rt)
+  registerHttp(ctx, rt, content)
+}
+
+async function startContent(rt: LocatorRuntime, content: ContentIndex, logger: { info(message: string): void }): Promise<void> {
+  try {
+    await rt.waitReady()
+    for (let i = 0; i < 900; i++) {
+      const status = await rt.engineStatus()
+      if (status === undefined || !status.scanning) break
+      await new Promise(resolve => setTimeout(resolve, 2_000))
+    }
+    content.start()
+    logger.info('content index started')
+  } catch (error) {
+    logger.info(`content index skipped: ${error instanceof Error ? error.message : String(error)}`)
+  }
 }