dsh-code-index 0.10.1 → 0.11.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dsh-code-index",
-  "version": "0.10.1",
-  "description": "Edit-ready Context Packs for DSH agents: bounded source excerpts, changes, related tests, and inclusion evidence — local-first.",
+  "version": "0.11.0",
+  "description": "Impact X-Ray for DSH: inspect potential code-change impact, provenance and real source evidence before AI edits.",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",

package/scripts/impact-xray-benchmark.mjs

@@ -0,0 +1,30 @@
+/** Measured local observations, not a performance promise. Run after build. */
+import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'
+import { execFileSync } from 'node:child_process'
+import path from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+if (process.argv[2] !== '--worker') {
+  const rows = []
+  const baseline = process.argv[2] === '--baseline' ? path.resolve(process.argv[3]) : null
+  for (const files of [6, 100, 600]) rows.push(JSON.parse(execFileSync(process.execPath, [fileURLToPath(import.meta.url), '--worker', String(files), ...(baseline ? [baseline] : [])], { encoding: 'utf8' })))
+  console.log(JSON.stringify({ method: 'Three isolated Node processes; generated real files; explicit file selection, not a Git-change or production-repository benchmark; one cold index and three warm analysis/projection samples per size', node: process.version, rows }, null, 2))
+} else {
+  const { buildIndex, buildChangeContext, buildImpactXray } = await import(process.argv[4] ? pathToFileURL(process.argv[4]).href : '../dist/index.js')
+  const count = Number(process.argv[3])
+  await mkdir('.dsh-code-index/impact-xray-mvp', { recursive: true })
+  const root = await mkdtemp(path.resolve('.dsh-code-index/impact-xray-mvp/bench-'))
+  await mkdir(path.join(root, '.git'))
+  await writeFile(path.join(root, 'core.ts'), 'export function normalize(input: string) { return input.trim() }\n')
+  for (let i = 1; i < count; i++) await writeFile(path.join(root, `caller${i}.ts`), `import { normalize } from './core'\nexport function caller${i}() { return normalize(' value ') }\nexport function entry${i}() { return caller${i}() }\n`)
+  const start = performance.now(), index = await buildIndex(root), indexMs = performance.now() - start
+  const samples = []
+  for (let i = 0; i < 3; i++) {
+    const cpu = process.cpuUsage(), before = performance.now()
+    const result = await buildChangeContext(index, { kind: 'files', files: ['core.ts'] }, { maxImpact: 200, maxPaths: 50 })
+    const analysisMs = performance.now() - before, project = performance.now(), view = buildImpactXray ? await buildImpactXray(index, result) : null
+    const cpuUsed = process.cpuUsage(cpu)
+    samples.push({ analysisMs, projectionMs: view ? performance.now() - project : null, cpuMs: (cpuUsed.user + cpuUsed.system) / 1000,
+      nodes: view?.nodes.length ?? 0, edges: view?.edges.length ?? 0, metadataBytes: view ? Buffer.byteLength(JSON.stringify(view)) : 0, truncated: view?.truncated ?? Object.values(result.truncated).some(Boolean) })
+  }
+  console.log(JSON.stringify({ files: index.files.length, symbols: index.files.reduce((n, f) => n + f.symbols.length, 0), indexMs, samples, rssBytes: process.memoryUsage().rss, processPeakRssBytes: process.resourceUsage().maxRSS * 1024 }))
+}

package/scripts/impact-xray-demo.mjs

@@ -0,0 +1,23 @@
+/** Reproducible real JavaScript repository for the Desktop demo. */
+import { access, mkdir, writeFile } from 'node:fs/promises'
+import { execFileSync } from 'node:child_process'
+import path from 'node:path'
+const root = path.resolve(process.argv[2] ?? '.dsh-code-index/impact-xray-mvp/demo-repo')
+try { await access(root); throw Error(`Refusing to overwrite ${root}`) } catch (e) { if (e.code !== 'ENOENT') throw e }
+await mkdir(path.join(root, 'src'), { recursive: true }); await mkdir(path.join(root, 'tests'))
+const files = {
+  'package.json': JSON.stringify({ name: 'impact-xray-demo', private: true, type: 'module', scripts: { test: 'node --test' } }, null, 2) + '\n',
+  'src/core.js': 'export function normalize(input) {\n  return input.trim()\n}\nexport function localUsage() { return normalize(" local ") }\n',
+  'src/startup.js': 'import { normalize } from "./core.js"\nexport function startup() { return normalize(" startup ") }\n',
+  'src/scoped.js': 'import * as core from "./core.js"\nexport function scopedUsage() { return core.normalize(" scoped ") }\n',
+  'src/loose.js': '// Deliberately unresolved name: a candidate, not a verified executable path.\nexport function looseUsage() { return normalize(" loose ") }\n',
+  'src/main.js': 'import { startup } from "./startup.js"\nexport function main() { return startup() }\n',
+  'tests/core.test.js': 'import test from "node:test"\nimport assert from "node:assert/strict"\nimport { normalize } from "../src/core.js"\nexport function checkNormalize() { return normalize(" test ") }\ntest("normalization preserves letter case", () => assert.equal(checkNormalize(), "test"))\n',
+}
+for (const [file, text] of Object.entries(files)) await writeFile(path.join(root, file), text)
+const git = (...args) => execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim()
+git('init', '-q'); git('add', '.'); git('-c', 'user.name=Impact X-Ray Demo', '-c', 'user.email=demo@example.invalid', 'commit', '-qm', 'real demo baseline')
+await writeFile(path.join(root, 'src/core.js'), files['src/core.js'].replace('input.trim()', 'input.trim().toUpperCase()'))
+console.log(JSON.stringify({ root, base: git('rev-parse', 'HEAD'), request: { repoRoot: root, outputFormat: 'xray' },
+  test: 'node --test (baseline passes; current case-conversion edit intentionally fails)',
+  uncertainty: 'looseUsage intentionally has no import; its name-only edge is a low-confidence candidate.' }, null, 2))

package/scripts/impact-xray-poc.mjs

@@ -0,0 +1,27 @@
+/** Create a fresh, real Git fixture; no host/profile changes or service. */
+import { mkdir, writeFile, access } from 'node:fs/promises'
+import { execFileSync } from 'node:child_process'
+import path from 'node:path'
+
+const root = path.resolve(process.argv[2] ?? '.dsh-code-index/impact-xray-poc/fixture')
+try { await access(root); throw Error(`Fixture directory already exists: ${root}`) }
+catch (error) { if (error.code !== 'ENOENT') throw error }
+await mkdir(path.join(root, 'src'), { recursive: true })
+await mkdir(path.join(root, 'tests'))
+const files = {
+  'src/core.ts': 'export function normalize(input: string) {\n  return input.trim()\n}\nexport function localUsage() { return normalize(" local ") }\n',
+  'src/startup.ts': 'import { normalize } from "./core"\nexport function startup() { return normalize(" startup ") }\n',
+  'src/scoped.ts': 'import * as core from "./core"\nexport function scopedUsage() { return core.normalize(" scoped ") }\n',
+  'src/loose.ts': '// Deliberately unresolved: a low-confidence name match, not an executable import.\nexport function looseUsage() { return normalize(" loose ") }\n',
+  'src/main.ts': 'import { startup } from "./startup"\nexport function main() { return startup() }\n',
+  'tests/core.test.ts': 'import { normalize } from "../src/core"\nexport function checkNormalize() { return normalize(" test ") === "test" }\n',
+}
+for (const [file, text] of Object.entries(files)) await writeFile(path.join(root, file), text)
+const git = (...args) => execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim()
+git('init', '-q')
+git('add', 'src', 'tests')
+git('-c', 'user.name=Impact X-Ray PoC', '-c', 'user.email=poc@example.invalid', 'commit', '-qm', 'fixture baseline')
+await writeFile(path.join(root, 'src/core.ts'), files['src/core.ts'].replace('input.trim()', 'input.trim().toUpperCase()'))
+console.log(JSON.stringify({ root, base: git('rev-parse', 'HEAD'), changed: 'src/core.ts:2',
+  request: { repoRoot: root, outputFormat: 'xray' },
+  notes: 'Tiny analysis fixture. loose.ts intentionally has no import; the test file is a lead, not a coverage claim.' }, null, 2))

package/scripts/impact-xray-rc-benchmark.mjs

@@ -0,0 +1,77 @@
+/** Stage-separated observations; never a speed or memory guarantee. No forced GC. */
+import { readFile, mkdir, writeFile } from 'node:fs/promises'
+import { execFileSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import path from 'node:path'
+import os from 'node:os'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+const sha = value => createHash('sha256').update(value).digest('hex')
+const script = fileURLToPath(import.meta.url)
+if (process.argv[2] === '--worker') {
+  const [moduleFile, root, file, mode] = process.argv.slice(3)
+  const api = await import(pathToFileURL(moduleFile).href)
+  const stages = []
+  const capture = (stage, ms = 0) => stages.push({ stage, ms, ...process.memoryUsage(), peakRssBytes: process.resourceUsage().maxRSS * 1024 })
+  capture('module-loaded')
+  let start = performance.now()
+  const index = await api.buildIndex(root)
+  capture('cold-index', performance.now() - start)
+  if (process.env.XRAY_BENCH_SETTLE_MS) {
+    await new Promise(resolve => setTimeout(resolve, Number(process.env.XRAY_BENCH_SETTLE_MS)))
+    capture('after-index-idle', Number(process.env.XRAY_BENCH_SETTLE_MS))
+  }
+  const samples = []
+  for (let i = 0; i < 3; i++) {
+    start = performance.now()
+    const result = await api.buildChangeContext(index, { kind: 'files', files: [file] }, { maxImpact: 200, maxPaths: 50 })
+    capture(`analysis-${i + 1}`, performance.now() - start)
+    const analysisSha256 = sha(JSON.stringify(result))
+    let view
+    if (mode === 'xray') {
+      start = performance.now()
+      view = await api.buildImpactXray(index, result)
+      capture(`projection-${i + 1}`, performance.now() - start)
+    }
+    samples.push({ analysisSha256, changed: result.changed.length, callers: result.directCallers.length, impact: result.impact.length,
+      tests: result.tests.length, nodes: view?.nodes.length, edges: view?.edges.length,
+      metadataBytes: view ? Buffer.byteLength(JSON.stringify(view)) : 0 })
+  }
+  console.log(JSON.stringify({ mode, files: index.files.length, symbols: index.files.reduce((n, f) => n + f.symbols.length, 0),
+    indexedSourceBytes: (await Promise.all(index.files.map(f => readFile(path.join(root, f.path))))).reduce((n, b) => n + b.length, 0), stages, samples }))
+} else {
+  const [baseline, mediumRoot, mediumFile = 'lib/utils.js', repeats = '3'] = process.argv.slice(2)
+  if (!baseline || !mediumRoot) throw Error('Usage: node scripts/impact-xray-rc-benchmark.mjs BASELINE_DIST MEDIUM_REPO [FILE] [REPEATS]')
+  const scratch = path.resolve('.dsh-code-index/impact-xray-rc')
+  const tiny = path.join(scratch, 'six-files')
+  await mkdir(tiny, { recursive: true })
+  await mkdir(path.join(tiny, '.git'), { recursive: true })
+  await writeFile(path.join(tiny, 'core.ts'), 'export function normalize(input: string) { return input.trim() }\n')
+  for (let i = 1; i < 6; i++) await writeFile(path.join(tiny, `caller${i}.ts`), `import { normalize } from './core'\nexport function caller${i}() { return normalize(' value ') }\nexport function entry${i}() { return caller${i}() }\n`)
+  const variants = [{ name: 'v0.10.2-analysis', module: path.resolve(baseline), mode: 'analysis' },
+    { name: 'candidate-analysis', module: path.resolve('dist/index.js'), mode: 'analysis' },
+    { name: 'candidate-xray', module: path.resolve('dist/index.js'), mode: 'xray' }]
+  const observations = []
+  for (const [fixture, root, file] of [['six-files', tiny, 'core.ts'], ['express-v5.1.0', path.resolve(mediumRoot), mediumFile]]) {
+    for (let repeat = 0; repeat < Number(repeats); repeat++) {
+      // Rotate order to reduce systematic order / machine-temperature bias.
+      const order = [...variants.slice(repeat % 3), ...variants.slice(0, repeat % 3)]
+      for (const v of order) {
+        const result = JSON.parse(execFileSync(process.execPath, [script, '--worker', v.module, root, file, v.mode], { encoding: 'utf8', maxBuffer: 4e6 }))
+        observations.push({ fixture, repeat: repeat + 1, variant: v.name, ...result })
+        await writeFile(path.join(scratch, 'performance-progress.json'), JSON.stringify(observations, null, 2) + '\n')
+      }
+    }
+  }
+  const byFixture = new Map()
+  for (const row of observations) {
+    const digests = row.samples.map(s => s.analysisSha256)
+    const expected = byFixture.get(row.fixture) ?? digests[0]
+    if (digests.some(d => d !== expected)) throw Error(`Analysis output changed: ${row.fixture}`)
+    byFixture.set(row.fixture, expected)
+  }
+  console.log(JSON.stringify({ node: process.version, platform: process.platform, cpu: os.cpus()[0]?.model, physicalMemoryBytes: os.totalmem(),
+    method: '18 isolated processes, three rotated repeats per fixture/variant. Identical files and explicit file selection, no fabricated diff. Cold index then three analysis samples; projection only in xray variant. Same analysis output hashes required. Process high-water RSS is cumulative, not a per-stage allocation metric. No forced GC; retained RSS is not proof of a leak.',
+    indexIdleMs: Number(process.env.XRAY_BENCH_SETTLE_MS ?? 0),
+    medium: { origin: execFileSync('git', ['-C', mediumRoot, 'remote', 'get-url', 'origin'], { encoding: 'utf8' }).trim(), commit: execFileSync('git', ['-C', mediumRoot, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), selectedFile: mediumFile },
+    engineSources: await Promise.all(['src/buildIndex.ts', 'src/extract.ts', 'src/refgraph.ts', 'src/change-context.ts'].map(async file => ({ file, sha256: sha(await readFile(file)) }))), observations }, null, 2))
+}

package/scripts/process-impact-xray-demo.mjs

@@ -0,0 +1,29 @@
+/** Trim/crop authentic Desktop footage. No generated UI, overlays or source text. */
+import { spawnSync } from 'node:child_process'
+import { readFile, stat, writeFile } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+import path from 'node:path'
+const [input, startArg, durationArg, crop = '-'] = process.argv.slice(2)
+const start = Number(startArg), duration = Number(durationArg)
+if (!input || !Number.isFinite(start) || start < 0 || !Number.isFinite(duration) || duration < 20 || duration > 30 ||
+  crop !== '-' && !/^\d+:\d+:\d+:\d+$/.test(crop)) throw Error('Usage: node scripts/process-impact-xray-demo.mjs INPUT START_SECONDS DURATION_20_TO_30 [WIDTH:HEIGHT:X:Y]. Set FFMPEG to an existing ffmpeg binary if needed.')
+const ffmpeg = process.env.FFMPEG || 'ffmpeg'
+const output = 'assets/impact-xray-mvp'
+const filters = [...(crop === '-' ? [] : [`crop=${crop}`]), "scale='min(1180,iw)':-2"]
+function run(args) {
+  const r = spawnSync(ffmpeg, ['-hide_banner', '-y', ...args], { encoding: 'utf8' })
+  if (r.status !== 0) throw Error(r.stderr || String(r.error))
+  return r.stderr
+}
+const log = run(['-ss', String(start), '-i', input, '-t', String(duration), '-vf', filters.join(','), '-an', '-r', '30', '-c:v', 'libx264', '-crf', '20', '-pix_fmt', 'yuv420p', '-movflags', '+faststart', `${output}.mp4`])
+if (!/frame=\s*[1-9]/.test(log)) throw Error('No footage encoded')
+run(['-i', `${output}.mp4`, '-vf', "fps=10,scale='min(1000,iw)':-2,split[a][b];[a]palettegen=max_colors=128[p];[b][p]paletteuse=dither=bayer:bayer_scale=3", '-loop', '0', `${output}.gif`])
+run(['-ss', '8', '-i', `${output}.mp4`, '-frames:v', '1', `${output}-poster.png`])
+const sha256 = async p => createHash('sha256').update(await readFile(p)).digest('hex')
+const files = []
+for (const suffix of ['mp4', 'gif', 'poster.png']) {
+  const file = suffix === 'poster.png' ? `${output}-poster.png` : `${output}.${suffix}`
+  files.push({ file, bytes: (await stat(file)).size, sha256: await sha256(file) })
+}
+await writeFile(`${output}-media.json`, JSON.stringify({ method: 'A chronological excerpt of actual DSH Desktop screen recording; rectangular crop/scale, frame-rate conversion, palette quantization and audio removal only. No fabricated UI, text, screenshots or latency claim.', raw: { filename: path.basename(input), sha256: await sha256(input) }, excerpt: { startSeconds: start, durationSeconds: duration, crop, speed: 1 }, files }, null, 2) + '\n')
+console.log(JSON.stringify(files, null, 2))

package/scripts/release-smoke.mjs

@@ -219,6 +219,23 @@ try {
   writeFileSync(path.join(arkRoot, 'Broken.ets'), '@Component\nstruct Broken { build() {')
   assert.match(await invoke('code_index', { repoRoot: arkRoot }), /index incomplete:/)
 
+  // Issue #5: exercise actual installed Rust signatures, not source imports.
+  const rustRoot = path.join(smokeDir, 'rust-repo')
+  mkdirSync(path.join(rustRoot, '.git'), { recursive: true })
+  writeFileSync(path.join(rustRoot, 'repro.rs'), [
+    'pub struct PublicOne { pub field: u32 }',
+    'struct PrivateOne { field: u32 }',
+    'pub enum PublicEnum { A }',
+    'enum PrivateEnum { A }',
+    'pub trait PublicTrait { fn run(&self); }',
+    'trait PrivateTrait { fn run(&self); }',
+    'pub fn public_function(a: u32) -> u32 { a }',
+    'fn private_function(a: u32) -> u32 { a }',
+    'pub(crate) struct CrateOnly;',
+    'pub(in crate) enum Restricted { A }',
+    'fn caller() { let value = PublicOne { field: public_function(1) }; }',
+  ].join('\n'))
+
   for (const dispose of disposers.reverse()) dispose()
 
   const compactRegistered = []
@@ -272,12 +289,56 @@ try {
       assert.equal(typeof result.value, outputFormat === 'pack' ? 'object' : 'string')
       if (outputFormat === 'pack') assert.ok(JSON.stringify(result.value).length <= 5000)
     }
+    await nativeInvoke('code_index', { action: 'build', repoRoot: rustRoot })
+    const rustSymbols = await nativeInvoke('code_symbols', { repoRoot: rustRoot })
+    assert.ok(rustSymbols.value.some(symbol => symbol.name === 'CrateOnly' && symbol.signature === 'CrateOnly' && !symbol.exported))
+    assert.ok(rustSymbols.value.some(symbol => symbol.name === 'Restricted' && symbol.signature === 'Restricted' && !symbol.exported))
+    const rustMap = await nativeInvoke('code_map', { repoRoot: rustRoot })
+    for (const name of ['PublicOne', 'PublicEnum', 'PublicTrait']) {
+      const search = await nativeInvoke('code_search', { query: name, repoRoot: rustRoot })
+      assert.equal(search.value[0]?.name, name)
+      assert.equal(search.value[0]?.signature, name)
+      assert.match(JSON.stringify(search.content), new RegExp(name))
+      assert.match(rustMap.value, new RegExp(name))
+      const refs = await nativeInvoke('code_refs', { symbol: name, repoRoot: rustRoot })
+      assert.match(refs.value, new RegExp(`${name} — repro.rs:`))
+      assert.match(refs.value, /callers \(0\)/) // no invented struct-literal/type-use edges
+    }
+    const rustFunction = await nativeInvoke('code_refs', { symbol: 'public_function', repoRoot: rustRoot })
+    assert.match(rustFunction.value, /callers \(1\)/)
+    const xrayRoot = path.join(smokeDir, 'xray-demo')
+    execFileSync(process.execPath, [path.join(smokeDir, 'node_modules', packageJson.name, 'scripts/impact-xray-demo.mjs'), xrayRoot], { encoding: 'utf8' })
+    const legacyChange = await nativeInvoke('code_change_context', { repoRoot: xrayRoot })
+    const xray = await nativeInvoke('code_change_context', { repoRoot: xrayRoot, outputFormat: 'xray' })
+    assert.equal(typeof legacyChange.value, 'string')
+    assert.deepEqual(xray.content, legacyChange.content)
+    assert.equal(xray.meta.card, 'impact-xray'); assert.equal(xray.meta.version, 2)
+    assert.equal(xray.meta.edges.filter(e => e.kind === 'call').length, 6)
+    assert.equal(xray.meta.edges.filter(e => e.kind === 'import').length, 3)
+    assert.equal(xray.meta.tests.length, 1)
+    assert.deepEqual(new Set(xray.meta.edges.map(e => e.resolution)), new Set(['exact', 'import-scoped', 'name-only']))
+    assert.ok(xray.meta.paths.some(p => p.edgeIds.length === 2 && p.resolution === 'exact' && !p.omitted))
+    assert.ok(Buffer.byteLength(JSON.stringify(xray.meta)) <= plugin.XRAY_LIMITS.metadataBytes)
+    for (const item of [...xray.meta.nodes, ...xray.meta.edges]) {
+      const e = item.evidence
+      assert.equal(typeof e.code, 'string')
+      const lines = readFileSync(path.join(xrayRoot, e.file), 'utf8').split(/\r?\n/)
+      assert.equal(e.code, lines.slice(e.startLine - 1, e.startLine - 1 + e.code.split('\n').length).join('\n'))
+    }
+    console.log('Impact X-Ray installed runtime: default text, call/import/test/path provenance and source windows passed')
+    for (const outputFormat of ['text', 'pack']) {
+      const result = await nativeInvoke('code_context', {
+        task: 'Explain PublicOne', repoRoot: rustRoot, outputFormat, budgetChars: 5000,
+      })
+      assert.match(JSON.stringify(result.content), /PublicOne/)
+      if (outputFormat === 'pack') assert.ok(result.value.items.some(item => item.name === 'PublicOne' && item.code?.includes('pub struct PublicOne')))
+    }
   } finally {
     for (const dispose of nativeDisposers.reverse()) dispose()
     await nativeContext.fiber.dispose()
   }
   console.log(`release smoke passed for ${packageJson.name}@${packageJson.version} with dsh-tools@${toolsVersion}`)
 } finally {
-  rmSync(smokeDir, { recursive: true, force: true })
+  rmSync(smokeDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 })
   if (!suppliedTarball) rmSync(tarball, { force: true })
 }