dsh-memu-memory 1.1.1 → 1.1.3

package/scripts/local-runtime-verify.mjs

@@ -0,0 +1,56 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import crypto from 'node:crypto'
+import { performance } from 'node:perf_hooks'
+import { getEmbeddingModel } from '../lib/embedding-catalog.js'
+import { startEmbeddingWorker } from '../lib/local-embedding-worker.js'
+
+const id = 'bge-small-zh-v1.5-int8'
+const root = path.resolve('.tmp-local-vector-20261030')
+const modelDir = path.join(root, 'model-root', id)
+const runtimeDir = path.join(root, 'runtime-root')
+const manifest = getEmbeddingModel(id)
+const token = crypto.randomBytes(32).toString('hex')
+const begin = performance.now()
+let worker
+const norm = a => Math.sqrt(a.reduce((n, x) => n + x * x, 0))
+const cosine = (a, b) => a.reduce((n, x, i) => n + x * b[i], 0) / norm(a) / norm(b)
+try {
+  worker = await startEmbeddingWorker({ modelDir, runtimeDir, manifest, token })
+  const readyMs = performance.now() - begin
+  const texts = ['一个人在海边散步，海风很轻。', '海边的微风吹着散步的人。', '数据库迁移必须保持事务一致性。']
+  async function request(input, encoding_format = 'float') {
+    const start = performance.now()
+    const response = await fetch(worker.url, { method: 'POST', headers: { authorization: 'Bearer ' + token, 'content-type': 'application/json' }, body: JSON.stringify({ model: worker.model, input, encoding_format }) })
+    const result = await response.json()
+    if (!response.ok) throw new Error(JSON.stringify(result))
+    return { result, ms: performance.now() - start }
+  }
+  const batch = await request(texts)
+  const single = await request(texts[0])
+  const single2 = await request(texts[0])
+  const base64 = await request(texts[0], 'base64')
+  const vectors = batch.result.data.map(row => row.embedding)
+  const buf = Buffer.from(base64.result.data[0].embedding, 'base64')
+  const decoded = Array.from({ length: buf.length / 4 }, (_, i) => buf.readFloatLE(i * 4))
+  if (vectors.some(v => v.length !== 512 || v.some(x => !Number.isFinite(x)) || Math.abs(norm(v) - 1) > 0.00001)) throw new Error('invalid real inference vector')
+  const batchSingleCosine = cosine(vectors[0], single.result.data[0].embedding)
+  const repeatedSingleCosine = cosine(single.result.data[0].embedding, single2.result.data[0].embedding)
+  console.log(JSON.stringify({ batchSingleCosine, repeatedSingleCosine, norms: vectors.map(norm) }))
+  if (batchSingleCosine < 0.999) throw new Error('batch padding changed embedding')
+  if (cosine(decoded, single.result.data[0].embedding) < 0.999999) throw new Error('base64 encoding mismatch')
+  const metadata = ['onnxruntime-node', 'onnxruntime-common', '@huggingface/tokenizers'].map(name => {
+    const file = path.join(runtimeDir, 'node_modules', name, 'package.json')
+    const bytes = fs.readFileSync(file)
+    const pkg = JSON.parse(bytes)
+    return { name, version: pkg.version, license: pkg.license, bytes: bytes.length, sha256: crypto.createHash('sha256').update(bytes).digest('hex') }
+  })
+  const report = { ok: true, model: worker.model, dimension: worker.dimension, readyMs, batchMs: batch.ms, singleMs: single.ms, norms: vectors.map(norm), relatedCosine: cosine(vectors[0], vectors[1]), unrelatedCosine: cosine(vectors[0], vectors[2]), batchSingleCosine, encodingCosine: cosine(decoded, single.result.data[0].embedding), metadata }
+  fs.writeFileSync(path.join(root, 'runtime-smoke.json'), JSON.stringify(report, null, 2) + '\n')
+  console.log(JSON.stringify(report, null, 2))
+} catch (error) {
+  console.error(error)
+  process.exitCode = 1
+} finally {
+  if (worker) await worker.close()
+}

package/scripts/local-vector-verify.mjs

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+// 本地向量目录/下载器的可复现核验脚本（§14.4）。
+//
+//   node scripts/local-vector-verify.mjs                  # 只读：校验目录、manifest、来源与已安装制品
+//   node scripts/local-vector-verify.mjs --download       # 显式联网下载（等同用户确认），默认落到临时目录
+//   node scripts/local-vector-verify.mjs --download --target <dir> --id <modelId>
+//
+// 边界：不读不写真实记忆库、不写 active profile；默认目标目录是仓库里的 .tmp-* 临时目录。
+
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import {
+  getEmbeddingCatalog,
+  getEmbeddingModel,
+  resolveEmbeddingSources,
+  validateEmbeddingManifest,
+} from '../lib/embedding-catalog.js'
+import { createEmbeddingDownloadManager } from '../lib/embedding-download.js'
+
+const HERE = path.dirname(fileURLToPath(import.meta.url))
+const REPO = path.resolve(HERE, '..')
+
+const argv = process.argv.slice(2)
+const flag = name => argv.includes('--' + name)
+const option = name => {
+  const index = argv.indexOf('--' + name)
+  return index >= 0 ? argv[index + 1] : undefined
+}
+
+const download = flag('download')
+const target = path.resolve(option('target') || path.join(REPO, '.tmp-local-vector-20261030', 'model-root'))
+const only = option('id')
+
+let failures = 0
+const log = line => console.log(line)
+const bad = line => { failures += 1; console.error('FAIL ' + line) }
+
+log('== 目录校验（manifest schema / 固定 revision / 逐文件 hash 声明）')
+for (const entry of getEmbeddingCatalog()) {
+  if (only && entry.id !== only) continue
+  const report = validateEmbeddingManifest(entry)
+  const source = resolveEmbeddingSources(entry)
+  log('-- ' + entry.id + ' [' + entry.availability + '] dim=' + entry.dimension + ' maxTokens=' + entry.maxTokens
+    + ' bytes=' + entry.totalBytes + ' fingerprint=' + entry.profileFingerprint.slice(0, 12))
+  if (report.errors.length) for (const error of report.errors) bad(entry.id + ' ' + error.code + ': ' + error.message)
+  else log('   manifest ok, ' + report.warnings.length + ' warning(s)')
+  for (const warning of report.warnings) log('   warn ' + warning.code + ': ' + warning.message)
+  if (entry.availability === 'available') {
+    if (!source.primary) bad(entry.id + ' has no usable pinned source')
+    else {
+      log('   source=' + source.primary.provider + ' repo=' + source.primary.repo + ' revision=' + source.primary.revision)
+      for (const file of source.primary.files) log('   file ' + file.path + ' ' + file.bytes + 'B ' + file.sha256.slice(0, 16) + '…')
+    }
+    for (const skipped of source.skipped) log('   skipped ' + skipped.provider + ': ' + skipped.reason)
+  } else {
+    log('   pending, 无可下载制品：' + entry.pendingReasons.join(' / '))
+  }
+}
+
+if (download) {
+  const entry = getEmbeddingModel(only || 'bge-small-zh-v1.5-int8')
+  if (!entry) { bad('unknown model id: ' + only); process.exit(1) }
+  log('')
+  log('== 显式下载 ' + entry.id + ' → ' + target)
+  const manager = createEmbeddingDownloadManager({ root: target })
+  try {
+    const status = await manager.download(entry.id, { confirmed: true })
+    log('installed=' + status.installed + ' state=' + status.state + ' revision=' + status.revision + ' provider=' + status.provider)
+    log('checks: ' + status.checks.join(', '))
+    const deep = manager.getStatus(entry.id, { deep: true })
+    log('deep hash re-verify: ' + deep.verified)
+    if (deep.verified !== true) bad('installed artifact does not re-verify')
+  } catch (error) {
+    bad('download failed: ' + (error.code || '') + ' ' + error.message)
+  }
+} else {
+  log('')
+  log('== 已安装状态（只读，deep=重新计算每个文件的 sha256）')
+  const manager = createEmbeddingDownloadManager({ root: target })
+  for (const entry of getEmbeddingCatalog()) {
+    if (only && entry.id !== only) continue
+    const status = manager.getStatus(entry.id, { deep: true })
+    log('-- ' + entry.id + ': state=' + status.state + ' installed=' + status.installed + ' active=' + status.active
+      + (status.verified === null ? '' : ' deep=' + status.verified)
+      + (status.error ? ' error=' + status.error.code : ''))
+    if (status.installed && status.verified === false) bad(entry.id + ' installed files no longer match their hashes')
+  }
+  log('（加 --download 才会联网；本脚本从不写 active profile）')
+}
+
+log('')
+log(failures === 0 ? 'RESULT ok' : 'RESULT ' + failures + ' failure(s)')
+process.exit(failures === 0 ? 0 : 1)