ai-code-review-kit 1.1.2

package/src/rag/index.js

@@ -0,0 +1,121 @@
+import fs from 'fs'
+import path from 'path'
+
+import { connectLanceDB } from './lancedb.js'
+import { collectTextFiles } from './fs.js'
+import { chunkText } from './text.js'
+
+function normalizePaths(value) {
+  if (Array.isArray(value)) return value.map(String).filter(Boolean)
+  if (typeof value === 'string') {
+    const trimmed = value.trim()
+    if (!trimmed) return []
+    return trimmed.split(',').map((s) => s.trim()).filter(Boolean)
+  }
+  return []
+}
+
+function ensureDir(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true })
+  }
+}
+
+export async function buildKnowledgeIndex({
+  repoRoot,
+  knowledgeBasePaths,
+  indexDir,
+  tableName,
+  embedder,
+  chunkSize = 1500,
+  chunkOverlap = 200,
+  maxFileSizeBytes = 512 * 1024,
+  batchSize = 64,
+}) {
+  if (!repoRoot) throw new Error('repoRoot is required')
+  if (!embedder?.embedTexts) throw new Error('embedder.embedTexts is required')
+
+  const paths = normalizePaths(knowledgeBasePaths)
+  if (paths.length === 0) throw new Error('knowledgeBasePaths is empty')
+
+  const files = collectTextFiles(repoRoot, paths, { maxFileSizeBytes })
+
+  const pending = []
+  for (const file of files) {
+    const chunks = chunkText(file.text, { chunkSize, overlap: chunkOverlap })
+    for (let i = 0; i < chunks.length; i++) {
+      pending.push({
+        id: `${file.relPath}::${i}`,
+        path: file.relPath,
+        chunkIndex: i,
+        text: chunks[i],
+      })
+    }
+  }
+
+  const rows = []
+  for (let i = 0; i < pending.length; i += batchSize) {
+    const slice = pending.slice(i, i + batchSize)
+    const vectors = await embedder.embedTexts(slice.map((x) => x.text))
+    if (!Array.isArray(vectors) || vectors.length !== slice.length) {
+      throw new Error('Embeddings vector count mismatch')
+    }
+    for (let j = 0; j < slice.length; j++) {
+      rows.push({ ...slice[j], vector: vectors[j] })
+    }
+  }
+
+  const absIndexDir = path.resolve(repoRoot, indexDir)
+  ensureDir(absIndexDir)
+  const db = await connectLanceDB(absIndexDir)
+  await db.createTable(tableName, rows, { mode: 'overwrite' })
+  // LanceDB keeps historical versions by default. Prune them to avoid the cache directory
+  // growing unbounded across repeated `index` runs.
+  try {
+    const table = await db.openTable(tableName)
+    if (typeof table?.optimize === 'function') {
+      await table.optimize({ cleanupOlderThan: new Date() })
+    }
+  } catch {
+    // Best-effort cleanup; ignore failures to keep indexing usable.
+  }
+
+  return { files: files.length, chunks: rows.length, indexDir: absIndexDir, tableName }
+}
+
+export async function retrieveKnowledgeContext({
+  repoRoot,
+  indexDir,
+  tableName,
+  queryText,
+  embedder,
+  topK = 6,
+  maxChars = 8000,
+}) {
+  if (!repoRoot) throw new Error('repoRoot is required')
+  if (!queryText) return []
+  if (!embedder?.embedTexts) throw new Error('embedder.embedTexts is required')
+
+  const absIndexDir = path.resolve(repoRoot, indexDir)
+  if (!fs.existsSync(absIndexDir)) return []
+
+  const [vector] = await embedder.embedTexts([queryText])
+  if (!vector) return []
+
+  const db = await connectLanceDB(absIndexDir)
+  const table = await db.openTable(tableName)
+  const results = await table.vectorSearch(vector).limit(topK).toArray()
+
+  const picked = []
+  let used = 0
+  for (const row of results || []) {
+    const relPath = row.path || row.file || 'unknown'
+    const text = String(row.text || '').trim()
+    if (!text) continue
+    const snippet = `[${relPath}]\n${text}`
+    if (used + snippet.length > maxChars) break
+    used += snippet.length
+    picked.push({ path: relPath, text })
+  }
+  return picked
+}

package/src/rag/lancedb.js

@@ -0,0 +1,14 @@
+function resolveConnectFn(mod) {
+  if (typeof mod?.connect === 'function') return mod.connect
+  if (typeof mod?.default?.connect === 'function') return mod.default.connect
+  return undefined
+}
+
+export async function connectLanceDB(uri) {
+  const mod = await import('@lancedb/lancedb')
+  const connect = resolveConnectFn(mod)
+  if (!connect) {
+    throw new Error('Unable to find lancedb.connect()')
+  }
+  return connect(uri)
+}

package/src/rag/text.js

@@ -0,0 +1,18 @@
+export function chunkText(text, { chunkSize = 1500, overlap = 200 } = {}) {
+  const input = String(text || '')
+  if (!input) return []
+
+  const size = Math.max(200, chunkSize)
+  const ov = Math.max(0, Math.min(overlap, Math.floor(size / 2)))
+
+  const chunks = []
+  let start = 0
+  while (start < input.length) {
+    const end = Math.min(input.length, start + size)
+    const piece = input.slice(start, end)
+    if (piece.trim()) chunks.push(piece)
+    if (end >= input.length) break
+    start = Math.max(0, end - ov)
+  }
+  return chunks
+}

package/src/utils/openai.js

@@ -0,0 +1,50 @@
+function stripTrailingSlashes(value) {
+  return String(value || '').replace(/\/+$/, '')
+}
+
+/**
+ * Normalize user-provided OpenAI-compatible base URLs.
+ *
+ * Users sometimes paste a full endpoint (e.g. `/v1/responses` or `/v1/chat/completions`)
+ * into `baseURL`. This helper strips known endpoint suffixes so the rest of the code can
+ * safely append API paths.
+ */
+export function normalizeOpenAIBaseURL(baseURL) {
+  const original = String(baseURL || '').trim()
+  if (!original) return ''
+
+  const url = stripTrailingSlashes(original)
+  const lower = url.toLowerCase()
+
+  const rules = [
+    { suffix: '/v1/chat/completions', replace: '/v1' },
+    { suffix: '/chat/completions', replace: '' },
+    { suffix: '/v1/embeddings', replace: '/v1' },
+    { suffix: '/embeddings', replace: '' },
+    { suffix: '/v1/responses', replace: '/v1' },
+    { suffix: '/responses', replace: '' },
+  ]
+
+  for (const rule of rules) {
+    if (lower.endsWith(rule.suffix)) {
+      return stripTrailingSlashes(url.slice(0, url.length - rule.suffix.length) + rule.replace)
+    }
+  }
+
+  return url
+}
+
+export function getOpenAIChatCompletionsPath(baseURL) {
+  const url = String(baseURL || '')
+  return /\/v\d+\/?$/.test(url) ? '/chat/completions' : '/v1/chat/completions'
+}
+
+export function getOpenAIEmbeddingsPath(baseURL) {
+  const url = String(baseURL || '')
+  return /\/v\d+\/?$/.test(url) ? '/embeddings' : '/v1/embeddings'
+}
+
+export function getOpenAIResponsesPath(baseURL) {
+  const url = String(baseURL || '')
+  return /\/v\d+\/?$/.test(url) ? '/responses' : '/v1/responses'
+}