@dotdotgod/cli 0.6.4 → 0.6.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dotdotgod/cli",
-  "version": "0.6.4",
+  "version": "0.6.5",
   "description": "CLI for dotdotgod validation, hybrid document/code-outline search, and graph indexing.",
   "type": "module",
   "license": "Elastic-2.0",

package/src/commands/query.mjs

@@ -76,7 +76,7 @@ export async function buildVectorIndex(root, embedOrOptions = {}) {
   chunks.forEach((chunk, index) => { const oldIndex = cachedOffsets.get(chunk.fingerprint); if (oldIndex === undefined || !cached) missing.push({ chunk, index }); else { const d = cached.manifest.dimensions; rows[index] = Array.from(cached.vectors.subarray(oldIndex * d, (oldIndex + 1) * d)); reused += 1; } });
   for (let offset = 0; offset < missing.length; offset += EMBED_BATCH_SIZE) {
     const batch = missing.slice(offset, offset + EMBED_BATCH_SIZE);
-    const embedded = await resolved.embed(batch.map(({ chunk }) => `passage: ${chunk.passage}`));
+    const embedded = await resolved.embed(batch.map(({ chunk }) => chunk.passage), 'document');
     embedded.forEach((vector, index) => { rows[batch[index].index] = vector; });
   }
   const dimensions = rows[0]?.length ?? cached?.manifest?.dimensions;
@@ -92,7 +92,7 @@ export async function buildVectorIndex(root, embedOrOptions = {}) {
 export async function queryDocumentation(root, query, options = {}) {
   const limit = options.limit ?? DEFAULT_LIMIT;
   const index = await buildVectorIndex(root, options);
-  const [queryVector] = await index.embedding.embed([`query: ${query}`]);
+  const [queryVector] = await index.embedding.embed([query], 'query');
   const dimensions = index.manifest.dimensions;
   assertVector(queryVector, dimensions, 'Query vector');
   const results = rankVectorFiles(queryVector, index).map((chunk) => ({ ...chunk, score: Number((chunk.vectorScore + lexicalBoost(query, chunk)).toFixed(6)), vectorScore: Number(chunk.vectorScore.toFixed(6)) }))
@@ -111,7 +111,7 @@ export async function queryProject(root, query, options = {}) {
     try {
       assertCodeEmbeddingConsent(root, options, corpus.chunks);
       index = await buildVectorIndex(root, { ...options, chunks: corpus.chunks });
-      const [queryVector] = await index.embedding.embed([`query: ${query}`]);
+      const [queryVector] = await index.embedding.embed([query], 'query');
       vector = index.chunks.map((chunk, position) => ({ ...chunk, vectorScore: cosineScore(queryVector, index.vectors, position * index.manifest.dimensions, index.manifest.dimensions) }))
         .sort((a, b) => b.vectorScore - a.vectorScore || a.id.localeCompare(b.id));
     } catch (error) {

package/src/impact/vector-overlay.mjs

@@ -24,7 +24,7 @@ export async function buildVectorImpactOverlay(root, index, changedPaths, option
       if (!profile) continue;
       const embed = vectorIndex.embedding?.embed ?? options.embed;
       if (!embed) throw new Error('Vector index did not provide an embedder.');
-      const [queryVector] = await embed([`query: ${profile.text}`]);
+      const [queryVector] = await embed([profile.text], 'query');
       if (!queryVector) throw new Error('Changed-file embedding was empty.');
       const matches = rankVectorFiles(queryVector, vectorIndex)
         .filter((match) => match.path !== changed && match.vectorScore >= threshold)

package/src/query/embedder.mjs

@@ -1,4 +1,4 @@
-import { embeddingProfileIdentity, resolveEmbeddingProfile } from './embedding-config.mjs';
+import { embeddingInputFormat, embeddingProfileIdentity, resolveEmbeddingProfile } from './embedding-config.mjs';
 import { importTransformers } from './embedding-runtime.mjs';
 
 const localPipelines = new Map();
@@ -37,7 +37,12 @@ async function postJson(url, body, profile) {
     const response = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', ...(key ? { authorization: `Bearer ${key}` } : {}) }, body: JSON.stringify(body), signal: controller.signal });
     if (!response.ok) {
       const category = response.status === 401 || response.status === 403 ? 'authentication' : response.status === 429 ? 'rate limit' : `HTTP ${response.status}`;
-      throw new Error(`Remote embedding ${category} failure.`);
+      const error = new Error(`Remote embedding ${category} failure.`);
+      if (response.status === 400 || response.status === 413) {
+        const body = (await response.text()).slice(0, 8192);
+        error.tokenLimit = /(?:exceed\w*[\s\S]{0,120}tokens?|tokens?[\s\S]{0,120}(?:exceed|limit|too (?:long|many))|maximum context length)/i.test(body);
+      }
+      throw error;
     }
     try { return await response.json(); } catch { throw new Error('Remote embedding returned malformed JSON.'); }
   } catch (error) {
@@ -60,21 +65,36 @@ function remoteEmbedder(profile) {
   };
 }
 
+function withInputFormat(embed, profile) {
+  const format = embeddingInputFormat(profile);
+  return (texts, role) => {
+    if (role !== undefined && role !== 'query' && role !== 'document') throw new Error('Embedding role must be query or document.');
+    return embed(role === undefined ? texts : texts.map((text) => format[`${role}Prefix`] + text));
+  };
+}
+
 export async function createEmbedder(profile) {
   const embed = profile.provider === 'local' ? await localEmbedder(profile) : remoteEmbedder(profile);
-  return async (texts) => {
+  async function embedBatch(texts) {
     if (texts.length === 0) return [];
-    const rows = await embed(texts);
+    let rows;
+    try { rows = await embed(texts); }
+    catch (error) {
+      if (!error.tokenLimit || texts.length === 1) throw error;
+      const middle = Math.ceil(texts.length / 2);
+      rows = [...await embedBatch(texts.slice(0, middle)), ...await embedBatch(texts.slice(middle))];
+    }
     if (!Array.isArray(rows) || rows.length !== texts.length) throw new Error('Embedding provider returned an unexpected vector count.');
     const dimensions = rows[0]?.length;
     if (!dimensions || rows.some((row) => row.length !== dimensions)) throw new Error('Embedding provider returned inconsistent vector dimensions.');
     return rows;
-  };
+  }
+  return withInputFormat(embedBatch, profile);
 }
 
 export async function resolveEmbedder(root, options = {}) {
   const resolved = options.profile ? { profile: options.profile, source: options.source ?? 'injected', path: null } : resolveEmbeddingProfile(root, options);
-  return { ...resolved, identity: embeddingProfileIdentity(resolved.profile), embed: options.embed ?? await createEmbedder(resolved.profile) };
+  return { ...resolved, identity: embeddingProfileIdentity(resolved.profile), embed: options.embed ? withInputFormat(options.embed, resolved.profile) : await createEmbedder(resolved.profile) };
 }
 
 export async function embedTexts(texts) {

package/src/query/embedding-config.mjs

@@ -4,6 +4,18 @@ import { join } from 'node:path';
 
 export const DEFAULT_EMBEDDING_PROFILE = Object.freeze({ provider: 'local', model: 'Xenova/multilingual-e5-small' });
 export const EMBEDDING_PROVIDERS = Object.freeze(['local', 'openai-compatible', 'ollama']);
+const DEFAULT_INPUT_FORMAT = Object.freeze({ queryPrefix: 'query: ', documentPrefix: 'passage: ' });
+
+export function embeddingInputFormat(profile, label = 'embedding') {
+  if (profile.inputFormat === undefined) return { ...DEFAULT_INPUT_FORMAT };
+  const format = profile.inputFormat;
+  if (!format || typeof format !== 'object' || Array.isArray(format)
+    || Object.keys(format).some((key) => !Object.hasOwn(DEFAULT_INPUT_FORMAT, key))
+    || Object.keys(DEFAULT_INPUT_FORMAT).some((key) => typeof format[key] !== 'string')) {
+    throw new Error(`${label}.inputFormat must contain exactly queryPrefix and documentPrefix strings.`);
+  }
+  return { queryPrefix: format.queryPrefix, documentPrefix: format.documentPrefix };
+}
 
 function readJson(path, label) {
   if (!existsSync(path)) return null;
@@ -13,7 +25,7 @@ function readJson(path, label) {
 
 export function validateEmbeddingProfile(profile, label = 'embedding') {
   if (!profile || typeof profile !== 'object' || Array.isArray(profile)) throw new Error(`${label} must be an object.`);
-  const allowed = new Set(['provider', 'model', 'baseUrl', 'apiKey', 'apiKeyEnv']);
+  const allowed = new Set(['provider', 'model', 'baseUrl', 'apiKey', 'apiKeyEnv', 'inputFormat']);
   const unknown = Object.keys(profile).find((key) => !allowed.has(key));
   if (unknown) throw new Error(`${label}.${unknown} is not supported.`);
   if (!EMBEDDING_PROVIDERS.includes(profile.provider)) throw new Error(`${label}.provider must be local, openai-compatible, or ollama.`);
@@ -27,6 +39,7 @@ export function validateEmbeddingProfile(profile, label = 'embedding') {
     if (!['http:', 'https:'].includes(url.protocol)) throw new Error(`${label}.baseUrl must use HTTP or HTTPS.`);
     if (url.username || url.password) throw new Error(`${label}.baseUrl must not contain credentials.`);
   }
+  embeddingInputFormat(profile, label);
   return { ...profile, model: profile.model.trim(), ...(profile.baseUrl ? { baseUrl: profile.baseUrl.replace(/\/$/, '') } : {}) };
 }
 
@@ -36,8 +49,11 @@ export function sanitizeEmbeddingProfile(profile) {
 }
 
 export function embeddingProfileIdentity(profile) {
-  const { apiKey, apiKeyEnv, ...identity } = profile;
-  return identity;
+  return {
+    provider: profile.provider, model: profile.model,
+    ...(profile.baseUrl !== undefined ? { baseUrl: profile.baseUrl } : {}),
+    inputFormatVersion: 1, inputFormat: embeddingInputFormat(profile),
+  };
 }
 
 export function resolveEmbeddingProfile(root = '.', options = {}) {