@dotdotgod/cli 0.6.3 → 0.6.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dotdotgod/cli",
-  "version": "0.6.3",
+  "version": "0.6.5",
   "description": "CLI for dotdotgod validation, hybrid document/code-outline search, and graph indexing.",
   "type": "module",
   "license": "Elastic-2.0",

package/src/commands/config.mjs

@@ -54,7 +54,6 @@ function formatConfigOutput(payload) {
   lines.push(`- load pinned paths: ${(payload.config.load?.pinnedPaths ?? []).join(', ') || 'none'}`);
   lines.push(`- load pinned bodies: ${(payload.config.load?.pinnedBodies ?? []).join(', ') || 'none'}`);
   lines.push(`- load documentation summary exclude: ${(payload.config.load?.documentationSummary?.exclude ?? []).join(', ') || 'none'}`);
-  lines.push(`- plan mode writable paths: ${(payload.config.planMode?.writablePaths ?? []).join(', ') || 'none'}`);
   const lowSignal = payload.config.referenceExpansion?.fuzzy?.lowSignal ?? { terms: [], add: [], remove: [] };
   lines.push(`- fuzzy low-signal terms: ${(lowSignal.terms ?? []).join(', ') || 'none'} (add: ${(lowSignal.add ?? []).join(', ') || 'none'}; remove: ${(lowSignal.remove ?? []).join(', ') || 'none'})`);
   if (errors.length > 0) {

package/src/commands/query.mjs

@@ -76,7 +76,7 @@ export async function buildVectorIndex(root, embedOrOptions = {}) {
   chunks.forEach((chunk, index) => { const oldIndex = cachedOffsets.get(chunk.fingerprint); if (oldIndex === undefined || !cached) missing.push({ chunk, index }); else { const d = cached.manifest.dimensions; rows[index] = Array.from(cached.vectors.subarray(oldIndex * d, (oldIndex + 1) * d)); reused += 1; } });
   for (let offset = 0; offset < missing.length; offset += EMBED_BATCH_SIZE) {
     const batch = missing.slice(offset, offset + EMBED_BATCH_SIZE);
-    const embedded = await resolved.embed(batch.map(({ chunk }) => `passage: ${chunk.passage}`));
+    const embedded = await resolved.embed(batch.map(({ chunk }) => chunk.passage), 'document');
     embedded.forEach((vector, index) => { rows[batch[index].index] = vector; });
   }
   const dimensions = rows[0]?.length ?? cached?.manifest?.dimensions;
@@ -92,7 +92,7 @@ export async function buildVectorIndex(root, embedOrOptions = {}) {
 export async function queryDocumentation(root, query, options = {}) {
   const limit = options.limit ?? DEFAULT_LIMIT;
   const index = await buildVectorIndex(root, options);
-  const [queryVector] = await index.embedding.embed([`query: ${query}`]);
+  const [queryVector] = await index.embedding.embed([query], 'query');
   const dimensions = index.manifest.dimensions;
   assertVector(queryVector, dimensions, 'Query vector');
   const results = rankVectorFiles(queryVector, index).map((chunk) => ({ ...chunk, score: Number((chunk.vectorScore + lexicalBoost(query, chunk)).toFixed(6)), vectorScore: Number(chunk.vectorScore.toFixed(6)) }))
@@ -111,7 +111,7 @@ export async function queryProject(root, query, options = {}) {
     try {
       assertCodeEmbeddingConsent(root, options, corpus.chunks);
       index = await buildVectorIndex(root, { ...options, chunks: corpus.chunks });
-      const [queryVector] = await index.embedding.embed([`query: ${query}`]);
+      const [queryVector] = await index.embedding.embed([query], 'query');
       vector = index.chunks.map((chunk, position) => ({ ...chunk, vectorScore: cosineScore(queryVector, index.vectors, position * index.manifest.dimensions, index.manifest.dimensions) }))
         .sort((a, b) => b.vectorScore - a.vectorScore || a.id.localeCompare(b.id));
     } catch (error) {

package/src/impact/vector-overlay.mjs

@@ -24,7 +24,7 @@ export async function buildVectorImpactOverlay(root, index, changedPaths, option
       if (!profile) continue;
       const embed = vectorIndex.embedding?.embed ?? options.embed;
       if (!embed) throw new Error('Vector index did not provide an embedder.');
-      const [queryVector] = await embed([`query: ${profile.text}`]);
+      const [queryVector] = await embed([profile.text], 'query');
       if (!queryVector) throw new Error('Changed-file embedding was empty.');
       const matches = rankVectorFiles(queryVector, vectorIndex)
         .filter((match) => match.path !== changed && match.vectorScore >= threshold)

package/src/memory/config.mjs

@@ -37,9 +37,7 @@ export const DEFAULT_LOAD_POLICY = {
     exclude: ['docs/plan', 'docs/archive'],
   },
 };
-export const DEFAULT_PLAN_MODE_POLICY = {
-  writablePaths: ['docs/plan/**', 'docs/archive/**'],
-};
+export const DEFAULT_PLAN_MODE_POLICY = {};
 export const DEFAULT_IMPACT_RANKING_POLICY = {
   connectionCap: 80,
   memoryCap: 20,
@@ -199,8 +197,8 @@ export function cloneLoadPolicy(policy = DEFAULT_LOAD_POLICY) {
   };
 }
 
-export function clonePlanModePolicy(policy = DEFAULT_PLAN_MODE_POLICY) {
-  return { writablePaths: [...(policy.writablePaths ?? DEFAULT_PLAN_MODE_POLICY.writablePaths)] };
+export function clonePlanModePolicy() {
+  return {};
 }
 
 export function normalizeDocumentationRoot(value = DEFAULT_DOCUMENTATION_ROOT) {
@@ -215,7 +213,7 @@ export function isValidDocumentationRoot(value) {
 }
 export function defaultMemoryConfig({ documentationRoot = DEFAULT_DOCUMENTATION_ROOT } = {}) {
   const root = normalizeDocumentationRoot(documentationRoot);
-  return { source: 'default', documentation: { root }, areas: memoryAreasForRoot(root).map(cloneArea), traceability: cloneTraceabilityPolicy({ ...DEFAULT_TRACEABILITY_POLICY, required: [`${root}/spec/**`] }), validation: cloneValidationPolicy(), impactRanking: cloneImpactRankingPolicy(), referenceExpansion: cloneReferenceExpansionPolicy(), load: cloneLoadPolicy({ ...DEFAULT_LOAD_POLICY, documentationSummary: { exclude: [`${root}/plan`, `${root}/archive`] } }), planMode: clonePlanModePolicy({ writablePaths: [`${root}/plan/**`, `${root}/archive/**`] }) };
+  return { source: 'default', documentation: { root }, areas: memoryAreasForRoot(root).map(cloneArea), traceability: cloneTraceabilityPolicy({ ...DEFAULT_TRACEABILITY_POLICY, required: [`${root}/spec/**`] }), validation: cloneValidationPolicy(), impactRanking: cloneImpactRankingPolicy(), referenceExpansion: cloneReferenceExpansionPolicy(), load: cloneLoadPolicy({ ...DEFAULT_LOAD_POLICY, documentationSummary: { exclude: [`${root}/plan`, `${root}/archive`] } }), planMode: clonePlanModePolicy() };
 }
 
 export function normalizePathPattern(value = '') {
@@ -289,12 +287,9 @@ export function isSecretLikePathPattern(value = '') {
   return /(^|\/)(\.env|\.npmrc|\.pypirc|id_rsa|id_dsa|id_ed25519|credentials?|secrets?)(\.|\/|$)/i.test(normalized);
 }
 
-function normalizePlanModePolicy(raw) {
-  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return clonePlanModePolicy();
-  const writablePaths = raw.writablePaths === undefined
-    ? DEFAULT_PLAN_MODE_POLICY.writablePaths
-    : [...new Set((Array.isArray(raw.writablePaths) ? raw.writablePaths : []).map(normalizePathPattern))];
-  return clonePlanModePolicy({ writablePaths });
+// Legacy planMode.writablePaths is accepted but discarded on normalization.
+function normalizePlanModePolicy(_raw) {
+  return clonePlanModePolicy();
 }
 
 function normalizeLoadPolicy(raw) {
@@ -437,10 +432,6 @@ export function validateMemoryConfigData(data, root = '.', file = 'dotdotgod.con
   if (planMode !== undefined) {
     if (!planMode || typeof planMode !== 'object' || Array.isArray(planMode)) {
       add('PLAN_MODE_CONFIG_INVALID', 'planMode', 'Expected an object.');
-    } else if (planMode.writablePaths !== undefined) {
-      const paths = planMode.writablePaths;
-      if (!Array.isArray(paths)) add('PLAN_MODE_CONFIG_INVALID_WRITABLE_PATHS', 'planMode.writablePaths', 'Expected an array of documentation path strings.');
-      else if (paths.some((value) => !isValidPathPattern(value) || typeof value !== 'string' || value.trim().startsWith('/') || (!normalizePathPattern(value).startsWith(`${normalizeDocumentationRoot(data.documentation?.root ?? DEFAULT_DOCUMENTATION_ROOT)}/`)) || normalizePathPattern(value).startsWith(`${normalizeDocumentationRoot(data.documentation?.root ?? DEFAULT_DOCUMENTATION_ROOT)}/.`) || isSecretLikePathPattern(value))) add('PLAN_MODE_CONFIG_INVALID_WRITABLE_PATHS', 'planMode.writablePaths', 'Expected safe repository-relative paths under documentation.root using exact paths or /** subtree patterns.');
     }
   }
   const referenceExpansion = data.referenceExpansion;

package/src/query/embedder.mjs

@@ -1,4 +1,4 @@
-import { embeddingProfileIdentity, resolveEmbeddingProfile } from './embedding-config.mjs';
+import { embeddingInputFormat, embeddingProfileIdentity, resolveEmbeddingProfile } from './embedding-config.mjs';
 import { importTransformers } from './embedding-runtime.mjs';
 
 const localPipelines = new Map();
@@ -37,7 +37,12 @@ async function postJson(url, body, profile) {
     const response = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', ...(key ? { authorization: `Bearer ${key}` } : {}) }, body: JSON.stringify(body), signal: controller.signal });
     if (!response.ok) {
       const category = response.status === 401 || response.status === 403 ? 'authentication' : response.status === 429 ? 'rate limit' : `HTTP ${response.status}`;
-      throw new Error(`Remote embedding ${category} failure.`);
+      const error = new Error(`Remote embedding ${category} failure.`);
+      if (response.status === 400 || response.status === 413) {
+        const body = (await response.text()).slice(0, 8192);
+        error.tokenLimit = /(?:exceed\w*[\s\S]{0,120}tokens?|tokens?[\s\S]{0,120}(?:exceed|limit|too (?:long|many))|maximum context length)/i.test(body);
+      }
+      throw error;
     }
     try { return await response.json(); } catch { throw new Error('Remote embedding returned malformed JSON.'); }
   } catch (error) {
@@ -60,21 +65,36 @@ function remoteEmbedder(profile) {
   };
 }
 
+function withInputFormat(embed, profile) {
+  const format = embeddingInputFormat(profile);
+  return (texts, role) => {
+    if (role !== undefined && role !== 'query' && role !== 'document') throw new Error('Embedding role must be query or document.');
+    return embed(role === undefined ? texts : texts.map((text) => format[`${role}Prefix`] + text));
+  };
+}
+
 export async function createEmbedder(profile) {
   const embed = profile.provider === 'local' ? await localEmbedder(profile) : remoteEmbedder(profile);
-  return async (texts) => {
+  async function embedBatch(texts) {
     if (texts.length === 0) return [];
-    const rows = await embed(texts);
+    let rows;
+    try { rows = await embed(texts); }
+    catch (error) {
+      if (!error.tokenLimit || texts.length === 1) throw error;
+      const middle = Math.ceil(texts.length / 2);
+      rows = [...await embedBatch(texts.slice(0, middle)), ...await embedBatch(texts.slice(middle))];
+    }
     if (!Array.isArray(rows) || rows.length !== texts.length) throw new Error('Embedding provider returned an unexpected vector count.');
     const dimensions = rows[0]?.length;
     if (!dimensions || rows.some((row) => row.length !== dimensions)) throw new Error('Embedding provider returned inconsistent vector dimensions.');
     return rows;
-  };
+  }
+  return withInputFormat(embedBatch, profile);
 }
 
 export async function resolveEmbedder(root, options = {}) {
   const resolved = options.profile ? { profile: options.profile, source: options.source ?? 'injected', path: null } : resolveEmbeddingProfile(root, options);
-  return { ...resolved, identity: embeddingProfileIdentity(resolved.profile), embed: options.embed ?? await createEmbedder(resolved.profile) };
+  return { ...resolved, identity: embeddingProfileIdentity(resolved.profile), embed: options.embed ? withInputFormat(options.embed, resolved.profile) : await createEmbedder(resolved.profile) };
 }
 
 export async function embedTexts(texts) {

package/src/query/embedding-config.mjs

@@ -4,6 +4,18 @@ import { join } from 'node:path';
 
 export const DEFAULT_EMBEDDING_PROFILE = Object.freeze({ provider: 'local', model: 'Xenova/multilingual-e5-small' });
 export const EMBEDDING_PROVIDERS = Object.freeze(['local', 'openai-compatible', 'ollama']);
+const DEFAULT_INPUT_FORMAT = Object.freeze({ queryPrefix: 'query: ', documentPrefix: 'passage: ' });
+
+export function embeddingInputFormat(profile, label = 'embedding') {
+  if (profile.inputFormat === undefined) return { ...DEFAULT_INPUT_FORMAT };
+  const format = profile.inputFormat;
+  if (!format || typeof format !== 'object' || Array.isArray(format)
+    || Object.keys(format).some((key) => !Object.hasOwn(DEFAULT_INPUT_FORMAT, key))
+    || Object.keys(DEFAULT_INPUT_FORMAT).some((key) => typeof format[key] !== 'string')) {
+    throw new Error(`${label}.inputFormat must contain exactly queryPrefix and documentPrefix strings.`);
+  }
+  return { queryPrefix: format.queryPrefix, documentPrefix: format.documentPrefix };
+}
 
 function readJson(path, label) {
   if (!existsSync(path)) return null;
@@ -13,7 +25,7 @@ function readJson(path, label) {
 
 export function validateEmbeddingProfile(profile, label = 'embedding') {
   if (!profile || typeof profile !== 'object' || Array.isArray(profile)) throw new Error(`${label} must be an object.`);
-  const allowed = new Set(['provider', 'model', 'baseUrl', 'apiKey', 'apiKeyEnv']);
+  const allowed = new Set(['provider', 'model', 'baseUrl', 'apiKey', 'apiKeyEnv', 'inputFormat']);
   const unknown = Object.keys(profile).find((key) => !allowed.has(key));
   if (unknown) throw new Error(`${label}.${unknown} is not supported.`);
   if (!EMBEDDING_PROVIDERS.includes(profile.provider)) throw new Error(`${label}.provider must be local, openai-compatible, or ollama.`);
@@ -27,6 +39,7 @@ export function validateEmbeddingProfile(profile, label = 'embedding') {
     if (!['http:', 'https:'].includes(url.protocol)) throw new Error(`${label}.baseUrl must use HTTP or HTTPS.`);
     if (url.username || url.password) throw new Error(`${label}.baseUrl must not contain credentials.`);
   }
+  embeddingInputFormat(profile, label);
   return { ...profile, model: profile.model.trim(), ...(profile.baseUrl ? { baseUrl: profile.baseUrl.replace(/\/$/, '') } : {}) };
 }
 
@@ -36,8 +49,11 @@ export function sanitizeEmbeddingProfile(profile) {
 }
 
 export function embeddingProfileIdentity(profile) {
-  const { apiKey, apiKeyEnv, ...identity } = profile;
-  return identity;
+  return {
+    provider: profile.provider, model: profile.model,
+    ...(profile.baseUrl !== undefined ? { baseUrl: profile.baseUrl } : {}),
+    inputFormatVersion: 1, inputFormat: embeddingInputFormat(profile),
+  };
 }
 
 export function resolveEmbeddingProfile(root = '.', options = {}) {