@dotdotgod/cli 0.6.4 → 0.6.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,5 +1,10 @@
1
1
  # @dotdotgod/cli
2
2
 
3
+ Without an explicit project or global `embedding` profile, search uses keywords
4
+ without loading/downloading models or creating vector indexes. Configured profiles
5
+ enable hybrid retrieval; `--search keyword` always skips embeddings and unconfigured
6
+ `--search vector` fails with setup guidance. Graph impact follows the same opt-in.
7
+
3
8
  [![npm version](https://img.shields.io/npm/v/@dotdotgod/cli.svg)](https://www.npmjs.com/package/@dotdotgod/cli) [![GitHub](https://img.shields.io/badge/GitHub-dotdotgod%2Fdotdotgod--kit-181717?logo=github)](https://github.com/dotdotgod/dotdotgod-kit/tree/main/packages/cli) [![License: Elastic 2.0](https://img.shields.io/badge/License-Elastic%202.0-blue.svg)](https://github.com/dotdotgod/dotdotgod-kit/blob/main/LICENSE)
4
9
 
5
10
  Command-line tools for dotdotgod project memory.
@@ -45,7 +50,7 @@ dotdotgod graph serve . --changed <path>
45
50
  - `init` creates `AGENTS.md`, thin agent entrypoints, docs indexes, active-plan space, archive map, local cache ignores, and the complete editable default config.
46
51
  - `validate` checks docs/project-memory rules; `--check-index` also checks graph and outline freshness without parsing, embedding or repair.
47
52
  - `index` builds graph shards and `.dotdotgod/outlines.json` with source hashes, declarations, comments and exact ranges.
48
- - `query` defaults to hybrid vector/keyword search over full Markdown and code outlines/comments, not implementation bodies. Use `--scope docs|code|all`, `--search hybrid|keyword|vector` and `--limit 30`. Keyword mode works without embeddings; hybrid reports vector failures and retains keyword results. Remote code-metadata embedding requires `--allow-code-embedding`. Local E5 remains the default when its optional runtime is installed. Tree-sitter coverage and boundaries: [CODE_OUTLINES.md](../../docs/spec/cli/CODE_OUTLINES.md).
53
+ - `query` defaults to hybrid vector/keyword search over full Markdown and code outlines/comments, not implementation bodies. Use `--scope docs|code|all`, `--search hybrid|keyword|vector` and `--limit 30`. Keyword mode works without embeddings; hybrid reports vector failures and retains keyword results. Selecting a remote embedding provider authorizes sending documentation and code metadata (names, signatures, and comments), without implementation bodies or an additional consent flag. Local E5 remains the default when its optional runtime is installed. Tree-sitter coverage and boundaries: [CODE_OUTLINES.md](../../docs/spec/cli/CODE_OUTLINES.md).
49
54
  - `resolve` and `expand` map explicit or high-signal prompt references to project files and can include related impact evidence.
50
55
  - `graph impact` ranks likely related specs, tests, docs, commands, and source files for one or more changed paths, with a combined ranking and per-file top five. Non-seed scores use fixed weighted PPR connection `80` plus memory policy `20`. A bounded request-local multilingual vector overlay participates in PPR when the query cache is available; vector preparation failures degrade to structural-only results.
51
56
  - `graph serve` starts a local read-only explorer rooted at one or more changed files. It keeps only connected shared structural nodes, lays them out by minimum structural hop, and supports persisted instant, layer, or one-by-one reveal modes.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dotdotgod/cli",
3
- "version": "0.6.4",
3
+ "version": "0.6.6",
4
4
  "description": "CLI for dotdotgod validation, hybrid document/code-outline search, and graph indexing.",
5
5
  "type": "module",
6
6
  "license": "Elastic-2.0",
@@ -0,0 +1,41 @@
1
+ import { existsSync, lstatSync, readFileSync, realpathSync, writeFileSync, mkdirSync } from 'node:fs';
2
+ import { dirname, join, relative, resolve, isAbsolute } from 'node:path';
3
+
4
+ // Preflight before initialization writes anything. Never follow settings symlinks.
5
+ export function prepareClaudePlans(root, documentationRoot) {
6
+ const path = join(root, '.claude', 'settings.json');
7
+ for (const target of [join(root, '.claude'), path]) {
8
+ try {
9
+ if (lstatSync(target).isSymbolicLink()) throw new Error(`Refusing symlink: ${target}`);
10
+ } catch (error) { if (error.code !== 'ENOENT') throw error; }
11
+ }
12
+ const base = resolve(root);
13
+ if (existsSync(dirname(path))) {
14
+ const rel = relative(realpathSync(base), realpathSync(dirname(path)));
15
+ if (rel.startsWith('..') || isAbsolute(rel)) throw new Error('Claude settings must stay inside the project.');
16
+ }
17
+ const current = existsSync(path) ? readFileSync(path, 'utf8') : null;
18
+ const settings = current === null ? {} : JSON.parse(current);
19
+ if (!settings || typeof settings !== 'object' || Array.isArray(settings)) throw new Error('Claude settings must be a JSON object.');
20
+ const value = `./${documentationRoot}/plan`;
21
+ if (Object.hasOwn(settings, 'plansDirectory')) return {
22
+ status: 'skipped', path, reason: 'Existing plansDirectory preserved.', value: settings.plansDirectory,
23
+ };
24
+ return { status: current === null ? 'created' : 'updated', path, value, current,
25
+ content: `${JSON.stringify({ ...settings, plansDirectory: value }, null, 2)}\n` };
26
+ }
27
+
28
+ export function applyClaudePlans(prepared, dryRun) {
29
+ const { content, current, ...result } = prepared;
30
+ if (!content) return result;
31
+ if (dryRun) return { ...result, status: `would_${result.status === 'created' ? 'create' : 'update'}` };
32
+ for (const target of [dirname(prepared.path), prepared.path]) {
33
+ try { if (lstatSync(target).isSymbolicLink()) throw new Error(`Refusing symlink: ${target}`); }
34
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
35
+ }
36
+ const latest = existsSync(prepared.path) ? readFileSync(prepared.path, 'utf8') : null;
37
+ if (latest !== current) throw new Error('Claude settings changed during initialization; retry without overwriting them.');
38
+ mkdirSync(dirname(prepared.path), { recursive: true });
39
+ writeFileSync(prepared.path, content, { flag: current === null ? 'wx' : 'w' });
40
+ return result;
41
+ }
package/src/cli/usage.mjs CHANGED
@@ -11,9 +11,10 @@ export function commandUsage(command = 'root') {
11
11
  dotdotgod validate <root> [--include-local-memory] [--check-index] [--max-lines n] [--max-chars n] [--no-link-check] [--json]`;
12
12
  case 'init':
13
13
  return `Usage:
14
- dotdotgod init <root> [--project-name NAME] [--template NAME] [--documentation-root PATH] [--dotdot-setting] [--dry-run] [--json]
14
+ dotdotgod init <root> [--project-name NAME] [--template NAME] [--documentation-root PATH] [--dotdot-setting] [--claude-plans] [--dry-run] [--json]
15
15
 
16
- Create AGENTS.md, agent entrypoints, docs indexes, a project config from an initialization template, and local memory gitignore entries.`;
16
+ Create AGENTS.md, agent entrypoints, docs indexes, a project config from an initialization template, and local memory gitignore entries.
17
+ Use --claude-plans only after opting into project-local native plan storage; existing Claude settings are preserved.`;
17
18
  case 'index':
18
19
  return `Usage:
19
20
  dotdotgod index <root> [--json]`;
@@ -33,9 +34,9 @@ Create dotdotgod.config.json from a bundled or user template. Without --template
33
34
  dotdotgod status <root> [--json]`;
34
35
  case 'query':
35
36
  return `Usage:
36
- dotdotgod query <root> <query> [--limit n] [--scope docs|code|all] [--search hybrid|keyword|vector] [--allow-code-embedding] [--json]
37
+ dotdotgod query <root> <query> [--limit n] [--scope docs|code|all] [--search hybrid|keyword|vector] [--json]
37
38
 
38
- Search full documentation and code outlines/comments. Default: hybrid docs+code. Remote code metadata requires explicit consent.`;
39
+ Search full documentation and code outlines/comments. Default: hybrid docs+code. Selecting a remote embedding profile authorizes code metadata transmission.`;
39
40
  case 'embedding':
40
41
  return `Usage:
41
42
  dotdotgod embedding status [<root>] [--json]
@@ -167,4 +168,3 @@ export function parseCommon(argv) {
167
168
  options.root = resolve(options.root);
168
169
  return options;
169
170
  }
170
-
@@ -42,7 +42,7 @@ function formatConfigOutput(payload) {
42
42
  lines.push(`- path: ${payload.path ?? 'none'}`);
43
43
  lines.push(`- documentation root: ${payload.config.documentation.root}`);
44
44
  lines.push(`- memory areas: ${payload.config.areas.length}`);
45
- lines.push(`- embedding: ${payload.embedding.profile.provider}/${payload.embedding.profile.model} (${payload.embedding.source})`);
45
+ lines.push(`- embedding: ${payload.embedding.profile ? `${payload.embedding.profile.provider}/${payload.embedding.profile.model}` : 'not configured; keyword search'} (${payload.embedding.source})`);
46
46
  lines.push(`- traceability required: ${(payload.config.traceability.required ?? []).join(', ') || 'none'}`);
47
47
  lines.push(`- traceability exclude: ${(payload.config.traceability.exclude ?? []).join(', ') || 'none'}`);
48
48
  lines.push('- traceability keys:');
@@ -92,7 +92,7 @@ export function runConfig(argv) {
92
92
  embedding = { source: resolved.source, path: resolved.path, profile: sanitizeEmbeddingProfile(resolved.profile) };
93
93
  } catch (error) {
94
94
  errors.push({ file: 'embedding config', code: 'EMBEDDING_CONFIG_INVALID', message: error instanceof Error ? error.message : String(error) });
95
- embedding = { source: 'invalid', path: null, profile: sanitizeEmbeddingProfile({ provider: 'local', model: 'Xenova/multilingual-e5-small' }) };
95
+ embedding = { source: 'invalid', path: null, profile: null };
96
96
  }
97
97
  const payload = { ok: errors.length === 0, command: 'config', root: options.root, source: config.source ?? 'default', path: config.source && config.source !== 'default' ? join(options.root, config.source) : null, embedding, config: memoryConfigSummary(config), errors };
98
98
  if (options.json) console.log(JSON.stringify(payload, null, 2));
@@ -19,6 +19,12 @@ export function runEmbedding(argv, options = {}) {
19
19
  const parsed = parse(argv);
20
20
  if (!['status', 'install'].includes(parsed.action)) usage(`Unknown embedding command: ${parsed.action}`, 'embedding');
21
21
  const resolved = resolveEmbeddingProfile(parsed.root, options);
22
+ if (!resolved.profile) {
23
+ if (parsed.action === 'install') throw new Error('Configure an explicit local embedding profile before installing its runtime.');
24
+ const result = { ok: true, command: 'embedding status', provider: null, required: false, installed: false, installAvailable: false, source: 'not-configured' };
25
+ console.log(parsed.json ? JSON.stringify(result, null, 2) : 'Embedding is not configured; keyword search is used.');
26
+ return result;
27
+ }
22
28
  if (resolved.profile.provider !== 'local') {
23
29
  const result = { ok: true, command: `embedding ${parsed.action}`, provider: resolved.profile.provider, required: false, installed: false, installAvailable: false };
24
30
  console.log(parsed.json ? JSON.stringify(result, null, 2) : 'Local embedding runtime is not required for the configured remote provider.');
@@ -3,16 +3,17 @@ import { usage } from '../cli/usage.mjs';
3
3
  import { readMemoryConfig } from '../memory/config.mjs';
4
4
  import { collectDocumentationChunks } from '../query/chunks.mjs';
5
5
  import { resolveEmbedder } from '../query/embedder.mjs';
6
+ import { resolveEmbeddingProfile } from '../query/embedding-config.mjs';
6
7
  import { isValidNormalizedVectorValues, profileFingerprint, readVectorCache, writeVectorCache } from '../query/store.mjs';
7
8
 
8
- import { projectCorpus, keywordCandidates, fuseCandidates, attachRelationships, assertCodeEmbeddingConsent } from '../query/hybrid.mjs';
9
+ import { projectCorpus, keywordCandidates, fuseCandidates, attachRelationships } from '../query/hybrid.mjs';
9
10
 
10
11
  const DEFAULT_LIMIT = 30;
11
12
  const MAX_LIMIT = 100;
12
13
  const EMBED_BATCH_SIZE = 16;
13
14
 
14
15
  export function parseQueryOptions(argv) {
15
- const options = { root: '.', query: '', limit: DEFAULT_LIMIT, json: false, scope: 'all', search: 'hybrid', allowCodeEmbedding: false };
16
+ const options = { root: '.', query: '', limit: DEFAULT_LIMIT, json: false, scope: 'all', search: 'hybrid' };
16
17
  let rootSet = false;
17
18
  const queryParts = [];
18
19
  for (let i = 0; i < argv.length; i += 1) {
@@ -20,7 +21,6 @@ export function parseQueryOptions(argv) {
20
21
  if (arg === '--json') options.json = true;
21
22
  else if (arg === '--scope') { options.scope = argv[++i]; if (!['docs', 'code', 'all'].includes(options.scope)) usage('--scope must be docs, code or all.', 'query'); }
22
23
  else if (arg === '--search') { options.search = argv[++i]; if (!['hybrid', 'keyword', 'vector'].includes(options.search)) usage('--search must be hybrid, keyword or vector.', 'query'); }
23
- else if (arg === '--allow-code-embedding') options.allowCodeEmbedding = true;
24
24
  else if (arg === '--limit') { const value = Number(argv[++i]); if (!Number.isInteger(value) || value < 1 || value > MAX_LIMIT) usage(`--limit must be an integer from 1 to ${MAX_LIMIT}.`, 'query'); options.limit = value; }
25
25
  else if (arg.startsWith('-')) usage(`Unknown option: ${arg}`, 'query');
26
26
  else if (!rootSet) { options.root = resolve(arg); rootSet = true; }
@@ -76,7 +76,7 @@ export async function buildVectorIndex(root, embedOrOptions = {}) {
76
76
  chunks.forEach((chunk, index) => { const oldIndex = cachedOffsets.get(chunk.fingerprint); if (oldIndex === undefined || !cached) missing.push({ chunk, index }); else { const d = cached.manifest.dimensions; rows[index] = Array.from(cached.vectors.subarray(oldIndex * d, (oldIndex + 1) * d)); reused += 1; } });
77
77
  for (let offset = 0; offset < missing.length; offset += EMBED_BATCH_SIZE) {
78
78
  const batch = missing.slice(offset, offset + EMBED_BATCH_SIZE);
79
- const embedded = await resolved.embed(batch.map(({ chunk }) => `passage: ${chunk.passage}`));
79
+ const embedded = await resolved.embed(batch.map(({ chunk }) => chunk.passage), 'document');
80
80
  embedded.forEach((vector, index) => { rows[batch[index].index] = vector; });
81
81
  }
82
82
  const dimensions = rows[0]?.length ?? cached?.manifest?.dimensions;
@@ -92,7 +92,7 @@ export async function buildVectorIndex(root, embedOrOptions = {}) {
92
92
  export async function queryDocumentation(root, query, options = {}) {
93
93
  const limit = options.limit ?? DEFAULT_LIMIT;
94
94
  const index = await buildVectorIndex(root, options);
95
- const [queryVector] = await index.embedding.embed([`query: ${query}`]);
95
+ const [queryVector] = await index.embedding.embed([query], 'query');
96
96
  const dimensions = index.manifest.dimensions;
97
97
  assertVector(queryVector, dimensions, 'Query vector');
98
98
  const results = rankVectorFiles(queryVector, index).map((chunk) => ({ ...chunk, score: Number((chunk.vectorScore + lexicalBoost(query, chunk)).toFixed(6)), vectorScore: Number(chunk.vectorScore.toFixed(6)) }))
@@ -107,11 +107,16 @@ export async function queryProject(root, query, options = {}) {
107
107
  const corpus = await projectCorpus(root, scope);
108
108
  const keyword = search === 'vector' ? [] : keywordCandidates(query, corpus.chunks);
109
109
  let vector = []; let index; let vectorError;
110
- if (search !== 'keyword' && corpus.chunks.length) {
110
+ let configured = false;
111
+ if (search !== 'keyword') {
112
+ try { configured = Boolean(options.profile || options.embed || resolveEmbeddingProfile(root, options).profile); }
113
+ catch (error) { if (search === 'vector') throw error; vectorError = error.message; }
114
+ if (!configured && !vectorError && search === 'vector') throw new Error('Embedding is not configured. Set embedding in project or global configuration to use vector search.');
115
+ }
116
+ if (search !== 'keyword' && configured && corpus.chunks.length) {
111
117
  try {
112
- assertCodeEmbeddingConsent(root, options, corpus.chunks);
113
118
  index = await buildVectorIndex(root, { ...options, chunks: corpus.chunks });
114
- const [queryVector] = await index.embedding.embed([`query: ${query}`]);
119
+ const [queryVector] = await index.embedding.embed([query], 'query');
115
120
  vector = index.chunks.map((chunk, position) => ({ ...chunk, vectorScore: cosineScore(queryVector, index.vectors, position * index.manifest.dimensions, index.manifest.dimensions) }))
116
121
  .sort((a, b) => b.vectorScore - a.vectorScore || a.id.localeCompare(b.id));
117
122
  } catch (error) {
@@ -1,4 +1,5 @@
1
1
  import { buildVectorIndex, rankVectorFiles } from '../commands/query.mjs';
2
+ import { resolveEmbeddingProfile } from '../query/embedding-config.mjs';
2
3
 
3
4
  import { buildChangedFileProfile, canonicalizeChangedPath } from './vector-profile.mjs';
4
5
 
@@ -16,6 +17,7 @@ export async function buildVectorImpactOverlay(root, index, changedPaths, option
16
17
  const threshold = semantic.threshold ?? 0.5;
17
18
  const topK = semantic.topKPerFile ?? 5;
18
19
  try {
20
+ if (!options.profile && !options.embed && !options.buildIndex && !resolveEmbeddingProfile(root, options).profile) return { status: 'disabled', edges: [], diagnostics: { enabled: false, reason: 'Embedding is not configured.' } };
19
21
  const vectorIndex = await (options.buildIndex ?? buildVectorIndex)(root, options);
20
22
  const edges = [];
21
23
  const normalizedChangedPaths = [...new Set(changedPaths.map((path) => canonicalizeChangedPath(root, path)).filter(Boolean))];
@@ -24,7 +26,7 @@ export async function buildVectorImpactOverlay(root, index, changedPaths, option
24
26
  if (!profile) continue;
25
27
  const embed = vectorIndex.embedding?.embed ?? options.embed;
26
28
  if (!embed) throw new Error('Vector index did not provide an embedder.');
27
- const [queryVector] = await embed([`query: ${profile.text}`]);
29
+ const [queryVector] = await embed([profile.text], 'query');
28
30
  if (!queryVector) throw new Error('Changed-file embedding was empty.');
29
31
  const matches = rankVectorFiles(queryVector, vectorIndex)
30
32
  .filter((match) => match.path !== changed && match.vectorScore >= threshold)
package/src/init.mjs CHANGED
@@ -2,6 +2,7 @@ import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'no
2
2
  import { basename, dirname, join, resolve } from 'node:path';
3
3
  import { builtInTemplateData, builtInTemplateScaffold, resolveInitializationTemplate } from './config/templates.mjs';
4
4
  import { isValidDocumentationRoot, normalizeDocumentationRoot, readMemoryConfig } from './memory/config.mjs';
5
+ import { prepareClaudePlans, applyClaudePlans } from './claude-plans.mjs';
5
6
 
6
7
  function action(status, path, extra = {}) {
7
8
  return { status, path, ...extra };
@@ -19,6 +20,7 @@ function parseInitOptions(argv) {
19
20
  if (arg === '--json') options.json = true;
20
21
  else if (arg === '--dotdot-setting') options.dotdotSetting = true;
21
22
  else if (arg === '--dry-run') options.dryRun = true;
23
+ else if (arg === '--claude-plans') options.claudePlans = true;
22
24
  else if (arg === '--project-name' || arg === '--template' || arg === '--documentation-root') {
23
25
  const value = argv[i + 1];
24
26
  if (!value || value.startsWith('-')) throw new Error(`${arg} requires a value`);
@@ -310,6 +312,15 @@ export function runInit(argv, usage) {
310
312
  }
311
313
  }
312
314
 
315
+ if (options.claudePlans && !options.documentationRootExplicit && existsSync(join(options.root, 'dotdotgod.config.json'))) {
316
+ options.documentationRoot = readMemoryConfig(options.root).documentation.root;
317
+ }
318
+ let claudePlans;
319
+ if (options.claudePlans) {
320
+ try { claudePlans = prepareClaudePlans(options.root, options.documentationRoot); }
321
+ catch (error) { initError(options, 'CLAUDE_SETTINGS_INVALID', error.message); }
322
+ }
323
+
313
324
  if (!options.dryRun) mkdirSync(options.root, { recursive: true });
314
325
 
315
326
  const configExists = existsSync(join(options.root, 'dotdotgod.config.json'));
@@ -335,6 +346,11 @@ export function runInit(argv, usage) {
335
346
 
336
347
  for (const entry of [`${options.documentationRoot}/plan`, `${options.documentationRoot}/archive`, '.dotdotgod']) ensureGitignoreEntry(options, entry, actions);
337
348
 
349
+ if (claudePlans) {
350
+ try { actions.push(applyClaudePlans(claudePlans, options.dryRun)); }
351
+ catch (error) { initError(options, 'CLAUDE_SETTINGS_CHANGED', error.message); }
352
+ }
353
+
338
354
  const payload = { ok: true, command: 'init', root: options.root, projectName: options.projectName, documentationRoot: options.documentationRoot, dryRun: options.dryRun, dotdotSetting: options.dotdotSetting, template: template ? { name: template.name, source: template.source, selectedBy: template.selectedBy } : null, actions };
339
355
  if (options.json) console.log(JSON.stringify(payload, null, 2));
340
356
  else {
@@ -1,4 +1,4 @@
1
- import { embeddingProfileIdentity, resolveEmbeddingProfile } from './embedding-config.mjs';
1
+ import { DEFAULT_EMBEDDING_PROFILE, embeddingInputFormat, embeddingProfileIdentity, resolveEmbeddingProfile } from './embedding-config.mjs';
2
2
  import { importTransformers } from './embedding-runtime.mjs';
3
3
 
4
4
  const localPipelines = new Map();
@@ -37,7 +37,12 @@ async function postJson(url, body, profile) {
37
37
  const response = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', ...(key ? { authorization: `Bearer ${key}` } : {}) }, body: JSON.stringify(body), signal: controller.signal });
38
38
  if (!response.ok) {
39
39
  const category = response.status === 401 || response.status === 403 ? 'authentication' : response.status === 429 ? 'rate limit' : `HTTP ${response.status}`;
40
- throw new Error(`Remote embedding ${category} failure.`);
40
+ const error = new Error(`Remote embedding ${category} failure.`);
41
+ if (response.status === 400 || response.status === 413) {
42
+ const body = (await response.text()).slice(0, 8192);
43
+ error.tokenLimit = /(?:exceed\w*[\s\S]{0,120}tokens?|tokens?[\s\S]{0,120}(?:exceed|limit|too (?:long|many))|maximum context length)/i.test(body);
44
+ }
45
+ throw error;
41
46
  }
42
47
  try { return await response.json(); } catch { throw new Error('Remote embedding returned malformed JSON.'); }
43
48
  } catch (error) {
@@ -60,21 +65,37 @@ function remoteEmbedder(profile) {
60
65
  };
61
66
  }
62
67
 
68
+ function withInputFormat(embed, profile) {
69
+ const format = embeddingInputFormat(profile);
70
+ return (texts, role) => {
71
+ if (role !== undefined && role !== 'query' && role !== 'document') throw new Error('Embedding role must be query or document.');
72
+ return embed(role === undefined ? texts : texts.map((text) => format[`${role}Prefix`] + text));
73
+ };
74
+ }
75
+
63
76
  export async function createEmbedder(profile) {
64
77
  const embed = profile.provider === 'local' ? await localEmbedder(profile) : remoteEmbedder(profile);
65
- return async (texts) => {
78
+ async function embedBatch(texts) {
66
79
  if (texts.length === 0) return [];
67
- const rows = await embed(texts);
80
+ let rows;
81
+ try { rows = await embed(texts); }
82
+ catch (error) {
83
+ if (!error.tokenLimit || texts.length === 1) throw error;
84
+ const middle = Math.ceil(texts.length / 2);
85
+ rows = [...await embedBatch(texts.slice(0, middle)), ...await embedBatch(texts.slice(middle))];
86
+ }
68
87
  if (!Array.isArray(rows) || rows.length !== texts.length) throw new Error('Embedding provider returned an unexpected vector count.');
69
88
  const dimensions = rows[0]?.length;
70
89
  if (!dimensions || rows.some((row) => row.length !== dimensions)) throw new Error('Embedding provider returned inconsistent vector dimensions.');
71
90
  return rows;
72
- };
91
+ }
92
+ return withInputFormat(embedBatch, profile);
73
93
  }
74
94
 
75
95
  export async function resolveEmbedder(root, options = {}) {
76
- const resolved = options.profile ? { profile: options.profile, source: options.source ?? 'injected', path: null } : resolveEmbeddingProfile(root, options);
77
- return { ...resolved, identity: embeddingProfileIdentity(resolved.profile), embed: options.embed ?? await createEmbedder(resolved.profile) };
96
+ const resolved = options.profile || options.embed ? { profile: options.profile ?? DEFAULT_EMBEDDING_PROFILE, source: options.source ?? 'injected', path: null } : resolveEmbeddingProfile(root, options);
97
+ if (!resolved.profile) throw new Error('Embedding is not configured. Set embedding in project or global configuration to use vector search.');
98
+ return { ...resolved, identity: embeddingProfileIdentity(resolved.profile), embed: options.embed ? withInputFormat(options.embed, resolved.profile) : await createEmbedder(resolved.profile) };
78
99
  }
79
100
 
80
101
  export async function embedTexts(texts) {
@@ -4,6 +4,18 @@ import { join } from 'node:path';
4
4
 
5
5
  export const DEFAULT_EMBEDDING_PROFILE = Object.freeze({ provider: 'local', model: 'Xenova/multilingual-e5-small' });
6
6
  export const EMBEDDING_PROVIDERS = Object.freeze(['local', 'openai-compatible', 'ollama']);
7
+ const DEFAULT_INPUT_FORMAT = Object.freeze({ queryPrefix: 'query: ', documentPrefix: 'passage: ' });
8
+
9
+ export function embeddingInputFormat(profile, label = 'embedding') {
10
+ if (profile.inputFormat === undefined) return { ...DEFAULT_INPUT_FORMAT };
11
+ const format = profile.inputFormat;
12
+ if (!format || typeof format !== 'object' || Array.isArray(format)
13
+ || Object.keys(format).some((key) => !Object.hasOwn(DEFAULT_INPUT_FORMAT, key))
14
+ || Object.keys(DEFAULT_INPUT_FORMAT).some((key) => typeof format[key] !== 'string')) {
15
+ throw new Error(`${label}.inputFormat must contain exactly queryPrefix and documentPrefix strings.`);
16
+ }
17
+ return { queryPrefix: format.queryPrefix, documentPrefix: format.documentPrefix };
18
+ }
7
19
 
8
20
  function readJson(path, label) {
9
21
  if (!existsSync(path)) return null;
@@ -13,7 +25,7 @@ function readJson(path, label) {
13
25
 
14
26
  export function validateEmbeddingProfile(profile, label = 'embedding') {
15
27
  if (!profile || typeof profile !== 'object' || Array.isArray(profile)) throw new Error(`${label} must be an object.`);
16
- const allowed = new Set(['provider', 'model', 'baseUrl', 'apiKey', 'apiKeyEnv']);
28
+ const allowed = new Set(['provider', 'model', 'baseUrl', 'apiKey', 'apiKeyEnv', 'inputFormat']);
17
29
  const unknown = Object.keys(profile).find((key) => !allowed.has(key));
18
30
  if (unknown) throw new Error(`${label}.${unknown} is not supported.`);
19
31
  if (!EMBEDDING_PROVIDERS.includes(profile.provider)) throw new Error(`${label}.provider must be local, openai-compatible, or ollama.`);
@@ -27,17 +39,22 @@ export function validateEmbeddingProfile(profile, label = 'embedding') {
27
39
  if (!['http:', 'https:'].includes(url.protocol)) throw new Error(`${label}.baseUrl must use HTTP or HTTPS.`);
28
40
  if (url.username || url.password) throw new Error(`${label}.baseUrl must not contain credentials.`);
29
41
  }
42
+ embeddingInputFormat(profile, label);
30
43
  return { ...profile, model: profile.model.trim(), ...(profile.baseUrl ? { baseUrl: profile.baseUrl.replace(/\/$/, '') } : {}) };
31
44
  }
32
45
 
33
46
  export function sanitizeEmbeddingProfile(profile) {
47
+ if (!profile) return null;
34
48
  const { apiKey, ...safe } = profile;
35
49
  return { ...safe, ...(apiKey ? { apiKey: '[redacted]' } : {}) };
36
50
  }
37
51
 
38
52
  export function embeddingProfileIdentity(profile) {
39
- const { apiKey, apiKeyEnv, ...identity } = profile;
40
- return identity;
53
+ return {
54
+ provider: profile.provider, model: profile.model,
55
+ ...(profile.baseUrl !== undefined ? { baseUrl: profile.baseUrl } : {}),
56
+ inputFormatVersion: 1, inputFormat: embeddingInputFormat(profile),
57
+ };
41
58
  }
42
59
 
43
60
  export function resolveEmbeddingProfile(root = '.', options = {}) {
@@ -48,5 +65,5 @@ export function resolveEmbeddingProfile(root = '.', options = {}) {
48
65
  if (projectConfig && Object.hasOwn(projectConfig, 'embedding')) return { profile: validateEmbeddingProfile(projectConfig.embedding, 'project embedding'), source: 'project', path: projectPath };
49
66
  const globalConfig = readJson(globalPath, 'global config');
50
67
  if (globalConfig && Object.hasOwn(globalConfig, 'embedding')) return { profile: validateEmbeddingProfile(globalConfig.embedding, 'global embedding'), source: 'global', path: globalPath };
51
- return { profile: { ...DEFAULT_EMBEDDING_PROFILE }, source: 'default', path: null };
68
+ return { profile: null, source: 'not-configured', path: null };
52
69
  }
@@ -1,6 +1,5 @@
1
1
  import { collectDocumentationChunks } from './chunks.mjs';
2
2
  import { readMemoryConfig } from '../memory/config.mjs';
3
- import { resolveEmbeddingProfile } from './embedding-config.mjs';
4
3
  import { buildOutlines, outlineChunks } from '../symbols/store.mjs';
5
4
  import { addOutlineGraph } from '../symbols/graph.mjs';
6
5
 
@@ -47,11 +46,6 @@ export async function projectCorpus(root, scope) {
47
46
  const outlines = scope === 'docs' ? null : await buildOutlines(root);
48
47
  return { chunks: [...docs, ...(outlines ? outlineChunks(outlines) : [])], outlines };
49
48
  }
50
- export function assertCodeEmbeddingConsent(root, options, chunks) {
51
- if (!chunks.some((chunk) => chunk.kind === 'symbol')) return;
52
- const profile = options.profile ?? resolveEmbeddingProfile(root).profile;
53
- if (profile.provider !== 'local' && options.allowCodeEmbedding !== true) throw new Error('Remote code-metadata embedding requires --allow-code-embedding; use --search keyword or --scope docs instead.');
54
- }
55
49
  export function attachRelationships(results, outlines) {
56
50
  if (!outlines) return results;
57
51
  const graph = addOutlineGraph({ nodes: [], edges: [] }, outlines);
@@ -11,7 +11,7 @@ export const SOURCE_EXTENSIONS = new Set([
11
11
  ...Object.values(LANGUAGE_EXTENSIONS).flat(), '.c', '.h', '.kt', '.kts', '.swift',
12
12
  '.scala', '.clj', '.cljs', '.ex', '.exs', '.erl', '.hrl', '.lua', '.pl', '.pm', '.r', '.R', '.sql', '.m', '.mm', '.zsh', '.fish',
13
13
  ]);
14
- export const EXTRACTOR_ID = 'outline-v3:vscode-tree-sitter-wasm-0.3.1';
14
+ export const EXTRACTOR_ID = 'outline-v4:vscode-0.3.1:typescript-import-types-1';
15
15
  export function languageForPath(path) {
16
16
  return Object.entries(LANGUAGE_EXTENSIONS).find(([, extensions]) => extensions.includes(extname(path)))?.[0] ?? null;
17
17
  }
@@ -1,5 +1,5 @@
1
- import { dirname, join } from 'node:path';
2
- import { parserEntry, loadParserApi } from './runtime.mjs';
1
+ import { dirname } from 'node:path';
2
+ import { parserEntry, loadParserApi, languageAsset } from './runtime.mjs';
3
3
  import { createHash } from 'node:crypto';
4
4
  import { COMMENT_TYPES, DECLARATIONS } from './languages.mjs';
5
5
 
@@ -13,7 +13,7 @@ async function parserFor(language) {
13
13
  return { api, directory: dirname(entry) };
14
14
  })();
15
15
  const { api, directory } = await runtime;
16
- if (!languages.has(language)) languages.set(language, api.Language.load(join(directory, `tree-sitter-${language}.wasm`)));
16
+ if (!languages.has(language)) languages.set(language, api.Language.load(languageAsset(language)));
17
17
  const loaded = await languages.get(language);
18
18
  const parser = new api.Parser();
19
19
  parser.setLanguage(loaded);
@@ -0,0 +1,24 @@
1
+ # Patched TypeScript Grammars
2
+
3
+ TypeScript and TSX WASM are generated from tree-sitter-typescript 0.23.2
4
+ (npm integrity sha512-e04JUUKxTT53/x3Uq1zIL45DoYKVfHH4CZqwgZhPg5qYROl5nQjV+85ruFzFGZxu+QeFVbRTPDRnqL9UbU4VeA==),
5
+ with tree-sitter-javascript 0.23.1, tree-sitter-cli 0.25.10, and emsdk 4.0.4.
6
+ The MIT license is retained as TYPESCRIPT_LICENSE.
7
+
8
+ The small source patch in ../typescript-import-grammar.mjs adds import types as
9
+ primary types and typeof operands with their own conflict/precedence handling.
10
+ It fixes generic typeof-import calls and qualified import-type arrays without
11
+ changing application text or hiding tree-sitter error nodes.
12
+
13
+ To regenerate, extract the pinned npm grammar into a disposable directory, install
14
+ its exact JavaScript dependency without lifecycle scripts
15
+ (`npm install --ignore-scripts --omit=dev --no-package-lock tree-sitter-javascript@0.23.1`), then run from checkout:
16
+
17
+ ```sh
18
+ TREE_SITTER_CLI=/path/to/tree-sitter-0.25.10 node scripts/build-typescript-parsers.mjs /path/to/extracted/package --docker
19
+ pnpm run generate
20
+ ```
21
+
22
+ Docker builds use emscripten/emsdk:4.0.4. Other language grammars and the runtime
23
+ remain supplied by @vscode/tree-sitter-wasm 0.3.1. Runtime and grammar hashes
24
+ participate in outline cache identity; adapters copy these same patched assets.
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2017 Max Brunsfeld
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -6,6 +6,10 @@ import { createHash } from 'node:crypto';
6
6
  import { LANGUAGE_EXTENSIONS } from './languages.mjs';
7
7
 
8
8
  const require = createRequire(import.meta.url);
9
+ export function languageAsset(language) {
10
+ const patched = fileURLToPath(new URL(`./parsers/tree-sitter-${language}.wasm`, import.meta.url));
11
+ return existsSync(patched) ? patched : join(dirname(parserEntry()), `tree-sitter-${language}.wasm`);
12
+ }
9
13
  export function parserEntry() {
10
14
  const bundled = fileURLToPath(new URL('./parsers/tree-sitter.cjs', import.meta.url));
11
15
  return existsSync(bundled) ? bundled : require.resolve('@vscode/tree-sitter-wasm');
@@ -15,7 +19,8 @@ export function parserIdentity() {
15
19
  if (!identity) {
16
20
  const entry = parserEntry();
17
21
  const hash = createHash('sha256').update(readFileSync(entry));
18
- for (const name of ['tree-sitter.wasm', ...Object.keys(LANGUAGE_EXTENSIONS).map((language) => `tree-sitter-${language}.wasm`)]) hash.update(name).update(readFileSync(join(dirname(entry), name)));
22
+ hash.update(readFileSync(join(dirname(entry), 'tree-sitter.wasm')));
23
+ for (const language of Object.keys(LANGUAGE_EXTENSIONS)) hash.update(language).update(readFileSync(languageAsset(language)));
19
24
  identity = hash.digest('hex');
20
25
  }
21
26
  return identity;
@@ -0,0 +1,13 @@
1
+ // Applied to pinned tree-sitter-typescript 0.23.2 before parser generation.
2
+ export function patchTypescriptGrammar(source) {
3
+ const replacements = [
4
+ ['conflicts: ($, previous) => previous.concat([', 'conflicts: ($, previous) => previous.concat([\n [$.import, $.import_type],'],
5
+ ['primary_type: $ => choice(', 'primary_type: $ => choice(\n $.import_type,'],
6
+ [" type_query: $ => prec.right(seq(\n 'typeof',\n choice(", " import_type: $ => prec.right('call', seq(\n 'import', '(', $.string, ')',\n repeat(seq('.', $._type_identifier)),\n optional($.type_arguments),\n )),\n\n type_query: $ => prec.right(seq(\n 'typeof',\n choice(\n $.import_type,"],
7
+ ];
8
+ for (const [before, after] of replacements) {
9
+ if (!source.includes(before)) throw new Error('Unsupported TypeScript grammar source.');
10
+ source = source.replace(before, after);
11
+ }
12
+ return source;
13
+ }