@dotdotgod/cli 0.2.19 → 0.2.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -51,6 +51,7 @@ dotdotgod validate .
51
51
  dotdotgod validate . --check-index
52
52
  dotdotgod config .
53
53
  dotdotgod config init .
54
+ dotdotgod config init . --template research
54
55
  dotdotgod status .
55
56
  dotdotgod index .
56
57
  dotdotgod query . "plan mode tools"
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dotdotgod/cli",
3
- "version": "0.2.19",
3
+ "version": "0.2.21",
4
4
  "description": "CLI for dotdotgod validation, local documentation search, and graph indexing.",
5
5
  "type": "module",
6
6
  "license": "Elastic-2.0",
package/src/cli/usage.mjs CHANGED
@@ -11,23 +11,23 @@ export function commandUsage(command = 'root') {
11
11
  dotdotgod validate <root> [--include-local-memory] [--check-index] [--max-lines n] [--max-chars n] [--no-link-check] [--json]`;
12
12
  case 'init':
13
13
  return `Usage:
14
- dotdotgod init <root> [--project-name NAME] [--dotdot-setting] [--dry-run] [--json]
14
+ dotdotgod init <root> [--project-name NAME] [--template NAME] [--dotdot-setting] [--dry-run] [--json]
15
15
 
16
- Create AGENTS.md, agent entrypoints, docs indexes, and local memory gitignore entries.`;
16
+ Create AGENTS.md, agent entrypoints, docs indexes, a project config from an initialization template, and local memory gitignore entries.`;
17
17
  case 'index':
18
18
  return `Usage:
19
19
  dotdotgod index <root> [--json]`;
20
20
  case 'config':
21
21
  return `Usage:
22
22
  dotdotgod config <root> [--json]
23
- dotdotgod config init <root> [--json]
23
+ dotdotgod config init <root> [--template NAME] [--json]
24
24
 
25
25
  Inspect or initialize the project-level dotdotgod config file.`;
26
26
  case 'config init':
27
27
  return `Usage:
28
- dotdotgod config init <root> [--json]
28
+ dotdotgod config init <root> [--template NAME] [--json]
29
29
 
30
- Create dotdotgod.config.json with the built-in default memory, traceability, validation, and impact ranking policy.`;
30
+ Create dotdotgod.config.json from a bundled or user template. Without --template, use the global defaultTemplate and then software.`;
31
31
  case 'status':
32
32
  return `Usage:
33
33
  dotdotgod status <root> [--json]`;
@@ -35,7 +35,7 @@ Create dotdotgod.config.json with the built-in default memory, traceability, val
35
35
  return `Usage:
36
36
  dotdotgod query <root> <query> [--limit n] [--json]
37
37
 
38
- Search shared project documentation with the local multilingual E5 vector index.`;
38
+ Search shared project documentation with the resolved local or remote embedding provider.`;
39
39
  case 'resolve':
40
40
  return `Usage:
41
41
  dotdotgod resolve <root> <ref> [--max-results n] [--include-archive] [--json]`;
@@ -1,15 +1,22 @@
1
1
  import { existsSync, statSync, writeFileSync } from 'node:fs';
2
2
  import { join, resolve } from 'node:path';
3
3
  import { usage } from '../cli/usage.mjs';
4
- import { defaultDotdotgodConfigText, memoryConfigSummary, readMemoryConfig } from '../memory/config.mjs';
4
+ import { memoryConfigSummary, readMemoryConfig } from '../memory/config.mjs';
5
+ import { resolveInitializationTemplate } from '../config/templates.mjs';
6
+ import { resolveEmbeddingProfile, sanitizeEmbeddingProfile } from '../query/embedding-config.mjs';
5
7
 
6
8
  function parseConfigOptions(argv, usageKey = 'config') {
7
- const options = { root: '.', json: false };
9
+ const options = { root: '.', json: false, template: null };
8
10
  let rootSet = false;
9
11
  for (let i = 0; i < argv.length; i += 1) {
10
12
  const arg = argv[i];
11
13
  if (arg === '--json') options.json = true;
12
- else if (!arg.startsWith('-') && !rootSet) {
14
+ else if (arg === '--template') {
15
+ const value = argv[i + 1];
16
+ if (!value || value.startsWith('-')) usage('--template requires a value', usageKey);
17
+ options.template = value;
18
+ i += 1;
19
+ } else if (!arg.startsWith('-') && !rootSet) {
13
20
  options.root = arg;
14
21
  rootSet = true;
15
22
  } else if (!arg.startsWith('-')) usage(`Unexpected argument: ${arg}`, usageKey);
@@ -34,6 +41,7 @@ function formatConfigOutput(payload) {
34
41
  const lines = [`dotdotgod config: ${payload.source}${errors.length > 0 ? ' (invalid; using defaults)' : ''}`];
35
42
  lines.push(`- path: ${payload.path ?? 'none'}`);
36
43
  lines.push(`- memory areas: ${payload.config.areas.length}`);
44
+ lines.push(`- embedding: ${payload.embedding.profile.provider}/${payload.embedding.profile.model} (${payload.embedding.source})`);
37
45
  lines.push(`- traceability required: ${(payload.config.traceability.required ?? []).join(', ') || 'none'}`);
38
46
  lines.push(`- traceability exclude: ${(payload.config.traceability.exclude ?? []).join(', ') || 'none'}`);
39
47
  lines.push('- traceability keys:');
@@ -67,16 +75,26 @@ export function runConfig(argv) {
67
75
  }
68
76
  const target = join(options.root, 'dotdotgod.config.json');
69
77
  if (existsSync(target)) configInitError(options, 'CONFIG_EXISTS', 'dotdotgod.config.json already exists.', target);
70
- writeFileSync(target, defaultDotdotgodConfigText());
71
- const payload = { ok: true, command: 'config init', root: options.root, path: target, created: true };
78
+ const template = resolveInitializationTemplate(options.template);
79
+ if (!template.ok) configInitError(options, template.error.code, template.error.message, template.error.path ?? target);
80
+ writeFileSync(target, template.text);
81
+ const payload = { ok: true, command: 'config init', root: options.root, path: target, created: true, template: { name: template.name, source: template.source, selectedBy: template.selectedBy } };
72
82
  if (options.json) console.log(JSON.stringify(payload, null, 2));
73
- else console.log(`dotdotgod config init: created ${target}`);
83
+ else console.log(`dotdotgod config init: created ${target} from ${template.source}:${template.name} (${template.selectedBy})`);
74
84
  return;
75
85
  }
76
86
 
77
87
  const config = readMemoryConfig(options.root);
78
88
  const errors = config.errors ?? [];
79
- const payload = { ok: errors.length === 0, command: 'config', root: options.root, source: config.source ?? 'default', path: config.source && config.source !== 'default' ? join(options.root, config.source) : null, config: memoryConfigSummary(config), errors };
89
+ let embedding;
90
+ try {
91
+ const resolved = resolveEmbeddingProfile(options.root);
92
+ embedding = { source: resolved.source, path: resolved.path, profile: sanitizeEmbeddingProfile(resolved.profile) };
93
+ } catch (error) {
94
+ errors.push({ file: 'embedding config', code: 'EMBEDDING_CONFIG_INVALID', message: error instanceof Error ? error.message : String(error) });
95
+ embedding = { source: 'invalid', path: null, profile: sanitizeEmbeddingProfile({ provider: 'local', model: 'Xenova/multilingual-e5-small' }) };
96
+ }
97
+ const payload = { ok: errors.length === 0, command: 'config', root: options.root, source: config.source ?? 'default', path: config.source && config.source !== 'default' ? join(options.root, config.source) : null, embedding, config: memoryConfigSummary(config), errors };
80
98
  if (options.json) console.log(JSON.stringify(payload, null, 2));
81
99
  else console.log(formatConfigOutput(payload));
82
100
  if (errors.length > 0) process.exit(1);
@@ -2,8 +2,8 @@ import { resolve } from 'node:path';
2
2
  import { usage } from '../cli/usage.mjs';
3
3
  import { readMemoryConfig } from '../memory/config.mjs';
4
4
  import { collectDocumentationChunks } from '../query/chunks.mjs';
5
- import { embedTexts } from '../query/embedder.mjs';
6
- import { isValidNormalizedVectorValues, readVectorCache, VECTOR_DIMENSIONS, VECTOR_MODEL, writeVectorCache } from '../query/store.mjs';
5
+ import { resolveEmbedder } from '../query/embedder.mjs';
6
+ import { isValidNormalizedVectorValues, profileFingerprint, readVectorCache, writeVectorCache } from '../query/store.mjs';
7
7
 
8
8
  const DEFAULT_LIMIT = 30;
9
9
  const MAX_LIMIT = 100;
@@ -16,137 +16,93 @@ export function parseQueryOptions(argv) {
16
16
  for (let i = 0; i < argv.length; i += 1) {
17
17
  const arg = argv[i];
18
18
  if (arg === '--json') options.json = true;
19
- else if (arg === '--limit') {
20
- const value = Number(argv[++i]);
21
- if (!Number.isInteger(value) || value < 1 || value > MAX_LIMIT) usage(`--limit must be an integer from 1 to ${MAX_LIMIT}.`, 'query');
22
- options.limit = value;
23
- } else if (arg.startsWith('-')) usage(`Unknown option: ${arg}`, 'query');
24
- else if (!rootSet) {
25
- options.root = resolve(arg);
26
- rootSet = true;
27
- } else queryParts.push(arg);
19
+ else if (arg === '--limit') { const value = Number(argv[++i]); if (!Number.isInteger(value) || value < 1 || value > MAX_LIMIT) usage(`--limit must be an integer from 1 to ${MAX_LIMIT}.`, 'query'); options.limit = value; }
20
+ else if (arg.startsWith('-')) usage(`Unknown option: ${arg}`, 'query');
21
+ else if (!rootSet) { options.root = resolve(arg); rootSet = true; }
22
+ else queryParts.push(arg);
28
23
  }
29
24
  options.query = queryParts.join(' ').trim();
30
25
  if (!options.query) usage('query requires search text.', 'query');
31
26
  return options;
32
27
  }
33
28
 
34
- function assertNormalizedVector(vector, label) {
35
- if (!vector || vector.length !== VECTOR_DIMENSIONS || !isValidNormalizedVectorValues(vector)) throw new Error(`${label} must be a finite normalized ${VECTOR_DIMENSIONS}-dimensional vector.`);
29
+ function assertVector(vector, dimensions, label) {
30
+ if (!vector || vector.length !== dimensions || !isValidNormalizedVectorValues(vector, dimensions)) throw new Error(`${label} must be a finite normalized ${dimensions}-dimensional vector.`);
36
31
  }
37
32
 
38
- export function cosineScore(left, vectors, offset = 0) {
39
- assertNormalizedVector(left, 'Query vector');
40
- const right = vectors?.subarray ? vectors.subarray(offset, offset + VECTOR_DIMENSIONS) : vectors?.slice(offset, offset + VECTOR_DIMENSIONS);
41
- assertNormalizedVector(right, 'Passage vector');
33
+ export function cosineScore(left, vectors, offset = 0, dimensions = left?.length) {
34
+ assertVector(left, dimensions, 'Query vector');
35
+ const right = vectors?.subarray ? vectors.subarray(offset, offset + dimensions) : vectors?.slice(offset, offset + dimensions);
36
+ assertVector(right, dimensions, 'Passage vector');
42
37
  let score = 0;
43
- for (let index = 0; index < VECTOR_DIMENSIONS; index += 1) score += left[index] * right[index];
44
- if (!Number.isFinite(score) || score < -1.000001 || score > 1.000001) throw new Error('Cosine similarity was outside the finite normalized range.');
38
+ for (let index = 0; index < dimensions; index += 1) score += left[index] * right[index];
45
39
  return Math.max(-1, Math.min(1, score));
46
40
  }
47
41
 
48
42
  export function rankVectorFiles(queryVector, index) {
49
- assertNormalizedVector(queryVector, 'Query vector');
50
- if (!index || !Array.isArray(index.chunks) || index.vectors?.length !== index.chunks.length * VECTOR_DIMENSIONS || !isValidNormalizedVectorValues(index.vectors)) throw new Error('Vector index contains invalid passage vectors.');
51
- const ranked = index.chunks.map((chunk, position) => ({ ...chunk, vectorScore: cosineScore(queryVector, index.vectors, position * VECTOR_DIMENSIONS) }))
43
+ const dimensions = index?.manifest?.dimensions ?? queryVector?.length;
44
+ assertVector(queryVector, dimensions, 'Query vector');
45
+ if (!index || !Array.isArray(index.chunks) || index.vectors?.length !== index.chunks.length * dimensions || !isValidNormalizedVectorValues(index.vectors, dimensions)) throw new Error('Vector index contains invalid passage vectors.');
46
+ const ranked = index.chunks.map((chunk, position) => ({ ...chunk, vectorScore: cosineScore(queryVector, index.vectors, position * dimensions, dimensions) }))
52
47
  .sort((left, right) => right.vectorScore - left.vectorScore || left.path.localeCompare(right.path) || left.id.localeCompare(right.id));
53
- const results = [];
54
- const seen = new Set();
55
- for (const chunk of ranked) {
56
- if (seen.has(chunk.path)) continue;
57
- seen.add(chunk.path);
58
- results.push(chunk);
59
- }
48
+ const results = []; const seen = new Set();
49
+ for (const chunk of ranked) if (!seen.has(chunk.path)) { seen.add(chunk.path); results.push(chunk); }
60
50
  return results;
61
51
  }
62
52
 
63
53
  function lexicalBoost(query, chunk) {
64
- const queryTerms = new Set(query.toLowerCase().split(/[^\p{L}\p{N}_.:/-]+/u).filter((term) => term.length > 1));
65
- if (queryTerms.size === 0) return 0;
54
+ const terms = new Set(query.toLowerCase().split(/[^\p{L}\p{N}_.:/-]+/u).filter((term) => term.length > 1));
55
+ if (!terms.size) return 0;
66
56
  const haystack = `${chunk.path} ${chunk.heading}`.toLowerCase();
67
- let matches = 0;
68
- for (const term of queryTerms) if (haystack.includes(term)) matches += 1;
69
- return Math.min(0.08, (matches / queryTerms.size) * 0.08);
57
+ let matches = 0; for (const term of terms) if (haystack.includes(term)) matches += 1;
58
+ return Math.min(0.08, (matches / terms.size) * 0.08);
70
59
  }
71
60
 
72
- export async function buildVectorIndex(root, embed = embedTexts) {
61
+ export async function buildVectorIndex(root, embedOrOptions = {}) {
62
+ const options = typeof embedOrOptions === 'function' ? { embed: embedOrOptions } : embedOrOptions;
63
+ const resolved = await resolveEmbedder(root, options);
73
64
  const config = readMemoryConfig(root);
74
65
  const exclude = config.load?.documentationSummary?.exclude ?? ['docs/plan', 'docs/archive'];
75
66
  const chunks = collectDocumentationChunks(root, exclude);
76
- const cached = readVectorCache(root);
77
- const cachedOffsets = new Map((cached?.chunks ?? []).map((chunk, index) => [chunk.fingerprint, index * VECTOR_DIMENSIONS]));
78
- const vectors = new Float32Array(chunks.length * VECTOR_DIMENSIONS);
79
- const missing = [];
80
- let reused = 0;
81
- chunks.forEach((chunk, index) => {
82
- const oldOffset = cachedOffsets.get(chunk.fingerprint);
83
- if (oldOffset === undefined || !cached) missing.push({ chunk, index });
84
- else {
85
- vectors.set(cached.vectors.subarray(oldOffset, oldOffset + VECTOR_DIMENSIONS), index * VECTOR_DIMENSIONS);
86
- reused += 1;
87
- }
88
- });
67
+ const cached = readVectorCache(root, resolved.identity);
68
+ const cachedOffsets = new Map((cached?.chunks ?? []).map((chunk, index) => [chunk.fingerprint, index]));
69
+ const rows = new Array(chunks.length);
70
+ const missing = []; let reused = 0;
71
+ chunks.forEach((chunk, index) => { const oldIndex = cachedOffsets.get(chunk.fingerprint); if (oldIndex === undefined || !cached) missing.push({ chunk, index }); else { const d = cached.manifest.dimensions; rows[index] = Array.from(cached.vectors.subarray(oldIndex * d, (oldIndex + 1) * d)); reused += 1; } });
89
72
  for (let offset = 0; offset < missing.length; offset += EMBED_BATCH_SIZE) {
90
73
  const batch = missing.slice(offset, offset + EMBED_BATCH_SIZE);
91
- const embedded = await embed(batch.map(({ chunk }) => `passage: ${chunk.passage}`));
92
- if (!Array.isArray(embedded) || embedded.length !== batch.length) throw new Error('Embedder returned an unexpected passage-vector count.');
93
- embedded.forEach((vector, index) => {
94
- assertNormalizedVector(vector, 'Passage vector');
95
- vectors.set(vector, batch[index].index * VECTOR_DIMENSIONS);
96
- });
74
+ const embedded = await resolved.embed(batch.map(({ chunk }) => `passage: ${chunk.passage}`));
75
+ embedded.forEach((vector, index) => { rows[batch[index].index] = vector; });
97
76
  }
77
+ const dimensions = rows[0]?.length ?? cached?.manifest?.dimensions;
78
+ if (!dimensions) throw new Error('No documentation chunks were available to establish vector dimensions.');
79
+ if (rows.some((row) => !row || row.length !== dimensions)) throw new Error('Embedder returned inconsistent passage-vector dimensions.');
80
+ const vectors = new Float32Array(chunks.length * dimensions);
81
+ rows.forEach((row, index) => { assertVector(row, dimensions, 'Passage vector'); vectors.set(row, index * dimensions); });
98
82
  const storedChunks = chunks.map(({ passage, ...chunk }) => chunk);
99
- const manifest = writeVectorCache(root, storedChunks, vectors, {
100
- excluded: exclude,
101
- embedded: missing.length,
102
- reused,
103
- });
104
- return { chunks: storedChunks, vectors, manifest };
83
+ const manifest = writeVectorCache(root, storedChunks, vectors, { provider: resolved.profile.provider, model: resolved.profile.model, dimensions, profileFingerprint: profileFingerprint(resolved.identity), excluded: exclude, embedded: missing.length, reused });
84
+ return { chunks: storedChunks, vectors, manifest, embedding: resolved };
105
85
  }
106
86
 
107
87
  export async function queryDocumentation(root, query, options = {}) {
108
- const embed = options.embed ?? embedTexts;
109
88
  const limit = options.limit ?? DEFAULT_LIMIT;
110
- const index = await buildVectorIndex(root, embed);
111
- const embeddedQuery = await embed([`query: ${query}`]);
112
- if (!Array.isArray(embeddedQuery) || embeddedQuery.length !== 1) throw new Error('Embedder returned an unexpected query-vector count.');
113
- const [queryVector] = embeddedQuery;
114
- assertNormalizedVector(queryVector, 'Query vector');
115
- const rankedChunks = index.chunks.map((chunk, position) => {
116
- const vectorScore = cosineScore(queryVector, index.vectors, position * VECTOR_DIMENSIONS);
117
- return { ...chunk, score: Number((vectorScore + lexicalBoost(query, chunk)).toFixed(6)), vectorScore: Number(vectorScore.toFixed(6)) };
118
- }).sort((left, right) => right.score - left.score || left.path.localeCompare(right.path));
119
- const results = [];
120
- const seenPaths = new Set();
121
- for (const chunk of rankedChunks) {
122
- if (seenPaths.has(chunk.path)) continue;
123
- seenPaths.add(chunk.path);
124
- results.push(chunk);
125
- if (results.length === limit) break;
126
- }
127
- return { ok: true, command: 'query', root, query, model: VECTOR_MODEL, dimensions: VECTOR_DIMENSIONS, limit, index: index.manifest, results };
89
+ const index = await buildVectorIndex(root, options);
90
+ const [queryVector] = await index.embedding.embed([`query: ${query}`]);
91
+ const dimensions = index.manifest.dimensions;
92
+ assertVector(queryVector, dimensions, 'Query vector');
93
+ const results = rankVectorFiles(queryVector, index).map((chunk) => ({ ...chunk, score: Number((chunk.vectorScore + lexicalBoost(query, chunk)).toFixed(6)), vectorScore: Number(chunk.vectorScore.toFixed(6)) }))
94
+ .sort((a, b) => b.score - a.score || a.path.localeCompare(b.path)).slice(0, limit);
95
+ return { ok: true, command: 'query', root, query, provider: index.embedding.profile.provider, model: index.embedding.profile.model, embeddingSource: index.embedding.source, dimensions, limit, index: index.manifest, results };
128
96
  }
129
97
 
130
98
  function formatQueryResult(payload) {
131
- const lines = [`dotdotgod query: ${payload.query}`, `- model: ${payload.model}`, `- results: ${payload.results.length}`];
132
- payload.results.forEach((result, index) => {
133
- lines.push(`${index + 1}. ${result.path}${result.heading ? ` — ${result.heading}` : ''} (${result.score.toFixed(3)})`);
134
- const excerpt = result.text.replace(/\s+/g, ' ').trim().slice(0, 240);
135
- if (excerpt) lines.push(` ${excerpt}${result.text.length > 240 ? '…' : ''}`);
136
- });
99
+ const lines = [`dotdotgod query: ${payload.query}`, `- embedding: ${payload.provider}/${payload.model} (${payload.embeddingSource})`, `- results: ${payload.results.length}`];
100
+ payload.results.forEach((result, index) => { lines.push(`${index + 1}. ${result.path}${result.heading ? ` — ${result.heading}` : ''} (${result.score.toFixed(3)})`); const excerpt = result.text.replace(/\s+/g, ' ').trim().slice(0, 240); if (excerpt) lines.push(` ${excerpt}${result.text.length > 240 ? '…' : ''}`); });
137
101
  return lines.join('\n');
138
102
  }
139
103
 
140
104
  export async function runQuery(argv) {
141
105
  const options = parseQueryOptions(argv);
142
- try {
143
- const payload = await queryDocumentation(options.root, options.query, options);
144
- if (options.json) console.log(JSON.stringify(payload, null, 2));
145
- else console.log(formatQueryResult(payload));
146
- } catch (error) {
147
- const message = `dotdotgod query failed: ${error instanceof Error ? error.message : String(error)}`;
148
- if (options.json) console.log(JSON.stringify({ ok: false, command: 'query', root: options.root, query: options.query, error: message }, null, 2));
149
- else console.error(message);
150
- process.exitCode = 1;
151
- }
106
+ try { const payload = await queryDocumentation(options.root, options.query, options); console.log(options.json ? JSON.stringify(payload, null, 2) : formatQueryResult(payload)); }
107
+ catch (error) { const message = `dotdotgod query failed: ${error instanceof Error ? error.message : String(error)}`; if (options.json) console.log(JSON.stringify({ ok: false, command: 'query', root: options.root, query: options.query, error: message }, null, 2)); else console.error(message); process.exitCode = 1; }
152
108
  }
@@ -0,0 +1,219 @@
1
+ import { existsSync, readFileSync } from 'node:fs';
2
+ import { homedir } from 'node:os';
3
+ import { join } from 'node:path';
4
+ import { defaultDotdotgodConfigData, validateMemoryConfigData } from '../memory/config.mjs';
5
+ import { isKebabCase } from '../docs/markdown.mjs';
6
+
7
+ export const DEFAULT_TEMPLATE_NAME = 'software';
8
+ export const BUILT_IN_TEMPLATE_NAMES = ['software', 'research', 'case-and-evidence', 'publication', 'portfolio', 'policy'];
9
+
10
+ const BUILT_IN_TEMPLATE_SCAFFOLDS = {
11
+ software: [],
12
+ research: [
13
+ ['docs/research/README.md', 'Research', 'Use this area for research notes, sources, and findings.'],
14
+ ['docs/record/README.md', 'Research Records', 'Use this area for dated measurements, experiments, and execution records.'],
15
+ ['docs/report/README.md', 'Reports', 'Use this area for research diagnoses, analyses, results, and performance reports.'],
16
+ ['outputs', null, null],
17
+ ],
18
+ 'case-and-evidence': [
19
+ ['docs/case/README.md', 'Case Records', 'Use this area for canonical case facts, questions, and decisions.'],
20
+ ['docs/evidence/README.md', 'Evidence', 'Use this area for factual, legal, and other supporting evidence.'],
21
+ ['docs/outputs/README.md', 'Case Outputs', 'Use this area for maintained case outputs.'],
22
+ ],
23
+ publication: [
24
+ ['docs/outline/README.md', 'Publication Outline', 'Use this area for publication structure and outlines.'],
25
+ ['docs/chapters/README.md', 'Chapter Plans', 'Use this area for chapter plans and direction.'],
26
+ ['docs/claims/README.md', 'Claims', 'Use this area for maintained claims and their support.'],
27
+ ['docs/research/README.md', 'Research Sources', 'Use this area for research sources and supporting notes.'],
28
+ ['book', null, null],
29
+ ],
30
+ portfolio: [
31
+ ['docs/strategy/README.md', 'Portfolio Strategy', 'Use this area for portfolio strategy and risk rules.'],
32
+ ['docs/positions/README.md', 'Position Theses', 'Use this area for position theses and investment conclusions.'],
33
+ ['docs/journal/README.md', 'Decision Journal', 'Use this area for dated investment decisions and reviews.'],
34
+ ['docs/report/README.md', 'Market Research', 'Use this area for maintained market research and analysis.'],
35
+ ['data', null, null],
36
+ ],
37
+ policy: [
38
+ ['docs/policy/README.md', 'Policy Documents', 'Use this area for integrated policy proposals and outputs.'],
39
+ ['docs/sections/README.md', 'Policy Sections', 'Use this area for source policy sections.'],
40
+ ['docs/evidence/README.md', 'Policy Evidence', 'Use this area for evidence supporting policy proposals.'],
41
+ ['docs/outputs/README.md', 'Submission Outputs', 'Use this area for maintained policy submission outputs.'],
42
+ ],
43
+ };
44
+
45
+ export function builtInTemplateScaffold(name) {
46
+ const entries = BUILT_IN_TEMPLATE_SCAFFOLDS[name];
47
+ if (!entries) return null;
48
+ return entries.map(([path, heading, description]) => ({
49
+ path,
50
+ type: heading ? 'file' : 'directory',
51
+ ...(heading ? { content: `# ${heading}\n\n${description}` } : {}),
52
+ }));
53
+ }
54
+
55
+ function area(id, label, paths, role, priority, includeBodiesByDefault = true, extra = {}) {
56
+ return { id, label, paths, excludePaths: [], scope: 'shared', freshness: 'fresh', role, priority, includeBodiesByDefault, ...extra };
57
+ }
58
+
59
+ function localAreas() {
60
+ return [
61
+ { ...area('active-plan', 'Active Plans', ['docs/plan/**'], 'active-task-intent', 95), scope: 'local' },
62
+ { ...area('archive-map', 'Archive Map', ['docs/archive/README.md'], 'historical-memory-map', 65), scope: 'local', freshness: 'stale' },
63
+ { ...area('archive-body', 'Archive Body', ['docs/archive/**'], 'historical-memory-body', 20, false), excludePaths: ['docs/archive/README.md'], scope: 'local', freshness: 'stale' },
64
+ ];
65
+ }
66
+
67
+ function baseAreas() {
68
+ return [
69
+ area('rules', 'Agent Rules', ['AGENTS.md'], 'agent-working-rules', 100),
70
+ area('agent-entrypoint', 'Agent Entrypoints', ['CLAUDE.md', 'CODEX.md'], 'agent-specific-entrypoint', 85),
71
+ area('project-overview', 'Project Overview', ['README.md'], 'project-map', 85),
72
+ area('docs-index', 'Docs Index', ['docs/README.md'], 'documentation-routing-map', 90),
73
+ ];
74
+ }
75
+
76
+ function traceability(required, keys) {
77
+ return { required, exclude: ['**/README.md'], keys };
78
+ }
79
+
80
+ const keys = {
81
+ software: [
82
+ ['implementedBy', 'Implemented by', 'Files that implement the behavior.', 'path', 'implemented_by', 4],
83
+ ['verifiedBy', 'Verified by', 'Tests or maintained verification documents.', 'path', 'verified_by', 4],
84
+ ['relatedDocs', 'Related docs', 'Documents needed to interpret the behavior.', 'path', 'related_doc', 3],
85
+ ['verificationCommands', 'Verification commands', 'Project-local verification commands.', 'command', 'verification_command', 3],
86
+ ],
87
+ research: [
88
+ ['informedBy', 'Informed by', 'Research sources or prior findings that justify the direction.', 'path', 'informed_by', 4],
89
+ ['assumptionsDefinedIn', 'Assumptions defined in', 'Documents defining assumptions that bound the claim.', 'path', 'assumptions_defined_in', 4],
90
+ ['evaluatedBy', 'Evaluated by', 'Evaluation protocols or measurement methods.', 'path', 'evaluated_by', 4],
91
+ ['evidenceRecordedIn', 'Evidence recorded in', 'Dated experiment, dataset, or validation records.', 'path', 'evidence_recorded_in', 4],
92
+ ],
93
+ case: [
94
+ ['basedOn', 'Based on', 'Canonical case records, evidence, or grounds supporting the document.', 'path', 'based_on', 4],
95
+ ['relatedDocs', 'Related docs', 'Documents needed to interpret the record.', 'path', 'related_doc', 2],
96
+ ],
97
+ publication: [
98
+ ['developedIn', 'Developed in', 'Outlines or manuscript drafts that develop this design.', 'path', 'developed_in', 4],
99
+ ['supportedBy', 'Supported by', 'Research notes and sources supporting the claims.', 'path', 'supported_by', 4],
100
+ ['framedBy', 'Framed by', 'Documents defining purpose, scope, or conceptual boundaries.', 'path', 'framed_by', 3],
101
+ ],
102
+ portfolio: [
103
+ ['supportedBy', 'Supported by', 'Data or research supporting the investment conclusion.', 'path', 'supported_by', 4],
104
+ ['recordedIn', 'Recorded in', 'Ledgers, snapshots, or journals recording the decision.', 'path', 'recorded_in', 4],
105
+ ['informs', 'Informs', 'Strategies, theses, or risk rules using this conclusion.', 'path', 'informs_decision', 3],
106
+ ],
107
+ policy: [
108
+ ['supportedBy', 'Supported by', 'Evidence, source sections, or research supporting the proposal.', 'path', 'supported_by', 4],
109
+ ['integratedInto', 'Integrated into', 'Integrated proposals or briefs that use this policy section.', 'path', 'integrated_into', 4],
110
+ ['relatedDocs', 'Related docs', 'Documents needed to interpret the policy.', 'path', 'related_doc', 2],
111
+ ],
112
+ };
113
+
114
+ function definitions(values) {
115
+ return values.map(([key, label, description, target, relation, weight]) => ({ key, label, description, target, relation, weight }));
116
+ }
117
+
118
+ function withPolicy(areas, required, definitionsList) {
119
+ const config = defaultDotdotgodConfigData();
120
+ config.memory.areas = [...baseAreas(), ...areas, ...localAreas(), area('docs', 'Project Documentation', ['docs/**'], 'project-documentation', 60)];
121
+ config.traceability = traceability(required, definitions(definitionsList));
122
+ return config;
123
+ }
124
+
125
+ export function builtInTemplateData(name) {
126
+ if (name === 'software') return defaultDotdotgodConfigData();
127
+ if (name === 'research') return withPolicy([
128
+ area('spec', 'Research Specifications', ['docs/spec/**'], 'research-contract', 85),
129
+ area('architecture', 'Architecture', ['docs/arch/**'], 'architecture-rationale', 82),
130
+ area('test', 'Verification', ['docs/test/**'], 'test-strategy', 78),
131
+ area('research', 'Research Notes', ['docs/research/**'], 'research-diary', 70),
132
+ area('record', 'Research Records', ['docs/record/**'], 'research-record', 68),
133
+ { ...area('report', 'Reports', ['docs/report/**'], 'report-background', 60), freshness: 'stale' },
134
+ { ...area('artifacts', 'Generated Artifacts', ['outputs/**'], 'artifact-body', 10, false), scope: 'local', freshness: 'stale' },
135
+ ], ['docs/spec/**', 'docs/report/**'], keys.research);
136
+ if (name === 'case-and-evidence') return withPolicy([
137
+ area('case', 'Case Record', ['docs/case/**'], 'case-truth-and-open-questions', 85),
138
+ area('evidence', 'Evidence and Grounds', ['docs/evidence/**'], 'factual-and-legal-support', 80),
139
+ area('outputs', 'Case Outputs', ['docs/outputs/**'], 'case-outputs', 75),
140
+ ], ['docs/case/**', 'docs/outputs/**'], keys.case);
141
+ if (name === 'publication') return withPolicy([
142
+ area('brief', 'Publication Brief', ['docs/BOOK_BRIEF.md'], 'publication-direction', 100),
143
+ area('outline', 'Publication Outline', ['docs/outline/**'], 'narrative-structure', 95),
144
+ area('chapters', 'Chapter Plans', ['docs/chapters/**'], 'chapter-direction', 90),
145
+ area('claims', 'Claims', ['docs/claims/**'], 'claim-registry', 90),
146
+ area('manuscript', 'Manuscript', ['book/**'], 'manuscript', 95),
147
+ area('research', 'Research Sources', ['docs/research/**'], 'evidence-library', 80),
148
+ ], ['docs/chapters/**'], keys.publication);
149
+ if (name === 'portfolio') return withPolicy([
150
+ area('strategy', 'Portfolio Strategy', ['docs/strategy/**'], 'portfolio-strategy', 95),
151
+ area('positions', 'Position Theses', ['docs/positions/**'], 'position-thesis', 95),
152
+ area('ledger', 'Trading Ledger', ['data/**'], 'trading-facts', 90, false, { excludePaths: ['data/raw/**'] }),
153
+ area('journal', 'Decision Journal', ['docs/journal/**'], 'decision-journal', 80),
154
+ { ...area('research', 'Research', ['docs/report/**'], 'market-research', 50), freshness: 'stale' },
155
+ ], ['docs/strategy/**', 'docs/positions/**'], keys.portfolio);
156
+ if (name === 'policy') return withPolicy([
157
+ area('policy', 'Policy Documents', ['docs/policy/**'], 'policy-output', 95),
158
+ area('sections', 'Policy Sections', ['docs/sections/**'], 'policy-source-sections', 90),
159
+ area('evidence', 'Policy Evidence', ['docs/evidence/**', 'docs/report/**'], 'policy-evidence', 80),
160
+ area('outputs', 'Submission Outputs', ['docs/outputs/**'], 'policy-submission-output', 75),
161
+ ], ['docs/sections/**', 'docs/outputs/**'], keys.policy);
162
+ return null;
163
+ }
164
+
165
+ export function builtInTemplateText(name) {
166
+ const data = builtInTemplateData(name);
167
+ return data ? `${JSON.stringify(data, null, 2)}\n` : null;
168
+ }
169
+
170
+ export function dotdotgodHome() {
171
+ return join(homedir(), '.dotdotgod');
172
+ }
173
+
174
+ function templateError(code, message, path = null) {
175
+ return { ok: false, error: { code, message, ...(path ? { path } : {}) } };
176
+ }
177
+
178
+ export function resolveInitializationTemplate(explicitName = null, home = dotdotgodHome()) {
179
+ let name = explicitName;
180
+ let selectedBy = explicitName ? 'explicit' : 'default';
181
+ const globalPath = join(home, 'config.json');
182
+ if (!name && existsSync(globalPath)) {
183
+ let globalConfig;
184
+ try {
185
+ globalConfig = JSON.parse(readFileSync(globalPath, 'utf8'));
186
+ } catch (error) {
187
+ return templateError('GLOBAL_CONFIG_INVALID_JSON', `Invalid global config JSON: ${error instanceof Error ? error.message : String(error)}`, globalPath);
188
+ }
189
+ if (!globalConfig || typeof globalConfig !== 'object' || Array.isArray(globalConfig)) {
190
+ return templateError('GLOBAL_CONFIG_INVALID', 'Global config must be a JSON object.', globalPath);
191
+ }
192
+ if (globalConfig.defaultTemplate !== undefined && !isKebabCase(globalConfig.defaultTemplate)) {
193
+ return templateError('GLOBAL_CONFIG_INVALID_DEFAULT_TEMPLATE', 'Global config defaultTemplate must be a kebab-case string.', globalPath);
194
+ }
195
+ if (globalConfig.defaultTemplate) {
196
+ name = globalConfig.defaultTemplate;
197
+ selectedBy = 'global';
198
+ }
199
+ }
200
+ name ??= DEFAULT_TEMPLATE_NAME;
201
+ if (!isKebabCase(name)) return templateError('TEMPLATE_INVALID_NAME', `Template name must be kebab-case: ${name}`);
202
+
203
+ const customPath = join(home, 'templates', `${name}.json`);
204
+ if (existsSync(customPath)) {
205
+ let data;
206
+ try {
207
+ data = JSON.parse(readFileSync(customPath, 'utf8'));
208
+ } catch (error) {
209
+ return templateError('TEMPLATE_INVALID_JSON', `Invalid template JSON: ${error instanceof Error ? error.message : String(error)}`, customPath);
210
+ }
211
+ const errors = validateMemoryConfigData(data, '.', customPath);
212
+ if (errors.length > 0) return { ok: false, error: { code: 'TEMPLATE_INVALID', message: `Custom template is invalid: ${name}`, path: customPath, errors } };
213
+ return { ok: true, name, source: 'custom', selectedBy, path: customPath, data, text: `${JSON.stringify(data, null, 2)}\n` };
214
+ }
215
+
216
+ const data = builtInTemplateData(name);
217
+ if (!data) return templateError('TEMPLATE_NOT_FOUND', `Template not found: ${name}`);
218
+ return { ok: true, name, source: 'bundled', selectedBy, path: null, data, text: `${JSON.stringify(data, null, 2)}\n` };
219
+ }
package/src/core.mjs CHANGED
@@ -14,6 +14,7 @@ export { rel } from './common/paths.mjs';
14
14
  export { extractAnchors, extractFirstHeading, extractLinks, headingToAnchor, isKebabCase, isNumberedSeriesFilename, isUpperSnakeMarkdown, removeCodeBlocks, suggestFilenameFromHeading } from './docs/markdown.mjs';
15
15
  export { extractDotdotgodTraceabilityBlocks, findTraceabilityLinksRegion, isLocalRelativeTraceabilityPath, renderCompactTraceabilityBlock, stripTraceabilityLinksRegion, syncTraceabilityLinksInContent, traceabilityExample, validateTraceabilityBlock, validateTraceabilityLinksRegion, validateTraceabilityPlacement } from './docs/traceability.mjs';
16
16
  export { DEFAULT_IMPACT_RANKING_POLICY, DEFAULT_LOAD_POLICY, DEFAULT_VALIDATION_POLICY, SEMANTIC_RELATIONS, cloneImpactRankingPolicy, cloneLoadPolicy, cloneReferenceExpansionPolicy, cloneValidationPolicy, defaultDotdotgodConfigData, defaultDotdotgodConfigText, defaultMemoryConfig, isMarkdownSizeExcluded, isSecretLikePathPattern, isValidPathPattern, matchPathPattern, memoryAreaForPath, memoryConfigSummary, normalizeLowSignalTerm, readMemoryConfig, requiresTraceability, resolveMemoryArea, validateMemoryConfigData } from './memory/config.mjs';
17
+ export { BUILT_IN_TEMPLATE_NAMES, DEFAULT_TEMPLATE_NAME, builtInTemplateData, builtInTemplateScaffold, builtInTemplateText, dotdotgodHome, resolveInitializationTemplate } from './config/templates.mjs';
17
18
  export { buildCommunities, relationWeight } from './graph/communities.mjs';
18
19
  export { addEdge, addNode, compactGraph, expandGraph, graphStats, jsonSize, shardFile, writeJson } from './graph/store.mjs';
19
20
  export { cacheFile, collectIndexFiles, fingerprint, shouldIndexPath } from './index/files.mjs';
@@ -30,7 +31,9 @@ export { formatCompactImpactOutput, formatYmlGraphImpactError, formatYmlImpactOu
30
31
  export { runConfig } from './commands/config.mjs';
31
32
  export { buildVectorIndex, cosineScore, parseQueryOptions, queryDocumentation, rankVectorFiles, runQuery } from './commands/query.mjs';
32
33
  export { chunkMarkdown, collectDocumentationChunks, collectDocumentationMarkdown, textFingerprint } from './query/chunks.mjs';
33
- export { readVectorCache, VECTOR_DIMENSIONS, VECTOR_MODEL, VECTOR_SCHEMA_VERSION, vectorCachePaths, writeVectorCache } from './query/store.mjs';
34
+ export { readVectorCache, VECTOR_DIMENSIONS, VECTOR_MODEL, VECTOR_SCHEMA_VERSION, profileFingerprint, vectorCachePaths, writeVectorCache } from './query/store.mjs';
35
+ export { DEFAULT_EMBEDDING_PROFILE, EMBEDDING_PROVIDERS, embeddingProfileIdentity, resolveEmbeddingProfile, sanitizeEmbeddingProfile, validateEmbeddingProfile } from './query/embedding-config.mjs';
36
+ export { createEmbedder, resolveEmbedder } from './query/embedder.mjs';
34
37
  export { extractBracketReferences, extractFuzzyReferences, normalizeReferenceAlias } from './reference/extract.mjs';
35
38
  export { resolveReferenceCandidates, runResolve, runExpand } from './reference/resolve.mjs';
36
39
  export { parseTraceabilityOptions, runTraceability } from './commands/traceability.mjs';
@@ -1,6 +1,5 @@
1
1
  import { buildVectorIndex, rankVectorFiles } from '../commands/query.mjs';
2
- import { embedTexts } from '../query/embedder.mjs';
3
- import { VECTOR_MODEL } from '../query/store.mjs';
2
+
4
3
  import { buildChangedFileProfile, canonicalizeChangedPath } from './vector-profile.mjs';
5
4
 
6
5
  export const VECTOR_RELATION_WEIGHT = 2;
@@ -14,16 +13,17 @@ function boundedEvidence(value, limit) {
14
13
  export async function buildVectorImpactOverlay(root, index, changedPaths, options = {}) {
15
14
  const semantic = index?.memoryConfig?.impactRanking?.semantic ?? {};
16
15
  if (semantic.enabled === false || semantic.topKPerFile === 0) return { status: 'disabled', edges: [], diagnostics: { enabled: false } };
17
- const embed = options.embed ?? embedTexts;
18
16
  const threshold = semantic.threshold ?? 0.5;
19
17
  const topK = semantic.topKPerFile ?? 5;
20
18
  try {
21
- const vectorIndex = await (options.buildIndex ?? buildVectorIndex)(root, embed);
19
+ const vectorIndex = await (options.buildIndex ?? buildVectorIndex)(root, options);
22
20
  const edges = [];
23
21
  const normalizedChangedPaths = [...new Set(changedPaths.map((path) => canonicalizeChangedPath(root, path)).filter(Boolean))];
24
22
  for (const changed of normalizedChangedPaths) {
25
23
  const profile = buildChangedFileProfile(root, changed, index?.graph);
26
24
  if (!profile) continue;
25
+ const embed = vectorIndex.embedding?.embed ?? options.embed;
26
+ if (!embed) throw new Error('Vector index did not provide an embedder.');
27
27
  const [queryVector] = await embed([`query: ${profile.text}`]);
28
28
  if (!queryVector) throw new Error('Changed-file embedding was empty.');
29
29
  const matches = rankVectorFiles(queryVector, vectorIndex)
@@ -42,7 +42,7 @@ export async function buildVectorImpactOverlay(root, index, changedPaths, option
42
42
  });
43
43
  }
44
44
  }
45
- return { status: 'available', edges, diagnostics: { enabled: true, model: VECTOR_MODEL, corpusChunks: vectorIndex.chunks.length, threshold, topK, embedded: vectorIndex.manifest?.embedded, reused: vectorIndex.manifest?.reused } };
45
+ return { status: 'available', edges, diagnostics: { enabled: true, provider: vectorIndex.embedding?.profile?.provider ?? 'injected', model: vectorIndex.embedding?.profile?.model ?? 'injected', source: vectorIndex.embedding?.source ?? 'injected', corpusChunks: vectorIndex.chunks.length, threshold, topK, embedded: vectorIndex.manifest?.embedded, reused: vectorIndex.manifest?.reused } };
46
46
  } catch (error) {
47
47
  return { status: 'unavailable', edges: [], diagnostics: { enabled: true, reason: error instanceof Error ? error.message : String(error) } };
48
48
  }
package/src/init.mjs CHANGED
@@ -1,6 +1,6 @@
1
1
  import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
2
2
  import { basename, dirname, join, resolve } from 'node:path';
3
- import { defaultDotdotgodConfigText } from './memory/config.mjs';
3
+ import { builtInTemplateScaffold, resolveInitializationTemplate } from './config/templates.mjs';
4
4
 
5
5
  function action(status, path, extra = {}) {
6
6
  return { status, path, ...extra };
@@ -12,16 +12,17 @@ function formatAction(item) {
12
12
  }
13
13
 
14
14
  function parseInitOptions(argv) {
15
- const options = { root: null, projectName: '', dotdotSetting: false, dryRun: false, json: false };
15
+ const options = { root: null, projectName: '', dotdotSetting: false, dryRun: false, json: false, template: null };
16
16
  for (let i = 0; i < argv.length; i += 1) {
17
17
  const arg = argv[i];
18
18
  if (arg === '--json') options.json = true;
19
19
  else if (arg === '--dotdot-setting') options.dotdotSetting = true;
20
20
  else if (arg === '--dry-run') options.dryRun = true;
21
- else if (arg === '--project-name') {
21
+ else if (arg === '--project-name' || arg === '--template') {
22
22
  const value = argv[i + 1];
23
- if (!value || value.startsWith('-')) throw new Error('--project-name requires a value');
24
- options.projectName = value;
23
+ if (!value || value.startsWith('-')) throw new Error(`${arg} requires a value`);
24
+ if (arg === '--project-name') options.projectName = value;
25
+ else options.template = value;
25
26
  i += 1;
26
27
  } else if (!arg.startsWith('-') && !options.root) {
27
28
  options.root = arg;
@@ -63,6 +64,20 @@ function writeInitFile(options, relativePath, content, actions) {
63
64
  actions.push(action('created', target));
64
65
  }
65
66
 
67
+ function ensureInitDirectory(options, relativePath, actions) {
68
+ const target = join(options.root, relativePath);
69
+ if (existsSync(target)) {
70
+ actions.push(action('skipped', target));
71
+ return;
72
+ }
73
+ if (options.dryRun) {
74
+ actions.push(action('would_create', target));
75
+ return;
76
+ }
77
+ mkdirSync(target, { recursive: true });
78
+ actions.push(action('created', target));
79
+ }
80
+
66
81
  function ensureGitignoreEntry(options, entry, actions) {
67
82
  const target = join(options.root, '.gitignore');
68
83
  const existed = existsSync(target);
@@ -294,14 +309,27 @@ export function runInit(argv, usage) {
294
309
 
295
310
  if (!options.dryRun) mkdirSync(options.root, { recursive: true });
296
311
 
312
+ const configExists = existsSync(join(options.root, 'dotdotgod.config.json'));
313
+ const template = configExists ? null : resolveInitializationTemplate(options.template);
314
+ if (template && !template.ok) initError(options, template.error.code, template.error.message);
315
+
297
316
  const actions = [];
298
317
  for (const [relativePath, content] of initFiles(options)) writeInitFile(options, relativePath, content, actions);
318
+ if (template?.source === 'bundled') {
319
+ for (const item of builtInTemplateScaffold(template.name) ?? []) {
320
+ if (item.type === 'directory') ensureInitDirectory(options, item.path, actions);
321
+ else writeInitFile(options, item.path, item.content, actions);
322
+ }
323
+ }
299
324
 
300
- writeInitFile(options, 'dotdotgod.config.json', defaultDotdotgodConfigText(), actions);
325
+ writeInitFile(options, 'dotdotgod.config.json', template?.text ?? '', actions);
301
326
 
302
327
  for (const entry of ['docs/plan', 'docs/archive', '.dotdotgod']) ensureGitignoreEntry(options, entry, actions);
303
328
 
304
- const payload = { ok: true, command: 'init', root: options.root, projectName: options.projectName, dryRun: options.dryRun, dotdotSetting: options.dotdotSetting, actions };
329
+ const payload = { ok: true, command: 'init', root: options.root, projectName: options.projectName, dryRun: options.dryRun, dotdotSetting: options.dotdotSetting, template: template ? { name: template.name, source: template.source, selectedBy: template.selectedBy } : null, actions };
305
330
  if (options.json) console.log(JSON.stringify(payload, null, 2));
306
- else for (const item of actions) console.log(formatAction(item));
331
+ else {
332
+ if (template) console.log(`template ${template.source}:${template.name} (${template.selectedBy})`);
333
+ for (const item of actions) console.log(formatAction(item));
334
+ }
307
335
  }
@@ -2,6 +2,7 @@ import { existsSync, readFileSync } from 'node:fs';
2
2
  import { join, resolve } from 'node:path';
3
3
  import { rel } from '../common/paths.mjs';
4
4
  import { isKebabCase } from '../docs/markdown.mjs';
5
+ import { validateEmbeddingProfile } from '../query/embedding-config.mjs';
5
6
 
6
7
  const MEMORY_CONFIG_FILE = 'dotdotgod.config.json';
7
8
  const MEMORY_SCOPES = new Set(['shared', 'local']);
@@ -335,6 +336,10 @@ export function validateMemoryConfigData(data, root = '.', file = 'dotdotgod.con
335
336
  add('MEMORY_CONFIG_INVALID', null, 'Config must be a JSON object.');
336
337
  return errors;
337
338
  }
339
+ if (data.embedding !== undefined) {
340
+ try { validateEmbeddingProfile(data.embedding, 'embedding'); }
341
+ catch (error) { add('EMBEDDING_CONFIG_INVALID', 'embedding', error instanceof Error ? error.message : String(error)); }
342
+ }
338
343
  const traceability = data.traceability;
339
344
  if (traceability !== undefined) {
340
345
  if (!traceability || typeof traceability !== 'object' || Array.isArray(traceability)) {
@@ -1,24 +1,82 @@
1
- import { VECTOR_DIMENSIONS, VECTOR_MODEL } from './store.mjs';
1
+ import { embeddingProfileIdentity, resolveEmbeddingProfile } from './embedding-config.mjs';
2
2
 
3
- let extractorPromise;
3
+ const localPipelines = new Map();
4
4
 
5
- async function extractor() {
6
- if (!extractorPromise) {
7
- extractorPromise = import('@huggingface/transformers').then(async ({ env, pipeline }) => {
5
+ function normalize(row) {
6
+ if (!Array.isArray(row) || row.length === 0 || row.some((value) => !Number.isFinite(value))) throw new Error('Embedding provider returned an invalid vector.');
7
+ const norm = Math.sqrt(row.reduce((sum, value) => sum + value * value, 0));
8
+ if (!Number.isFinite(norm) || norm === 0) throw new Error('Embedding provider returned a zero-length vector.');
9
+ return row.map((value) => value / norm);
10
+ }
11
+
12
+ async function localEmbedder(profile) {
13
+ if (!localPipelines.has(profile.model)) {
14
+ localPipelines.set(profile.model, import('@huggingface/transformers').then(async ({ env, pipeline }) => {
8
15
  env.allowLocalModels = true;
9
- return pipeline('feature-extraction', VECTOR_MODEL);
10
- });
16
+ return pipeline('feature-extraction', profile.model);
17
+ }));
11
18
  }
12
- return extractorPromise;
19
+ const model = await localPipelines.get(profile.model);
20
+ return async (texts) => (await model(texts, { pooling: 'mean', normalize: true })).tolist().map(normalize);
21
+ }
22
+
23
+ function credential(profile) {
24
+ if (profile.apiKey) return profile.apiKey;
25
+ if (!profile.apiKeyEnv) return null;
26
+ const value = process.env[profile.apiKeyEnv];
27
+ if (!value) throw new Error(`Embedding credential environment variable is not set: ${profile.apiKeyEnv}`);
28
+ return value;
29
+ }
30
+
31
+ async function postJson(url, body, profile) {
32
+ const controller = new AbortController();
33
+ const timer = setTimeout(() => controller.abort(), 30000);
34
+ try {
35
+ const key = credential(profile);
36
+ const response = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', ...(key ? { authorization: `Bearer ${key}` } : {}) }, body: JSON.stringify(body), signal: controller.signal });
37
+ if (!response.ok) {
38
+ const category = response.status === 401 || response.status === 403 ? 'authentication' : response.status === 429 ? 'rate limit' : `HTTP ${response.status}`;
39
+ throw new Error(`Remote embedding ${category} failure.`);
40
+ }
41
+ try { return await response.json(); } catch { throw new Error('Remote embedding returned malformed JSON.'); }
42
+ } catch (error) {
43
+ if (error?.name === 'AbortError') throw new Error('Remote embedding request timed out.');
44
+ throw error;
45
+ } finally { clearTimeout(timer); }
46
+ }
47
+
48
+ function remoteEmbedder(profile) {
49
+ if (profile.provider === 'ollama') return async (texts) => {
50
+ const data = await postJson(`${profile.baseUrl ?? 'http://localhost:11434'}/api/embed`, { model: profile.model, input: texts }, profile);
51
+ if (!Array.isArray(data.embeddings)) throw new Error('Ollama embedding response is missing embeddings.');
52
+ return data.embeddings.map(normalize);
53
+ };
54
+ return async (texts) => {
55
+ const data = await postJson(`${profile.baseUrl ?? 'https://api.openai.com/v1'}/embeddings`, { model: profile.model, input: texts }, profile);
56
+ if (!Array.isArray(data.data)) throw new Error('OpenAI-compatible embedding response is missing data.');
57
+ const ordered = [...data.data].sort((a, b) => a.index - b.index).map((item) => item.embedding);
58
+ return ordered.map(normalize);
59
+ };
60
+ }
61
+
62
+ export async function createEmbedder(profile) {
63
+ const embed = profile.provider === 'local' ? await localEmbedder(profile) : remoteEmbedder(profile);
64
+ return async (texts) => {
65
+ if (texts.length === 0) return [];
66
+ const rows = await embed(texts);
67
+ if (!Array.isArray(rows) || rows.length !== texts.length) throw new Error('Embedding provider returned an unexpected vector count.');
68
+ const dimensions = rows[0]?.length;
69
+ if (!dimensions || rows.some((row) => row.length !== dimensions)) throw new Error('Embedding provider returned inconsistent vector dimensions.');
70
+ return rows;
71
+ };
72
+ }
73
+
74
+ export async function resolveEmbedder(root, options = {}) {
75
+ const resolved = options.profile ? { profile: options.profile, source: options.source ?? 'injected', path: null } : resolveEmbeddingProfile(root, options);
76
+ return { ...resolved, identity: embeddingProfileIdentity(resolved.profile), embed: options.embed ?? await createEmbedder(resolved.profile) };
13
77
  }
14
78
 
15
79
  export async function embedTexts(texts) {
16
- if (texts.length === 0) return [];
17
- const model = await extractor();
18
- const output = await model(texts, { pooling: 'mean', normalize: true });
19
- const rows = output.tolist();
20
- if (!Array.isArray(rows) || rows.some((row) => !Array.isArray(row) || row.length !== VECTOR_DIMENSIONS)) {
21
- throw new Error(`Unexpected ${VECTOR_MODEL} embedding shape.`);
22
- }
23
- return rows;
80
+ const { embed } = await resolveEmbedder('.');
81
+ return embed(texts);
24
82
  }
@@ -0,0 +1,52 @@
1
+ import { existsSync, readFileSync } from 'node:fs';
2
+ import { homedir } from 'node:os';
3
+ import { join } from 'node:path';
4
+
5
+ export const DEFAULT_EMBEDDING_PROFILE = Object.freeze({ provider: 'local', model: 'Xenova/multilingual-e5-small' });
6
+ export const EMBEDDING_PROVIDERS = Object.freeze(['local', 'openai-compatible', 'ollama']);
7
+
8
+ function readJson(path, label) {
9
+ if (!existsSync(path)) return null;
10
+ try { return JSON.parse(readFileSync(path, 'utf8')); }
11
+ catch (error) { throw new Error(`Invalid ${label} JSON: ${error instanceof Error ? error.message : String(error)}`); }
12
+ }
13
+
14
+ export function validateEmbeddingProfile(profile, label = 'embedding') {
15
+ if (!profile || typeof profile !== 'object' || Array.isArray(profile)) throw new Error(`${label} must be an object.`);
16
+ const allowed = new Set(['provider', 'model', 'baseUrl', 'apiKey', 'apiKeyEnv']);
17
+ const unknown = Object.keys(profile).find((key) => !allowed.has(key));
18
+ if (unknown) throw new Error(`${label}.${unknown} is not supported.`);
19
+ if (!EMBEDDING_PROVIDERS.includes(profile.provider)) throw new Error(`${label}.provider must be local, openai-compatible, or ollama.`);
20
+ if (typeof profile.model !== 'string' || !profile.model.trim()) throw new Error(`${label}.model must be a non-empty string.`);
21
+ for (const key of ['baseUrl', 'apiKey', 'apiKeyEnv']) if (profile[key] !== undefined && (typeof profile[key] !== 'string' || !profile[key].trim())) throw new Error(`${label}.${key} must be a non-empty string.`);
22
+ if (profile.apiKey !== undefined && profile.apiKeyEnv !== undefined) throw new Error(`${label}.apiKey and ${label}.apiKeyEnv are mutually exclusive.`);
23
+ if (profile.provider === 'local' && ['baseUrl', 'apiKey', 'apiKeyEnv'].some((key) => profile[key] !== undefined)) throw new Error(`${label} local provider does not accept remote connection fields.`);
24
+ if (profile.baseUrl !== undefined) {
25
+ let url;
26
+ try { url = new URL(profile.baseUrl); } catch { throw new Error(`${label}.baseUrl must be an absolute HTTP(S) URL.`); }
27
+ if (!['http:', 'https:'].includes(url.protocol)) throw new Error(`${label}.baseUrl must use HTTP or HTTPS.`);
28
+ if (url.username || url.password) throw new Error(`${label}.baseUrl must not contain credentials.`);
29
+ }
30
+ return { ...profile, model: profile.model.trim(), ...(profile.baseUrl ? { baseUrl: profile.baseUrl.replace(/\/$/, '') } : {}) };
31
+ }
32
+
33
+ export function sanitizeEmbeddingProfile(profile) {
34
+ const { apiKey, ...safe } = profile;
35
+ return { ...safe, ...(apiKey ? { apiKey: '[redacted]' } : {}) };
36
+ }
37
+
38
+ export function embeddingProfileIdentity(profile) {
39
+ const { apiKey, apiKeyEnv, ...identity } = profile;
40
+ return identity;
41
+ }
42
+
43
+ export function resolveEmbeddingProfile(root = '.', options = {}) {
44
+ const home = options.home ?? homedir();
45
+ const globalPath = join(home, '.dotdotgod', 'config.json');
46
+ const projectPath = join(root, 'dotdotgod.config.json');
47
+ const projectConfig = readJson(projectPath, 'project config');
48
+ if (projectConfig && Object.hasOwn(projectConfig, 'embedding')) return { profile: validateEmbeddingProfile(projectConfig.embedding, 'project embedding'), source: 'project', path: projectPath };
49
+ const globalConfig = readJson(globalPath, 'global config');
50
+ if (globalConfig && Object.hasOwn(globalConfig, 'embedding')) return { profile: validateEmbeddingProfile(globalConfig.embedding, 'global embedding'), source: 'global', path: globalPath };
51
+ return { profile: { ...DEFAULT_EMBEDDING_PROFILE }, source: 'default', path: null };
52
+ }
@@ -1,15 +1,20 @@
1
+ import { createHash } from 'node:crypto';
1
2
  import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
2
3
  import { join } from 'node:path';
3
4
 
4
- export const VECTOR_SCHEMA_VERSION = 1;
5
+ export const VECTOR_SCHEMA_VERSION = 2;
5
6
  export const VECTOR_MODEL = 'Xenova/multilingual-e5-small';
6
7
  export const VECTOR_DIMENSIONS = 384;
7
8
 
8
- export function isValidNormalizedVectorValues(values) {
9
- if (!values || values.length % VECTOR_DIMENSIONS !== 0) return false;
10
- for (let offset = 0; offset < values.length; offset += VECTOR_DIMENSIONS) {
9
+ export function profileFingerprint(identity) {
10
+ return createHash('sha256').update(JSON.stringify(identity)).digest('hex');
11
+ }
12
+
13
+ export function isValidNormalizedVectorValues(values, dimensions = VECTOR_DIMENSIONS) {
14
+ if (!Number.isInteger(dimensions) || dimensions < 1 || !values || values.length % dimensions !== 0) return false;
15
+ for (let offset = 0; offset < values.length; offset += dimensions) {
11
16
  let squaredNorm = 0;
12
- for (let index = 0; index < VECTOR_DIMENSIONS; index += 1) {
17
+ for (let index = 0; index < dimensions; index += 1) {
13
18
  const value = values[offset + index];
14
19
  if (!Number.isFinite(value) || value < -1 || value > 1) return false;
15
20
  squaredNorm += value * value;
@@ -21,29 +26,23 @@ export function isValidNormalizedVectorValues(values) {
21
26
 
22
27
  export function vectorCachePaths(root) {
23
28
  const directory = join(root, '.dotdotgod', 'vectors');
24
- return {
25
- directory,
26
- manifest: join(directory, 'manifest.json'),
27
- chunks: join(directory, 'chunks.jsonl'),
28
- embeddings: join(directory, 'embeddings.f32'),
29
- };
29
+ return { directory, manifest: join(directory, 'manifest.json'), chunks: join(directory, 'chunks.jsonl'), embeddings: join(directory, 'embeddings.f32') };
30
30
  }
31
31
 
32
- export function readVectorCache(root) {
32
+ export function readVectorCache(root, identity = null) {
33
33
  const paths = vectorCachePaths(root);
34
34
  if (!existsSync(paths.manifest) || !existsSync(paths.chunks) || !existsSync(paths.embeddings)) return null;
35
35
  try {
36
36
  const manifest = JSON.parse(readFileSync(paths.manifest, 'utf8'));
37
- if (manifest.schemaVersion !== VECTOR_SCHEMA_VERSION || manifest.model !== VECTOR_MODEL || manifest.dimensions !== VECTOR_DIMENSIONS) return null;
37
+ if (manifest.schemaVersion !== VECTOR_SCHEMA_VERSION || !Number.isInteger(manifest.dimensions) || manifest.dimensions < 1) return null;
38
+ if (identity && manifest.profileFingerprint !== profileFingerprint(identity)) return null;
38
39
  const chunks = readFileSync(paths.chunks, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line));
39
40
  const buffer = readFileSync(paths.embeddings);
40
- if (buffer.byteLength !== chunks.length * VECTOR_DIMENSIONS * 4) return null;
41
+ if (buffer.byteLength !== chunks.length * manifest.dimensions * 4) return null;
41
42
  const vectors = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
42
- if (!isValidNormalizedVectorValues(vectors)) return null;
43
+ if (!isValidNormalizedVectorValues(vectors, manifest.dimensions)) return null;
43
44
  return { manifest, chunks, vectors: new Float32Array(vectors) };
44
- } catch {
45
- return null;
46
- }
45
+ } catch { return null; }
47
46
  }
48
47
 
49
48
  function atomicWrite(path, data) {
@@ -54,23 +53,16 @@ function atomicWrite(path, data) {
54
53
 
55
54
  export function writeVectorCache(root, chunks, vectors, metadata = {}) {
56
55
  const paths = vectorCachePaths(root);
56
+ const dimensions = metadata.dimensions;
57
+ if (!Number.isInteger(dimensions) || dimensions < 1) throw new Error('Vector dimensions must be a positive integer.');
57
58
  mkdirSync(paths.directory, { recursive: true });
58
- if (vectors.length !== chunks.length * VECTOR_DIMENSIONS) throw new Error('Vector count does not match chunk metadata.');
59
- if (!isValidNormalizedVectorValues(vectors)) throw new Error('Vectors must contain finite normalized values.');
60
- const manifest = {
61
- schemaVersion: VECTOR_SCHEMA_VERSION,
62
- model: VECTOR_MODEL,
63
- dimensions: VECTOR_DIMENSIONS,
64
- chunks: chunks.length,
65
- generatedAt: new Date().toISOString(),
66
- ...metadata,
67
- };
59
+ if (vectors.length !== chunks.length * dimensions) throw new Error('Vector count does not match chunk metadata.');
60
+ if (!isValidNormalizedVectorValues(vectors, dimensions)) throw new Error('Vectors must contain finite normalized values.');
61
+ const manifest = { schemaVersion: VECTOR_SCHEMA_VERSION, chunks: chunks.length, generatedAt: new Date().toISOString(), ...metadata };
68
62
  atomicWrite(paths.chunks, `${chunks.map((chunk) => JSON.stringify(chunk)).join('\n')}\n`);
69
63
  atomicWrite(paths.embeddings, Buffer.from(vectors.buffer, vectors.byteOffset, vectors.byteLength));
70
64
  atomicWrite(paths.manifest, `${JSON.stringify(manifest, null, 2)}\n`);
71
65
  return manifest;
72
66
  }
73
67
 
74
- export function removeVectorCache(root) {
75
- rmSync(vectorCachePaths(root).directory, { recursive: true, force: true });
76
- }
68
+ export function removeVectorCache(root) { rmSync(vectorCachePaths(root).directory, { recursive: true, force: true }); }