@dougharness/cli 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/ablate.d.ts +96 -0
- package/dist/ablate.js +440 -0
- package/dist/ablate.js.map +1 -0
- package/dist/apply.d.ts +2 -0
- package/dist/apply.js +19 -0
- package/dist/apply.js.map +1 -0
- package/dist/bin.d.ts +2 -0
- package/dist/bin.js +80 -0
- package/dist/bin.js.map +1 -0
- package/dist/board-page.d.ts +15 -0
- package/dist/board-page.js +27 -0
- package/dist/board-page.js.map +1 -0
- package/dist/board-serve-process.d.ts +24 -0
- package/dist/board-serve-process.js +227 -0
- package/dist/board-serve-process.js.map +1 -0
- package/dist/board-serve.d.ts +9 -0
- package/dist/board-serve.js +302 -0
- package/dist/board-serve.js.map +1 -0
- package/dist/board.d.ts +6 -0
- package/dist/board.js +515 -0
- package/dist/board.js.map +1 -0
- package/dist/detect/fs.d.ts +6 -0
- package/dist/detect/fs.js +50 -0
- package/dist/detect/fs.js.map +1 -0
- package/dist/detect/index.d.ts +3 -0
- package/dist/detect/index.js +13 -0
- package/dist/detect/index.js.map +1 -0
- package/dist/detect/node.d.ts +3 -0
- package/dist/detect/node.js +179 -0
- package/dist/detect/node.js.map +1 -0
- package/dist/detect/python.d.ts +2 -0
- package/dist/detect/python.js +388 -0
- package/dist/detect/python.js.map +1 -0
- package/dist/detect/repo.d.ts +2 -0
- package/dist/detect/repo.js +102 -0
- package/dist/detect/repo.js.map +1 -0
- package/dist/detect/types.d.ts +98 -0
- package/dist/detect/types.js +2 -0
- package/dist/detect/types.js.map +1 -0
- package/dist/diff.d.ts +12 -0
- package/dist/diff.js +105 -0
- package/dist/diff.js.map +1 -0
- package/dist/generate/agents.d.ts +19 -0
- package/dist/generate/agents.js +184 -0
- package/dist/generate/agents.js.map +1 -0
- package/dist/generate/claude-md.d.ts +9 -0
- package/dist/generate/claude-md.js +111 -0
- package/dist/generate/claude-md.js.map +1 -0
- package/dist/generate/config.d.ts +51 -0
- package/dist/generate/config.js +155 -0
- package/dist/generate/config.js.map +1 -0
- package/dist/generate/proposal.d.ts +11 -0
- package/dist/generate/proposal.js +175 -0
- package/dist/generate/proposal.js.map +1 -0
- package/dist/generate/settings.d.ts +35 -0
- package/dist/generate/settings.js +213 -0
- package/dist/generate/settings.js.map +1 -0
- package/dist/generate/skills.d.ts +8 -0
- package/dist/generate/skills.js +264 -0
- package/dist/generate/skills.js.map +1 -0
- package/dist/init.d.ts +10 -0
- package/dist/init.js +69 -0
- package/dist/init.js.map +1 -0
- package/dist/trace.d.ts +9 -0
- package/dist/trace.js +106 -0
- package/dist/trace.js.map +1 -0
- package/node_modules/@dougharness/flow/.claude-plugin/plugin.json +10 -0
- package/node_modules/@dougharness/flow/LICENSE +21 -0
- package/node_modules/@dougharness/flow/agents/adversary-claude.md +33 -0
- package/node_modules/@dougharness/flow/agents/adversary.md +29 -0
- package/node_modules/@dougharness/flow/agents/implementer.md +23 -0
- package/node_modules/@dougharness/flow/agents/lead.md +35 -0
- package/node_modules/@dougharness/flow/agents/planner.md +70 -0
- package/node_modules/@dougharness/flow/agents/researcher.md +31 -0
- package/node_modules/@dougharness/flow/agents/reviewer.md +26 -0
- package/node_modules/@dougharness/flow/agents/verifier.md +24 -0
- package/node_modules/@dougharness/flow/lib/board.mjs +840 -0
- package/node_modules/@dougharness/flow/lib/code-index.mjs +629 -0
- package/node_modules/@dougharness/flow/lib/cost.mjs +392 -0
- package/node_modules/@dougharness/flow/lib/decisions.mjs +314 -0
- package/node_modules/@dougharness/flow/lib/embeddings.mjs +539 -0
- package/node_modules/@dougharness/flow/lib/land.mjs +219 -0
- package/node_modules/@dougharness/flow/lib/learn.mjs +742 -0
- package/node_modules/@dougharness/flow/lib/memory-import.mjs +145 -0
- package/node_modules/@dougharness/flow/lib/memory.mjs +1641 -0
- package/node_modules/@dougharness/flow/lib/models.mjs +148 -0
- package/node_modules/@dougharness/flow/lib/plan.mjs +773 -0
- package/node_modules/@dougharness/flow/lib/rehearse.mjs +825 -0
- package/node_modules/@dougharness/flow/lib/worktrees.mjs +215 -0
- package/node_modules/@dougharness/flow/package.json +19 -0
- package/node_modules/@dougharness/flow/scripts/board.mjs +260 -0
- package/node_modules/@dougharness/flow/scripts/cost.mjs +66 -0
- package/node_modules/@dougharness/flow/scripts/learn.mjs +161 -0
- package/node_modules/@dougharness/flow/scripts/memory.mjs +1274 -0
- package/node_modules/@dougharness/flow/scripts/plan.mjs +547 -0
- package/node_modules/@dougharness/flow/scripts/rehearse.mjs +129 -0
- package/node_modules/@dougharness/flow/skills/core-next/SKILL.md +92 -0
- package/node_modules/@dougharness/flow/skills/doug-approve/SKILL.md +15 -0
- package/node_modules/@dougharness/flow/skills/doug-decide/SKILL.md +26 -0
- package/node_modules/@dougharness/flow/skills/doug-implement/SKILL.md +22 -0
- package/node_modules/@dougharness/flow/skills/doug-learn/SKILL.md +20 -0
- package/node_modules/@dougharness/flow/skills/doug-next/SKILL.md +87 -0
- package/node_modules/@dougharness/flow/skills/doug-plan/SKILL.md +22 -0
- package/node_modules/@dougharness/flow/skills/doug-swarm/SKILL.md +55 -0
- package/node_modules/@dougharness/flow/skills/harness-fix/SKILL.md +85 -0
- package/node_modules/@dougharness/flow/skills/research/SKILL.md +41 -0
- package/node_modules/@dougharness/flow/skills/run-report/SKILL.md +51 -0
- package/node_modules/@dougharness/flow/skills/swarm-launch/SKILL.md +27 -0
- package/node_modules/@dougharness/flow/workflows/doug-implement.js +2687 -0
- package/node_modules/@dougharness/gates/.claude-plugin/plugin.json +10 -0
- package/node_modules/@dougharness/gates/LICENSE +21 -0
- package/node_modules/@dougharness/gates/hooks/hooks.json +184 -0
- package/node_modules/@dougharness/gates/lib/agent-context.mjs +111 -0
- package/node_modules/@dougharness/gates/lib/anchor.mjs +110 -0
- package/node_modules/@dougharness/gates/lib/baseline.mjs +160 -0
- package/node_modules/@dougharness/gates/lib/bash-rules.mjs +1522 -0
- package/node_modules/@dougharness/gates/lib/block-ledger.mjs +127 -0
- package/node_modules/@dougharness/gates/lib/checkpoint.mjs +97 -0
- package/node_modules/@dougharness/gates/lib/config.mjs +168 -0
- package/node_modules/@dougharness/gates/lib/evidence.mjs +59 -0
- package/node_modules/@dougharness/gates/lib/glob.mjs +86 -0
- package/node_modules/@dougharness/gates/lib/io.mjs +91 -0
- package/node_modules/@dougharness/gates/lib/paths.mjs +101 -0
- package/node_modules/@dougharness/gates/lib/proposals.mjs +201 -0
- package/node_modules/@dougharness/gates/lib/run.mjs +147 -0
- package/node_modules/@dougharness/gates/lib/scope.mjs +117 -0
- package/node_modules/@dougharness/gates/lib/secret-rules.mjs +268 -0
- package/node_modules/@dougharness/gates/lib/state.mjs +114 -0
- package/node_modules/@dougharness/gates/lib/trace.mjs +196 -0
- package/node_modules/@dougharness/gates/package.json +17 -0
- package/node_modules/@dougharness/gates/scripts/edit-loop.mjs +39 -0
- package/node_modules/@dougharness/gates/scripts/format-on-edit.mjs +45 -0
- package/node_modules/@dougharness/gates/scripts/guard-bash.mjs +32 -0
- package/node_modules/@dougharness/gates/scripts/protect-paths.mjs +49 -0
- package/node_modules/@dougharness/gates/scripts/reanchor.mjs +81 -0
- package/node_modules/@dougharness/gates/scripts/research-cap.mjs +295 -0
- package/node_modules/@dougharness/gates/scripts/secret-scan.mjs +26 -0
- package/node_modules/@dougharness/gates/scripts/session-start-baseline.mjs +57 -0
- package/node_modules/@dougharness/gates/scripts/statusline.mjs +133 -0
- package/node_modules/@dougharness/gates/scripts/stop-gate.mjs +1422 -0
- package/node_modules/@dougharness/gates/scripts/trace.mjs +75 -0
- package/package.json +33 -0
- package/templates/board-app.js +533 -0
|
@@ -0,0 +1,539 @@
|
|
|
1
|
+
// Embedding provider layer for the memory store (card memory-recall), over fetch, no install weight: no ONNX
|
|
2
|
+
// runtime, no native dependency, no new package. Two adapters share the same shape, an OpenAI-compatible
|
|
3
|
+
// /v1/embeddings endpoint (Ollama, llama-server, vLLM, LM Studio, a hosted OpenAI-shaped key) and Voyage AI's
|
|
4
|
+
// own endpoint. Product first: memory.embeddings in .doug/config.json is null by default, and every function
|
|
5
|
+
// here treats "no provider" or "provider unreachable" as a normal outcome, never a thrown error.
|
|
6
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
7
|
+
import { join, dirname } from "node:path";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
9
|
+
|
|
10
|
+
const CONFIG_RELPATH = ".doug/config.json";
|
|
11
|
+
const VALID_PROVIDERS = new Set(["openai-compatible", "voyage"]);
|
|
12
|
+
const DEFAULT_TIMEOUT_MS = 2000;
|
|
13
|
+
const OLLAMA_BASE_URL = "http://localhost:11434";
|
|
14
|
+
const DEFAULT_MODEL = "embeddinggemma";
|
|
15
|
+
|
|
16
|
+
function isValidEmbeddingsConfig(e) {
|
|
17
|
+
return !!(
|
|
18
|
+
e &&
|
|
19
|
+
typeof e === "object" &&
|
|
20
|
+
VALID_PROVIDERS.has(e.provider) &&
|
|
21
|
+
typeof e.baseUrl === "string" &&
|
|
22
|
+
e.baseUrl.trim() &&
|
|
23
|
+
typeof e.model === "string" &&
|
|
24
|
+
e.model.trim() &&
|
|
25
|
+
typeof e.dims === "number" &&
|
|
26
|
+
Number.isFinite(e.dims) &&
|
|
27
|
+
e.dims > 0 &&
|
|
28
|
+
(e.apiKeyEnv === undefined || typeof e.apiKeyEnv === "string")
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Tolerant: a missing or unreadable .doug/config.json, or a missing/malformed memory.embeddings key, is
|
|
33
|
+
// { embeddings: null } (keyword-only). An embeddings key that is present but the wrong shape is reported once
|
|
34
|
+
// on stderr and also treated as null, so a typo in the config never throws mid-run.
|
|
35
|
+
export function readMemoryConfig(dir) {
|
|
36
|
+
const file = join(dir, CONFIG_RELPATH);
|
|
37
|
+
if (!existsSync(file)) return { embeddings: null };
|
|
38
|
+
let raw;
|
|
39
|
+
try {
|
|
40
|
+
raw = JSON.parse(readFileSync(file, "utf8"));
|
|
41
|
+
} catch {
|
|
42
|
+
return { embeddings: null };
|
|
43
|
+
}
|
|
44
|
+
const memory = raw && typeof raw === "object" ? raw.memory : undefined;
|
|
45
|
+
const e = memory && typeof memory === "object" ? memory.embeddings : undefined;
|
|
46
|
+
if (e === null || e === undefined) return { embeddings: null };
|
|
47
|
+
if (!isValidEmbeddingsConfig(e)) {
|
|
48
|
+
process.stderr.write(`[doug] ${CONFIG_RELPATH} memory.embeddings is not a valid shape; treating as unset (keyword-only)\n`);
|
|
49
|
+
return { embeddings: null };
|
|
50
|
+
}
|
|
51
|
+
return { embeddings: { provider: e.provider, baseUrl: e.baseUrl, model: e.model, dims: e.dims, apiKeyEnv: e.apiKeyEnv } };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function isValidIndexConfig(idx) {
|
|
55
|
+
return !!(
|
|
56
|
+
idx &&
|
|
57
|
+
typeof idx === "object" &&
|
|
58
|
+
typeof idx.enabled === "boolean" &&
|
|
59
|
+
(idx.include === undefined || (Array.isArray(idx.include) && idx.include.every((s) => typeof s === "string"))) &&
|
|
60
|
+
(idx.exclude === undefined || (Array.isArray(idx.exclude) && idx.exclude.every((s) => typeof s === "string"))) &&
|
|
61
|
+
(idx.chunkLines === undefined || (Number.isInteger(idx.chunkLines) && idx.chunkLines > 0))
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const INDEX_CONFIG_OFF = { enabled: false, include: [], exclude: [], chunkLines: 60 };
|
|
66
|
+
|
|
67
|
+
// memory.index (card semantic-index): the opt-in code index's own config, read the same tolerant way
|
|
68
|
+
// readMemoryConfig reads memory.embeddings — missing/unreadable config, an absent/null key, or an invalid shape
|
|
69
|
+
// all come back as the same "off" shape, an invalid shape warning once on stderr; this never throws. `enabled`
|
|
70
|
+
// only gates the flow's own use of the index (brief B); the `memory.mjs index` CLI commands run regardless,
|
|
71
|
+
// since running one is itself the opt-in.
|
|
72
|
+
export function readIndexConfig(dir) {
|
|
73
|
+
const file = join(dir, CONFIG_RELPATH);
|
|
74
|
+
if (!existsSync(file)) return { ...INDEX_CONFIG_OFF };
|
|
75
|
+
let raw;
|
|
76
|
+
try {
|
|
77
|
+
raw = JSON.parse(readFileSync(file, "utf8"));
|
|
78
|
+
} catch {
|
|
79
|
+
return { ...INDEX_CONFIG_OFF };
|
|
80
|
+
}
|
|
81
|
+
const memory = raw && typeof raw === "object" ? raw.memory : undefined;
|
|
82
|
+
const idx = memory && typeof memory === "object" ? memory.index : undefined;
|
|
83
|
+
if (idx === null || idx === undefined) return { ...INDEX_CONFIG_OFF };
|
|
84
|
+
if (!isValidIndexConfig(idx)) {
|
|
85
|
+
process.stderr.write(`[doug] ${CONFIG_RELPATH} memory.index is not a valid shape; treating as off\n`);
|
|
86
|
+
return { ...INDEX_CONFIG_OFF };
|
|
87
|
+
}
|
|
88
|
+
return {
|
|
89
|
+
enabled: idx.enabled,
|
|
90
|
+
include: Array.isArray(idx.include) ? idx.include : [],
|
|
91
|
+
exclude: Array.isArray(idx.exclude) ? idx.exclude : [],
|
|
92
|
+
chunkLines: typeof idx.chunkLines === "number" ? idx.chunkLines : 60,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// memory.staleDays (card memory-docs-drift #2): how many days a lesson may go unconfirmed before recallLessons
|
|
97
|
+
// excludes it as a candidate (recallLessons' own default is 30, mirrored here). Tolerant like readMemoryConfig
|
|
98
|
+
// above: a missing/unreadable config or an unset key is silently 30; a present value that is not a positive
|
|
99
|
+
// finite number warns once on stderr and still returns 30 — this must never throw or exit nonzero.
|
|
100
|
+
export function readMemoryStaleDays(dir) {
|
|
101
|
+
const file = join(dir, CONFIG_RELPATH);
|
|
102
|
+
if (!existsSync(file)) return 30;
|
|
103
|
+
let raw;
|
|
104
|
+
try {
|
|
105
|
+
raw = JSON.parse(readFileSync(file, "utf8"));
|
|
106
|
+
} catch {
|
|
107
|
+
return 30;
|
|
108
|
+
}
|
|
109
|
+
const memory = raw && typeof raw === "object" ? raw.memory : undefined;
|
|
110
|
+
const value = memory && typeof memory === "object" ? memory.staleDays : undefined;
|
|
111
|
+
if (value === undefined) return 30;
|
|
112
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
113
|
+
process.stderr.write(`[doug] ${CONFIG_RELPATH} memory.staleDays is not a positive number; using 30\n`);
|
|
114
|
+
return 30;
|
|
115
|
+
}
|
|
116
|
+
return value;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function l2normalize(vec) {
|
|
120
|
+
let sumSq = 0;
|
|
121
|
+
for (let i = 0; i < vec.length; i++) sumSq += vec[i] * vec[i];
|
|
122
|
+
const norm = Math.sqrt(sumSq);
|
|
123
|
+
if (!Number.isFinite(norm) || norm === 0) return Float32Array.from(vec);
|
|
124
|
+
const out = new Float32Array(vec.length);
|
|
125
|
+
for (let i = 0; i < vec.length; i++) out[i] = vec[i] / norm;
|
|
126
|
+
return out;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Cosine sanity check (card memory-measure #3, ruvLLM #655: a non-semantic embedder scored paraphrase and
|
|
130
|
+
// unrelated pairs both 91-100%, making retrieval worse than keyword-only, silently). A dozen known paraphrase
|
|
131
|
+
// pairs and a dozen known-unrelated pairs, embedded together, must separate by a stated margin or the
|
|
132
|
+
// configured embedder is not doing semantic work. Failing loudly here catches that before reembed commits to
|
|
133
|
+
// it.
|
|
134
|
+
// This path lies outside plugins/doug-flow, at the repo root — so it is outside a Doug plugin's own `files`
|
|
135
|
+
// list. A standalone-published copy of this plugin (installed on its own, without the rest of this repo
|
|
136
|
+
// checked out alongside it) will not have it; pass `--pairs <file>` in that case.
|
|
137
|
+
export const DEFAULT_SANITY_PAIRS = join(dirname(fileURLToPath(import.meta.url)), "../../../evals/memory/pairs.json");
|
|
138
|
+
|
|
139
|
+
function meanOf(nums) {
|
|
140
|
+
return nums.reduce((a, b) => a + b, 0) / nums.length;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Vectors from embed() are L2-normalized (l2normalize), so cosine similarity is just the dot product.
|
|
144
|
+
function dot(a, b) {
|
|
145
|
+
let sum = 0;
|
|
146
|
+
for (let i = 0; i < a.length; i++) sum += a[i] * b[i];
|
|
147
|
+
return sum;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function isPairArray(value, name, file) {
|
|
151
|
+
if (!Array.isArray(value) || value.length !== 12) {
|
|
152
|
+
throw new Error(`sanity pairs file ${file}: "${name}" must be an array of exactly 12 pairs`);
|
|
153
|
+
}
|
|
154
|
+
for (const pair of value) {
|
|
155
|
+
if (
|
|
156
|
+
!Array.isArray(pair) ||
|
|
157
|
+
pair.length !== 2 ||
|
|
158
|
+
typeof pair[0] !== "string" ||
|
|
159
|
+
typeof pair[1] !== "string" ||
|
|
160
|
+
!pair[0].trim() ||
|
|
161
|
+
!pair[1].trim()
|
|
162
|
+
) {
|
|
163
|
+
throw new Error(`sanity pairs file ${file}: "${name}" entries must each be a [sentenceA, sentenceB] pair of non-empty strings`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Loads and validates the sanity pairs file's shape (throws a clear message on anything else): 12 paraphrase
|
|
169
|
+
// pairs, 12 unrelated pairs, a numeric margin. Never guesses a default margin — a malformed file is a hard stop.
|
|
170
|
+
export function loadSanityPairs(file) {
|
|
171
|
+
let raw;
|
|
172
|
+
try {
|
|
173
|
+
raw = JSON.parse(readFileSync(file, "utf8"));
|
|
174
|
+
} catch (err) {
|
|
175
|
+
throw new Error(`cannot read sanity pairs file ${file}: ${err.message}`);
|
|
176
|
+
}
|
|
177
|
+
if (!raw || typeof raw !== "object") throw new Error(`sanity pairs file ${file} must contain a JSON object`);
|
|
178
|
+
if (typeof raw.margin !== "number" || !Number.isFinite(raw.margin)) {
|
|
179
|
+
throw new Error(`sanity pairs file ${file}: "margin" must be a number`);
|
|
180
|
+
}
|
|
181
|
+
isPairArray(raw.paraphrase, "paraphrase", file);
|
|
182
|
+
isPairArray(raw.unrelated, "unrelated", file);
|
|
183
|
+
return { margin: raw.margin, paraphrase: raw.paraphrase, unrelated: raw.unrelated };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Embeds every sentence from both pair lists in one embed() call (inputType "document", matching how lessons
|
|
187
|
+
// are stored), then scores each pair by cosine. ok iff the paraphrase mean clears the unrelated mean by at
|
|
188
|
+
// least `margin`, AND no single paraphrase pair scores below the unrelated mean (a good mean hiding one dead
|
|
189
|
+
// pair is still a red flag). `failures` lists only that second kind of offender, each with its cosine — a
|
|
190
|
+
// margin-only failure (e.g. every vector identical) reports through `reason` with an empty `failures`, since no
|
|
191
|
+
// individual pair is uniquely at fault. A provider error (unreachable, timeout, bad response) short-circuits to
|
|
192
|
+
// `{ ok:false, reason }` before any scoring.
|
|
193
|
+
export async function checkEmbeddingSanity(provider, pairs) {
|
|
194
|
+
const { margin, paraphrase, unrelated } = pairs;
|
|
195
|
+
const sentences = [];
|
|
196
|
+
for (const pair of paraphrase) sentences.push(pair[0], pair[1]);
|
|
197
|
+
for (const pair of unrelated) sentences.push(pair[0], pair[1]);
|
|
198
|
+
const result = await provider.embed(sentences, { inputType: "document" });
|
|
199
|
+
if (!result.ok) return { ok: false, reason: result.reason };
|
|
200
|
+
const vectors = result.vectors;
|
|
201
|
+
const paraScores = paraphrase.map((_, i) => dot(vectors[i * 2], vectors[i * 2 + 1]));
|
|
202
|
+
const unrelatedBase = paraphrase.length * 2;
|
|
203
|
+
const unrelatedScores = unrelated.map((_, i) => dot(vectors[unrelatedBase + i * 2], vectors[unrelatedBase + i * 2 + 1]));
|
|
204
|
+
const paraphraseMean = meanOf(paraScores);
|
|
205
|
+
const paraphraseMin = Math.min(...paraScores);
|
|
206
|
+
const unrelatedMean = meanOf(unrelatedScores);
|
|
207
|
+
const unrelatedMax = Math.max(...unrelatedScores);
|
|
208
|
+
const gap = paraphraseMean - unrelatedMean;
|
|
209
|
+
const failures = [];
|
|
210
|
+
for (let i = 0; i < paraScores.length; i++) {
|
|
211
|
+
if (paraScores[i] < unrelatedMean) {
|
|
212
|
+
failures.push({ pair: paraphrase[i], cosine: paraScores[i] });
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
const marginOk = gap >= margin;
|
|
216
|
+
const ok = marginOk && failures.length === 0;
|
|
217
|
+
const reason = ok
|
|
218
|
+
? null
|
|
219
|
+
: marginOk
|
|
220
|
+
? `${failures.length} paraphrase pair(s) scored below the unrelated mean ${unrelatedMean.toFixed(4)}`
|
|
221
|
+
: `gap ${gap.toFixed(4)} is below the margin ${margin}`;
|
|
222
|
+
return {
|
|
223
|
+
ok,
|
|
224
|
+
margin,
|
|
225
|
+
gap,
|
|
226
|
+
paraphrase: { mean: paraphraseMean, min: paraphraseMin },
|
|
227
|
+
unrelated: { mean: unrelatedMean, max: unrelatedMax },
|
|
228
|
+
failures,
|
|
229
|
+
reason,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// Task prefixes the provider layer must add; the server never adds them. Only the two models named in the
|
|
234
|
+
// research note carry a prefix; every other model gets none.
|
|
235
|
+
function prefixFor(model, inputType) {
|
|
236
|
+
const m = String(model || "");
|
|
237
|
+
if (m.startsWith("embeddinggemma")) {
|
|
238
|
+
return inputType === "query" ? "task: search result | query: " : "title: none | text: ";
|
|
239
|
+
}
|
|
240
|
+
if (m.startsWith("nomic")) {
|
|
241
|
+
return inputType === "query" ? "search_query: " : "search_document: ";
|
|
242
|
+
}
|
|
243
|
+
return "";
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function isAbortLike(err) {
|
|
247
|
+
return err && (err.name === "AbortError" || err.name === "TimeoutError");
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
async function readBodySnippet(res) {
|
|
251
|
+
try {
|
|
252
|
+
return (await res.text()).slice(0, 200);
|
|
253
|
+
} catch {
|
|
254
|
+
return "";
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async function embedOpenAICompatible(e, texts, inputType, { fetchImpl, timeoutMs }) {
|
|
259
|
+
const prefix = prefixFor(e.model, inputType);
|
|
260
|
+
const input = texts.map((t) => prefix + t);
|
|
261
|
+
const url = `${e.baseUrl.replace(/\/$/, "")}/v1/embeddings`;
|
|
262
|
+
const headers = { "content-type": "application/json" };
|
|
263
|
+
const key = process.env[e.apiKeyEnv || "OPENAI_API_KEY"];
|
|
264
|
+
if (key) headers.authorization = `Bearer ${key}`;
|
|
265
|
+
let res;
|
|
266
|
+
try {
|
|
267
|
+
res = await fetchImpl(url, {
|
|
268
|
+
method: "POST",
|
|
269
|
+
headers,
|
|
270
|
+
body: JSON.stringify({ model: e.model, input, dimensions: e.dims }),
|
|
271
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
272
|
+
});
|
|
273
|
+
} catch (err) {
|
|
274
|
+
return { ok: false, reason: isAbortLike(err) ? "timeout" : `request failed: ${err.message}` };
|
|
275
|
+
}
|
|
276
|
+
if (!res.ok) {
|
|
277
|
+
const body = await readBodySnippet(res);
|
|
278
|
+
return { ok: false, reason: `HTTP ${res.status}${body ? `: ${body}` : ""}` };
|
|
279
|
+
}
|
|
280
|
+
let json;
|
|
281
|
+
try {
|
|
282
|
+
json = await res.json();
|
|
283
|
+
} catch (err) {
|
|
284
|
+
return { ok: false, reason: `invalid JSON response: ${err.message}` };
|
|
285
|
+
}
|
|
286
|
+
if (!json || !Array.isArray(json.data)) return { ok: false, reason: "response missing data[]" };
|
|
287
|
+
const ordered = new Array(texts.length);
|
|
288
|
+
for (const item of json.data) {
|
|
289
|
+
if (item && typeof item.index === "number" && Array.isArray(item.embedding)) ordered[item.index] = item.embedding;
|
|
290
|
+
}
|
|
291
|
+
const vectors = [];
|
|
292
|
+
for (let i = 0; i < texts.length; i++) {
|
|
293
|
+
let vec = ordered[i];
|
|
294
|
+
if (!Array.isArray(vec)) return { ok: false, reason: "response missing an embedding" };
|
|
295
|
+
// Older Ollama and llama-server ignore `dimensions` and return the native width; truncate then re-normalize.
|
|
296
|
+
if (vec.length > e.dims) vec = vec.slice(0, e.dims);
|
|
297
|
+
if (vec.length !== e.dims) return { ok: false, reason: `wrong dims: got ${vec.length}, expected ${e.dims}` };
|
|
298
|
+
vectors.push(l2normalize(Float32Array.from(vec)));
|
|
299
|
+
}
|
|
300
|
+
return { ok: true, vectors };
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
async function embedVoyage(e, texts, inputType, { fetchImpl, timeoutMs }) {
|
|
304
|
+
const key = process.env[e.apiKeyEnv || "VOYAGE_API_KEY"];
|
|
305
|
+
if (!key) return { ok: false, reason: `missing API key: set ${e.apiKeyEnv || "VOYAGE_API_KEY"}` };
|
|
306
|
+
const base = e.baseUrl && e.baseUrl.trim() ? e.baseUrl : "https://api.voyageai.com";
|
|
307
|
+
const url = `${base.replace(/\/$/, "")}/v1/embeddings`;
|
|
308
|
+
let res;
|
|
309
|
+
try {
|
|
310
|
+
res = await fetchImpl(url, {
|
|
311
|
+
method: "POST",
|
|
312
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${key}` },
|
|
313
|
+
body: JSON.stringify({
|
|
314
|
+
input: texts,
|
|
315
|
+
model: e.model,
|
|
316
|
+
input_type: inputType === "query" ? "query" : "document",
|
|
317
|
+
output_dimension: e.dims,
|
|
318
|
+
}),
|
|
319
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
320
|
+
});
|
|
321
|
+
} catch (err) {
|
|
322
|
+
return { ok: false, reason: isAbortLike(err) ? "timeout" : `request failed: ${err.message}` };
|
|
323
|
+
}
|
|
324
|
+
if (!res.ok) {
|
|
325
|
+
const body = await readBodySnippet(res);
|
|
326
|
+
return { ok: false, reason: `HTTP ${res.status}${body ? `: ${body}` : ""}` };
|
|
327
|
+
}
|
|
328
|
+
let json;
|
|
329
|
+
try {
|
|
330
|
+
json = await res.json();
|
|
331
|
+
} catch (err) {
|
|
332
|
+
return { ok: false, reason: `invalid JSON response: ${err.message}` };
|
|
333
|
+
}
|
|
334
|
+
if (!json || !Array.isArray(json.data)) return { ok: false, reason: "response missing data[]" };
|
|
335
|
+
const ordered = new Array(texts.length);
|
|
336
|
+
for (const item of json.data) {
|
|
337
|
+
if (item && typeof item.index === "number" && Array.isArray(item.embedding)) ordered[item.index] = item.embedding;
|
|
338
|
+
}
|
|
339
|
+
const vectors = [];
|
|
340
|
+
for (let i = 0; i < texts.length; i++) {
|
|
341
|
+
let vec = ordered[i];
|
|
342
|
+
if (!Array.isArray(vec)) return { ok: false, reason: "response missing an embedding" };
|
|
343
|
+
// Same rule as openai-compatible: a server that ignores output_dimension gets truncated then re-normalized;
|
|
344
|
+
// anything still the wrong length after that is a real error, not silently mislabeled with the wrong dims.
|
|
345
|
+
if (vec.length > e.dims) vec = vec.slice(0, e.dims);
|
|
346
|
+
if (vec.length !== e.dims) return { ok: false, reason: `wrong dims: got ${vec.length}, expected ${e.dims}` };
|
|
347
|
+
vectors.push(l2normalize(Float32Array.from(vec)));
|
|
348
|
+
}
|
|
349
|
+
return { ok: true, vectors };
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// Splits into batches of at most 100 texts per request; a failure on any batch stops and reports that batch's
|
|
353
|
+
// reason (embed() never throws or hangs past timeoutMs per request).
|
|
354
|
+
async function embedBatched(texts, inputType, sendBatch) {
|
|
355
|
+
const vectors = [];
|
|
356
|
+
for (let i = 0; i < texts.length; i += 100) {
|
|
357
|
+
const result = await sendBatch(texts.slice(i, i + 100), inputType);
|
|
358
|
+
if (!result.ok) return result;
|
|
359
|
+
vectors.push(...result.vectors);
|
|
360
|
+
}
|
|
361
|
+
return { ok: true, vectors };
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// cfg is the { embeddings } shape readMemoryConfig returns. Returns null when nothing is configured (the
|
|
365
|
+
// caller's cue to run keyword-only); otherwise { name, model, dims, embed(texts, { inputType }) }.
|
|
366
|
+
export function createProvider(cfg, opts = {}) {
|
|
367
|
+
const e = cfg && cfg.embeddings;
|
|
368
|
+
if (!e) return null;
|
|
369
|
+
const fetchImpl = opts.fetch || globalThis.fetch;
|
|
370
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
371
|
+
const sendBatch =
|
|
372
|
+
e.provider === "voyage"
|
|
373
|
+
? (texts, inputType) => embedVoyage(e, texts, inputType, { fetchImpl, timeoutMs })
|
|
374
|
+
: (texts, inputType) => embedOpenAICompatible(e, texts, inputType, { fetchImpl, timeoutMs });
|
|
375
|
+
return {
|
|
376
|
+
name: e.provider,
|
|
377
|
+
model: e.model,
|
|
378
|
+
dims: e.dims,
|
|
379
|
+
async embed(texts, { inputType = "document" } = {}) {
|
|
380
|
+
try {
|
|
381
|
+
return await embedBatched(texts, inputType, sendBatch);
|
|
382
|
+
} catch (err) {
|
|
383
|
+
return { ok: false, reason: `unexpected error: ${err.message}` };
|
|
384
|
+
}
|
|
385
|
+
},
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// Ollama lists a pulled model with a ":tag" suffix (embeddinggemma:latest) even when the config names it bare
|
|
390
|
+
// (embeddinggemma); match on the exact name or the part before ":" on either side.
|
|
391
|
+
function baseModelName(name) {
|
|
392
|
+
const s = String(name ?? "");
|
|
393
|
+
const i = s.indexOf(":");
|
|
394
|
+
return i === -1 ? s : s.slice(0, i);
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
function modelNameMatches(entry, wantModel) {
|
|
398
|
+
if (!entry) return false;
|
|
399
|
+
const want = baseModelName(wantModel);
|
|
400
|
+
return [entry.name, entry.model].some((n) => n !== undefined && n !== null && (n === wantModel || baseModelName(n) === want));
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// True for the Ollama preset's host:port (localhost or 127.0.0.1 on 11434) regardless of scheme or trailing
|
|
404
|
+
// slash — used only to decide whether an unreachable openai-compatible provider is still worth telling the
|
|
405
|
+
// user to go install Ollama for, never to change what gets fetched.
|
|
406
|
+
function isOllamaPresetUrl(baseUrl) {
|
|
407
|
+
try {
|
|
408
|
+
const u = new URL(baseUrl);
|
|
409
|
+
return (u.hostname === "localhost" || u.hostname === "127.0.0.1") && u.port === "11434";
|
|
410
|
+
} catch {
|
|
411
|
+
return false;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// Probes a configured provider, or a local Ollama when nothing is configured. GET /api/version (then /api/tags)
|
|
416
|
+
// is tried first; only when it succeeds is this reported as kind "ollama" with a model list. When /api/version
|
|
417
|
+
// fails — a working llama-server, vLLM, LM Studio, or hosted key never serves it — this falls back to one
|
|
418
|
+
// embed(["probe"]) call and reports reachability from that instead, as kind "openai-compatible": a real,
|
|
419
|
+
// reachable non-Ollama provider must never be told to go install Ollama. The one exception: an openai-compatible
|
|
420
|
+
// provider configured at the Ollama preset host:port (localhost/127.0.0.1:11434) that is NOT reachable even via
|
|
421
|
+
// the embed fallback is still reported as kind "ollama" — that combination means a down local Ollama, the one
|
|
422
|
+
// case doctor exists to catch, not a different, genuinely unreachable server. Never throws; every branch has a
|
|
423
|
+
// 2s timeout.
|
|
424
|
+
export async function probeProvider(cfg, opts = {}) {
|
|
425
|
+
const fetchImpl = opts.fetch || globalThis.fetch;
|
|
426
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
427
|
+
const e = cfg && cfg.embeddings;
|
|
428
|
+
const baseUrl = e && e.baseUrl ? e.baseUrl : OLLAMA_BASE_URL;
|
|
429
|
+
const wantModel = e && e.model ? e.model : DEFAULT_MODEL;
|
|
430
|
+
const root = baseUrl.replace(/\/$/, "");
|
|
431
|
+
const isPresetHost = (!e || e.provider === "openai-compatible") && isOllamaPresetUrl(baseUrl);
|
|
432
|
+
|
|
433
|
+
let version = null;
|
|
434
|
+
let versionReachable = false;
|
|
435
|
+
let versionReason = null;
|
|
436
|
+
try {
|
|
437
|
+
const res = await fetchImpl(`${root}/api/version`, { signal: AbortSignal.timeout(timeoutMs) });
|
|
438
|
+
if (res.ok) {
|
|
439
|
+
const json = await res.json();
|
|
440
|
+
version = json && typeof json.version === "string" ? json.version : null;
|
|
441
|
+
versionReachable = true;
|
|
442
|
+
} else {
|
|
443
|
+
versionReason = `HTTP ${res.status}`;
|
|
444
|
+
}
|
|
445
|
+
} catch (err) {
|
|
446
|
+
versionReason = isAbortLike(err) ? "timeout" : `request failed: ${err.message}`;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
if (versionReachable) {
|
|
450
|
+
let models = null;
|
|
451
|
+
try {
|
|
452
|
+
const res = await fetchImpl(`${root}/api/tags`, { signal: AbortSignal.timeout(timeoutMs) });
|
|
453
|
+
if (res.ok) {
|
|
454
|
+
const json = await res.json();
|
|
455
|
+
models = Array.isArray(json && json.models) ? json.models : [];
|
|
456
|
+
}
|
|
457
|
+
} catch {
|
|
458
|
+
models = null;
|
|
459
|
+
}
|
|
460
|
+
const modelPresent = Array.isArray(models) ? models.some((mo) => modelNameMatches(mo, wantModel)) : false;
|
|
461
|
+
return { reachable: true, kind: "ollama", version, models, modelPresent, embedOk: null, reason: null };
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
const provider2 = createProvider(cfg, { fetch: fetchImpl, timeoutMs });
|
|
465
|
+
if (!provider2) {
|
|
466
|
+
return { reachable: false, kind: "ollama", version: null, models: null, modelPresent: false, embedOk: false, reason: versionReason || "no provider configured" };
|
|
467
|
+
}
|
|
468
|
+
const result = await provider2.embed(["probe"]);
|
|
469
|
+
return {
|
|
470
|
+
reachable: result.ok,
|
|
471
|
+
kind: !result.ok && isPresetHost ? "ollama" : "openai-compatible",
|
|
472
|
+
version: null,
|
|
473
|
+
models: null,
|
|
474
|
+
modelPresent: null,
|
|
475
|
+
embedOk: result.ok,
|
|
476
|
+
reason: result.ok ? null : result.reason,
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
const INSTALL_COMMANDS = {
|
|
481
|
+
darwin: "brew install ollama",
|
|
482
|
+
linux: "curl -fsSL https://ollama.com/install.sh | sh",
|
|
483
|
+
win32: "winget install Ollama.Ollama",
|
|
484
|
+
};
|
|
485
|
+
|
|
486
|
+
const PRESET_SNIPPET =
|
|
487
|
+
'"memory": { "embeddings": { "provider": "openai-compatible", "baseUrl": "http://localhost:11434", "model": "embeddinggemma", "dims": 512 } }';
|
|
488
|
+
|
|
489
|
+
function installCommandFor(platform) {
|
|
490
|
+
return INSTALL_COMMANDS[platform] || INSTALL_COMMANDS.linux;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
// Renders the text `memory.mjs doctor` prints, from a probeProvider() result: what was found, the model it
|
|
494
|
+
// would use, and the one command to install what is missing for the given platform (default process.platform).
|
|
495
|
+
// cfg is the { embeddings } shape readMemoryConfig returns — the caller reads the config once and passes it in,
|
|
496
|
+
// so an invalid memory.embeddings shape is reported on stderr exactly once, not once per doctorReport call.
|
|
497
|
+
export function doctorReport(cfg, probe, platform = process.platform) {
|
|
498
|
+
const embeddings = cfg && cfg.embeddings;
|
|
499
|
+
const installCmd = installCommandFor(platform);
|
|
500
|
+
const lines = [];
|
|
501
|
+
|
|
502
|
+
if (!embeddings) {
|
|
503
|
+
lines.push("memory.embeddings is not configured in .doug/config.json; the store is keyword-only.");
|
|
504
|
+
if (probe.reachable) {
|
|
505
|
+
lines.push(`A local Ollama was found at ${OLLAMA_BASE_URL} (version ${probe.version || "unknown"}).`);
|
|
506
|
+
lines.push(
|
|
507
|
+
probe.modelPresent
|
|
508
|
+
? `The model it would use, ${DEFAULT_MODEL}, is already pulled.`
|
|
509
|
+
: `Pull the model it would use with: ollama pull ${DEFAULT_MODEL}`
|
|
510
|
+
);
|
|
511
|
+
} else {
|
|
512
|
+
lines.push(`No local Ollama found at ${OLLAMA_BASE_URL} (${probe.reason || "not reachable"}).`);
|
|
513
|
+
lines.push(`Install it with: ${installCmd}`);
|
|
514
|
+
lines.push(`Then: ollama pull ${DEFAULT_MODEL}`);
|
|
515
|
+
}
|
|
516
|
+
lines.push("Enable it by adding to .doug/config.json:");
|
|
517
|
+
lines.push(` ${PRESET_SNIPPET}`);
|
|
518
|
+
return lines.join("\n");
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
lines.push(`memory.embeddings is configured: ${embeddings.provider} ${embeddings.model}/${embeddings.dims} at ${embeddings.baseUrl}.`);
|
|
522
|
+
if (!probe.reachable) {
|
|
523
|
+
lines.push(`Not reachable: ${probe.reason || "unknown error"}.`);
|
|
524
|
+
// Only an Ollama-shaped miss (or nothing detected at all) gets install instructions — a real, reachable
|
|
525
|
+
// llama-server/vLLM/hosted key that merely lacks /api/version must never be told to go install Ollama.
|
|
526
|
+
if (probe.kind === "ollama") {
|
|
527
|
+
lines.push(`Install it with: ${installCmd}`);
|
|
528
|
+
lines.push(`Then: ollama pull ${embeddings.model}`);
|
|
529
|
+
}
|
|
530
|
+
} else if (probe.modelPresent === false) {
|
|
531
|
+
lines.push(`Reachable (version ${probe.version || "unknown"}) but ${embeddings.model} is not pulled.`);
|
|
532
|
+
lines.push(`Pull it with: ollama pull ${embeddings.model}`);
|
|
533
|
+
} else if (probe.embedOk === false) {
|
|
534
|
+
lines.push(`Reachable but an embed call failed: ${probe.reason || "unknown error"}.`);
|
|
535
|
+
} else {
|
|
536
|
+
lines.push(`Reachable; would use ${embeddings.model}/${embeddings.dims}.`);
|
|
537
|
+
}
|
|
538
|
+
return lines.join("\n");
|
|
539
|
+
}
|