@dzhechkov/harness-core 0.3.140 → 0.3.142

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/backlog.ts ADDED
@@ -0,0 +1,978 @@
1
+ /**
2
+ * `dz backlog` — the Smart Backlog: a personal, goal-directed idea pipeline (feature smart-backlog).
3
+ *
4
+ * Capture an idea → semantic dedup against existing ideas via the EXISTING Brain vector engine
5
+ * (agentdb + the shared embedder — ADR-001; NO second store) → score alignment against a GoalMap
6
+ * (ADR-003) → weighted-roulette pick (ADR-004) → stage an idea2prd enrich hand-off → draft a Jira
7
+ * issue through a configurable, stub-first adapter seam (ADR-006).
8
+ *
9
+ * ARCHITECTURE (05): every load-bearing decision is a PURE function tested on a layer-1 deterministic
10
+ * test; the CLI handler (`cmdBacklog`) is a thin arg-parse + call. Semantic work (embedding, cosine,
11
+ * vector search) is an OUTBOUND PORT to harness-core's Brain engine — this file defines NO embedder,
12
+ * NO cosine, and imports NO MCP client (ADR-001 / ADR-006, both grep-guarded).
13
+ *
14
+ * Ideas carry `task_type:'dz-backlog'` in the shared `.dz/agentdb.db`, so they never surface in
15
+ * `dz recall`'s lesson namespace (ADR-005 T-005b, the discriminating isolation test). Structured
16
+ * records live in a dedicated `.dz/backlog/ideas.jsonl`, never in `.dz/memory/patterns.*`.
17
+ */
18
+
19
+ import { createHash } from 'node:crypto';
20
+ import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
21
+ import { dirname, join } from 'node:path';
22
+
23
+ // ── Outbound ports to the Brain engine (ADR-001): reuse VERBATIM, define none locally. ──
24
+ // Backlog vectors are written DIRECTLY to the agentdb `dz-backlog` namespace via
25
+ // `importVectorsToAgentdb` (NOT the configurable `mirrorEntriesToVector` engine), so write and read
26
+ // ALWAYS hit the SAME store regardless of `memory.vector.engine` (ADR-001/005 — the single-store
27
+ // property; routing through the generic engine let `engine:'rvf'` split writes from reads).
28
+ import { cosineSimilarity, deleteAgentdbByDzIds, importVectorsToAgentdb, resolveAgentdbEmbedder, searchAgentdbPatterns } from './agentdb-index.js';
29
+ import { resolveEmbedModel } from './embedding-config.js';
30
+ // Seeded PRNG — reuse the ONE mulberry32 already in the repo (no second RNG; ADR-004 determinism).
31
+ import { mulberry32 } from './compounding.js';
32
+
33
+ /** The vector-store namespace that isolates ideas from lessons (ADR-001/005). */
34
+ export const BACKLOG_TASK_TYPE = 'dz-backlog';
35
+
36
+ /* ================================================================== */
37
+ /* DOMAIN TYPES (04 Domain Model) */
38
+ /* ================================================================== */
39
+
40
+ export type IdeaStatus = 'new' | 'enriched' | 'in-progress' | 'shipped' | 'dropped';
41
+
42
+ /** The aggregate root — one captured idea. `id` is content-addressed (no `Date.now()` in identity). */
43
+ export interface IdeaRecord {
44
+ readonly id: string;
45
+ readonly text: string;
46
+ status: IdeaStatus;
47
+ readonly createdTs: string;
48
+ effort: number; // 1..5, user estimate; default 3
49
+ goalId: string | null; // argmax goal (ADR-003)
50
+ goalAlignment: number; // [0,1] weighted-max cosine
51
+ relatedIds: string[]; // top-K RELATED ideas (ADR-002)
52
+ uses: number; // reinforcement count — a DUPLICATE bumps the existing root's uses (ADR-002 T-002b)
53
+ proposal?: string; // agent-authored (FR-3.2); the CLI NEVER fabricates this
54
+ enrichedPath?: string; // features/<slug>/ once enriched (FR-5)
55
+ jiraKey?: string; // IssueRef.key or the outbox ref (ADR-006)
56
+ tags: string[];
57
+ }
58
+
59
+ /** The compass (ADR-003) — user-owned, edited as a file. */
60
+ export interface Goal {
61
+ readonly id: string;
62
+ readonly statement: string;
63
+ readonly weight: number; // (0,1]
64
+ readonly keywords: readonly string[];
65
+ }
66
+ export interface GoalMap {
67
+ readonly version: number;
68
+ readonly goals: readonly Goal[];
69
+ }
70
+
71
+ export type DedupAction = 'duplicate' | 'related' | 'new';
72
+
73
+ /** Pure output of the classifier (04) — consumed by capture. */
74
+ export interface DedupVerdict {
75
+ readonly action: DedupAction;
76
+ /** Top-1 raw cosine (ADR-002 — never an RRF score). `-1` when there is nothing to compare against. */
77
+ readonly cosine: number;
78
+ readonly matchedId: string | undefined;
79
+ readonly relatedIds: readonly string[];
80
+ /** True when the embedder was unavailable and dedup degraded to exact-text (ADR-002 §degrade). */
81
+ readonly exactTextOnly: boolean;
82
+ }
83
+
84
+ /* ================================================================== */
85
+ /* CONFIG (readBacklogConfig) — defensive, Number.isFinite clamps. */
86
+ /* ================================================================== */
87
+
88
+ export const DEFAULT_DUPLICATE_THRESHOLD = 0.92; // measured house constant (02 §R-D)
89
+ export const DEFAULT_RELATEDNESS_FLOOR = 0.35; // GROUND_VECTOR_SIMILARITY_FLOOR (02 §R-D)
90
+ export const DEFAULT_ROULETTE_ALPHA = 1.5;
91
+ export const DEFAULT_HALF_LIFE_DAYS = 30;
92
+ export const DEFAULT_RECENCY_FLOOR = 0.3;
93
+ export const DEFAULT_EFFORT = 3;
94
+
95
+ export interface BacklogConfig {
96
+ readonly dedup: { readonly duplicateThreshold: number; readonly relatednessFloor: number };
97
+ readonly roulette: {
98
+ readonly alpha: number;
99
+ readonly halfLifeDays: number;
100
+ readonly recencyFloor: number;
101
+ readonly defaultEffort: number;
102
+ };
103
+ readonly jira: { readonly adapter: BacklogBackend };
104
+ }
105
+
106
+ /** Finite-and-in-range or the fallback (the recurring Infinity-clamp lesson). */
107
+ function clampNum(v: unknown, lo: number, hi: number, fallback: number, opts: { intOnly?: boolean } = {}): number {
108
+ if (typeof v !== 'number' || !Number.isFinite(v) || v < lo || v > hi) return fallback;
109
+ return opts.intOnly ? Math.floor(v) : v;
110
+ }
111
+
112
+ export function readBacklogConfig(projectRoot: string): BacklogConfig {
113
+ const fallback: BacklogConfig = {
114
+ dedup: { duplicateThreshold: DEFAULT_DUPLICATE_THRESHOLD, relatednessFloor: DEFAULT_RELATEDNESS_FLOOR },
115
+ roulette: {
116
+ alpha: DEFAULT_ROULETTE_ALPHA,
117
+ halfLifeDays: DEFAULT_HALF_LIFE_DAYS,
118
+ recencyFloor: DEFAULT_RECENCY_FLOOR,
119
+ defaultEffort: DEFAULT_EFFORT,
120
+ },
121
+ jira: { adapter: 'none' },
122
+ };
123
+ const configPath = join(projectRoot, '.dz', 'config.json');
124
+ if (!existsSync(configPath)) return fallback;
125
+ try {
126
+ const parsed = JSON.parse(readFileSync(configPath, 'utf-8')) as {
127
+ backlog?: {
128
+ dedup?: { duplicateThreshold?: unknown; relatednessFloor?: unknown };
129
+ roulette?: { alpha?: unknown; halfLifeDays?: unknown; recencyFloor?: unknown; defaultEffort?: unknown };
130
+ jira?: { adapter?: unknown };
131
+ };
132
+ };
133
+ const b = parsed.backlog ?? {};
134
+ // `> 0 && <= 1` — a value of 2 or NaN falls back to 0.92 (ADR-002 T-002d).
135
+ const dup = clampNum(b.dedup?.duplicateThreshold, Number.MIN_VALUE, 1, DEFAULT_DUPLICATE_THRESHOLD);
136
+ // The floor must sit in [0, dup]. MED-7: re-range-check the ADJUSTED value (clamp the RESULT, not
137
+ // just the input) — a tiny `dup` (e.g. 5e-324) previously drove `dup - EPSILON` NEGATIVE, so a
138
+ // cosine of -1e-16 fell into the RELATED band. Force the repaired floor back into [0, dup].
139
+ let floor = clampNum(b.dedup?.relatednessFloor, 0, 1, DEFAULT_RELATEDNESS_FLOOR);
140
+ if (floor >= dup) floor = DEFAULT_RELATEDNESS_FLOOR < dup ? DEFAULT_RELATEDNESS_FLOOR : dup;
141
+ floor = Math.min(Math.max(floor, 0), dup);
142
+ const adapter = isBacklogBackend(b.jira?.adapter) ? b.jira!.adapter : 'none'; // unknown ⇒ none (ADR-006 T-006d)
143
+ return {
144
+ dedup: { duplicateThreshold: dup, relatednessFloor: floor },
145
+ roulette: {
146
+ alpha: clampNum(b.roulette?.alpha, Number.MIN_VALUE, Number.MAX_VALUE, DEFAULT_ROULETTE_ALPHA),
147
+ halfLifeDays: clampNum(b.roulette?.halfLifeDays, Number.MIN_VALUE, Number.MAX_VALUE, DEFAULT_HALF_LIFE_DAYS),
148
+ recencyFloor: clampNum(b.roulette?.recencyFloor, Number.MIN_VALUE, 1, DEFAULT_RECENCY_FLOOR),
149
+ defaultEffort: clampNum(b.roulette?.defaultEffort, 1, 5, DEFAULT_EFFORT, { intOnly: true }),
150
+ },
151
+ jira: { adapter },
152
+ };
153
+ } catch {
154
+ return fallback;
155
+ }
156
+ }
157
+
158
+ /* ================================================================== */
159
+ /* STORE (AM-1) — .dz/backlog/ideas.jsonl (append-only JSONL, ADR-005) */
160
+ /* ================================================================== */
161
+
162
+ export function backlogDir(projectRoot: string): string {
163
+ return join(projectRoot, '.dz', 'backlog');
164
+ }
165
+ export function ideasPath(projectRoot: string): string {
166
+ return join(backlogDir(projectRoot), 'ideas.jsonl');
167
+ }
168
+ export function goalsPath(projectRoot: string): string {
169
+ return join(backlogDir(projectRoot), 'goals.json');
170
+ }
171
+ export function jiraOutboxDir(projectRoot: string): string {
172
+ return join(backlogDir(projectRoot), 'jira-outbox');
173
+ }
174
+
175
+ /** Content-addressed id — `sha1(text|createdTs).slice(0,16)`; NO `Date.now()` (mirrors patterns.ts). */
176
+ export function ideaId(text: string, createdTs: string): string {
177
+ return createHash('sha1').update(`${text}|${createdTs}`).digest('hex').slice(0, 16);
178
+ }
179
+
180
+ /**
181
+ * Strict id whitelist (mirrors `dz score`'s slug guard) — the ONLY shape that may be interpolated into
182
+ * a filesystem path (jira-outbox/<id>.json, enrich slug fallback). Rejects `.`/`..`/`/`/`\` and any
183
+ * traversal, so an idea id like `../../../owned` from a hand-edited store can NEVER escape its dir
184
+ * (HIGH-1). A content-addressed `ideaId()` (16 hex) always passes; anything else is refused, not sanitised
185
+ * into a surprising path.
186
+ */
187
+ export function isSafeId(id: unknown): id is string {
188
+ return typeof id === 'string' && id.length > 0 && id.length <= 64 && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id) && !id.includes('..');
189
+ }
190
+
191
+ /** Strip an id to a filesystem-safe token (defence-in-depth for a slug fallback — never a path). */
192
+ function safeIdToken(id: string): string {
193
+ const t = id.replace(/[^A-Za-z0-9]/g, '').slice(0, 12);
194
+ return t === '' ? 'idea' : t;
195
+ }
196
+
197
+ /** Normalise a partial/loaded object into a full IdeaRecord (defensive: bad fields ⇒ safe defaults). */
198
+ function normaliseIdea(raw: Record<string, unknown>): IdeaRecord | undefined {
199
+ // HIGH-1: an id that is not filesystem-safe can NEVER enter the store — it would later interpolate
200
+ // into an outbox filename or an enrich slug and escape its directory. Drop the record entirely.
201
+ if (!isSafeId(raw.id) || typeof raw.text !== 'string') return undefined;
202
+ const status = ['new', 'enriched', 'in-progress', 'shipped', 'dropped'].includes(raw.status as string)
203
+ ? (raw.status as IdeaStatus)
204
+ : 'new';
205
+ // MED-6: a bad createdTs (`Date.parse` ⇒ NaN) would poison the roulette weight; coerce it to a
206
+ // valid, maximally-old ISO timestamp so recency degrades to the neutral floor, never NaN.
207
+ const createdRaw = typeof raw.createdTs === 'string' && Number.isFinite(Date.parse(raw.createdTs)) ? raw.createdTs : new Date(0).toISOString();
208
+ const rec: IdeaRecord = {
209
+ id: raw.id,
210
+ text: raw.text,
211
+ status,
212
+ createdTs: createdRaw,
213
+ effort: clampNum(raw.effort, 1, 5, DEFAULT_EFFORT, { intOnly: true }),
214
+ goalId: typeof raw.goalId === 'string' ? raw.goalId : null,
215
+ goalAlignment: clampNum(raw.goalAlignment, 0, 1, 0),
216
+ relatedIds: Array.isArray(raw.relatedIds) ? raw.relatedIds.filter((x): x is string => typeof x === 'string') : [],
217
+ uses: clampNum(raw.uses, 0, Number.MAX_VALUE, 0, { intOnly: true }),
218
+ tags: Array.isArray(raw.tags) ? raw.tags.filter((x): x is string => typeof x === 'string') : [],
219
+ };
220
+ if (typeof raw.proposal === 'string') rec.proposal = raw.proposal;
221
+ if (typeof raw.enrichedPath === 'string') rec.enrichedPath = raw.enrichedPath;
222
+ if (typeof raw.jiraKey === 'string') rec.jiraKey = raw.jiraKey;
223
+ return rec;
224
+ }
225
+
226
+ /** Read the append-only store. A corrupt line is SKIPPED (never fatal) — the whole store never throws. */
227
+ export function readIdeas(projectRoot: string): IdeaRecord[] {
228
+ const path = ideasPath(projectRoot);
229
+ if (!existsSync(path)) return [];
230
+ let text: string;
231
+ try {
232
+ text = readFileSync(path, 'utf-8');
233
+ } catch {
234
+ return [];
235
+ }
236
+ const out: IdeaRecord[] = [];
237
+ for (const line of text.split('\n')) {
238
+ const trimmed = line.trim();
239
+ if (trimmed === '') continue;
240
+ try {
241
+ const rec = normaliseIdea(JSON.parse(trimmed) as Record<string, unknown>);
242
+ if (rec !== undefined) out.push(rec);
243
+ } catch {
244
+ /* skip corrupt line */
245
+ }
246
+ }
247
+ return out;
248
+ }
249
+
250
+ /** Atomic full rewrite (tmp + rename) so a crash mid-write never truncates the store. */
251
+ export function writeIdeas(projectRoot: string, ideas: readonly IdeaRecord[]): void {
252
+ const path = ideasPath(projectRoot);
253
+ mkdirSync(dirname(path), { recursive: true });
254
+ const body = ideas.map((i) => JSON.stringify(i)).join('\n') + (ideas.length > 0 ? '\n' : '');
255
+ const tmp = `${path}.tmp-${process.pid}`;
256
+ writeFileSync(tmp, body);
257
+ renameSync(tmp, path);
258
+ }
259
+
260
+ /** Pre-mutation snapshot (NFR-6) — mirrors `snapshotStore`. A failed snapshot returns `{error}` so
261
+ * the caller ABORTS the mutation (no partial merge). */
262
+ export interface SnapshotResult {
263
+ readonly path: string;
264
+ readonly count: number;
265
+ readonly error?: string;
266
+ }
267
+ export function snapshotIdeas(projectRoot: string, dest: string): SnapshotResult {
268
+ try {
269
+ const ideas = readIdeas(projectRoot);
270
+ mkdirSync(dirname(dest), { recursive: true });
271
+ writeFileSync(dest, ideas.map((i) => JSON.stringify(i)).join('\n') + (ideas.length > 0 ? '\n' : ''));
272
+ return { path: dest, count: ideas.length };
273
+ } catch (err) {
274
+ return { path: dest, count: 0, error: `snapshot failed: ${err instanceof Error ? err.message : String(err)}` };
275
+ }
276
+ }
277
+
278
+ /* ================================================================== */
279
+ /* DEDUP (AM-2 / ADR-002) — raw cosine bands, NOT RRF. */
280
+ /* ================================================================== */
281
+
282
+ /** One existing-idea comparison candidate: its id and the RAW cosine of the new idea against it. */
283
+ export interface DedupCandidate {
284
+ readonly id: string;
285
+ readonly cosine: number;
286
+ }
287
+
288
+ /**
289
+ * THE LOAD-BEARING CLASSIFIER (ADR-002 T-002a) — PURE. Bands the top-1 raw cosine:
290
+ * DUPLICATE cosine ≥ duplicateThreshold (default 0.92)
291
+ * RELATED relatednessFloor ≤ cosine < dup (default [0.35, 0.92)) ⇒ create + attach relatedIds
292
+ * NEW cosine < relatednessFloor
293
+ * Flip either cut and exactly one boundary fixture crosses a band — the test REDS.
294
+ */
295
+ export function classifyDedup(
296
+ candidates: readonly DedupCandidate[],
297
+ cfg: BacklogConfig['dedup'],
298
+ opts: { exactTextOnly?: boolean } = {},
299
+ ): DedupVerdict {
300
+ // HIGH-4: a non-finite cosine (NaN/±Infinity) sorts unpredictably and can shove a real 0.97 duplicate
301
+ // out of the top slot → misclassified NEW. Drop non-finite candidates BEFORE sorting/banding (the
302
+ // recurring repo `Number.isFinite` lesson).
303
+ const sorted = candidates.filter((c) => Number.isFinite(c.cosine)).sort((a, b) => b.cosine - a.cosine);
304
+ const top = sorted[0];
305
+ const exactTextOnly = opts.exactTextOnly === true;
306
+ if (top === undefined) return { action: 'new', cosine: -1, matchedId: undefined, relatedIds: [], exactTextOnly };
307
+ if (top.cosine >= cfg.duplicateThreshold) {
308
+ return { action: 'duplicate', cosine: top.cosine, matchedId: top.id, relatedIds: [], exactTextOnly };
309
+ }
310
+ const related = sorted.filter((c) => c.cosine >= cfg.relatednessFloor && c.cosine < cfg.duplicateThreshold);
311
+ if (related.length > 0) {
312
+ return { action: 'related', cosine: top.cosine, matchedId: undefined, relatedIds: related.map((c) => c.id), exactTextOnly };
313
+ }
314
+ return { action: 'new', cosine: top.cosine, matchedId: undefined, relatedIds: [], exactTextOnly };
315
+ }
316
+
317
+ /** Injectable deps so the production dedup path is testable without a live agentdb. */
318
+ export interface DedupDeps {
319
+ /** Semantic search over the `dz-backlog` namespace — defaults to the real `searchAgentdbPatterns`. */
320
+ readonly search?: (
321
+ projectRoot: string,
322
+ query: string,
323
+ ) => Promise<{ hits: { dzId?: string | undefined; similarity: number }[]; error?: string | undefined }>;
324
+ /** Existing structured ideas (for the exact-text degrade path) — defaults to `readIdeas`. */
325
+ readonly ideas?: readonly IdeaRecord[];
326
+ }
327
+
328
+ /**
329
+ * Production dedup: embed+search the `dz-backlog` vectors (RAW COSINE via `searchAgentdbPatterns`,
330
+ * NEVER the RRF `recallHybrid().score` — ADR-002 T-002c), then band via {@link classifyDedup}. If the
331
+ * embedder/search is unavailable it DEGRADES to exact-text dedup (identical text ⇒ DUPLICATE), never
332
+ * blocking capture (NFR-2 / ADR-002 T-002e).
333
+ */
334
+ export async function dedupIdea(projectRoot: string, text: string, cfg: BacklogConfig, deps: DedupDeps = {}): Promise<DedupVerdict> {
335
+ const ideas = deps.ideas ?? readIdeas(projectRoot);
336
+ const search =
337
+ deps.search ??
338
+ ((root: string, query: string) => searchAgentdbPatterns(root, query, { taskTypes: [BACKLOG_TASK_TYPE], limit: 20 }));
339
+ // Match the stored embed form `${taskType}: ${text}` (02 §R-B) so query and row vectors co-locate.
340
+ const result = await search(projectRoot, `${BACKLOG_TASK_TYPE}: ${text}`);
341
+ // MED-5: build the VALID candidate set FIRST (a hit must carry a dzId AND a finite cosine). Only
342
+ // then decide — a malformed hit (missing dzId / NaN cosine) must NOT bypass the exact-text net.
343
+ // HIGH-A: a hit whose dzId is NOT a member of the CURRENT ideas.jsonl is an ORPHAN vector (its
344
+ // structured record was removed, e.g. by `harmonize --apply`); matching it would report a DUPLICATE
345
+ // of a nonexistent idea (a dead match). Drop orphans before deciding — defense in depth alongside the
346
+ // prune-on-removal in `harmonizeBacklog`.
347
+ const liveIds = new Set(ideas.map((i) => i.id));
348
+ const candidates: DedupCandidate[] = [];
349
+ for (const h of result.hits) {
350
+ if (typeof h.dzId === 'string' && liveIds.has(h.dzId) && Number.isFinite(h.similarity)) candidates.push({ id: h.dzId, cosine: h.similarity });
351
+ }
352
+ if (result.error !== undefined || candidates.length === 0) {
353
+ // No usable semantic signal (embedder unavailable, nothing mirrored, or only malformed hits) ⇒
354
+ // EXACT-text safety net: identical text among existing ideas is still a DUPLICATE (content-addressed
355
+ // idempotency, ADR-002 §degrade). Otherwise NEW — the RELATED band needs cosine and is skipped.
356
+ const match = ideas.find((i) => i.text === text);
357
+ if (match !== undefined) return { action: 'duplicate', cosine: 1, matchedId: match.id, relatedIds: [], exactTextOnly: true };
358
+ return { action: 'new', cosine: -1, matchedId: undefined, relatedIds: [], exactTextOnly: result.error !== undefined };
359
+ }
360
+ return classifyDedup(candidates, cfg.dedup);
361
+ }
362
+
363
+ /* ================================================================== */
364
+ /* ALIGNMENT (AM-3 / ADR-003) — weighted-MAX cosine over the GoalMap. */
365
+ /* ================================================================== */
366
+
367
+ /** Defensive GoalMap reader — never throws; a missing/corrupt file ⇒ empty compass. */
368
+ export function readGoalMap(projectRoot: string): GoalMap {
369
+ const path = goalsPath(projectRoot);
370
+ if (!existsSync(path)) return { version: 1, goals: [] };
371
+ try {
372
+ const parsed = JSON.parse(readFileSync(path, 'utf-8')) as { version?: unknown; goals?: unknown };
373
+ const goals: Goal[] = Array.isArray(parsed.goals)
374
+ ? parsed.goals
375
+ .map((g): Goal | undefined => {
376
+ const o = g as Record<string, unknown>;
377
+ if (typeof o.id !== 'string' || typeof o.statement !== 'string') return undefined;
378
+ return {
379
+ id: o.id,
380
+ statement: o.statement,
381
+ weight: clampNum(o.weight, Number.MIN_VALUE, 1, 1),
382
+ keywords: Array.isArray(o.keywords) ? o.keywords.filter((k): k is string => typeof k === 'string') : [],
383
+ };
384
+ })
385
+ .filter((g): g is Goal => g !== undefined)
386
+ : [];
387
+ return { version: typeof parsed.version === 'number' ? parsed.version : 1, goals };
388
+ } catch {
389
+ return { version: 1, goals: [] };
390
+ }
391
+ }
392
+
393
+ /** The text embedded for a goal: statement + keywords (same convention across cache + score). */
394
+ export function goalEmbedText(goal: Goal): string {
395
+ return goal.keywords.length > 0 ? `${goal.statement} ${goal.keywords.join(' ')}` : goal.statement;
396
+ }
397
+
398
+ export interface AlignmentResult {
399
+ readonly goalId: string | null;
400
+ readonly goalAlignment: number;
401
+ }
402
+
403
+ /** One goal with its precomputed embedding + weight — the input to the pure scorer. */
404
+ export interface GoalVector {
405
+ readonly id: string;
406
+ readonly vec: Float32Array;
407
+ readonly weight: number;
408
+ }
409
+
410
+ /**
411
+ * THE LOAD-BEARING ALIGNMENT SCORER (ADR-003 T-003a) — PURE. Alignment =
412
+ * `max over goals of ( cosine(ideaVec, goalVec) * weight )`, clamped to [0,1]. Weighted-MAX (not
413
+ * sum/mean) so a focused idea advancing ONE goal strongly scores high. No goals ⇒ 0 / null.
414
+ */
415
+ export function scoreAlignment(ideaVec: Float32Array, goals: readonly GoalVector[]): AlignmentResult {
416
+ let bestId: string | null = null;
417
+ let best = 0;
418
+ for (const g of goals) {
419
+ const raw = Math.max(0, cosineSimilarity(ideaVec, g.vec)) * g.weight;
420
+ if (raw > best) {
421
+ best = raw;
422
+ bestId = g.id;
423
+ }
424
+ }
425
+ return { goalId: bestId, goalAlignment: Math.min(1, best) };
426
+ }
427
+
428
+ /** Injectable embedder so the production alignment path is testable. */
429
+ export interface AlignDeps {
430
+ readonly embed?: (text: string) => Promise<Float32Array>;
431
+ }
432
+
433
+ /**
434
+ * Production alignment: embed the idea + each goal (same reused embedder) and score. No embedder or no
435
+ * GoalMap ⇒ `{goalId:null, goalAlignment:0}` — capture and roulette still work (ADR-003 T-003d).
436
+ */
437
+ export async function alignIdea(projectRoot: string, text: string, goalMap: GoalMap, deps: AlignDeps = {}): Promise<AlignmentResult> {
438
+ if (goalMap.goals.length === 0) return { goalId: null, goalAlignment: 0 };
439
+ let embed = deps.embed;
440
+ if (embed === undefined) {
441
+ const resolved = await resolveAgentdbEmbedder(projectRoot);
442
+ if ('error' in resolved) return { goalId: null, goalAlignment: 0 };
443
+ embed = resolved.embed;
444
+ }
445
+ try {
446
+ const ideaVec = await embed(`${BACKLOG_TASK_TYPE}: ${text}`);
447
+ // LOW-10: USE the goal-embed cache (the ADR-003 design) — goal vectors are stable across captures,
448
+ // so re-embedding them every `add` is waste. The cache is keyed by embed model+dim (a model change
449
+ // invalidates it, T-003e) and by each goal's text hash. Cache use is best-effort: any failure just
450
+ // falls through to a live embed, never breaks alignment.
451
+ const model = resolveEmbedModel(projectRoot);
452
+ const useCache = !('error' in model);
453
+ const cache = useCache ? readGoalEmbedCache(projectRoot) : undefined;
454
+ const nextGoals: GoalEmbedCache['goals'] = {};
455
+ const goalVecs: GoalVector[] = [];
456
+ let cacheChanged = false;
457
+ for (const g of goalMap.goals) {
458
+ let vec = useCache && !('error' in model) ? goalCacheHit(cache, model.model, model.dim, g) : undefined;
459
+ if (vec === undefined) {
460
+ vec = await embed(`${BACKLOG_TASK_TYPE}: ${goalEmbedText(g)}`);
461
+ cacheChanged = true;
462
+ }
463
+ goalVecs.push({ id: g.id, vec, weight: g.weight });
464
+ if (useCache) nextGoals[g.id] = { hash: goalTextHash(g), vec: Array.from(vec) };
465
+ }
466
+ if (useCache && !('error' in model) && cacheChanged) {
467
+ try {
468
+ writeGoalEmbedCache(projectRoot, { model: model.model, dim: model.dim, goals: nextGoals });
469
+ } catch {
470
+ /* cache write is best-effort — a failure never affects the score */
471
+ }
472
+ }
473
+ return scoreAlignment(ideaVec, goalVecs);
474
+ } catch {
475
+ return { goalId: null, goalAlignment: 0 };
476
+ }
477
+ }
478
+
479
+ /* ── Goal-embed cache + manifest invalidation (ADR-003 T-003e) ── */
480
+
481
+ export interface GoalEmbedCache {
482
+ readonly model: string;
483
+ readonly dim: number;
484
+ /** id → { hash of the embed text, vector as a plain number[] } */
485
+ readonly goals: Record<string, { readonly hash: string; readonly vec: number[] }>;
486
+ }
487
+
488
+ function goalCachePath(projectRoot: string): string {
489
+ return join(backlogDir(projectRoot), 'goal-embeds.json');
490
+ }
491
+
492
+ function goalTextHash(goal: Goal): string {
493
+ return createHash('sha1').update(goalEmbedText(goal)).digest('hex').slice(0, 16);
494
+ }
495
+
496
+ /**
497
+ * PURE cache validity check (ADR-003 T-003e): a cache is valid for a goal ONLY when the embed
498
+ * model+dim manifest matches AND the goal's embed-text hash is unchanged. A model change (different
499
+ * `model`/`dim`) invalidates EVERY cached goal vector — forcing a recompute, never a stale alignment.
500
+ */
501
+ export function goalCacheHit(cache: GoalEmbedCache | undefined, model: string, dim: number, goal: Goal): Float32Array | undefined {
502
+ if (cache === undefined || cache.model !== model || cache.dim !== dim) return undefined;
503
+ const entry = cache.goals[goal.id];
504
+ if (entry === undefined || entry.hash !== goalTextHash(goal)) return undefined;
505
+ return Float32Array.from(entry.vec);
506
+ }
507
+
508
+ export function readGoalEmbedCache(projectRoot: string): GoalEmbedCache | undefined {
509
+ const path = goalCachePath(projectRoot);
510
+ if (!existsSync(path)) return undefined;
511
+ try {
512
+ const parsed = JSON.parse(readFileSync(path, 'utf-8')) as GoalEmbedCache;
513
+ if (typeof parsed.model !== 'string' || typeof parsed.dim !== 'number' || typeof parsed.goals !== 'object') return undefined;
514
+ return parsed;
515
+ } catch {
516
+ return undefined;
517
+ }
518
+ }
519
+
520
+ export function writeGoalEmbedCache(projectRoot: string, cache: GoalEmbedCache): void {
521
+ const path = goalCachePath(projectRoot);
522
+ mkdirSync(dirname(path), { recursive: true });
523
+ writeFileSync(path, JSON.stringify(cache, null, 2));
524
+ }
525
+
526
+ /* ================================================================== */
527
+ /* ROULETTE (AM-5 / ADR-004) — WEIGHTED, seeded, no starvation. */
528
+ /* ================================================================== */
529
+
530
+ /** Tiny base added to the compass term so an unaligned idea keeps a non-zero weight (no starvation). */
531
+ export const ROULETTE_EPSILON = 0.02;
532
+ const DAY_MS = 24 * 60 * 60 * 1000;
533
+
534
+ /** Ideas eligible for a spin: only `new` and `enriched` (ADR-004). */
535
+ export function eligibleIdeas(ideas: readonly IdeaRecord[]): IdeaRecord[] {
536
+ return ideas.filter((i) => i.status === 'new' || i.status === 'enriched');
537
+ }
538
+
539
+ /** Bounded recency decay ∈ [floor, 1]: `2^(-ageDays/halfLife)`, floored so old ideas are down-weighted, never zero. */
540
+ export function recencyDecay(ageMs: number, halfLifeDays: number, floor: number): number {
541
+ const ageDays = Math.max(0, ageMs) / DAY_MS;
542
+ const decay = Math.pow(2, -ageDays / halfLifeDays);
543
+ return Math.max(floor, Math.min(1, decay));
544
+ }
545
+
546
+ /**
547
+ * THE selection weight (ADR-004): `(alignment^alpha + EPS) · recencyDecay(age) · (1/effort)`. Strictly
548
+ * positive for every idea (EPS>0, floor>0, effort≥1) ⇒ no permanent starvation (T-004c). Equal
549
+ * alignment+age+effort ⇒ equal weights ⇒ a uniform draw (the T-004a control).
550
+ */
551
+ export function ideaWeight(idea: IdeaRecord, cfg: BacklogConfig['roulette'], nowMs: number): number {
552
+ // MED-6: EVERY factor is Number.isFinite-guarded so one bad field (NaN alignment, an unparseable
553
+ // createdTs) degrades to a neutral contribution — it can NEVER poison the whole spin's total (which
554
+ // would make the draw always pick the last item). A bad timestamp ⇒ neutral (floor) recency.
555
+ const alignment = Number.isFinite(idea.goalAlignment) ? Math.max(0, Math.min(1, idea.goalAlignment)) : 0;
556
+ const compass = Math.pow(alignment, cfg.alpha) + ROULETTE_EPSILON;
557
+ const parsed = Date.parse(idea.createdTs);
558
+ const age = Number.isFinite(parsed) ? nowMs - parsed : Number.POSITIVE_INFINITY; // bad ts ⇒ maximally old ⇒ floor
559
+ const recency = recencyDecay(age, cfg.halfLifeDays, cfg.recencyFloor);
560
+ const effort = Number.isFinite(idea.effort) && idea.effort >= 1 ? Math.max(1, Math.min(5, idea.effort)) : cfg.defaultEffort;
561
+ const w = compass * recency * (1 / effort);
562
+ return Number.isFinite(w) && w > 0 ? w : ROULETTE_EPSILON; // last-resort finite floor
563
+ }
564
+
565
+ /** Deterministic ranked shortlist (`--pick N`): by weight desc, id asc — no RNG. */
566
+ export function rankRoulette(ideas: readonly IdeaRecord[], cfg: BacklogConfig['roulette'], nowMs: number): IdeaRecord[] {
567
+ return eligibleIdeas(ideas)
568
+ .map((i) => ({ i, w: ideaWeight(i, cfg, nowMs) }))
569
+ .sort((a, b) => b.w - a.w || (a.i.id < b.i.id ? -1 : 1))
570
+ .map((x) => x.i);
571
+ }
572
+
573
+ /**
574
+ * A single WEIGHTED-RANDOM spin over normalised weights, using an injected seeded RNG (`rng()∈[0,1)`).
575
+ * Same seed ⇒ identical pick (ADR-004 T-004b determinism). Returns `undefined` when nothing is eligible.
576
+ */
577
+ export function spinRoulette(
578
+ ideas: readonly IdeaRecord[],
579
+ cfg: BacklogConfig['roulette'],
580
+ rng: () => number,
581
+ nowMs: number,
582
+ ): IdeaRecord | undefined {
583
+ const pool = eligibleIdeas(ideas);
584
+ if (pool.length === 0) return undefined;
585
+ const weights = pool.map((i) => ideaWeight(i, cfg, nowMs));
586
+ const total = weights.reduce((a, b) => a + b, 0);
587
+ if (total <= 0) return pool[0];
588
+ let r = rng() * total;
589
+ for (let k = 0; k < pool.length; k += 1) {
590
+ r -= weights[k]!;
591
+ if (r < 0) return pool[k];
592
+ }
593
+ return pool[pool.length - 1];
594
+ }
595
+
596
+ /** Build a seeded RNG from a `--seed` integer (reuses the one repo mulberry32). */
597
+ export function seededRng(seed: number): () => number {
598
+ return mulberry32(Number.isFinite(seed) ? Math.floor(seed) : 0);
599
+ }
600
+
601
+ /* ================================================================== */
602
+ /* ENRICH (AM-7 / FR-5) — stage the idea2prd hand-off, do NOT expand. */
603
+ /* ================================================================== */
604
+
605
+ /** kebab-case, Latin-only, ≤40 chars (feature-adr slug convention). */
606
+ export function ideaSlug(idea: IdeaRecord): string {
607
+ const base = idea.text
608
+ .toLowerCase()
609
+ .replace(/[^a-z0-9]+/g, '-')
610
+ .replace(/^-+|-+$/g, '')
611
+ .slice(0, 40)
612
+ .replace(/-+$/g, '');
613
+ // HIGH-1: the fallback must be a filesystem-safe token, NOT the raw id — a hand-edited id like
614
+ // `../../../owned` would otherwise make the slug traverse out of `features/`.
615
+ return base === '' ? `idea-${safeIdToken(idea.id)}` : base;
616
+ }
617
+
618
+ export interface EnrichmentStaging {
619
+ readonly slug: string;
620
+ readonly scaffoldPath: string;
621
+ }
622
+
623
+ /**
624
+ * Stage the idea2prd INPUT and hand off to the `idea2prd-manual` skill (an AGENT phase — 02 §R-G).
625
+ * The CLI writes the scaffold ONLY; it never fabricates a PRD (idea2prd is a skill, not a synchronous
626
+ * transform). Returns the hand-off target so the agent can pick it up.
627
+ */
628
+ export function stageEnrichment(
629
+ projectRoot: string,
630
+ idea: IdeaRecord,
631
+ related: readonly IdeaRecord[],
632
+ goalMap: GoalMap,
633
+ ): EnrichmentStaging {
634
+ const slug = ideaSlug(idea);
635
+ const dir = join(projectRoot, 'features', slug);
636
+ const scaffoldPath = join(dir, 'idea2prd-input.md');
637
+ const goal = idea.goalId !== null ? goalMap.goals.find((g) => g.id === idea.goalId) : undefined;
638
+ const lines = [
639
+ `# idea2prd input — ${slug}`,
640
+ '',
641
+ '> STAGED by `dz backlog enrich`. This is the HAND-OFF scaffold, not a PRD. Run the',
642
+ '> `idea2prd-manual` skill (agent phase) to expand it — the CLI never fabricates the PRD.',
643
+ '',
644
+ '## Idea',
645
+ '',
646
+ idea.text,
647
+ '',
648
+ `- id: \`${idea.id}\``,
649
+ `- effort: ${idea.effort}/5`,
650
+ `- goal alignment: ${idea.goalAlignment.toFixed(3)}${goal !== undefined ? ` (top goal: ${goal.id} — ${goal.statement})` : ' (no aligned goal)'}`,
651
+ '',
652
+ '## Related ideas (dedup RELATED band)',
653
+ '',
654
+ related.length > 0 ? related.map((r) => `- \`${r.id}\` — ${r.text}`).join('\n') : '_none_',
655
+ '',
656
+ '## Goal context (the compass)',
657
+ '',
658
+ goalMap.goals.length > 0 ? goalMap.goals.map((g) => `- \`${g.id}\` (w=${g.weight}): ${g.statement}`).join('\n') : '_no GoalMap_',
659
+ '',
660
+ '## Verification & honesty (owned at the BACKLOG layer, per the user steer)',
661
+ '',
662
+ '- Apply claim-check discipline: every accuracy claim MEASURED with a reproducer.',
663
+ '- Run an adversarial red-team pass on the resulting PRD/ADRs.',
664
+ '- Recall Brain lessons before asserting what exists (`dz recall`).',
665
+ '',
666
+ ];
667
+ mkdirSync(dir, { recursive: true });
668
+ writeFileSync(scaffoldPath, lines.join('\n'));
669
+ return { slug, scaffoldPath };
670
+ }
671
+
672
+ /* ================================================================== */
673
+ /* JIRA PORT (AM-6 / ADR-006) — closed vocab + registry + real stub. */
674
+ /* ⚠ NO MCP CLIENT IMPORT ANYWHERE IN THIS SEAM (grep-guard T-006b). */
675
+ /* ================================================================== */
676
+
677
+ export const BACKLOG_BACKENDS = ['jira-mcp', 'copilot-mcp', 'none'] as const;
678
+ export type BacklogBackend = (typeof BACKLOG_BACKENDS)[number];
679
+
680
+ export function isBacklogBackend(v: unknown): v is BacklogBackend {
681
+ return typeof v === 'string' && (BACKLOG_BACKENDS as readonly string[]).includes(v);
682
+ }
683
+
684
+ /** Built by the domain from an IdeaRecord (+ enrichedPath if any). */
685
+ export interface JiraIssueDraft {
686
+ readonly summary: string;
687
+ readonly description: string;
688
+ readonly labels: readonly string[];
689
+ readonly sourceIdeaId: string;
690
+ }
691
+ export interface IssueRef {
692
+ readonly backend: BacklogBackend;
693
+ readonly key: string | null;
694
+ readonly url?: string;
695
+ readonly stub: boolean;
696
+ readonly outboxPath: string;
697
+ }
698
+ export type VerifyForm = 'full' | 'manual';
699
+ export interface JiraVerifyResult {
700
+ readonly form: VerifyForm;
701
+ readonly ready: boolean;
702
+ /** The exact wiring step the user must run — non-empty for a `manual` backend (T-006e). */
703
+ readonly instruction: string;
704
+ }
705
+
706
+ /** The I/O the adapters need — injected so `createIssue` is testable without touching disk. */
707
+ export interface BacklogIO {
708
+ /** Persist an outbox payload for `<ideaId>`; returns the path written. */
709
+ writeOutbox(ideaId: string, payload: unknown): string;
710
+ }
711
+
712
+ /** THE SEAM (ADR-006): a port with pure methods; NO MCP client is imported here or by its impls. */
713
+ export interface JiraPort {
714
+ readonly backend: BacklogBackend;
715
+ createIssue(draft: JiraIssueDraft, io: BacklogIO): Promise<IssueRef>;
716
+ verify(): Promise<JiraVerifyResult>;
717
+ }
718
+
719
+ /** Pure draft builder from an idea. */
720
+ export function buildJiraDraft(idea: IdeaRecord, goalMap: GoalMap): JiraIssueDraft {
721
+ const goal = idea.goalId !== null ? goalMap.goals.find((g) => g.id === idea.goalId) : undefined;
722
+ const descLines = [
723
+ idea.proposal ?? idea.text,
724
+ '',
725
+ `Source idea: ${idea.id} (effort ${idea.effort}/5, alignment ${idea.goalAlignment.toFixed(3)})`,
726
+ goal !== undefined ? `Top goal: ${goal.id} — ${goal.statement}` : 'No aligned goal',
727
+ idea.relatedIds.length > 0 ? `Related ideas: ${idea.relatedIds.join(', ')}` : '',
728
+ idea.enrichedPath !== undefined ? `Enriched: ${idea.enrichedPath}` : '',
729
+ ].filter((l) => l !== '');
730
+ return {
731
+ summary: idea.text.length > 120 ? `${idea.text.slice(0, 117)}...` : idea.text,
732
+ description: descLines.join('\n'),
733
+ labels: ['dz-backlog', ...idea.tags],
734
+ sourceIdeaId: idea.id,
735
+ };
736
+ }
737
+
738
+ /** The `none` adapter — a REAL stub (not an absence check): writes the full draft to the outbox. */
739
+ const noneAdapter: JiraPort = {
740
+ backend: 'none',
741
+ async createIssue(draft, io) {
742
+ const outboxPath = io.writeOutbox(draft.sourceIdeaId, { backend: 'none', draft });
743
+ return { backend: 'none', key: null, stub: true, outboxPath };
744
+ },
745
+ async verify() {
746
+ return {
747
+ form: 'manual',
748
+ ready: true, // the stub IS ready — it writes an auditable outbox with no external wiring
749
+ instruction: 'The `none` backend writes .dz/backlog/jira-outbox/<id>.json — no external wiring needed.',
750
+ };
751
+ },
752
+ };
753
+
754
+ /** A declared-not-wired MCP seam: builds+persists the SAME outbox payload; verify() reports `manual`. */
755
+ function declaredMcpAdapter(backend: 'jira-mcp' | 'copilot-mcp', instruction: string): JiraPort {
756
+ return {
757
+ backend,
758
+ async createIssue(draft, io) {
759
+ // v1: build + persist the same payload; NO live MCP call (FR-8.2). Honest stub — the wire is a TODO.
760
+ const outboxPath = io.writeOutbox(draft.sourceIdeaId, { backend, draft, note: 'declared-not-wired seam (v1)' });
761
+ return { backend, key: null, stub: true, outboxPath };
762
+ },
763
+ async verify() {
764
+ return { form: 'manual', ready: false, instruction };
765
+ },
766
+ };
767
+ }
768
+
769
+ /** The registry — coverage-tested (ADR-006 T-006a): keys ≡ BACKLOG_BACKENDS as a set. */
770
+ export const JIRA_ADAPTERS: Record<BacklogBackend, JiraPort> = {
771
+ 'jira-mcp': declaredMcpAdapter(
772
+ 'jira-mcp',
773
+ 'Wire a Jira MCP server: `claude mcp add jira ...` then add it to .mcp.json `mcpServers`. Live wiring is a post-v1 adapter.',
774
+ ),
775
+ 'copilot-mcp': declaredMcpAdapter(
776
+ 'copilot-mcp',
777
+ 'Wire a Copilot MCP server into .mcp.json `mcpServers`. Live wiring is a post-v1 adapter.',
778
+ ),
779
+ none: noneAdapter,
780
+ };
781
+
782
+ /** Factory: pick the configured adapter; an unknown value fell back to `none` in readBacklogConfig. */
783
+ export function resolveJiraAdapter(cfg: BacklogConfig): JiraPort {
784
+ return JIRA_ADAPTERS[cfg.jira.adapter] ?? JIRA_ADAPTERS.none;
785
+ }
786
+
787
+ /** Production BacklogIO — writes `.dz/backlog/jira-outbox/<id>.json`. */
788
+ export function makeBacklogIO(projectRoot: string): BacklogIO {
789
+ return {
790
+ writeOutbox(id, payload) {
791
+ // HIGH-1: refuse any id that is not filesystem-safe — the filename is derived from it. A
792
+ // traversal id can never write outside jira-outbox/. Legit content-addressed ids always pass.
793
+ if (!isSafeId(id)) throw new Error(`refusing unsafe idea id for outbox path: ${JSON.stringify(id)}`);
794
+ const dir = jiraOutboxDir(projectRoot);
795
+ mkdirSync(dir, { recursive: true });
796
+ const path = join(dir, `${id}.json`);
797
+ writeFileSync(path, JSON.stringify(payload, null, 2));
798
+ return path;
799
+ },
800
+ };
801
+ }
802
+
803
+ /* ================================================================== */
804
+ /* HARMONIZE (AM-7) — batch semantic dedup of the backlog ideas. */
805
+ /* ================================================================== */
806
+
807
+ export interface BacklogHarmonizeReport {
808
+ readonly mode: 'dry-run' | 'apply';
809
+ /** True when no embedder was available and clustering fell back to EXACT text. */
810
+ readonly fellBackToExact: boolean;
811
+ readonly threshold: number;
812
+ /** One cluster per group of size ≥ 2: the surviving keeper id + the merged-away ids. */
813
+ readonly clusters: readonly { readonly keep: string; readonly drops: readonly string[] }[];
814
+ readonly kept: number;
815
+ readonly dropped: number;
816
+ readonly unique: number;
817
+ readonly snapshotPath?: string;
818
+ readonly error?: string;
819
+ /**
820
+ * MED-E: set when the structured ideas were removed but their agentdb `dz-backlog` vectors could NOT be
821
+ * pruned (locked/unavailable store). Backlog dedup stays correct (the membership guard drops orphans),
822
+ * but the store is left with dangling vectors — a non-clean outcome the caller MUST surface, never a
823
+ * silent success. `dz backlog harmonize` prints this as a warning; the idea removal still stands.
824
+ */
825
+ readonly pruneError?: string;
826
+ }
827
+
828
+ /** Deterministic keeper within a cluster: most uses → oldest → smallest id (stable, testable). */
829
+ function pickKeeper(members: readonly IdeaRecord[]): IdeaRecord {
830
+ return [...members].sort(
831
+ (a, b) => b.uses - a.uses || Date.parse(a.createdTs) - Date.parse(b.createdTs) || (a.id < b.id ? -1 : 1),
832
+ )[0]!;
833
+ }
834
+
835
+ /**
836
+ * Batch-dedup the backlog: cluster near-duplicate ideas (cosine ≥ threshold when an embedder is
837
+ * available, EXACT text otherwise), keep one deterministic keeper per cluster, merge the others'
838
+ * `uses` into it. DRY-RUN by default (writes nothing); `--apply` SNAPSHOTS FIRST then mutates and
839
+ * ABORTS the mutation if the snapshot fails (NFR-6). Injectable `embed` for tests.
840
+ */
841
+ export async function harmonizeBacklog(
842
+ projectRoot: string,
843
+ opts: { apply?: boolean; threshold?: number; embed?: ((t: string) => Promise<Float32Array>) | null } = {},
844
+ ): Promise<BacklogHarmonizeReport> {
845
+ const apply = opts.apply === true;
846
+ const threshold = opts.threshold !== undefined && opts.threshold > 0 && opts.threshold <= 1 ? opts.threshold : DEFAULT_DUPLICATE_THRESHOLD;
847
+ const ideas = readIdeas(projectRoot);
848
+
849
+ let embed = opts.embed;
850
+ if (embed === undefined) {
851
+ const resolved = await resolveAgentdbEmbedder(projectRoot);
852
+ embed = 'error' in resolved ? null : resolved.embed;
853
+ }
854
+ let fellBackToExact = embed === null;
855
+
856
+ // Build clusters (union-find over ≥threshold cosine, or exact-text groups).
857
+ const parent = ideas.map((_, i) => i);
858
+ const find = (x: number): number => {
859
+ while (parent[x] !== x) {
860
+ parent[x] = parent[parent[x]!]!;
861
+ x = parent[x]!;
862
+ }
863
+ return x;
864
+ };
865
+ const union = (a: number, b: number): void => {
866
+ parent[find(a)] = find(b);
867
+ };
868
+ if (embed !== null && ideas.length >= 2) {
869
+ try {
870
+ const vecs = await Promise.all(ideas.map((i) => embed!(`${BACKLOG_TASK_TYPE}: ${i.text}`)));
871
+ for (let a = 0; a < ideas.length; a += 1) {
872
+ for (let b = a + 1; b < ideas.length; b += 1) {
873
+ if (cosineSimilarity(vecs[a]!, vecs[b]!) >= threshold) union(a, b);
874
+ }
875
+ }
876
+ } catch {
877
+ fellBackToExact = true;
878
+ }
879
+ }
880
+ if (fellBackToExact) {
881
+ const byText = new Map<string, number>();
882
+ ideas.forEach((idea, i) => {
883
+ const first = byText.get(idea.text);
884
+ if (first === undefined) byText.set(idea.text, i);
885
+ else union(i, first);
886
+ });
887
+ }
888
+
889
+ const groups = new Map<number, IdeaRecord[]>();
890
+ ideas.forEach((idea, i) => {
891
+ const root = find(i);
892
+ (groups.get(root) ?? groups.set(root, []).get(root)!).push(idea);
893
+ });
894
+
895
+ const clusters: { keep: string; drops: string[] }[] = [];
896
+ const keepById = new Map<string, IdeaRecord>();
897
+ const dropIds = new Set<string>();
898
+ for (const members of groups.values()) {
899
+ if (members.length < 2) continue;
900
+ const keeper = pickKeeper(members);
901
+ const drops = members.filter((m) => m.id !== keeper.id);
902
+ clusters.push({ keep: keeper.id, drops: drops.map((d) => d.id) });
903
+ keepById.set(keeper.id, keeper);
904
+ for (const d of drops) dropIds.add(d.id);
905
+ }
906
+
907
+ const report: BacklogHarmonizeReport = {
908
+ mode: apply ? 'apply' : 'dry-run',
909
+ fellBackToExact,
910
+ threshold,
911
+ clusters,
912
+ kept: clusters.length,
913
+ dropped: dropIds.size,
914
+ unique: ideas.length - clusters.length - dropIds.size,
915
+ };
916
+ if (!apply || dropIds.size === 0) return report;
917
+
918
+ // --apply: snapshot FIRST, then merge uses into keepers and drop the rest.
919
+ const snapDest = join(backlogDir(projectRoot), `ideas.pre-harmonize-${Date.now()}.jsonl`);
920
+ const snap = snapshotIdeas(projectRoot, snapDest);
921
+ if (snap.error !== undefined) return { ...report, error: `backup failed — drop aborted: ${snap.error}` };
922
+ const merged = new Map<string, IdeaRecord>();
923
+ for (const [id, keeper] of keepById) merged.set(id, { ...keeper });
924
+ for (const c of clusters) {
925
+ const keeper = merged.get(c.keep)!;
926
+ keeper.uses += c.drops.length;
927
+ }
928
+ const survivors = ideas
929
+ .filter((i) => !dropIds.has(i.id))
930
+ .map((i) => merged.get(i.id) ?? i);
931
+ writeIdeas(projectRoot, survivors);
932
+ // HIGH-A: prune the agentdb `dz-backlog` vectors for every removed idea, so a later `add` can't match
933
+ // an ORPHAN dzId (a DUPLICATE of a nonexistent idea). Best-effort for the structured store (already
934
+ // snapshotted + written) — but MED-E: the prune outcome must NOT be swallowed. If it failed, the store
935
+ // has dangling vectors; report it (a non-clean status) so the user knows, rather than a false success.
936
+ const prune = deleteAgentdbByDzIds(projectRoot, [...dropIds], { taskTypes: [BACKLOG_TASK_TYPE] });
937
+ return {
938
+ ...report,
939
+ snapshotPath: snapDest,
940
+ ...(prune.error !== undefined
941
+ ? { pruneError: `${dropIds.size} idea(s) removed from ideas.jsonl, but their agentdb vectors were NOT pruned: ${prune.error}` }
942
+ : {}),
943
+ };
944
+ }
945
+
946
+ /* ================================================================== */
947
+ /* MIRROR seam (ADR-001) — write idea vectors through the ONE seam. */
948
+ /* ================================================================== */
949
+
950
+ /**
951
+ * Mirror one idea's vector into the SHARED `.dz/agentdb.db` under `task_type:'dz-backlog'`, written
952
+ * DIRECTLY through `importVectorsToAgentdb` (upsert-by-dzId) — NOT the configurable vector engine.
953
+ * This is the load-bearing single-store guarantee (ADR-001/005): dedup ALWAYS searches agentdb, so
954
+ * the write must ALWAYS land in agentdb, no matter what `memory.vector.engine` says. Upsert-by-dzId
955
+ * makes a re-mirror idempotent (0 duplicate rows). `guardEmbedSpace` inside `importVectorsToAgentdb`
956
+ * covers the write (ADR-001 T-001c). Best-effort: NEVER blocks capture (I-1) — honest `{error}`.
957
+ */
958
+ export async function mirrorIdeaVector(projectRoot: string, idea: IdeaRecord): Promise<{ mirrored: number; error?: string | undefined }> {
959
+ const emb = await resolveAgentdbEmbedder(projectRoot);
960
+ if ('error' in emb) return { mirrored: 0, error: emb.error };
961
+ let vector: Float32Array;
962
+ try {
963
+ vector = await emb.embed(`${BACKLOG_TASK_TYPE}: ${idea.text}`); // SAME embed form the dedup query uses
964
+ } catch (err) {
965
+ return { mirrored: 0, error: `embed failed: ${err instanceof Error ? err.message : String(err)}` };
966
+ }
967
+ const res = await importVectorsToAgentdb(projectRoot, [
968
+ {
969
+ dzId: idea.id,
970
+ vector,
971
+ text: idea.text,
972
+ taskType: BACKLOG_TASK_TYPE,
973
+ score: Math.max(0, Math.min(1, Number.isFinite(idea.goalAlignment) ? idea.goalAlignment : 0)),
974
+ metadata: { kind: 'dz-backlog-idea' },
975
+ },
976
+ ]);
977
+ return { mirrored: res.imported, ...(res.error !== undefined ? { error: res.error } : {}) };
978
+ }