@dzhechkov/harness-core 0.3.138 → 0.3.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dz-manifest.json +69 -17
- package/dist/agentdb-index.d.ts +36 -0
- package/dist/agentdb-index.d.ts.map +1 -1
- package/dist/agentdb-index.js +146 -2
- package/dist/agentdb-index.js.map +1 -1
- package/dist/backlog.d.ts +310 -0
- package/dist/backlog.d.ts.map +1 -0
- package/dist/backlog.js +755 -0
- package/dist/backlog.js.map +1 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -1
- package/dist/index.js.map +1 -1
- package/dist/score.d.ts +44 -0
- package/dist/score.d.ts.map +1 -0
- package/dist/score.js +148 -0
- package/dist/score.js.map +1 -0
- package/dist/vector-tier.d.ts +9 -0
- package/dist/vector-tier.d.ts.map +1 -1
- package/dist/vector-tier.js +38 -6
- package/dist/vector-tier.js.map +1 -1
- package/package.json +6 -6
- package/sbom.json +146 -16
- package/src/agentdb-index.ts +145 -2
- package/src/backlog.ts +978 -0
- package/src/index.ts +10 -1
- package/src/score.ts +220 -0
- package/src/vector-tier.ts +47 -4
package/src/backlog.ts
ADDED
|
@@ -0,0 +1,978 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `dz backlog` — the Smart Backlog: a personal, goal-directed idea pipeline (feature smart-backlog).
|
|
3
|
+
*
|
|
4
|
+
* Capture an idea → semantic dedup against existing ideas via the EXISTING Brain vector engine
|
|
5
|
+
* (agentdb + the shared embedder — ADR-001; NO second store) → score alignment against a GoalMap
|
|
6
|
+
* (ADR-003) → weighted-roulette pick (ADR-004) → stage an idea2prd enrich hand-off → draft a Jira
|
|
7
|
+
* issue through a configurable, stub-first adapter seam (ADR-006).
|
|
8
|
+
*
|
|
9
|
+
* ARCHITECTURE (05): every load-bearing decision is a PURE function tested on a layer-1 deterministic
|
|
10
|
+
* test; the CLI handler (`cmdBacklog`) is a thin arg-parse + call. Semantic work (embedding, cosine,
|
|
11
|
+
* vector search) is an OUTBOUND PORT to harness-core's Brain engine — this file defines NO embedder,
|
|
12
|
+
* NO cosine, and imports NO MCP client (ADR-001 / ADR-006, both grep-guarded).
|
|
13
|
+
*
|
|
14
|
+
* Ideas carry `task_type:'dz-backlog'` in the shared `.dz/agentdb.db`, so they never surface in
|
|
15
|
+
* `dz recall`'s lesson namespace (ADR-005 T-005b, the discriminating isolation test). Structured
|
|
16
|
+
* records live in a dedicated `.dz/backlog/ideas.jsonl`, never in `.dz/memory/patterns.*`.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { createHash } from 'node:crypto';
|
|
20
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
21
|
+
import { dirname, join } from 'node:path';
|
|
22
|
+
|
|
23
|
+
// ── Outbound ports to the Brain engine (ADR-001): reuse VERBATIM, define none locally. ──
|
|
24
|
+
// Backlog vectors are written DIRECTLY to the agentdb `dz-backlog` namespace via
|
|
25
|
+
// `importVectorsToAgentdb` (NOT the configurable `mirrorEntriesToVector` engine), so write and read
|
|
26
|
+
// ALWAYS hit the SAME store regardless of `memory.vector.engine` (ADR-001/005 — the single-store
|
|
27
|
+
// property; routing through the generic engine let `engine:'rvf'` split writes from reads).
|
|
28
|
+
import { cosineSimilarity, deleteAgentdbByDzIds, importVectorsToAgentdb, resolveAgentdbEmbedder, searchAgentdbPatterns } from './agentdb-index.js';
|
|
29
|
+
import { resolveEmbedModel } from './embedding-config.js';
|
|
30
|
+
// Seeded PRNG — reuse the ONE mulberry32 already in the repo (no second RNG; ADR-004 determinism).
|
|
31
|
+
import { mulberry32 } from './compounding.js';
|
|
32
|
+
|
|
33
|
+
/** The vector-store namespace that isolates ideas from lessons (ADR-001/005). */
|
|
34
|
+
export const BACKLOG_TASK_TYPE = 'dz-backlog';
|
|
35
|
+
|
|
36
|
+
/* ================================================================== */
|
|
37
|
+
/* DOMAIN TYPES (04 Domain Model) */
|
|
38
|
+
/* ================================================================== */
|
|
39
|
+
|
|
40
|
+
export type IdeaStatus = 'new' | 'enriched' | 'in-progress' | 'shipped' | 'dropped';
|
|
41
|
+
|
|
42
|
+
/** The aggregate root — one captured idea. `id` is content-addressed (no `Date.now()` in identity). */
|
|
43
|
+
export interface IdeaRecord {
|
|
44
|
+
readonly id: string;
|
|
45
|
+
readonly text: string;
|
|
46
|
+
status: IdeaStatus;
|
|
47
|
+
readonly createdTs: string;
|
|
48
|
+
effort: number; // 1..5, user estimate; default 3
|
|
49
|
+
goalId: string | null; // argmax goal (ADR-003)
|
|
50
|
+
goalAlignment: number; // [0,1] weighted-max cosine
|
|
51
|
+
relatedIds: string[]; // top-K RELATED ideas (ADR-002)
|
|
52
|
+
uses: number; // reinforcement count — a DUPLICATE bumps the existing root's uses (ADR-002 T-002b)
|
|
53
|
+
proposal?: string; // agent-authored (FR-3.2); the CLI NEVER fabricates this
|
|
54
|
+
enrichedPath?: string; // features/<slug>/ once enriched (FR-5)
|
|
55
|
+
jiraKey?: string; // IssueRef.key or the outbox ref (ADR-006)
|
|
56
|
+
tags: string[];
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** The compass (ADR-003) — user-owned, edited as a file. */
|
|
60
|
+
export interface Goal {
|
|
61
|
+
readonly id: string;
|
|
62
|
+
readonly statement: string;
|
|
63
|
+
readonly weight: number; // (0,1]
|
|
64
|
+
readonly keywords: readonly string[];
|
|
65
|
+
}
|
|
66
|
+
export interface GoalMap {
|
|
67
|
+
readonly version: number;
|
|
68
|
+
readonly goals: readonly Goal[];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export type DedupAction = 'duplicate' | 'related' | 'new';
|
|
72
|
+
|
|
73
|
+
/** Pure output of the classifier (04) — consumed by capture. */
|
|
74
|
+
export interface DedupVerdict {
|
|
75
|
+
readonly action: DedupAction;
|
|
76
|
+
/** Top-1 raw cosine (ADR-002 — never an RRF score). `-1` when there is nothing to compare against. */
|
|
77
|
+
readonly cosine: number;
|
|
78
|
+
readonly matchedId: string | undefined;
|
|
79
|
+
readonly relatedIds: readonly string[];
|
|
80
|
+
/** True when the embedder was unavailable and dedup degraded to exact-text (ADR-002 §degrade). */
|
|
81
|
+
readonly exactTextOnly: boolean;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/* ================================================================== */
|
|
85
|
+
/* CONFIG (readBacklogConfig) — defensive, Number.isFinite clamps. */
|
|
86
|
+
/* ================================================================== */
|
|
87
|
+
|
|
88
|
+
export const DEFAULT_DUPLICATE_THRESHOLD = 0.92; // measured house constant (02 §R-D)
|
|
89
|
+
export const DEFAULT_RELATEDNESS_FLOOR = 0.35; // GROUND_VECTOR_SIMILARITY_FLOOR (02 §R-D)
|
|
90
|
+
export const DEFAULT_ROULETTE_ALPHA = 1.5;
|
|
91
|
+
export const DEFAULT_HALF_LIFE_DAYS = 30;
|
|
92
|
+
export const DEFAULT_RECENCY_FLOOR = 0.3;
|
|
93
|
+
export const DEFAULT_EFFORT = 3;
|
|
94
|
+
|
|
95
|
+
export interface BacklogConfig {
|
|
96
|
+
readonly dedup: { readonly duplicateThreshold: number; readonly relatednessFloor: number };
|
|
97
|
+
readonly roulette: {
|
|
98
|
+
readonly alpha: number;
|
|
99
|
+
readonly halfLifeDays: number;
|
|
100
|
+
readonly recencyFloor: number;
|
|
101
|
+
readonly defaultEffort: number;
|
|
102
|
+
};
|
|
103
|
+
readonly jira: { readonly adapter: BacklogBackend };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Finite-and-in-range or the fallback (the recurring Infinity-clamp lesson). */
|
|
107
|
+
function clampNum(v: unknown, lo: number, hi: number, fallback: number, opts: { intOnly?: boolean } = {}): number {
|
|
108
|
+
if (typeof v !== 'number' || !Number.isFinite(v) || v < lo || v > hi) return fallback;
|
|
109
|
+
return opts.intOnly ? Math.floor(v) : v;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function readBacklogConfig(projectRoot: string): BacklogConfig {
|
|
113
|
+
const fallback: BacklogConfig = {
|
|
114
|
+
dedup: { duplicateThreshold: DEFAULT_DUPLICATE_THRESHOLD, relatednessFloor: DEFAULT_RELATEDNESS_FLOOR },
|
|
115
|
+
roulette: {
|
|
116
|
+
alpha: DEFAULT_ROULETTE_ALPHA,
|
|
117
|
+
halfLifeDays: DEFAULT_HALF_LIFE_DAYS,
|
|
118
|
+
recencyFloor: DEFAULT_RECENCY_FLOOR,
|
|
119
|
+
defaultEffort: DEFAULT_EFFORT,
|
|
120
|
+
},
|
|
121
|
+
jira: { adapter: 'none' },
|
|
122
|
+
};
|
|
123
|
+
const configPath = join(projectRoot, '.dz', 'config.json');
|
|
124
|
+
if (!existsSync(configPath)) return fallback;
|
|
125
|
+
try {
|
|
126
|
+
const parsed = JSON.parse(readFileSync(configPath, 'utf-8')) as {
|
|
127
|
+
backlog?: {
|
|
128
|
+
dedup?: { duplicateThreshold?: unknown; relatednessFloor?: unknown };
|
|
129
|
+
roulette?: { alpha?: unknown; halfLifeDays?: unknown; recencyFloor?: unknown; defaultEffort?: unknown };
|
|
130
|
+
jira?: { adapter?: unknown };
|
|
131
|
+
};
|
|
132
|
+
};
|
|
133
|
+
const b = parsed.backlog ?? {};
|
|
134
|
+
// `> 0 && <= 1` — a value of 2 or NaN falls back to 0.92 (ADR-002 T-002d).
|
|
135
|
+
const dup = clampNum(b.dedup?.duplicateThreshold, Number.MIN_VALUE, 1, DEFAULT_DUPLICATE_THRESHOLD);
|
|
136
|
+
// The floor must sit in [0, dup]. MED-7: re-range-check the ADJUSTED value (clamp the RESULT, not
|
|
137
|
+
// just the input) — a tiny `dup` (e.g. 5e-324) previously drove `dup - EPSILON` NEGATIVE, so a
|
|
138
|
+
// cosine of -1e-16 fell into the RELATED band. Force the repaired floor back into [0, dup].
|
|
139
|
+
let floor = clampNum(b.dedup?.relatednessFloor, 0, 1, DEFAULT_RELATEDNESS_FLOOR);
|
|
140
|
+
if (floor >= dup) floor = DEFAULT_RELATEDNESS_FLOOR < dup ? DEFAULT_RELATEDNESS_FLOOR : dup;
|
|
141
|
+
floor = Math.min(Math.max(floor, 0), dup);
|
|
142
|
+
const adapter = isBacklogBackend(b.jira?.adapter) ? b.jira!.adapter : 'none'; // unknown ⇒ none (ADR-006 T-006d)
|
|
143
|
+
return {
|
|
144
|
+
dedup: { duplicateThreshold: dup, relatednessFloor: floor },
|
|
145
|
+
roulette: {
|
|
146
|
+
alpha: clampNum(b.roulette?.alpha, Number.MIN_VALUE, Number.MAX_VALUE, DEFAULT_ROULETTE_ALPHA),
|
|
147
|
+
halfLifeDays: clampNum(b.roulette?.halfLifeDays, Number.MIN_VALUE, Number.MAX_VALUE, DEFAULT_HALF_LIFE_DAYS),
|
|
148
|
+
recencyFloor: clampNum(b.roulette?.recencyFloor, Number.MIN_VALUE, 1, DEFAULT_RECENCY_FLOOR),
|
|
149
|
+
defaultEffort: clampNum(b.roulette?.defaultEffort, 1, 5, DEFAULT_EFFORT, { intOnly: true }),
|
|
150
|
+
},
|
|
151
|
+
jira: { adapter },
|
|
152
|
+
};
|
|
153
|
+
} catch {
|
|
154
|
+
return fallback;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/* ================================================================== */
|
|
159
|
+
/* STORE (AM-1) — .dz/backlog/ideas.jsonl (append-only JSONL, ADR-005) */
|
|
160
|
+
/* ================================================================== */
|
|
161
|
+
|
|
162
|
+
export function backlogDir(projectRoot: string): string {
|
|
163
|
+
return join(projectRoot, '.dz', 'backlog');
|
|
164
|
+
}
|
|
165
|
+
export function ideasPath(projectRoot: string): string {
|
|
166
|
+
return join(backlogDir(projectRoot), 'ideas.jsonl');
|
|
167
|
+
}
|
|
168
|
+
export function goalsPath(projectRoot: string): string {
|
|
169
|
+
return join(backlogDir(projectRoot), 'goals.json');
|
|
170
|
+
}
|
|
171
|
+
export function jiraOutboxDir(projectRoot: string): string {
|
|
172
|
+
return join(backlogDir(projectRoot), 'jira-outbox');
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Content-addressed id — `sha1(text|createdTs).slice(0,16)`; NO `Date.now()` (mirrors patterns.ts). */
|
|
176
|
+
export function ideaId(text: string, createdTs: string): string {
|
|
177
|
+
return createHash('sha1').update(`${text}|${createdTs}`).digest('hex').slice(0, 16);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Strict id whitelist (mirrors `dz score`'s slug guard) — the ONLY shape that may be interpolated into
|
|
182
|
+
* a filesystem path (jira-outbox/<id>.json, enrich slug fallback). Rejects `.`/`..`/`/`/`\` and any
|
|
183
|
+
* traversal, so an idea id like `../../../owned` from a hand-edited store can NEVER escape its dir
|
|
184
|
+
* (HIGH-1). A content-addressed `ideaId()` (16 hex) always passes; anything else is refused, not sanitised
|
|
185
|
+
* into a surprising path.
|
|
186
|
+
*/
|
|
187
|
+
export function isSafeId(id: unknown): id is string {
|
|
188
|
+
return typeof id === 'string' && id.length > 0 && id.length <= 64 && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id) && !id.includes('..');
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** Strip an id to a filesystem-safe token (defence-in-depth for a slug fallback — never a path). */
|
|
192
|
+
function safeIdToken(id: string): string {
|
|
193
|
+
const t = id.replace(/[^A-Za-z0-9]/g, '').slice(0, 12);
|
|
194
|
+
return t === '' ? 'idea' : t;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Normalise a partial/loaded object into a full IdeaRecord (defensive: bad fields ⇒ safe defaults). */
|
|
198
|
+
function normaliseIdea(raw: Record<string, unknown>): IdeaRecord | undefined {
|
|
199
|
+
// HIGH-1: an id that is not filesystem-safe can NEVER enter the store — it would later interpolate
|
|
200
|
+
// into an outbox filename or an enrich slug and escape its directory. Drop the record entirely.
|
|
201
|
+
if (!isSafeId(raw.id) || typeof raw.text !== 'string') return undefined;
|
|
202
|
+
const status = ['new', 'enriched', 'in-progress', 'shipped', 'dropped'].includes(raw.status as string)
|
|
203
|
+
? (raw.status as IdeaStatus)
|
|
204
|
+
: 'new';
|
|
205
|
+
// MED-6: a bad createdTs (`Date.parse` ⇒ NaN) would poison the roulette weight; coerce it to a
|
|
206
|
+
// valid, maximally-old ISO timestamp so recency degrades to the neutral floor, never NaN.
|
|
207
|
+
const createdRaw = typeof raw.createdTs === 'string' && Number.isFinite(Date.parse(raw.createdTs)) ? raw.createdTs : new Date(0).toISOString();
|
|
208
|
+
const rec: IdeaRecord = {
|
|
209
|
+
id: raw.id,
|
|
210
|
+
text: raw.text,
|
|
211
|
+
status,
|
|
212
|
+
createdTs: createdRaw,
|
|
213
|
+
effort: clampNum(raw.effort, 1, 5, DEFAULT_EFFORT, { intOnly: true }),
|
|
214
|
+
goalId: typeof raw.goalId === 'string' ? raw.goalId : null,
|
|
215
|
+
goalAlignment: clampNum(raw.goalAlignment, 0, 1, 0),
|
|
216
|
+
relatedIds: Array.isArray(raw.relatedIds) ? raw.relatedIds.filter((x): x is string => typeof x === 'string') : [],
|
|
217
|
+
uses: clampNum(raw.uses, 0, Number.MAX_VALUE, 0, { intOnly: true }),
|
|
218
|
+
tags: Array.isArray(raw.tags) ? raw.tags.filter((x): x is string => typeof x === 'string') : [],
|
|
219
|
+
};
|
|
220
|
+
if (typeof raw.proposal === 'string') rec.proposal = raw.proposal;
|
|
221
|
+
if (typeof raw.enrichedPath === 'string') rec.enrichedPath = raw.enrichedPath;
|
|
222
|
+
if (typeof raw.jiraKey === 'string') rec.jiraKey = raw.jiraKey;
|
|
223
|
+
return rec;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Read the append-only store. A corrupt line is SKIPPED (never fatal) — the whole store never throws. */
|
|
227
|
+
export function readIdeas(projectRoot: string): IdeaRecord[] {
|
|
228
|
+
const path = ideasPath(projectRoot);
|
|
229
|
+
if (!existsSync(path)) return [];
|
|
230
|
+
let text: string;
|
|
231
|
+
try {
|
|
232
|
+
text = readFileSync(path, 'utf-8');
|
|
233
|
+
} catch {
|
|
234
|
+
return [];
|
|
235
|
+
}
|
|
236
|
+
const out: IdeaRecord[] = [];
|
|
237
|
+
for (const line of text.split('\n')) {
|
|
238
|
+
const trimmed = line.trim();
|
|
239
|
+
if (trimmed === '') continue;
|
|
240
|
+
try {
|
|
241
|
+
const rec = normaliseIdea(JSON.parse(trimmed) as Record<string, unknown>);
|
|
242
|
+
if (rec !== undefined) out.push(rec);
|
|
243
|
+
} catch {
|
|
244
|
+
/* skip corrupt line */
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return out;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** Atomic full rewrite (tmp + rename) so a crash mid-write never truncates the store. */
|
|
251
|
+
export function writeIdeas(projectRoot: string, ideas: readonly IdeaRecord[]): void {
|
|
252
|
+
const path = ideasPath(projectRoot);
|
|
253
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
254
|
+
const body = ideas.map((i) => JSON.stringify(i)).join('\n') + (ideas.length > 0 ? '\n' : '');
|
|
255
|
+
const tmp = `${path}.tmp-${process.pid}`;
|
|
256
|
+
writeFileSync(tmp, body);
|
|
257
|
+
renameSync(tmp, path);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** Pre-mutation snapshot (NFR-6) — mirrors `snapshotStore`. A failed snapshot returns `{error}` so
|
|
261
|
+
* the caller ABORTS the mutation (no partial merge). */
|
|
262
|
+
export interface SnapshotResult {
|
|
263
|
+
readonly path: string;
|
|
264
|
+
readonly count: number;
|
|
265
|
+
readonly error?: string;
|
|
266
|
+
}
|
|
267
|
+
export function snapshotIdeas(projectRoot: string, dest: string): SnapshotResult {
|
|
268
|
+
try {
|
|
269
|
+
const ideas = readIdeas(projectRoot);
|
|
270
|
+
mkdirSync(dirname(dest), { recursive: true });
|
|
271
|
+
writeFileSync(dest, ideas.map((i) => JSON.stringify(i)).join('\n') + (ideas.length > 0 ? '\n' : ''));
|
|
272
|
+
return { path: dest, count: ideas.length };
|
|
273
|
+
} catch (err) {
|
|
274
|
+
return { path: dest, count: 0, error: `snapshot failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/* ================================================================== */
|
|
279
|
+
/* DEDUP (AM-2 / ADR-002) — raw cosine bands, NOT RRF. */
|
|
280
|
+
/* ================================================================== */
|
|
281
|
+
|
|
282
|
+
/** One existing-idea comparison candidate: its id and the RAW cosine of the new idea against it. */
|
|
283
|
+
export interface DedupCandidate {
|
|
284
|
+
readonly id: string;
|
|
285
|
+
readonly cosine: number;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* THE LOAD-BEARING CLASSIFIER (ADR-002 T-002a) — PURE. Bands the top-1 raw cosine:
|
|
290
|
+
* DUPLICATE cosine ≥ duplicateThreshold (default 0.92)
|
|
291
|
+
* RELATED relatednessFloor ≤ cosine < dup (default [0.35, 0.92)) ⇒ create + attach relatedIds
|
|
292
|
+
* NEW cosine < relatednessFloor
|
|
293
|
+
* Flip either cut and exactly one boundary fixture crosses a band — the test REDS.
|
|
294
|
+
*/
|
|
295
|
+
export function classifyDedup(
|
|
296
|
+
candidates: readonly DedupCandidate[],
|
|
297
|
+
cfg: BacklogConfig['dedup'],
|
|
298
|
+
opts: { exactTextOnly?: boolean } = {},
|
|
299
|
+
): DedupVerdict {
|
|
300
|
+
// HIGH-4: a non-finite cosine (NaN/±Infinity) sorts unpredictably and can shove a real 0.97 duplicate
|
|
301
|
+
// out of the top slot → misclassified NEW. Drop non-finite candidates BEFORE sorting/banding (the
|
|
302
|
+
// recurring repo `Number.isFinite` lesson).
|
|
303
|
+
const sorted = candidates.filter((c) => Number.isFinite(c.cosine)).sort((a, b) => b.cosine - a.cosine);
|
|
304
|
+
const top = sorted[0];
|
|
305
|
+
const exactTextOnly = opts.exactTextOnly === true;
|
|
306
|
+
if (top === undefined) return { action: 'new', cosine: -1, matchedId: undefined, relatedIds: [], exactTextOnly };
|
|
307
|
+
if (top.cosine >= cfg.duplicateThreshold) {
|
|
308
|
+
return { action: 'duplicate', cosine: top.cosine, matchedId: top.id, relatedIds: [], exactTextOnly };
|
|
309
|
+
}
|
|
310
|
+
const related = sorted.filter((c) => c.cosine >= cfg.relatednessFloor && c.cosine < cfg.duplicateThreshold);
|
|
311
|
+
if (related.length > 0) {
|
|
312
|
+
return { action: 'related', cosine: top.cosine, matchedId: undefined, relatedIds: related.map((c) => c.id), exactTextOnly };
|
|
313
|
+
}
|
|
314
|
+
return { action: 'new', cosine: top.cosine, matchedId: undefined, relatedIds: [], exactTextOnly };
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** Injectable deps so the production dedup path is testable without a live agentdb. */
|
|
318
|
+
export interface DedupDeps {
|
|
319
|
+
/** Semantic search over the `dz-backlog` namespace — defaults to the real `searchAgentdbPatterns`. */
|
|
320
|
+
readonly search?: (
|
|
321
|
+
projectRoot: string,
|
|
322
|
+
query: string,
|
|
323
|
+
) => Promise<{ hits: { dzId?: string | undefined; similarity: number }[]; error?: string | undefined }>;
|
|
324
|
+
/** Existing structured ideas (for the exact-text degrade path) — defaults to `readIdeas`. */
|
|
325
|
+
readonly ideas?: readonly IdeaRecord[];
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* Production dedup: embed+search the `dz-backlog` vectors (RAW COSINE via `searchAgentdbPatterns`,
|
|
330
|
+
* NEVER the RRF `recallHybrid().score` — ADR-002 T-002c), then band via {@link classifyDedup}. If the
|
|
331
|
+
* embedder/search is unavailable it DEGRADES to exact-text dedup (identical text ⇒ DUPLICATE), never
|
|
332
|
+
* blocking capture (NFR-2 / ADR-002 T-002e).
|
|
333
|
+
*/
|
|
334
|
+
export async function dedupIdea(projectRoot: string, text: string, cfg: BacklogConfig, deps: DedupDeps = {}): Promise<DedupVerdict> {
|
|
335
|
+
const ideas = deps.ideas ?? readIdeas(projectRoot);
|
|
336
|
+
const search =
|
|
337
|
+
deps.search ??
|
|
338
|
+
((root: string, query: string) => searchAgentdbPatterns(root, query, { taskTypes: [BACKLOG_TASK_TYPE], limit: 20 }));
|
|
339
|
+
// Match the stored embed form `${taskType}: ${text}` (02 §R-B) so query and row vectors co-locate.
|
|
340
|
+
const result = await search(projectRoot, `${BACKLOG_TASK_TYPE}: ${text}`);
|
|
341
|
+
// MED-5: build the VALID candidate set FIRST (a hit must carry a dzId AND a finite cosine). Only
|
|
342
|
+
// then decide — a malformed hit (missing dzId / NaN cosine) must NOT bypass the exact-text net.
|
|
343
|
+
// HIGH-A: a hit whose dzId is NOT a member of the CURRENT ideas.jsonl is an ORPHAN vector (its
|
|
344
|
+
// structured record was removed, e.g. by `harmonize --apply`); matching it would report a DUPLICATE
|
|
345
|
+
// of a nonexistent idea (a dead match). Drop orphans before deciding — defense in depth alongside the
|
|
346
|
+
// prune-on-removal in `harmonizeBacklog`.
|
|
347
|
+
const liveIds = new Set(ideas.map((i) => i.id));
|
|
348
|
+
const candidates: DedupCandidate[] = [];
|
|
349
|
+
for (const h of result.hits) {
|
|
350
|
+
if (typeof h.dzId === 'string' && liveIds.has(h.dzId) && Number.isFinite(h.similarity)) candidates.push({ id: h.dzId, cosine: h.similarity });
|
|
351
|
+
}
|
|
352
|
+
if (result.error !== undefined || candidates.length === 0) {
|
|
353
|
+
// No usable semantic signal (embedder unavailable, nothing mirrored, or only malformed hits) ⇒
|
|
354
|
+
// EXACT-text safety net: identical text among existing ideas is still a DUPLICATE (content-addressed
|
|
355
|
+
// idempotency, ADR-002 §degrade). Otherwise NEW — the RELATED band needs cosine and is skipped.
|
|
356
|
+
const match = ideas.find((i) => i.text === text);
|
|
357
|
+
if (match !== undefined) return { action: 'duplicate', cosine: 1, matchedId: match.id, relatedIds: [], exactTextOnly: true };
|
|
358
|
+
return { action: 'new', cosine: -1, matchedId: undefined, relatedIds: [], exactTextOnly: result.error !== undefined };
|
|
359
|
+
}
|
|
360
|
+
return classifyDedup(candidates, cfg.dedup);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/* ================================================================== */
|
|
364
|
+
/* ALIGNMENT (AM-3 / ADR-003) — weighted-MAX cosine over the GoalMap. */
|
|
365
|
+
/* ================================================================== */
|
|
366
|
+
|
|
367
|
+
/** Defensive GoalMap reader — never throws; a missing/corrupt file ⇒ empty compass. */
|
|
368
|
+
export function readGoalMap(projectRoot: string): GoalMap {
|
|
369
|
+
const path = goalsPath(projectRoot);
|
|
370
|
+
if (!existsSync(path)) return { version: 1, goals: [] };
|
|
371
|
+
try {
|
|
372
|
+
const parsed = JSON.parse(readFileSync(path, 'utf-8')) as { version?: unknown; goals?: unknown };
|
|
373
|
+
const goals: Goal[] = Array.isArray(parsed.goals)
|
|
374
|
+
? parsed.goals
|
|
375
|
+
.map((g): Goal | undefined => {
|
|
376
|
+
const o = g as Record<string, unknown>;
|
|
377
|
+
if (typeof o.id !== 'string' || typeof o.statement !== 'string') return undefined;
|
|
378
|
+
return {
|
|
379
|
+
id: o.id,
|
|
380
|
+
statement: o.statement,
|
|
381
|
+
weight: clampNum(o.weight, Number.MIN_VALUE, 1, 1),
|
|
382
|
+
keywords: Array.isArray(o.keywords) ? o.keywords.filter((k): k is string => typeof k === 'string') : [],
|
|
383
|
+
};
|
|
384
|
+
})
|
|
385
|
+
.filter((g): g is Goal => g !== undefined)
|
|
386
|
+
: [];
|
|
387
|
+
return { version: typeof parsed.version === 'number' ? parsed.version : 1, goals };
|
|
388
|
+
} catch {
|
|
389
|
+
return { version: 1, goals: [] };
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** The text embedded for a goal: statement + keywords (same convention across cache + score). */
|
|
394
|
+
export function goalEmbedText(goal: Goal): string {
|
|
395
|
+
return goal.keywords.length > 0 ? `${goal.statement} ${goal.keywords.join(' ')}` : goal.statement;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
export interface AlignmentResult {
|
|
399
|
+
readonly goalId: string | null;
|
|
400
|
+
readonly goalAlignment: number;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/** One goal with its precomputed embedding + weight — the input to the pure scorer. */
|
|
404
|
+
export interface GoalVector {
|
|
405
|
+
readonly id: string;
|
|
406
|
+
readonly vec: Float32Array;
|
|
407
|
+
readonly weight: number;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* THE LOAD-BEARING ALIGNMENT SCORER (ADR-003 T-003a) — PURE. Alignment =
|
|
412
|
+
* `max over goals of ( cosine(ideaVec, goalVec) * weight )`, clamped to [0,1]. Weighted-MAX (not
|
|
413
|
+
* sum/mean) so a focused idea advancing ONE goal strongly scores high. No goals ⇒ 0 / null.
|
|
414
|
+
*/
|
|
415
|
+
export function scoreAlignment(ideaVec: Float32Array, goals: readonly GoalVector[]): AlignmentResult {
|
|
416
|
+
let bestId: string | null = null;
|
|
417
|
+
let best = 0;
|
|
418
|
+
for (const g of goals) {
|
|
419
|
+
const raw = Math.max(0, cosineSimilarity(ideaVec, g.vec)) * g.weight;
|
|
420
|
+
if (raw > best) {
|
|
421
|
+
best = raw;
|
|
422
|
+
bestId = g.id;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
return { goalId: bestId, goalAlignment: Math.min(1, best) };
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/** Injectable embedder so the production alignment path is testable. */
|
|
429
|
+
export interface AlignDeps {
|
|
430
|
+
readonly embed?: (text: string) => Promise<Float32Array>;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Production alignment: embed the idea + each goal (same reused embedder) and score. No embedder or no
|
|
435
|
+
* GoalMap ⇒ `{goalId:null, goalAlignment:0}` — capture and roulette still work (ADR-003 T-003d).
|
|
436
|
+
*/
|
|
437
|
+
export async function alignIdea(projectRoot: string, text: string, goalMap: GoalMap, deps: AlignDeps = {}): Promise<AlignmentResult> {
|
|
438
|
+
if (goalMap.goals.length === 0) return { goalId: null, goalAlignment: 0 };
|
|
439
|
+
let embed = deps.embed;
|
|
440
|
+
if (embed === undefined) {
|
|
441
|
+
const resolved = await resolveAgentdbEmbedder(projectRoot);
|
|
442
|
+
if ('error' in resolved) return { goalId: null, goalAlignment: 0 };
|
|
443
|
+
embed = resolved.embed;
|
|
444
|
+
}
|
|
445
|
+
try {
|
|
446
|
+
const ideaVec = await embed(`${BACKLOG_TASK_TYPE}: ${text}`);
|
|
447
|
+
// LOW-10: USE the goal-embed cache (the ADR-003 design) — goal vectors are stable across captures,
|
|
448
|
+
// so re-embedding them every `add` is waste. The cache is keyed by embed model+dim (a model change
|
|
449
|
+
// invalidates it, T-003e) and by each goal's text hash. Cache use is best-effort: any failure just
|
|
450
|
+
// falls through to a live embed, never breaks alignment.
|
|
451
|
+
const model = resolveEmbedModel(projectRoot);
|
|
452
|
+
const useCache = !('error' in model);
|
|
453
|
+
const cache = useCache ? readGoalEmbedCache(projectRoot) : undefined;
|
|
454
|
+
const nextGoals: GoalEmbedCache['goals'] = {};
|
|
455
|
+
const goalVecs: GoalVector[] = [];
|
|
456
|
+
let cacheChanged = false;
|
|
457
|
+
for (const g of goalMap.goals) {
|
|
458
|
+
let vec = useCache && !('error' in model) ? goalCacheHit(cache, model.model, model.dim, g) : undefined;
|
|
459
|
+
if (vec === undefined) {
|
|
460
|
+
vec = await embed(`${BACKLOG_TASK_TYPE}: ${goalEmbedText(g)}`);
|
|
461
|
+
cacheChanged = true;
|
|
462
|
+
}
|
|
463
|
+
goalVecs.push({ id: g.id, vec, weight: g.weight });
|
|
464
|
+
if (useCache) nextGoals[g.id] = { hash: goalTextHash(g), vec: Array.from(vec) };
|
|
465
|
+
}
|
|
466
|
+
if (useCache && !('error' in model) && cacheChanged) {
|
|
467
|
+
try {
|
|
468
|
+
writeGoalEmbedCache(projectRoot, { model: model.model, dim: model.dim, goals: nextGoals });
|
|
469
|
+
} catch {
|
|
470
|
+
/* cache write is best-effort — a failure never affects the score */
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
return scoreAlignment(ideaVec, goalVecs);
|
|
474
|
+
} catch {
|
|
475
|
+
return { goalId: null, goalAlignment: 0 };
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/* ── Goal-embed cache + manifest invalidation (ADR-003 T-003e) ── */
|
|
480
|
+
|
|
481
|
+
export interface GoalEmbedCache {
|
|
482
|
+
readonly model: string;
|
|
483
|
+
readonly dim: number;
|
|
484
|
+
/** id → { hash of the embed text, vector as a plain number[] } */
|
|
485
|
+
readonly goals: Record<string, { readonly hash: string; readonly vec: number[] }>;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
function goalCachePath(projectRoot: string): string {
|
|
489
|
+
return join(backlogDir(projectRoot), 'goal-embeds.json');
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
function goalTextHash(goal: Goal): string {
|
|
493
|
+
return createHash('sha1').update(goalEmbedText(goal)).digest('hex').slice(0, 16);
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
/**
|
|
497
|
+
* PURE cache validity check (ADR-003 T-003e): a cache is valid for a goal ONLY when the embed
|
|
498
|
+
* model+dim manifest matches AND the goal's embed-text hash is unchanged. A model change (different
|
|
499
|
+
* `model`/`dim`) invalidates EVERY cached goal vector — forcing a recompute, never a stale alignment.
|
|
500
|
+
*/
|
|
501
|
+
export function goalCacheHit(cache: GoalEmbedCache | undefined, model: string, dim: number, goal: Goal): Float32Array | undefined {
|
|
502
|
+
if (cache === undefined || cache.model !== model || cache.dim !== dim) return undefined;
|
|
503
|
+
const entry = cache.goals[goal.id];
|
|
504
|
+
if (entry === undefined || entry.hash !== goalTextHash(goal)) return undefined;
|
|
505
|
+
return Float32Array.from(entry.vec);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
export function readGoalEmbedCache(projectRoot: string): GoalEmbedCache | undefined {
|
|
509
|
+
const path = goalCachePath(projectRoot);
|
|
510
|
+
if (!existsSync(path)) return undefined;
|
|
511
|
+
try {
|
|
512
|
+
const parsed = JSON.parse(readFileSync(path, 'utf-8')) as GoalEmbedCache;
|
|
513
|
+
if (typeof parsed.model !== 'string' || typeof parsed.dim !== 'number' || typeof parsed.goals !== 'object') return undefined;
|
|
514
|
+
return parsed;
|
|
515
|
+
} catch {
|
|
516
|
+
return undefined;
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
export function writeGoalEmbedCache(projectRoot: string, cache: GoalEmbedCache): void {
|
|
521
|
+
const path = goalCachePath(projectRoot);
|
|
522
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
523
|
+
writeFileSync(path, JSON.stringify(cache, null, 2));
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/* ================================================================== */
|
|
527
|
+
/* ROULETTE (AM-5 / ADR-004) — WEIGHTED, seeded, no starvation. */
|
|
528
|
+
/* ================================================================== */
|
|
529
|
+
|
|
530
|
+
/** Tiny base added to the compass term so an unaligned idea keeps a non-zero weight (no starvation). */
|
|
531
|
+
export const ROULETTE_EPSILON = 0.02;
|
|
532
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
533
|
+
|
|
534
|
+
/** Ideas eligible for a spin: only `new` and `enriched` (ADR-004). */
|
|
535
|
+
export function eligibleIdeas(ideas: readonly IdeaRecord[]): IdeaRecord[] {
|
|
536
|
+
return ideas.filter((i) => i.status === 'new' || i.status === 'enriched');
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** Bounded recency decay ∈ [floor, 1]: `2^(-ageDays/halfLife)`, floored so old ideas are down-weighted, never zero. */
|
|
540
|
+
export function recencyDecay(ageMs: number, halfLifeDays: number, floor: number): number {
|
|
541
|
+
const ageDays = Math.max(0, ageMs) / DAY_MS;
|
|
542
|
+
const decay = Math.pow(2, -ageDays / halfLifeDays);
|
|
543
|
+
return Math.max(floor, Math.min(1, decay));
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
/**
|
|
547
|
+
* THE selection weight (ADR-004): `(alignment^alpha + EPS) · recencyDecay(age) · (1/effort)`. Strictly
|
|
548
|
+
* positive for every idea (EPS>0, floor>0, effort≥1) ⇒ no permanent starvation (T-004c). Equal
|
|
549
|
+
* alignment+age+effort ⇒ equal weights ⇒ a uniform draw (the T-004a control).
|
|
550
|
+
*/
|
|
551
|
+
export function ideaWeight(idea: IdeaRecord, cfg: BacklogConfig['roulette'], nowMs: number): number {
|
|
552
|
+
// MED-6: EVERY factor is Number.isFinite-guarded so one bad field (NaN alignment, an unparseable
|
|
553
|
+
// createdTs) degrades to a neutral contribution — it can NEVER poison the whole spin's total (which
|
|
554
|
+
// would make the draw always pick the last item). A bad timestamp ⇒ neutral (floor) recency.
|
|
555
|
+
const alignment = Number.isFinite(idea.goalAlignment) ? Math.max(0, Math.min(1, idea.goalAlignment)) : 0;
|
|
556
|
+
const compass = Math.pow(alignment, cfg.alpha) + ROULETTE_EPSILON;
|
|
557
|
+
const parsed = Date.parse(idea.createdTs);
|
|
558
|
+
const age = Number.isFinite(parsed) ? nowMs - parsed : Number.POSITIVE_INFINITY; // bad ts ⇒ maximally old ⇒ floor
|
|
559
|
+
const recency = recencyDecay(age, cfg.halfLifeDays, cfg.recencyFloor);
|
|
560
|
+
const effort = Number.isFinite(idea.effort) && idea.effort >= 1 ? Math.max(1, Math.min(5, idea.effort)) : cfg.defaultEffort;
|
|
561
|
+
const w = compass * recency * (1 / effort);
|
|
562
|
+
return Number.isFinite(w) && w > 0 ? w : ROULETTE_EPSILON; // last-resort finite floor
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/** Deterministic ranked shortlist (`--pick N`): by weight desc, id asc — no RNG. */
|
|
566
|
+
export function rankRoulette(ideas: readonly IdeaRecord[], cfg: BacklogConfig['roulette'], nowMs: number): IdeaRecord[] {
|
|
567
|
+
return eligibleIdeas(ideas)
|
|
568
|
+
.map((i) => ({ i, w: ideaWeight(i, cfg, nowMs) }))
|
|
569
|
+
.sort((a, b) => b.w - a.w || (a.i.id < b.i.id ? -1 : 1))
|
|
570
|
+
.map((x) => x.i);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* A single WEIGHTED-RANDOM spin over normalised weights, using an injected seeded RNG (`rng()∈[0,1)`).
|
|
575
|
+
* Same seed ⇒ identical pick (ADR-004 T-004b determinism). Returns `undefined` when nothing is eligible.
|
|
576
|
+
*/
|
|
577
|
+
export function spinRoulette(
|
|
578
|
+
ideas: readonly IdeaRecord[],
|
|
579
|
+
cfg: BacklogConfig['roulette'],
|
|
580
|
+
rng: () => number,
|
|
581
|
+
nowMs: number,
|
|
582
|
+
): IdeaRecord | undefined {
|
|
583
|
+
const pool = eligibleIdeas(ideas);
|
|
584
|
+
if (pool.length === 0) return undefined;
|
|
585
|
+
const weights = pool.map((i) => ideaWeight(i, cfg, nowMs));
|
|
586
|
+
const total = weights.reduce((a, b) => a + b, 0);
|
|
587
|
+
if (total <= 0) return pool[0];
|
|
588
|
+
let r = rng() * total;
|
|
589
|
+
for (let k = 0; k < pool.length; k += 1) {
|
|
590
|
+
r -= weights[k]!;
|
|
591
|
+
if (r < 0) return pool[k];
|
|
592
|
+
}
|
|
593
|
+
return pool[pool.length - 1];
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/** Build a seeded RNG from a `--seed` integer (reuses the one repo mulberry32). */
|
|
597
|
+
export function seededRng(seed: number): () => number {
|
|
598
|
+
return mulberry32(Number.isFinite(seed) ? Math.floor(seed) : 0);
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
/* ================================================================== */
|
|
602
|
+
/* ENRICH (AM-7 / FR-5) — stage the idea2prd hand-off, do NOT expand. */
|
|
603
|
+
/* ================================================================== */
|
|
604
|
+
|
|
605
|
+
/** kebab-case, Latin-only, ≤40 chars (feature-adr slug convention). */
|
|
606
|
+
export function ideaSlug(idea: IdeaRecord): string {
|
|
607
|
+
const base = idea.text
|
|
608
|
+
.toLowerCase()
|
|
609
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
610
|
+
.replace(/^-+|-+$/g, '')
|
|
611
|
+
.slice(0, 40)
|
|
612
|
+
.replace(/-+$/g, '');
|
|
613
|
+
// HIGH-1: the fallback must be a filesystem-safe token, NOT the raw id — a hand-edited id like
|
|
614
|
+
// `../../../owned` would otherwise make the slug traverse out of `features/`.
|
|
615
|
+
return base === '' ? `idea-${safeIdToken(idea.id)}` : base;
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
export interface EnrichmentStaging {
|
|
619
|
+
readonly slug: string;
|
|
620
|
+
readonly scaffoldPath: string;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Stage the idea2prd INPUT and hand off to the `idea2prd-manual` skill (an AGENT phase — 02 §R-G).
|
|
625
|
+
* The CLI writes the scaffold ONLY; it never fabricates a PRD (idea2prd is a skill, not a synchronous
|
|
626
|
+
* transform). Returns the hand-off target so the agent can pick it up.
|
|
627
|
+
*/
|
|
628
|
+
export function stageEnrichment(
|
|
629
|
+
projectRoot: string,
|
|
630
|
+
idea: IdeaRecord,
|
|
631
|
+
related: readonly IdeaRecord[],
|
|
632
|
+
goalMap: GoalMap,
|
|
633
|
+
): EnrichmentStaging {
|
|
634
|
+
const slug = ideaSlug(idea);
|
|
635
|
+
const dir = join(projectRoot, 'features', slug);
|
|
636
|
+
const scaffoldPath = join(dir, 'idea2prd-input.md');
|
|
637
|
+
const goal = idea.goalId !== null ? goalMap.goals.find((g) => g.id === idea.goalId) : undefined;
|
|
638
|
+
const lines = [
|
|
639
|
+
`# idea2prd input — ${slug}`,
|
|
640
|
+
'',
|
|
641
|
+
'> STAGED by `dz backlog enrich`. This is the HAND-OFF scaffold, not a PRD. Run the',
|
|
642
|
+
'> `idea2prd-manual` skill (agent phase) to expand it — the CLI never fabricates the PRD.',
|
|
643
|
+
'',
|
|
644
|
+
'## Idea',
|
|
645
|
+
'',
|
|
646
|
+
idea.text,
|
|
647
|
+
'',
|
|
648
|
+
`- id: \`${idea.id}\``,
|
|
649
|
+
`- effort: ${idea.effort}/5`,
|
|
650
|
+
`- goal alignment: ${idea.goalAlignment.toFixed(3)}${goal !== undefined ? ` (top goal: ${goal.id} — ${goal.statement})` : ' (no aligned goal)'}`,
|
|
651
|
+
'',
|
|
652
|
+
'## Related ideas (dedup RELATED band)',
|
|
653
|
+
'',
|
|
654
|
+
related.length > 0 ? related.map((r) => `- \`${r.id}\` — ${r.text}`).join('\n') : '_none_',
|
|
655
|
+
'',
|
|
656
|
+
'## Goal context (the compass)',
|
|
657
|
+
'',
|
|
658
|
+
goalMap.goals.length > 0 ? goalMap.goals.map((g) => `- \`${g.id}\` (w=${g.weight}): ${g.statement}`).join('\n') : '_no GoalMap_',
|
|
659
|
+
'',
|
|
660
|
+
'## Verification & honesty (owned at the BACKLOG layer, per the user steer)',
|
|
661
|
+
'',
|
|
662
|
+
'- Apply claim-check discipline: every accuracy claim MEASURED with a reproducer.',
|
|
663
|
+
'- Run an adversarial red-team pass on the resulting PRD/ADRs.',
|
|
664
|
+
'- Recall Brain lessons before asserting what exists (`dz recall`).',
|
|
665
|
+
'',
|
|
666
|
+
];
|
|
667
|
+
mkdirSync(dir, { recursive: true });
|
|
668
|
+
writeFileSync(scaffoldPath, lines.join('\n'));
|
|
669
|
+
return { slug, scaffoldPath };
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
/* ================================================================== */
|
|
673
|
+
/* JIRA PORT (AM-6 / ADR-006) — closed vocab + registry + real stub. */
|
|
674
|
+
/* ⚠ NO MCP CLIENT IMPORT ANYWHERE IN THIS SEAM (grep-guard T-006b). */
|
|
675
|
+
/* ================================================================== */
|
|
676
|
+
|
|
677
|
+
export const BACKLOG_BACKENDS = ['jira-mcp', 'copilot-mcp', 'none'] as const;
|
|
678
|
+
export type BacklogBackend = (typeof BACKLOG_BACKENDS)[number];
|
|
679
|
+
|
|
680
|
+
export function isBacklogBackend(v: unknown): v is BacklogBackend {
|
|
681
|
+
return typeof v === 'string' && (BACKLOG_BACKENDS as readonly string[]).includes(v);
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/** Built by the domain from an IdeaRecord (+ enrichedPath if any). */
|
|
685
|
+
export interface JiraIssueDraft {
|
|
686
|
+
readonly summary: string;
|
|
687
|
+
readonly description: string;
|
|
688
|
+
readonly labels: readonly string[];
|
|
689
|
+
readonly sourceIdeaId: string;
|
|
690
|
+
}
|
|
691
|
+
export interface IssueRef {
|
|
692
|
+
readonly backend: BacklogBackend;
|
|
693
|
+
readonly key: string | null;
|
|
694
|
+
readonly url?: string;
|
|
695
|
+
readonly stub: boolean;
|
|
696
|
+
readonly outboxPath: string;
|
|
697
|
+
}
|
|
698
|
+
export type VerifyForm = 'full' | 'manual';
|
|
699
|
+
export interface JiraVerifyResult {
|
|
700
|
+
readonly form: VerifyForm;
|
|
701
|
+
readonly ready: boolean;
|
|
702
|
+
/** The exact wiring step the user must run — non-empty for a `manual` backend (T-006e). */
|
|
703
|
+
readonly instruction: string;
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
/** The I/O the adapters need — injected so `createIssue` is testable without touching disk. */
|
|
707
|
+
export interface BacklogIO {
|
|
708
|
+
/** Persist an outbox payload for `<ideaId>`; returns the path written. */
|
|
709
|
+
writeOutbox(ideaId: string, payload: unknown): string;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/** THE SEAM (ADR-006): a port with pure methods; NO MCP client is imported here or by its impls. */
|
|
713
|
+
export interface JiraPort {
|
|
714
|
+
readonly backend: BacklogBackend;
|
|
715
|
+
createIssue(draft: JiraIssueDraft, io: BacklogIO): Promise<IssueRef>;
|
|
716
|
+
verify(): Promise<JiraVerifyResult>;
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
/** Pure draft builder from an idea. */
|
|
720
|
+
export function buildJiraDraft(idea: IdeaRecord, goalMap: GoalMap): JiraIssueDraft {
|
|
721
|
+
const goal = idea.goalId !== null ? goalMap.goals.find((g) => g.id === idea.goalId) : undefined;
|
|
722
|
+
const descLines = [
|
|
723
|
+
idea.proposal ?? idea.text,
|
|
724
|
+
'',
|
|
725
|
+
`Source idea: ${idea.id} (effort ${idea.effort}/5, alignment ${idea.goalAlignment.toFixed(3)})`,
|
|
726
|
+
goal !== undefined ? `Top goal: ${goal.id} — ${goal.statement}` : 'No aligned goal',
|
|
727
|
+
idea.relatedIds.length > 0 ? `Related ideas: ${idea.relatedIds.join(', ')}` : '',
|
|
728
|
+
idea.enrichedPath !== undefined ? `Enriched: ${idea.enrichedPath}` : '',
|
|
729
|
+
].filter((l) => l !== '');
|
|
730
|
+
return {
|
|
731
|
+
summary: idea.text.length > 120 ? `${idea.text.slice(0, 117)}...` : idea.text,
|
|
732
|
+
description: descLines.join('\n'),
|
|
733
|
+
labels: ['dz-backlog', ...idea.tags],
|
|
734
|
+
sourceIdeaId: idea.id,
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
/** The `none` adapter — a REAL stub (not an absence check): writes the full draft to the outbox. */
|
|
739
|
+
const noneAdapter: JiraPort = {
|
|
740
|
+
backend: 'none',
|
|
741
|
+
async createIssue(draft, io) {
|
|
742
|
+
const outboxPath = io.writeOutbox(draft.sourceIdeaId, { backend: 'none', draft });
|
|
743
|
+
return { backend: 'none', key: null, stub: true, outboxPath };
|
|
744
|
+
},
|
|
745
|
+
async verify() {
|
|
746
|
+
return {
|
|
747
|
+
form: 'manual',
|
|
748
|
+
ready: true, // the stub IS ready — it writes an auditable outbox with no external wiring
|
|
749
|
+
instruction: 'The `none` backend writes .dz/backlog/jira-outbox/<id>.json — no external wiring needed.',
|
|
750
|
+
};
|
|
751
|
+
},
|
|
752
|
+
};
|
|
753
|
+
|
|
754
|
+
/** A declared-not-wired MCP seam: builds+persists the SAME outbox payload; verify() reports `manual`. */
|
|
755
|
+
function declaredMcpAdapter(backend: 'jira-mcp' | 'copilot-mcp', instruction: string): JiraPort {
|
|
756
|
+
return {
|
|
757
|
+
backend,
|
|
758
|
+
async createIssue(draft, io) {
|
|
759
|
+
// v1: build + persist the same payload; NO live MCP call (FR-8.2). Honest stub — the wire is a TODO.
|
|
760
|
+
const outboxPath = io.writeOutbox(draft.sourceIdeaId, { backend, draft, note: 'declared-not-wired seam (v1)' });
|
|
761
|
+
return { backend, key: null, stub: true, outboxPath };
|
|
762
|
+
},
|
|
763
|
+
async verify() {
|
|
764
|
+
return { form: 'manual', ready: false, instruction };
|
|
765
|
+
},
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
/** The registry — coverage-tested (ADR-006 T-006a): keys ≡ BACKLOG_BACKENDS as a set. */
|
|
770
|
+
export const JIRA_ADAPTERS: Record<BacklogBackend, JiraPort> = {
|
|
771
|
+
'jira-mcp': declaredMcpAdapter(
|
|
772
|
+
'jira-mcp',
|
|
773
|
+
'Wire a Jira MCP server: `claude mcp add jira ...` then add it to .mcp.json `mcpServers`. Live wiring is a post-v1 adapter.',
|
|
774
|
+
),
|
|
775
|
+
'copilot-mcp': declaredMcpAdapter(
|
|
776
|
+
'copilot-mcp',
|
|
777
|
+
'Wire a Copilot MCP server into .mcp.json `mcpServers`. Live wiring is a post-v1 adapter.',
|
|
778
|
+
),
|
|
779
|
+
none: noneAdapter,
|
|
780
|
+
};
|
|
781
|
+
|
|
782
|
+
/** Factory: pick the configured adapter; an unknown value fell back to `none` in readBacklogConfig. */
|
|
783
|
+
export function resolveJiraAdapter(cfg: BacklogConfig): JiraPort {
|
|
784
|
+
return JIRA_ADAPTERS[cfg.jira.adapter] ?? JIRA_ADAPTERS.none;
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
/** Production BacklogIO — writes `.dz/backlog/jira-outbox/<id>.json`. */
|
|
788
|
+
export function makeBacklogIO(projectRoot: string): BacklogIO {
|
|
789
|
+
return {
|
|
790
|
+
writeOutbox(id, payload) {
|
|
791
|
+
// HIGH-1: refuse any id that is not filesystem-safe — the filename is derived from it. A
|
|
792
|
+
// traversal id can never write outside jira-outbox/. Legit content-addressed ids always pass.
|
|
793
|
+
if (!isSafeId(id)) throw new Error(`refusing unsafe idea id for outbox path: ${JSON.stringify(id)}`);
|
|
794
|
+
const dir = jiraOutboxDir(projectRoot);
|
|
795
|
+
mkdirSync(dir, { recursive: true });
|
|
796
|
+
const path = join(dir, `${id}.json`);
|
|
797
|
+
writeFileSync(path, JSON.stringify(payload, null, 2));
|
|
798
|
+
return path;
|
|
799
|
+
},
|
|
800
|
+
};
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
/* ================================================================== */
|
|
804
|
+
/* HARMONIZE (AM-7) — batch semantic dedup of the backlog ideas. */
|
|
805
|
+
/* ================================================================== */
|
|
806
|
+
|
|
807
|
+
export interface BacklogHarmonizeReport {
|
|
808
|
+
readonly mode: 'dry-run' | 'apply';
|
|
809
|
+
/** True when no embedder was available and clustering fell back to EXACT text. */
|
|
810
|
+
readonly fellBackToExact: boolean;
|
|
811
|
+
readonly threshold: number;
|
|
812
|
+
/** One cluster per group of size ≥ 2: the surviving keeper id + the merged-away ids. */
|
|
813
|
+
readonly clusters: readonly { readonly keep: string; readonly drops: readonly string[] }[];
|
|
814
|
+
readonly kept: number;
|
|
815
|
+
readonly dropped: number;
|
|
816
|
+
readonly unique: number;
|
|
817
|
+
readonly snapshotPath?: string;
|
|
818
|
+
readonly error?: string;
|
|
819
|
+
/**
|
|
820
|
+
* MED-E: set when the structured ideas were removed but their agentdb `dz-backlog` vectors could NOT be
|
|
821
|
+
* pruned (locked/unavailable store). Backlog dedup stays correct (the membership guard drops orphans),
|
|
822
|
+
* but the store is left with dangling vectors — a non-clean outcome the caller MUST surface, never a
|
|
823
|
+
* silent success. `dz backlog harmonize` prints this as a warning; the idea removal still stands.
|
|
824
|
+
*/
|
|
825
|
+
readonly pruneError?: string;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
/** Deterministic keeper within a cluster: most uses → oldest → smallest id (stable, testable). */
|
|
829
|
+
function pickKeeper(members: readonly IdeaRecord[]): IdeaRecord {
|
|
830
|
+
return [...members].sort(
|
|
831
|
+
(a, b) => b.uses - a.uses || Date.parse(a.createdTs) - Date.parse(b.createdTs) || (a.id < b.id ? -1 : 1),
|
|
832
|
+
)[0]!;
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
/**
|
|
836
|
+
* Batch-dedup the backlog: cluster near-duplicate ideas (cosine ≥ threshold when an embedder is
|
|
837
|
+
* available, EXACT text otherwise), keep one deterministic keeper per cluster, merge the others'
|
|
838
|
+
* `uses` into it. DRY-RUN by default (writes nothing); `--apply` SNAPSHOTS FIRST then mutates and
|
|
839
|
+
* ABORTS the mutation if the snapshot fails (NFR-6). Injectable `embed` for tests.
|
|
840
|
+
*/
|
|
841
|
+
export async function harmonizeBacklog(
|
|
842
|
+
projectRoot: string,
|
|
843
|
+
opts: { apply?: boolean; threshold?: number; embed?: ((t: string) => Promise<Float32Array>) | null } = {},
|
|
844
|
+
): Promise<BacklogHarmonizeReport> {
|
|
845
|
+
const apply = opts.apply === true;
|
|
846
|
+
const threshold = opts.threshold !== undefined && opts.threshold > 0 && opts.threshold <= 1 ? opts.threshold : DEFAULT_DUPLICATE_THRESHOLD;
|
|
847
|
+
const ideas = readIdeas(projectRoot);
|
|
848
|
+
|
|
849
|
+
let embed = opts.embed;
|
|
850
|
+
if (embed === undefined) {
|
|
851
|
+
const resolved = await resolveAgentdbEmbedder(projectRoot);
|
|
852
|
+
embed = 'error' in resolved ? null : resolved.embed;
|
|
853
|
+
}
|
|
854
|
+
let fellBackToExact = embed === null;
|
|
855
|
+
|
|
856
|
+
// Build clusters (union-find over ≥threshold cosine, or exact-text groups).
|
|
857
|
+
const parent = ideas.map((_, i) => i);
|
|
858
|
+
const find = (x: number): number => {
|
|
859
|
+
while (parent[x] !== x) {
|
|
860
|
+
parent[x] = parent[parent[x]!]!;
|
|
861
|
+
x = parent[x]!;
|
|
862
|
+
}
|
|
863
|
+
return x;
|
|
864
|
+
};
|
|
865
|
+
const union = (a: number, b: number): void => {
|
|
866
|
+
parent[find(a)] = find(b);
|
|
867
|
+
};
|
|
868
|
+
if (embed !== null && ideas.length >= 2) {
|
|
869
|
+
try {
|
|
870
|
+
const vecs = await Promise.all(ideas.map((i) => embed!(`${BACKLOG_TASK_TYPE}: ${i.text}`)));
|
|
871
|
+
for (let a = 0; a < ideas.length; a += 1) {
|
|
872
|
+
for (let b = a + 1; b < ideas.length; b += 1) {
|
|
873
|
+
if (cosineSimilarity(vecs[a]!, vecs[b]!) >= threshold) union(a, b);
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
} catch {
|
|
877
|
+
fellBackToExact = true;
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
if (fellBackToExact) {
|
|
881
|
+
const byText = new Map<string, number>();
|
|
882
|
+
ideas.forEach((idea, i) => {
|
|
883
|
+
const first = byText.get(idea.text);
|
|
884
|
+
if (first === undefined) byText.set(idea.text, i);
|
|
885
|
+
else union(i, first);
|
|
886
|
+
});
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
const groups = new Map<number, IdeaRecord[]>();
|
|
890
|
+
ideas.forEach((idea, i) => {
|
|
891
|
+
const root = find(i);
|
|
892
|
+
(groups.get(root) ?? groups.set(root, []).get(root)!).push(idea);
|
|
893
|
+
});
|
|
894
|
+
|
|
895
|
+
const clusters: { keep: string; drops: string[] }[] = [];
|
|
896
|
+
const keepById = new Map<string, IdeaRecord>();
|
|
897
|
+
const dropIds = new Set<string>();
|
|
898
|
+
for (const members of groups.values()) {
|
|
899
|
+
if (members.length < 2) continue;
|
|
900
|
+
const keeper = pickKeeper(members);
|
|
901
|
+
const drops = members.filter((m) => m.id !== keeper.id);
|
|
902
|
+
clusters.push({ keep: keeper.id, drops: drops.map((d) => d.id) });
|
|
903
|
+
keepById.set(keeper.id, keeper);
|
|
904
|
+
for (const d of drops) dropIds.add(d.id);
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
const report: BacklogHarmonizeReport = {
|
|
908
|
+
mode: apply ? 'apply' : 'dry-run',
|
|
909
|
+
fellBackToExact,
|
|
910
|
+
threshold,
|
|
911
|
+
clusters,
|
|
912
|
+
kept: clusters.length,
|
|
913
|
+
dropped: dropIds.size,
|
|
914
|
+
unique: ideas.length - clusters.length - dropIds.size,
|
|
915
|
+
};
|
|
916
|
+
if (!apply || dropIds.size === 0) return report;
|
|
917
|
+
|
|
918
|
+
// --apply: snapshot FIRST, then merge uses into keepers and drop the rest.
|
|
919
|
+
const snapDest = join(backlogDir(projectRoot), `ideas.pre-harmonize-${Date.now()}.jsonl`);
|
|
920
|
+
const snap = snapshotIdeas(projectRoot, snapDest);
|
|
921
|
+
if (snap.error !== undefined) return { ...report, error: `backup failed — drop aborted: ${snap.error}` };
|
|
922
|
+
const merged = new Map<string, IdeaRecord>();
|
|
923
|
+
for (const [id, keeper] of keepById) merged.set(id, { ...keeper });
|
|
924
|
+
for (const c of clusters) {
|
|
925
|
+
const keeper = merged.get(c.keep)!;
|
|
926
|
+
keeper.uses += c.drops.length;
|
|
927
|
+
}
|
|
928
|
+
const survivors = ideas
|
|
929
|
+
.filter((i) => !dropIds.has(i.id))
|
|
930
|
+
.map((i) => merged.get(i.id) ?? i);
|
|
931
|
+
writeIdeas(projectRoot, survivors);
|
|
932
|
+
// HIGH-A: prune the agentdb `dz-backlog` vectors for every removed idea, so a later `add` can't match
|
|
933
|
+
// an ORPHAN dzId (a DUPLICATE of a nonexistent idea). Best-effort for the structured store (already
|
|
934
|
+
// snapshotted + written) — but MED-E: the prune outcome must NOT be swallowed. If it failed, the store
|
|
935
|
+
// has dangling vectors; report it (a non-clean status) so the user knows, rather than a false success.
|
|
936
|
+
const prune = deleteAgentdbByDzIds(projectRoot, [...dropIds], { taskTypes: [BACKLOG_TASK_TYPE] });
|
|
937
|
+
return {
|
|
938
|
+
...report,
|
|
939
|
+
snapshotPath: snapDest,
|
|
940
|
+
...(prune.error !== undefined
|
|
941
|
+
? { pruneError: `${dropIds.size} idea(s) removed from ideas.jsonl, but their agentdb vectors were NOT pruned: ${prune.error}` }
|
|
942
|
+
: {}),
|
|
943
|
+
};
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
/* ================================================================== */
|
|
947
|
+
/* MIRROR seam (ADR-001) — write idea vectors through the ONE seam. */
|
|
948
|
+
/* ================================================================== */
|
|
949
|
+
|
|
950
|
+
/**
|
|
951
|
+
* Mirror one idea's vector into the SHARED `.dz/agentdb.db` under `task_type:'dz-backlog'`, written
|
|
952
|
+
* DIRECTLY through `importVectorsToAgentdb` (upsert-by-dzId) — NOT the configurable vector engine.
|
|
953
|
+
* This is the load-bearing single-store guarantee (ADR-001/005): dedup ALWAYS searches agentdb, so
|
|
954
|
+
* the write must ALWAYS land in agentdb, no matter what `memory.vector.engine` says. Upsert-by-dzId
|
|
955
|
+
* makes a re-mirror idempotent (0 duplicate rows). `guardEmbedSpace` inside `importVectorsToAgentdb`
|
|
956
|
+
* covers the write (ADR-001 T-001c). Best-effort: NEVER blocks capture (I-1) — honest `{error}`.
|
|
957
|
+
*/
|
|
958
|
+
export async function mirrorIdeaVector(projectRoot: string, idea: IdeaRecord): Promise<{ mirrored: number; error?: string | undefined }> {
|
|
959
|
+
const emb = await resolveAgentdbEmbedder(projectRoot);
|
|
960
|
+
if ('error' in emb) return { mirrored: 0, error: emb.error };
|
|
961
|
+
let vector: Float32Array;
|
|
962
|
+
try {
|
|
963
|
+
vector = await emb.embed(`${BACKLOG_TASK_TYPE}: ${idea.text}`); // SAME embed form the dedup query uses
|
|
964
|
+
} catch (err) {
|
|
965
|
+
return { mirrored: 0, error: `embed failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
966
|
+
}
|
|
967
|
+
const res = await importVectorsToAgentdb(projectRoot, [
|
|
968
|
+
{
|
|
969
|
+
dzId: idea.id,
|
|
970
|
+
vector,
|
|
971
|
+
text: idea.text,
|
|
972
|
+
taskType: BACKLOG_TASK_TYPE,
|
|
973
|
+
score: Math.max(0, Math.min(1, Number.isFinite(idea.goalAlignment) ? idea.goalAlignment : 0)),
|
|
974
|
+
metadata: { kind: 'dz-backlog-idea' },
|
|
975
|
+
},
|
|
976
|
+
]);
|
|
977
|
+
return { mirrored: res.imported, ...(res.error !== undefined ? { error: res.error } : {}) };
|
|
978
|
+
}
|