@dzhechkov/harness-core 0.3.78 → 0.3.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agentdb-index.d.ts +35 -0
- package/dist/agentdb-index.d.ts.map +1 -1
- package/dist/agentdb-index.js +67 -2
- package/dist/agentdb-index.js.map +1 -1
- package/dist/index.d.ts +6 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/patterns.d.ts +32 -0
- package/dist/patterns.d.ts.map +1 -1
- package/dist/patterns.js +0 -0
- package/dist/patterns.js.map +1 -1
- package/dist/vector-tier.d.ts +139 -0
- package/dist/vector-tier.d.ts.map +1 -1
- package/dist/vector-tier.js +400 -3
- package/dist/vector-tier.js.map +1 -1
- package/package.json +4 -4
- package/src/agentdb-index.ts +100 -2
- package/src/index.ts +16 -4
- package/src/patterns.ts +0 -0
- package/src/vector-tier.ts +503 -2
package/dist/vector-tier.d.ts
CHANGED
|
@@ -68,6 +68,15 @@ export interface MirrorReceipt {
|
|
|
68
68
|
readonly engine?: VectorEngineKind | undefined;
|
|
69
69
|
readonly error?: string | undefined;
|
|
70
70
|
}
|
|
71
|
+
/** One precomputed vector to upsert by its content-addressed `dzId` (the `dz vector import` row). */
|
|
72
|
+
export interface ImportVectorRow {
|
|
73
|
+
readonly dzId: string;
|
|
74
|
+
readonly vector: Float32Array;
|
|
75
|
+
readonly text: string;
|
|
76
|
+
readonly taskType: string;
|
|
77
|
+
readonly score: number;
|
|
78
|
+
readonly metadata?: Record<string, unknown> | undefined;
|
|
79
|
+
}
|
|
71
80
|
/** The engine PORT — both adapters implement exactly this surface (04 §4.1). */
|
|
72
81
|
export interface VectorEngine {
|
|
73
82
|
readonly kind: VectorEngineKind;
|
|
@@ -87,6 +96,15 @@ export interface VectorEngine {
|
|
|
87
96
|
exportCheckpoint?(dest: string): Promise<{
|
|
88
97
|
error?: string | undefined;
|
|
89
98
|
}>;
|
|
99
|
+
/**
|
|
100
|
+
* Write precomputed `{ dzId, vector }` rows by id (`dz vector import`) — UPSERT-BY-dzId, never a
|
|
101
|
+
* blind whole-store overwrite. Optional (like {@link VectorEngine.exportCheckpoint}): an engine
|
|
102
|
+
* that cannot take a precomputed vector reports an honest reason; import degrades, never throws.
|
|
103
|
+
*/
|
|
104
|
+
importVectors?(rows: readonly ImportVectorRow[]): Promise<{
|
|
105
|
+
imported: number;
|
|
106
|
+
error?: string | undefined;
|
|
107
|
+
}>;
|
|
90
108
|
}
|
|
91
109
|
/** Outcome of {@link resolveVectorEngine}: an engine, or an honest reason why not. */
|
|
92
110
|
export interface ResolvedVectorEngine {
|
|
@@ -125,6 +143,85 @@ export interface VectorTierStatus {
|
|
|
125
143
|
}
|
|
126
144
|
/** Wall-time bound applied to EVERY engine call, read and write legs alike (ADR R1 + NC1). */
|
|
127
145
|
export declare const DEFAULT_VECTOR_TIMEOUT_MS = 10000;
|
|
146
|
+
/** Default cosine cutoff for near-duplicate clustering (`--threshold` / config overrides). */
|
|
147
|
+
export declare const DEFAULT_HARMONIZE_THRESHOLD = 0.92;
|
|
148
|
+
/** One record in the harmonize pool — a lexical-store record mapped to its dzId + reward + ts. */
|
|
149
|
+
export interface HarmonizeItem {
|
|
150
|
+
readonly dzId: string;
|
|
151
|
+
readonly text: string;
|
|
152
|
+
readonly reward: number;
|
|
153
|
+
readonly ts: string;
|
|
154
|
+
readonly taskType: string;
|
|
155
|
+
}
|
|
156
|
+
/** One near-duplicate cluster: the surviving keeper + the members that would be / were dropped. */
|
|
157
|
+
export interface HarmonizeCluster {
|
|
158
|
+
readonly keep: {
|
|
159
|
+
readonly dzId: string;
|
|
160
|
+
readonly text: string;
|
|
161
|
+
readonly reward: number;
|
|
162
|
+
readonly ts: string;
|
|
163
|
+
};
|
|
164
|
+
readonly drops: readonly {
|
|
165
|
+
readonly dzId: string;
|
|
166
|
+
readonly text: string;
|
|
167
|
+
readonly reward: number;
|
|
168
|
+
readonly cos: number;
|
|
169
|
+
}[];
|
|
170
|
+
}
|
|
171
|
+
/** Outcome of {@link harmonizeVectorStore}. */
|
|
172
|
+
export interface HarmonizeReport {
|
|
173
|
+
readonly mode: 'dry-run' | 'apply';
|
|
174
|
+
/** Resolved engine kind, or `'none'` when there is no engine. */
|
|
175
|
+
readonly engine: string;
|
|
176
|
+
/** True when semantic dedup was unavailable and the store was harmonized by EXACT text only. */
|
|
177
|
+
readonly fellBackToExact: boolean;
|
|
178
|
+
readonly threshold: number;
|
|
179
|
+
readonly clusters: readonly HarmonizeCluster[];
|
|
180
|
+
/** Number of clusters (size ≥ 2) — one keeper survives per cluster. */
|
|
181
|
+
readonly kept: number;
|
|
182
|
+
/** Total non-keeper members (previewed in dry-run, removed on `--apply`). */
|
|
183
|
+
readonly dropped: number;
|
|
184
|
+
/** Singleton (non-duplicate) patterns — NEVER touched. */
|
|
185
|
+
readonly unique: number;
|
|
186
|
+
/** Backup path written before an `--apply` drop (restorable via `dz teach --from-json`). */
|
|
187
|
+
readonly backupPath?: string | undefined;
|
|
188
|
+
/** Honest reason on failure (e.g. a backup write failed and the drop was aborted). */
|
|
189
|
+
readonly error?: string | undefined;
|
|
190
|
+
}
|
|
191
|
+
/** Options for {@link harmonizeVectorStore}. */
|
|
192
|
+
export interface HarmonizeOptions extends VectorServiceOptions {
|
|
193
|
+
/** Perform the drop (default `false` — dry-run previews and writes nothing). */
|
|
194
|
+
readonly apply?: boolean | undefined;
|
|
195
|
+
/** Cosine cutoff in `(0, 1]`; overrides config + the {@link DEFAULT_HARMONIZE_THRESHOLD} default. */
|
|
196
|
+
readonly threshold?: number | undefined;
|
|
197
|
+
/**
|
|
198
|
+
* Inject an embedder (tests): a function ⇒ semantic path with these embeddings; `null` ⇒ force the
|
|
199
|
+
* exact-text fallback; `undefined` ⇒ resolve the project's agentdb embedder.
|
|
200
|
+
*/
|
|
201
|
+
readonly embed?: ((text: string) => Promise<Float32Array>) | null | undefined;
|
|
202
|
+
}
|
|
203
|
+
/** Outcome of {@link importRvfCheckpoint}. */
|
|
204
|
+
export interface ImportReport {
|
|
205
|
+
/** Vectors upserted by dzId (new + replaced). */
|
|
206
|
+
readonly imported: number;
|
|
207
|
+
/** Source dzIds skipped because no local pattern exists (text must be imported first). */
|
|
208
|
+
readonly skippedOrphans: number;
|
|
209
|
+
/** Resolved target engine kind, or `'none'`. */
|
|
210
|
+
readonly engine: string;
|
|
211
|
+
/** The source `.rvf` path. */
|
|
212
|
+
readonly source: string;
|
|
213
|
+
readonly error?: string | undefined;
|
|
214
|
+
}
|
|
215
|
+
/** Options for {@link importRvfCheckpoint}. */
|
|
216
|
+
export interface ImportOptions extends VectorServiceOptions {
|
|
217
|
+
/** Inject the source `{ dzId, vector }` rows (tests) — bypasses the `.rvf`/idmap file reads. */
|
|
218
|
+
readonly sourceRows?: readonly {
|
|
219
|
+
readonly dzId: string;
|
|
220
|
+
readonly vector: Float32Array;
|
|
221
|
+
}[] | undefined;
|
|
222
|
+
/** Inject an embedder (tests) for the local-text re-embed; else the project's agentdb embedder. */
|
|
223
|
+
readonly embed?: ((text: string) => Promise<Float32Array>) | undefined;
|
|
224
|
+
}
|
|
128
225
|
/**
|
|
129
226
|
* Bound `promise` to `ms` wall-clock milliseconds. On timeout, resolve with `onTimeout()`
|
|
130
227
|
* instead — the underlying operation keeps running detached (its eventual write is later
|
|
@@ -150,6 +247,11 @@ export declare function dreamVectorEntry(d: DreamPattern): VectorEntry | undefin
|
|
|
150
247
|
export declare function memoryRecordVectorEntry(r: MemoryRecord): VectorEntry | undefined;
|
|
151
248
|
/** Read `memory.vector.engine` from `.dz/config.json`. Absent/corrupt ⇒ `auto` (never throws). */
|
|
152
249
|
export declare function readVectorEngineMode(projectRoot: string): VectorEngineMode;
|
|
250
|
+
/**
|
|
251
|
+
* Read `memory.vector.harmonizeThreshold` from `.dz/config.json`. Absent/corrupt/out-of-range ⇒
|
|
252
|
+
* {@link DEFAULT_HARMONIZE_THRESHOLD} (never throws). `--threshold` overrides this at the call site.
|
|
253
|
+
*/
|
|
254
|
+
export declare function readHarmonizeThreshold(projectRoot: string): number;
|
|
153
255
|
/**
|
|
154
256
|
* Should `dz teach` attempt the best-effort vector mirror at all? True when the project opted
|
|
155
257
|
* into the agentdb memory backend (`memory.backend === 'agentdb'`, the same gate consolidate
|
|
@@ -217,6 +319,43 @@ export declare function recallHybrid(projectRoot: string, query: string, opts?:
|
|
|
217
319
|
}): Promise<HybridRecall>;
|
|
218
320
|
/** Field observability: engine availability + mirrored-vs-lexical counts + queue size. */
|
|
219
321
|
export declare function vectorTierStatus(projectRoot: string, opts?: VectorServiceOptions): Promise<VectorTierStatus>;
|
|
322
|
+
/**
|
|
323
|
+
* Deterministic keeper INDEX within a near-dup cluster (a TOTAL order over fixed inputs — NFR-7):
|
|
324
|
+
* (1) highest reward → (2) longer / more-specific text → (3) newer `ts` → (4) `dzId` (stable
|
|
325
|
+
* final tiebreak). Pure — no I/O. The keeper survives; the other members are the drop set.
|
|
326
|
+
*/
|
|
327
|
+
export declare function selectClusterKeeper(members: readonly HarmonizeItem[]): number;
|
|
328
|
+
/**
|
|
329
|
+
* SEMANTIC dedup of the learned-pattern store (`dz vector harmonize` / `dz teach --harmonize`) —
|
|
330
|
+
* **NON-DESTRUCTIVE by contract**. Finds near-duplicate PAIRS via pairwise cosine over the embedder
|
|
331
|
+
* both adapters share (θ default {@link DEFAULT_HARMONIZE_THRESHOLD}), union-finds them into clusters,
|
|
332
|
+
* and within each cluster KEEPs the highest-signal member ({@link selectClusterKeeper}), dropping the
|
|
333
|
+
* rest. Modes:
|
|
334
|
+
*
|
|
335
|
+
* - **dry-run (default)**: previews the clusters and returns — writes NOTHING (the store is
|
|
336
|
+
* byte-identical after).
|
|
337
|
+
* - **`--apply`**: writes a restorable backup FIRST (`.dz/memory/patterns.pre-harmonize.json`); a
|
|
338
|
+
* failed backup ABORTS the drop (no partial mutation). Then removes the non-keepers from BOTH
|
|
339
|
+
* lexical tiers via {@link removePatternsByIds}. A UNIQUE (singleton) pattern is NEVER a drop.
|
|
340
|
+
*
|
|
341
|
+
* Degrades honestly: with no engine/embedder it falls back to EXACT-text dedup + a `fellBackToExact`
|
|
342
|
+
* note, exits without throwing (dry-run still writes nothing). Reversal: `dz teach --from-json <backup>`.
|
|
343
|
+
*/
|
|
344
|
+
export declare function harmonizeVectorStore(projectRoot: string, opts?: HarmonizeOptions): Promise<HarmonizeReport>;
|
|
345
|
+
/**
|
|
346
|
+
* Ingest an external `.rvf` checkpoint's vectors into THIS project's vector store, **UPSERT-BY-dzId,
|
|
347
|
+
* NON-DESTRUCTIVE** (`dz vector import <file.rvf>`). The `.idmap.json` sidecar is the dzId authority
|
|
348
|
+
* (the shipped `@ruvector/rvf` SDK exposes no vector read-out — see rUv `rvf-backend-blocker.md`), so
|
|
349
|
+
* for each checkpoint dzId that exists in the LOCAL lexical store the vector is reproduced by
|
|
350
|
+
* re-embedding the local text (D7 — under the manifest guard the same model over the same text yields
|
|
351
|
+
* the checkpoint's vector) and upserted by dzId via {@link VectorEngine.importVectors}. dzIds absent
|
|
352
|
+
* locally are ORPHANS — skipped + counted (their text must be imported first via `dz teach --from-json`).
|
|
353
|
+
*
|
|
354
|
+
* Non-destructive: only the imported dzIds are inserted/replaced; re-importing the same file adds 0
|
|
355
|
+
* duplicates and deletes nothing. A model/dim manifest mismatch is REFUSED (no cross-space merge). All
|
|
356
|
+
* failure modes return an honest `{ error }`, never a throw.
|
|
357
|
+
*/
|
|
358
|
+
export declare function importRvfCheckpoint(projectRoot: string, source: string, opts?: ImportOptions): Promise<ImportReport>;
|
|
220
359
|
interface RvfStoreHandle {
|
|
221
360
|
ingest: (id: string, vec: Float32Array) => Promise<unknown> | unknown;
|
|
222
361
|
query: (vec: Float32Array, k: number) => Promise<unknown> | unknown;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-tier.d.ts","sourceRoot":"","sources":["../src/vector-tier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAQH,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEpE,OAAO,
|
|
1
|
+
{"version":3,"file":"vector-tier.d.ts","sourceRoot":"","sources":["../src/vector-tier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAQH,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEpE,OAAO,EASL,KAAK,aAAa,EAClB,KAAK,SAAS,EACf,MAAM,eAAe,CAAC;AAcvB,0CAA0C;AAC1C,MAAM,MAAM,gBAAgB,GAAG,SAAS,GAAG,KAAK,CAAC;AAEjD,wFAAwF;AACxF,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,SAAS,GAAG,KAAK,GAAG,KAAK,CAAC;AAElE,4FAA4F;AAC5F,MAAM,WAAW,WAAW;IAC1B,kGAAkG;IAClG,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,4DAA4D;IAC5D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qGAAqG;IACrG,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAC;IAC9C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CACzD;AAED,6FAA6F;AAC7F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC;AAED,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B,2DAA2D;IAC3D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,oFAAoF;IACpF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,0FAA0F;IAC1F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;IAC/C,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC;AAED,qGAAqG;AACrG,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CACzD;AAED,gFAAgF;AAChF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,MAAM,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;IAClG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;IACjG,OAAO,IAAI,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;IAClE,+EAA+E;IAC/E,gBAAgB,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;IACzE;;;;OAIG;IACH,aAAa,CAAC,CAAC,IAAI,EAAE,SAAS,eAAe,EAAE,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC;CAC7G;AAED,sFAAsF;AACtF,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IAC3C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC;AAED,iHAAiH;AACjH,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,UAAU,GAAG,SAAS,CAAC;AAEjE,+FAA+F;AAC/F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;IACvC,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,qGAAqG;AACrG,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;IAC3B,QAAQ,CAAC,cAAc,EAAE,QAAQ,GAAG,MAAM,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,gBAAgB,GAAG,MAAM,CAAC;IACjD,0EAA0E;IAC1E,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,6FAA6F;IAC7F,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C;AAED,yEAAyE;AACzE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,IAAI,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;IAC7C,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,8FAA8F;AAC9F,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAMhD,8FAA8F;AAC9F,eAAO,MAAM,2BAA2B,OAAO,CAAC;AAMhD,kGAAkG;AAClG,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,mGAAmG;AACnG,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9G,QAAQ,CAAC,KAAK,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC5H;AAED,+CAA+C;AAC/C,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC;IACnC,iEAAiE;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,gGAAgG;IAChG,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAC/C,uEAAuE;IACvE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,4FAA4F;IAC5F,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,sFAAsF;IACtF,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC;AAED,gDAAgD;AAChD,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB;IAC5D,gFAAgF;IAChF,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACrC,qGAAqG;IACrG,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;;OAGG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;CAC/E;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,iDAAiD;IACjD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,0FAA0F;IAC1F,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,gDAAgD;IAChD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC;AAED,+CAA+C;AAC/C,MAAM,WAAW,aAAc,SAAQ,oBAAoB;IACzD,gGAAgG;IAChG,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAA;KAAE,EAAE,GAAG,SAAS,CAAC;IACtG,mGAAmG;IACnG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC,GAAG,SAAS,CAAC;CACxE;AAMD;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAa1G;AAsBD,+FAA+F;AAC/F,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEnD;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,SAAa,GAAG,WAAW,GAAG,SAAS,CAWjG;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,GAAG,WAAW,GAAG,SAAS,CAWzE;AAED,gGAAgG;AAChG,wBAAgB,uBAAuB,CAAC,CAAC,EAAE,YAAY,GAAG,WAAW,GAAG,SAAS,CAUhF;AAMD,kGAAkG;AAClG,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,gBAAgB,CAU1E;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAUlE;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAWhE;AAyBD,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG,oBAAoB,CAiB7E;AAuED,2FAA2F;AAC3F,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,IAAI,GAAG,SAAS,CAAC;IAClD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC;AAQD;;;;;;;;;;;;GAYG;AACH,wBAAsB,qBAAqB,CACzC,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,IAAI,GAAE,oBAAyB,GAC9B,OAAO,CAAC,aAAa,CAAC,CA8DxB;AAED,iGAAiG;AACjG,wBAAsB,sBAAsB,CAC1C,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,SAAS,aAAa,EAAE,EAClC,MAAM,SAAa,EACnB,IAAI,GAAE,oBAAyB,GAC9B,OAAO,CAAC,aAAa,CAAC,CAUxB;AAED;;;;;GAKG;AACH,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,MAAM,EACnB,IAAI,GAAE,oBAAoB,GAAG;IAAE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAAO,GAC7E,OAAO,CAAC,aAAa,CAAC,CA+BxB;AAMD,8FAA8F;AAC9F,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;CACxC;AAID;;;;;GAKG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,SAAS,aAAa,EAAE,EACjC,QAAQ,EAAE,SAAS,aAAa,EAAE,EAClC,IAAI,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,GAC7E,SAAS,EAAE,CAwBb;AAED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,oBAAoB,GAAG;IAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAA;CAAO,GACtH,OAAO,CAAC,YAAY,CAAC,CA2EvB;AAMD,0FAA0F;AAC1F,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,IAAI,GAAE,oBAAyB,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAqC3B;AAmBD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,GAAG,MAAM,CAM7E;AA+FD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAmGrH;AAMD;;;;;;;;;;;;GAYG;AACH,wBAAsB,mBAAmB,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CAuF9H;AAyFD,UAAU,cAAc;IACtB,MAAM,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,YAAY,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IACtE,KAAK,EAAE,CAAC,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IACpE,KAAK,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,SAAS,CAAC;IACjD,gBAAgB,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC;CAC/E;AAED;;;;;;;;GAQG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA4B9I"}
|
package/dist/vector-tier.js
CHANGED
|
@@ -32,14 +32,19 @@
|
|
|
32
32
|
* @packageDocumentation
|
|
33
33
|
*/
|
|
34
34
|
import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, renameSync, appendFileSync, copyFileSync } from 'node:fs';
|
|
35
|
-
import { dirname, join } from 'node:path';
|
|
35
|
+
import { basename, dirname, join } from 'node:path';
|
|
36
36
|
import { pathToFileURL } from 'node:url';
|
|
37
37
|
import { createRequire } from 'node:module';
|
|
38
38
|
import { isNoiseInsight } from '@dzhechkov/memory';
|
|
39
|
-
import { recallPatterns, recordToPattern, patternRecordId, patternIdentityOf, dreamRecordId, loadStoreRecords, } from './patterns.js';
|
|
40
|
-
import { indexPatternsToAgentdb, searchAgentdbPatterns, listAgentdbDzIds, resolveAgentdbEmbedder } from './agentdb-index.js';
|
|
39
|
+
import { recallPatterns, recordToPattern, patternRecordId, patternIdentityOf, dreamRecordId, loadStoreRecords, removePatternsByIds, snapshotStore, } from './patterns.js';
|
|
40
|
+
import { indexPatternsToAgentdb, searchAgentdbPatterns, listAgentdbDzIds, resolveAgentdbEmbedder, cosineSimilarity, importVectorsToAgentdb, } from './agentdb-index.js';
|
|
41
41
|
/** Wall-time bound applied to EVERY engine call, read and write legs alike (ADR R1 + NC1). */
|
|
42
42
|
export const DEFAULT_VECTOR_TIMEOUT_MS = 10_000;
|
|
43
|
+
/** The pinned local embedding model + dimension (agentdb's `EmbeddingService`; the RVF manifest space). */
|
|
44
|
+
const LOCAL_EMBED_MODEL = 'Xenova/all-MiniLM-L6-v2';
|
|
45
|
+
const LOCAL_EMBED_DIM = 384;
|
|
46
|
+
/** Default cosine cutoff for near-duplicate clustering (`--threshold` / config overrides). */
|
|
47
|
+
export const DEFAULT_HARMONIZE_THRESHOLD = 0.92;
|
|
43
48
|
/* ------------------------------------------------------------------ */
|
|
44
49
|
/* Timeout wrapper (both legs — NC1/QR-1) */
|
|
45
50
|
/* ------------------------------------------------------------------ */
|
|
@@ -150,6 +155,20 @@ export function readVectorEngineMode(projectRoot) {
|
|
|
150
155
|
return 'auto';
|
|
151
156
|
}
|
|
152
157
|
}
|
|
158
|
+
/**
|
|
159
|
+
* Read `memory.vector.harmonizeThreshold` from `.dz/config.json`. Absent/corrupt/out-of-range ⇒
|
|
160
|
+
* {@link DEFAULT_HARMONIZE_THRESHOLD} (never throws). `--threshold` overrides this at the call site.
|
|
161
|
+
*/
|
|
162
|
+
export function readHarmonizeThreshold(projectRoot) {
|
|
163
|
+
try {
|
|
164
|
+
const cfg = JSON.parse(readFileSync(join(projectRoot, '.dz', 'config.json'), 'utf-8'));
|
|
165
|
+
const t = cfg.memory?.vector?.harmonizeThreshold;
|
|
166
|
+
return typeof t === 'number' && t > 0 && t <= 1 ? t : DEFAULT_HARMONIZE_THRESHOLD;
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
return DEFAULT_HARMONIZE_THRESHOLD;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
153
172
|
/**
|
|
154
173
|
* Should `dz teach` attempt the best-effort vector mirror at all? True when the project opted
|
|
155
174
|
* into the agentdb memory backend (`memory.backend === 'agentdb'`, the same gate consolidate
|
|
@@ -546,6 +565,351 @@ export async function vectorTierStatus(projectRoot, opts = {}) {
|
|
|
546
565
|
};
|
|
547
566
|
}
|
|
548
567
|
/* ------------------------------------------------------------------ */
|
|
568
|
+
/* Harmonize — SEMANTIC dedup of the lexical store (05 §2.1) */
|
|
569
|
+
/* ------------------------------------------------------------------ */
|
|
570
|
+
/** Bounded, honest embed of one text — a throw/timeout surfaces as `{ error }`, never propagates. */
|
|
571
|
+
async function boundedEmbed(embed, text, timeoutMs) {
|
|
572
|
+
return withVectorTimeout(safeEngineCall(() => embed(text), (m) => ({ error: m })), timeoutMs, () => ({ error: 'embed timed out' }));
|
|
573
|
+
}
|
|
574
|
+
/**
|
|
575
|
+
* Deterministic keeper INDEX within a near-dup cluster (a TOTAL order over fixed inputs — NFR-7):
|
|
576
|
+
* (1) highest reward → (2) longer / more-specific text → (3) newer `ts` → (4) `dzId` (stable
|
|
577
|
+
* final tiebreak). Pure — no I/O. The keeper survives; the other members are the drop set.
|
|
578
|
+
*/
|
|
579
|
+
export function selectClusterKeeper(members) {
|
|
580
|
+
let best = 0;
|
|
581
|
+
for (let i = 1; i < members.length; i += 1) {
|
|
582
|
+
if (isBetterKeeper(members[i], members[best]))
|
|
583
|
+
best = i;
|
|
584
|
+
}
|
|
585
|
+
return best;
|
|
586
|
+
}
|
|
587
|
+
function isBetterKeeper(a, b) {
|
|
588
|
+
if (a.reward !== b.reward)
|
|
589
|
+
return a.reward > b.reward; // (1) highest reward
|
|
590
|
+
if (a.text.length !== b.text.length)
|
|
591
|
+
return a.text.length > b.text.length; // (2) longer / more specific
|
|
592
|
+
if (a.ts !== b.ts)
|
|
593
|
+
return a.ts > b.ts; // (3) newer
|
|
594
|
+
return a.dzId < b.dzId; // (4) stable, deterministic final tiebreak
|
|
595
|
+
}
|
|
596
|
+
/** Connected components over undirected `edges` (union-find) — transitive clusters (A~B,B~C ⇒ {A,B,C}). */
|
|
597
|
+
function connectedComponents(n, edges) {
|
|
598
|
+
const parent = Array.from({ length: n }, (_, i) => i);
|
|
599
|
+
const find = (x) => {
|
|
600
|
+
let r = x;
|
|
601
|
+
while (parent[r] !== r)
|
|
602
|
+
r = parent[r];
|
|
603
|
+
while (parent[x] !== r) {
|
|
604
|
+
const next = parent[x];
|
|
605
|
+
parent[x] = r;
|
|
606
|
+
x = next;
|
|
607
|
+
}
|
|
608
|
+
return r;
|
|
609
|
+
};
|
|
610
|
+
for (const [a, b] of edges) {
|
|
611
|
+
const ra = find(a);
|
|
612
|
+
const rb = find(b);
|
|
613
|
+
if (ra !== rb)
|
|
614
|
+
parent[ra] = rb;
|
|
615
|
+
}
|
|
616
|
+
const groups = new Map();
|
|
617
|
+
for (let i = 0; i < n; i += 1) {
|
|
618
|
+
const r = find(i);
|
|
619
|
+
const g = groups.get(r);
|
|
620
|
+
if (g === undefined)
|
|
621
|
+
groups.set(r, [i]);
|
|
622
|
+
else
|
|
623
|
+
g.push(i);
|
|
624
|
+
}
|
|
625
|
+
return [...groups.values()];
|
|
626
|
+
}
|
|
627
|
+
/** Build a {@link HarmonizeCluster} from a component's item indices + keeper's cosine to each drop. */
|
|
628
|
+
function buildCluster(items, indices, cosToKeeper) {
|
|
629
|
+
const members = indices.map((i) => items[i]);
|
|
630
|
+
const keeperIdx = indices[selectClusterKeeper(members)];
|
|
631
|
+
const keeper = items[keeperIdx];
|
|
632
|
+
const drops = indices
|
|
633
|
+
.filter((i) => i !== keeperIdx)
|
|
634
|
+
.map((i) => ({ dzId: items[i].dzId, text: items[i].text, reward: items[i].reward, cos: cosToKeeper(i, keeperIdx) }));
|
|
635
|
+
return { keep: { dzId: keeper.dzId, text: keeper.text, reward: keeper.reward, ts: keeper.ts }, drops };
|
|
636
|
+
}
|
|
637
|
+
/** Semantic clusters: pairwise cosine ≥ θ (i<j) ⇒ union-find edge; components of size ≥ 2 are clusters. */
|
|
638
|
+
function semanticClusters(items, vecs, threshold) {
|
|
639
|
+
const n = items.length;
|
|
640
|
+
const edges = [];
|
|
641
|
+
for (let i = 0; i < n; i += 1) {
|
|
642
|
+
for (let j = i + 1; j < n; j += 1) {
|
|
643
|
+
if (cosineSimilarity(vecs[i], vecs[j]) >= threshold)
|
|
644
|
+
edges.push([i, j]);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
const clusters = [];
|
|
648
|
+
for (const comp of connectedComponents(n, edges)) {
|
|
649
|
+
if (comp.length < 2)
|
|
650
|
+
continue;
|
|
651
|
+
clusters.push(buildCluster(items, comp, (d, k) => cosineSimilarity(vecs[d], vecs[k])));
|
|
652
|
+
}
|
|
653
|
+
return clusters;
|
|
654
|
+
}
|
|
655
|
+
/** Exact-text fallback: group by RAW pattern text (the identity `teach --from-json` dedups on); cos = 1.0. */
|
|
656
|
+
function exactClusters(items) {
|
|
657
|
+
const byText = new Map();
|
|
658
|
+
items.forEach((it, i) => {
|
|
659
|
+
const g = byText.get(it.text);
|
|
660
|
+
if (g === undefined)
|
|
661
|
+
byText.set(it.text, [i]);
|
|
662
|
+
else
|
|
663
|
+
g.push(i);
|
|
664
|
+
});
|
|
665
|
+
const clusters = [];
|
|
666
|
+
for (const indices of byText.values()) {
|
|
667
|
+
if (indices.length < 2)
|
|
668
|
+
continue;
|
|
669
|
+
clusters.push(buildCluster(items, indices, () => 1.0));
|
|
670
|
+
}
|
|
671
|
+
return clusters;
|
|
672
|
+
}
|
|
673
|
+
/** Honest note next to the session telemetry (mirrors {@link logMirrorNote}). */
|
|
674
|
+
function logHarmonizeNote(projectRoot, info) {
|
|
675
|
+
try {
|
|
676
|
+
appendFileSync(join(projectRoot, '.dz', 'sessions.jsonl'), JSON.stringify({ event: 'harmonize', ts: new Date().toISOString(), ...info }) + '\n');
|
|
677
|
+
}
|
|
678
|
+
catch { /* best-effort */ }
|
|
679
|
+
}
|
|
680
|
+
/**
|
|
681
|
+
* SEMANTIC dedup of the learned-pattern store (`dz vector harmonize` / `dz teach --harmonize`) —
|
|
682
|
+
* **NON-DESTRUCTIVE by contract**. Finds near-duplicate PAIRS via pairwise cosine over the embedder
|
|
683
|
+
* both adapters share (θ default {@link DEFAULT_HARMONIZE_THRESHOLD}), union-finds them into clusters,
|
|
684
|
+
* and within each cluster KEEPs the highest-signal member ({@link selectClusterKeeper}), dropping the
|
|
685
|
+
* rest. Modes:
|
|
686
|
+
*
|
|
687
|
+
* - **dry-run (default)**: previews the clusters and returns — writes NOTHING (the store is
|
|
688
|
+
* byte-identical after).
|
|
689
|
+
* - **`--apply`**: writes a restorable backup FIRST (`.dz/memory/patterns.pre-harmonize.json`); a
|
|
690
|
+
* failed backup ABORTS the drop (no partial mutation). Then removes the non-keepers from BOTH
|
|
691
|
+
* lexical tiers via {@link removePatternsByIds}. A UNIQUE (singleton) pattern is NEVER a drop.
|
|
692
|
+
*
|
|
693
|
+
* Degrades honestly: with no engine/embedder it falls back to EXACT-text dedup + a `fellBackToExact`
|
|
694
|
+
* note, exits without throwing (dry-run still writes nothing). Reversal: `dz teach --from-json <backup>`.
|
|
695
|
+
*/
|
|
696
|
+
export async function harmonizeVectorStore(projectRoot, opts = {}) {
|
|
697
|
+
const apply = opts.apply === true;
|
|
698
|
+
const threshold = opts.threshold !== undefined && opts.threshold > 0 && opts.threshold <= 1
|
|
699
|
+
? opts.threshold
|
|
700
|
+
: readHarmonizeThreshold(projectRoot);
|
|
701
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS;
|
|
702
|
+
// 1. LOAD the pool from the lexical source of truth (id = dzId).
|
|
703
|
+
let records;
|
|
704
|
+
try {
|
|
705
|
+
records = loadStoreRecords(projectRoot);
|
|
706
|
+
}
|
|
707
|
+
catch {
|
|
708
|
+
records = [];
|
|
709
|
+
}
|
|
710
|
+
const items = records.map((r) => ({
|
|
711
|
+
dzId: r.id,
|
|
712
|
+
text: r.text,
|
|
713
|
+
reward: r.score,
|
|
714
|
+
ts: r.timestamp,
|
|
715
|
+
taskType: r.id.startsWith('dream:') ? 'dz-learning' : 'dz-teach',
|
|
716
|
+
}));
|
|
717
|
+
// 2. GATE: an embedder ⇒ SEMANTIC clustering; absence/failure ⇒ EXACT-text fallback (D4).
|
|
718
|
+
let embed;
|
|
719
|
+
let engineKind = 'none';
|
|
720
|
+
let fellBackToExact = false;
|
|
721
|
+
if (opts.embed === null) {
|
|
722
|
+
fellBackToExact = true;
|
|
723
|
+
}
|
|
724
|
+
else if (opts.embed !== undefined) {
|
|
725
|
+
embed = opts.embed;
|
|
726
|
+
engineKind = 'agentdb';
|
|
727
|
+
}
|
|
728
|
+
else {
|
|
729
|
+
const resolved = pickEngine(projectRoot, opts);
|
|
730
|
+
if (resolved.engine === undefined) {
|
|
731
|
+
fellBackToExact = true;
|
|
732
|
+
}
|
|
733
|
+
else {
|
|
734
|
+
engineKind = resolved.engine.kind;
|
|
735
|
+
const emb = await resolveAgentdbEmbedder(projectRoot);
|
|
736
|
+
if ('error' in emb)
|
|
737
|
+
fellBackToExact = true;
|
|
738
|
+
else
|
|
739
|
+
embed = (t) => emb.embed(t);
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
// 3. CLUSTER (nothing to cluster ⇒ no clusters, everything unique).
|
|
743
|
+
let clusters;
|
|
744
|
+
if (items.length >= 2 && !fellBackToExact && embed !== undefined) {
|
|
745
|
+
const vecs = [];
|
|
746
|
+
let ok = true;
|
|
747
|
+
for (const it of items) {
|
|
748
|
+
const v = await boundedEmbed(embed, `${it.taskType}: ${it.text}`, timeoutMs);
|
|
749
|
+
if (!(v instanceof Float32Array)) {
|
|
750
|
+
ok = false;
|
|
751
|
+
break;
|
|
752
|
+
}
|
|
753
|
+
vecs.push(v);
|
|
754
|
+
}
|
|
755
|
+
if (ok)
|
|
756
|
+
clusters = semanticClusters(items, vecs, threshold);
|
|
757
|
+
else
|
|
758
|
+
fellBackToExact = true; // embed failed/timed out — fall back to exact
|
|
759
|
+
}
|
|
760
|
+
if (clusters === undefined) {
|
|
761
|
+
fellBackToExact = fellBackToExact || embed === undefined;
|
|
762
|
+
clusters = items.length >= 2 ? exactClusters(items) : [];
|
|
763
|
+
}
|
|
764
|
+
// 4. TOTALS (a unique = a singleton; never a member of a drop set).
|
|
765
|
+
const dropDzIds = new Set();
|
|
766
|
+
for (const c of clusters)
|
|
767
|
+
for (const d of c.drops)
|
|
768
|
+
dropDzIds.add(d.dzId);
|
|
769
|
+
const kept = clusters.length;
|
|
770
|
+
const dropped = dropDzIds.size;
|
|
771
|
+
const unique = items.length - kept - dropped;
|
|
772
|
+
const base = {
|
|
773
|
+
mode: apply ? 'apply' : 'dry-run',
|
|
774
|
+
engine: engineKind,
|
|
775
|
+
fellBackToExact,
|
|
776
|
+
threshold,
|
|
777
|
+
clusters,
|
|
778
|
+
kept,
|
|
779
|
+
dropped,
|
|
780
|
+
unique,
|
|
781
|
+
};
|
|
782
|
+
// 5a. DRY-RUN (default): return — ZERO writes (the store is byte-identical after).
|
|
783
|
+
if (!apply)
|
|
784
|
+
return base;
|
|
785
|
+
// 5b. --apply: BACKUP FIRST, then drop the non-keepers. Nothing to drop ⇒ no backup, no mutation.
|
|
786
|
+
if (dropped === 0) {
|
|
787
|
+
logHarmonizeNote(projectRoot, { dropped: 0, kept, engine: engineKind });
|
|
788
|
+
return base;
|
|
789
|
+
}
|
|
790
|
+
const backupPath = join(projectRoot, '.dz', 'memory', 'patterns.pre-harmonize.json');
|
|
791
|
+
const snap = snapshotStore(projectRoot, backupPath);
|
|
792
|
+
if (snap.error !== undefined) {
|
|
793
|
+
// Backup write failed ⇒ ABORT the drop (no partial mutation — the store is untouched).
|
|
794
|
+
return { ...base, error: `backup failed — drop aborted: ${snap.error}` };
|
|
795
|
+
}
|
|
796
|
+
const removal = removePatternsByIds(projectRoot, dropDzIds);
|
|
797
|
+
logHarmonizeNote(projectRoot, { dropped: removal.removed, kept, engine: engineKind, error: removal.error });
|
|
798
|
+
return { ...base, backupPath, ...(removal.error !== undefined ? { error: removal.error } : {}) };
|
|
799
|
+
}
|
|
800
|
+
/* ------------------------------------------------------------------ */
|
|
801
|
+
/* Import — RVF checkpoint ingest, UPSERT-BY-dzId (05 §2.2) */
|
|
802
|
+
/* ------------------------------------------------------------------ */
|
|
803
|
+
/**
|
|
804
|
+
* Ingest an external `.rvf` checkpoint's vectors into THIS project's vector store, **UPSERT-BY-dzId,
|
|
805
|
+
* NON-DESTRUCTIVE** (`dz vector import <file.rvf>`). The `.idmap.json` sidecar is the dzId authority
|
|
806
|
+
* (the shipped `@ruvector/rvf` SDK exposes no vector read-out — see rUv `rvf-backend-blocker.md`), so
|
|
807
|
+
* for each checkpoint dzId that exists in the LOCAL lexical store the vector is reproduced by
|
|
808
|
+
* re-embedding the local text (D7 — under the manifest guard the same model over the same text yields
|
|
809
|
+
* the checkpoint's vector) and upserted by dzId via {@link VectorEngine.importVectors}. dzIds absent
|
|
810
|
+
* locally are ORPHANS — skipped + counted (their text must be imported first via `dz teach --from-json`).
|
|
811
|
+
*
|
|
812
|
+
* Non-destructive: only the imported dzIds are inserted/replaced; re-importing the same file adds 0
|
|
813
|
+
* duplicates and deletes nothing. A model/dim manifest mismatch is REFUSED (no cross-space merge). All
|
|
814
|
+
* failure modes return an honest `{ error }`, never a throw.
|
|
815
|
+
*/
|
|
816
|
+
export async function importRvfCheckpoint(projectRoot, source, opts = {}) {
|
|
817
|
+
const fail = (error, engine = 'none') => ({ imported: 0, skippedOrphans: 0, engine, source, error });
|
|
818
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS;
|
|
819
|
+
// 1. Source dzIds: the `.idmap.json` sidecar (dzId authority) — or injected rows (tests).
|
|
820
|
+
const injected = new Map();
|
|
821
|
+
let sourceDzIds;
|
|
822
|
+
if (opts.sourceRows !== undefined) {
|
|
823
|
+
for (const r of opts.sourceRows)
|
|
824
|
+
injected.set(r.dzId, r.vector);
|
|
825
|
+
sourceDzIds = [...injected.keys()];
|
|
826
|
+
}
|
|
827
|
+
else {
|
|
828
|
+
if (!existsSync(source))
|
|
829
|
+
return fail(`no such file: ${source}`);
|
|
830
|
+
const idmapPath = `${source}.idmap.json`;
|
|
831
|
+
if (!existsSync(idmapPath)) {
|
|
832
|
+
return fail(`missing sidecar ${basename(idmapPath)} — export writes it next to the .rvf (re-run: dz vector export)`);
|
|
833
|
+
}
|
|
834
|
+
let idmap;
|
|
835
|
+
try {
|
|
836
|
+
const parsed = JSON.parse(readFileSync(idmapPath, 'utf-8'));
|
|
837
|
+
idmap = typeof parsed === 'object' && parsed !== null && typeof parsed.slots === 'object' ? parsed : { version: 1, slots: {} };
|
|
838
|
+
}
|
|
839
|
+
catch {
|
|
840
|
+
return fail(`unreadable idmap sidecar: ${basename(idmapPath)}`);
|
|
841
|
+
}
|
|
842
|
+
// Manifest guard (R-i1): refuse a foreign embedding model/dim — no silent cross-space merge.
|
|
843
|
+
const manifestPath = `${source}.manifest.json`;
|
|
844
|
+
if (existsSync(manifestPath)) {
|
|
845
|
+
try {
|
|
846
|
+
const m = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
847
|
+
if ((typeof m.model === 'string' && m.model !== LOCAL_EMBED_MODEL) || (typeof m.dim === 'number' && m.dim !== LOCAL_EMBED_DIM)) {
|
|
848
|
+
return fail(`manifest mismatch: checkpoint (${String(m.model)}/${String(m.dim)}) ≠ local (${LOCAL_EMBED_MODEL}/${LOCAL_EMBED_DIM}) — refusing a cross-embedding-space merge`);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
catch { /* unreadable manifest — tolerate; the idmap is the authority */ }
|
|
852
|
+
}
|
|
853
|
+
sourceDzIds = [...new Set(Object.values(idmap.slots))];
|
|
854
|
+
}
|
|
855
|
+
// 2. Resolve the TARGET engine (agentdb default, or rvf if configured).
|
|
856
|
+
const resolved = pickEngine(projectRoot, opts);
|
|
857
|
+
if (resolved.engine === undefined)
|
|
858
|
+
return fail(resolved.reason ?? 'no vector engine available');
|
|
859
|
+
const engine = resolved.engine;
|
|
860
|
+
if (engine.importVectors === undefined) {
|
|
861
|
+
return { imported: 0, skippedOrphans: 0, engine: engine.kind, source, error: `the ${engine.kind} engine cannot import precomputed vectors` };
|
|
862
|
+
}
|
|
863
|
+
// 3. ORPHAN GATE against the lexical source of truth.
|
|
864
|
+
let records;
|
|
865
|
+
try {
|
|
866
|
+
records = loadStoreRecords(projectRoot);
|
|
867
|
+
}
|
|
868
|
+
catch {
|
|
869
|
+
records = [];
|
|
870
|
+
}
|
|
871
|
+
const byId = new Map();
|
|
872
|
+
for (const r of records)
|
|
873
|
+
byId.set(r.id, r);
|
|
874
|
+
let skippedOrphans = 0;
|
|
875
|
+
const kept = [];
|
|
876
|
+
for (const dzId of sourceDzIds) {
|
|
877
|
+
const rec = byId.get(dzId);
|
|
878
|
+
if (rec === undefined)
|
|
879
|
+
skippedOrphans += 1;
|
|
880
|
+
else
|
|
881
|
+
kept.push({ dzId, rec });
|
|
882
|
+
}
|
|
883
|
+
if (kept.length === 0)
|
|
884
|
+
return { imported: 0, skippedOrphans, engine: engine.kind, source };
|
|
885
|
+
// 4. VECTOR per kept dzId: injected verbatim vector, else RE-EMBED the local text (D7).
|
|
886
|
+
let embed = opts.embed;
|
|
887
|
+
if (embed === undefined && kept.some((k) => !injected.has(k.dzId))) {
|
|
888
|
+
const emb = await resolveAgentdbEmbedder(projectRoot);
|
|
889
|
+
if ('error' in emb)
|
|
890
|
+
return { imported: 0, skippedOrphans, engine: engine.kind, source, error: emb.error };
|
|
891
|
+
embed = (t) => emb.embed(t);
|
|
892
|
+
}
|
|
893
|
+
const rows = [];
|
|
894
|
+
for (const { dzId, rec } of kept) {
|
|
895
|
+
const taskType = dzId.startsWith('dream:') ? 'dz-learning' : 'dz-teach';
|
|
896
|
+
let vector = injected.get(dzId);
|
|
897
|
+
if (vector === undefined) {
|
|
898
|
+
const v = await boundedEmbed(embed, `${taskType}: ${rec.text}`, timeoutMs);
|
|
899
|
+
if (!(v instanceof Float32Array)) {
|
|
900
|
+
return { imported: 0, skippedOrphans, engine: engine.kind, source, error: `embed failed: ${v.error}` };
|
|
901
|
+
}
|
|
902
|
+
vector = v;
|
|
903
|
+
}
|
|
904
|
+
rows.push({ dzId, vector, text: rec.text, taskType, score: rec.score, metadata: { dzId } });
|
|
905
|
+
}
|
|
906
|
+
// 5. UPSERT-BY-dzId (re-import of the same dzIds REPLACEs in place — 0 new rows, nothing deleted).
|
|
907
|
+
const up = await engine.importVectors(rows);
|
|
908
|
+
if (up.error !== undefined)
|
|
909
|
+
return { imported: up.imported, skippedOrphans, engine: engine.kind, source, error: up.error };
|
|
910
|
+
return { imported: up.imported, skippedOrphans, engine: engine.kind, source };
|
|
911
|
+
}
|
|
912
|
+
/* ------------------------------------------------------------------ */
|
|
549
913
|
/* Adapter A (default): AgentdbVectorEngine */
|
|
550
914
|
/* ------------------------------------------------------------------ */
|
|
551
915
|
/**
|
|
@@ -579,6 +943,16 @@ function agentdbVectorEngine(projectRoot) {
|
|
|
579
943
|
async listIds() {
|
|
580
944
|
return listAgentdbDzIds(projectRoot);
|
|
581
945
|
},
|
|
946
|
+
async importVectors(rows) {
|
|
947
|
+
return importVectorsToAgentdb(projectRoot, rows.map((r) => ({
|
|
948
|
+
dzId: r.dzId,
|
|
949
|
+
vector: r.vector,
|
|
950
|
+
text: r.text,
|
|
951
|
+
taskType: r.taskType,
|
|
952
|
+
score: r.score,
|
|
953
|
+
...(r.metadata !== undefined ? { metadata: r.metadata } : {}),
|
|
954
|
+
})));
|
|
955
|
+
},
|
|
582
956
|
};
|
|
583
957
|
}
|
|
584
958
|
function rvfBase(projectRoot) {
|
|
@@ -722,6 +1096,29 @@ function rvfVectorEngine(projectRoot) {
|
|
|
722
1096
|
// Sidecar-only read — no SDK load needed for observability/dedup.
|
|
723
1097
|
return { ids: [...new Set(Object.values(readRvfIdmap(projectRoot).slots))] };
|
|
724
1098
|
},
|
|
1099
|
+
async importVectors(rows) {
|
|
1100
|
+
const loaded = await loadRvfModule(projectRoot);
|
|
1101
|
+
if (!loaded.ok)
|
|
1102
|
+
return { imported: 0, error: loaded.error };
|
|
1103
|
+
const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), 384);
|
|
1104
|
+
if ('error' in store)
|
|
1105
|
+
return { imported: 0, error: store.error };
|
|
1106
|
+
try {
|
|
1107
|
+
const idmap = readRvfIdmap(projectRoot);
|
|
1108
|
+
let imported = 0;
|
|
1109
|
+
for (const r of rows) {
|
|
1110
|
+
await store.ingest(r.dzId, r.vector); // RVF ingest is upsert-by-id (id = dzId) — no duplicates
|
|
1111
|
+
idmap.slots[r.dzId] = r.dzId;
|
|
1112
|
+
imported += 1;
|
|
1113
|
+
}
|
|
1114
|
+
await store.close?.();
|
|
1115
|
+
writeRvfSidecars(projectRoot, idmap);
|
|
1116
|
+
return { imported };
|
|
1117
|
+
}
|
|
1118
|
+
catch (err) {
|
|
1119
|
+
return { imported: 0, error: `rvf import failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
1120
|
+
}
|
|
1121
|
+
},
|
|
725
1122
|
async exportCheckpoint(dest) {
|
|
726
1123
|
try {
|
|
727
1124
|
const base = rvfBase(projectRoot);
|