osborn 0.9.221 → 0.9.222
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embedder.js +2 -2
- package/dist/session-store.d.ts +3 -0
- package/dist/session-store.js +43 -1
- package/package.json +1 -1
package/dist/embedder.js
CHANGED
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
*
|
|
16
16
|
* Model cache honors TRANSFORMERS_CACHE / HF_HOME so it can be baked into the image.
|
|
17
17
|
*/
|
|
18
|
-
import { EMBED_DIM } from './session-store.js';
|
|
19
|
-
const MODEL_ID =
|
|
18
|
+
import { EMBED_DIM, EMBED_MODEL } from './session-store.js';
|
|
19
|
+
const MODEL_ID = EMBED_MODEL;
|
|
20
20
|
let pipelinePromise = null;
|
|
21
21
|
let disabled = false;
|
|
22
22
|
/** Quantize a normalized float32 embedding to int8[dim] (×127, clamped). */
|
package/dist/session-store.d.ts
CHANGED
|
@@ -30,6 +30,9 @@
|
|
|
30
30
|
import Database from 'better-sqlite3';
|
|
31
31
|
export declare const STORE_VERSION = 1;
|
|
32
32
|
export declare const EMBED_DIM = 384;
|
|
33
|
+
/** Canonical embed model id — the single source of truth (embedder.ts imports this).
|
|
34
|
+
* Recorded in meta.embed_model so a model swap is detected as a stale store. */
|
|
35
|
+
export declare const EMBED_MODEL: string;
|
|
33
36
|
/** An embedder turns text into int8[EMBED_DIM] vectors (quantized, normalized). */
|
|
34
37
|
export type Embedder = (texts: string[]) => Promise<Int8Array[] | null>;
|
|
35
38
|
export interface RecallHit {
|
package/dist/session-store.js
CHANGED
|
@@ -39,6 +39,9 @@ import { getSessionPaths, getSessionSubAgents, projectPathToSlug } from './sessi
|
|
|
39
39
|
// ============================================================
|
|
40
40
|
export const STORE_VERSION = 1;
|
|
41
41
|
export const EMBED_DIM = 384;
|
|
42
|
+
/** Canonical embed model id — the single source of truth (embedder.ts imports this).
|
|
43
|
+
* Recorded in meta.embed_model so a model swap is detected as a stale store. */
|
|
44
|
+
export const EMBED_MODEL = process.env.OSBORN_EMBED_MODEL || 'Xenova/all-MiniLM-L6-v2';
|
|
42
45
|
// ============================================================
|
|
43
46
|
// PATHS
|
|
44
47
|
// ============================================================
|
|
@@ -64,6 +67,41 @@ export function openStore(dbPath) {
|
|
|
64
67
|
sqliteVec.load(db);
|
|
65
68
|
db.pragma('journal_mode = WAL');
|
|
66
69
|
db.pragma('synchronous = NORMAL');
|
|
70
|
+
// ── Migration gate ──────────────────────────────────────────────
|
|
71
|
+
// The derived tables (content/fts/vec/sources) are a re-derivable cache of the
|
|
72
|
+
// JSONL, which is the source of truth. If this DB was built by a different
|
|
73
|
+
// STORE_VERSION, EMBED_DIM, or embed model, the cache is stale — most fatally the
|
|
74
|
+
// vec table is fixed at its old dimension, so queries embedded at the new dim throw
|
|
75
|
+
// or return garbage. Detect that and drop the derived tables + reset offsets; the
|
|
76
|
+
// next updateSessionStore() rebuilds them fresh from the JSONL at offset 0.
|
|
77
|
+
//
|
|
78
|
+
// A store with NO recorded embed_model (i.e. predating this stamp) is treated as
|
|
79
|
+
// NOT stale — otherwise the first rollout of this code would wipe every existing DB.
|
|
80
|
+
// We only wipe on a real mismatch, never on absence.
|
|
81
|
+
const hasMeta = db
|
|
82
|
+
.prepare(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='meta'`)
|
|
83
|
+
.get();
|
|
84
|
+
if (hasMeta) {
|
|
85
|
+
const readMeta = db.prepare('SELECT value FROM meta WHERE key = ?');
|
|
86
|
+
const prevVersion = readMeta.get('version')?.value ?? null;
|
|
87
|
+
const prevDim = readMeta.get('embed_dim')?.value ?? null;
|
|
88
|
+
const prevModel = readMeta.get('embed_model')?.value ?? null;
|
|
89
|
+
const stale = (prevVersion != null && prevVersion !== String(STORE_VERSION)) ||
|
|
90
|
+
(prevDim != null && prevDim !== String(EMBED_DIM)) ||
|
|
91
|
+
(prevModel != null && prevModel !== EMBED_MODEL);
|
|
92
|
+
if (stale) {
|
|
93
|
+
console.warn(`⚠️ recall store ${dbPath} is stale ` +
|
|
94
|
+
`(version ${prevVersion}→${STORE_VERSION}, dim ${prevDim}→${EMBED_DIM}, ` +
|
|
95
|
+
`model ${prevModel}→${EMBED_MODEL}) — rebuilding derived tables from JSONL`);
|
|
96
|
+
db.exec(`
|
|
97
|
+
DROP TABLE IF EXISTS content;
|
|
98
|
+
DROP TABLE IF EXISTS fts;
|
|
99
|
+
DROP TABLE IF EXISTS vec;
|
|
100
|
+
DROP TABLE IF EXISTS sources;
|
|
101
|
+
DELETE FROM meta WHERE key IN ('version', 'embed_dim', 'embed_model');
|
|
102
|
+
`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
67
105
|
db.exec(`
|
|
68
106
|
CREATE TABLE IF NOT EXISTS content (
|
|
69
107
|
id INTEGER PRIMARY KEY,
|
|
@@ -94,9 +132,13 @@ export function openStore(dbPath) {
|
|
|
94
132
|
|
|
95
133
|
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
|
|
96
134
|
`);
|
|
97
|
-
|
|
135
|
+
// INSERT OR REPLACE (not IGNORE) so a rebuilt store restamps its identity; the
|
|
136
|
+
// migration gate above cleared these rows on a stale open, so this writes the
|
|
137
|
+
// current version/dim/model that the freshly-created tables were built for.
|
|
138
|
+
const setMeta = db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)');
|
|
98
139
|
setMeta.run('version', String(STORE_VERSION));
|
|
99
140
|
setMeta.run('embed_dim', String(EMBED_DIM));
|
|
141
|
+
setMeta.run('embed_model', EMBED_MODEL);
|
|
100
142
|
return db;
|
|
101
143
|
}
|
|
102
144
|
// ============================================================
|