onbuzz 4.10.0 → 5.0.0

package/src/services/embeddings/__tests__/inMemoryJsonStore.test.js

@@ -1,207 +1,207 @@
-/**
- * Conformance tests for InMemoryJsonStore. Most of the test surface lives
- * in `runStoreContractTests`, which any future VectorStore implementation
- * also has to pass (e.g. sqliteVecStore later). This file adds a few
- * implementation-specific tests on top: atomic-write behavior, the
- * substring path, and the RRF fusion helper directly.
- */
-
-import { describe, test, expect, beforeEach, afterEach } from '@jest/globals';
-import os from 'os';
-import fsp from 'fs/promises';
-import path from 'path';
-import { InMemoryJsonStore } from '../vectorStore/inMemoryJsonStore.js';
-import { reciprocalRankFusion } from '../vectorStore/storeInterface.js';
-import { runStoreContractTests } from '../vectorStore/storeContractTests.js';
-
-// Shared conformance contract — every VectorStore impl runs through these.
-runStoreContractTests((opts) => new InMemoryJsonStore(opts));
-
-describe('reciprocalRankFusion', () => {
-  test('items present in both lists rank higher than items in only one', () => {
-    const out = reciprocalRankFusion([
-      [{ id: 'both' }, { id: 'only-semantic' }],
-      [{ id: 'both' }, { id: 'only-substring' }],
-    ]);
-    expect(out[0].id).toBe('both');
-  });
-
-  test('preserves input ranking when only one list has results', () => {
-    const out = reciprocalRankFusion([
-      [{ id: 'a' }, { id: 'b' }, { id: 'c' }],
-      [],
-    ]);
-    expect(out.map(x => x.id)).toEqual(['a', 'b', 'c']);
-  });
-
-  test('topK trims output', () => {
-    const out = reciprocalRankFusion(
-      [[{ id: 'a' }, { id: 'b' }, { id: 'c' }]],
-      { topK: 2 },
-    );
-    expect(out).toHaveLength(2);
-  });
-
-  test('empty input lists produce empty output', () => {
-    expect(reciprocalRankFusion([])).toEqual([]);
-    expect(reciprocalRankFusion([[], []])).toEqual([]);
-  });
-
-  test('ignores non-array entries gracefully', () => {
-    expect(() => reciprocalRankFusion([null, undefined, [{ id: 'x' }]])).not.toThrow();
-    const out = reciprocalRankFusion([null, undefined, [{ id: 'x' }]]);
-    expect(out.map(o => o.id)).toEqual(['x']);
-  });
-});
-
-describe('InMemoryJsonStore — atomic persistence', () => {
-  let tmpDir, filePath, store;
-
-  beforeEach(async () => {
-    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-vec-atomic-'));
-    filePath = path.join(tmpDir, 'store.vec.json');
-    store = new InMemoryJsonStore({
-      filePath,
-      dimensions: 4,
-      modelFingerprint: 'test:fp',
-    });
-    await store.load();
-  });
-
-  afterEach(async () => {
-    await fsp.rm(tmpDir, { recursive: true, force: true });
-  });
-
-  test('writes via .tmp + rename (no partial file on disk after success)', async () => {
-    await store.upsert('a', Float32Array.from([1, 0, 0, 0]), { text: 'a' });
-    const entries = await fsp.readdir(tmpDir);
-    expect(entries).toContain('store.vec.json');
-    expect(entries.filter(e => e.endsWith('.tmp'))).toEqual([]);
-  });
-
-  test('drop() removes the file from disk', async () => {
-    await store.upsert('a', Float32Array.from([1, 0, 0, 0]), {});
-    await store.drop();
-    await expect(fsp.access(filePath)).rejects.toMatchObject({ code: 'ENOENT' });
-  });
-
-  test('writes parent directory if missing', async () => {
-    const nested = path.join(tmpDir, 'nested', 'dirs', 'store.vec.json');
-    const nestedStore = new InMemoryJsonStore({
-      filePath: nested,
-      dimensions: 4,
-      modelFingerprint: 'test:fp',
-    });
-    await nestedStore.load();
-    await nestedStore.upsert('a', Float32Array.from([1, 0, 0, 0]), {});
-    await expect(fsp.access(nested)).resolves.toBeUndefined();
-  });
-
-  test('corrupt JSON on load → starts empty, no throw', async () => {
-    await fsp.writeFile(filePath, 'not json{{');
-    const broken = new InMemoryJsonStore({
-      filePath,
-      dimensions: 4,
-      modelFingerprint: 'test:fp',
-    });
-    await expect(broken.load()).resolves.toBeUndefined();
-    expect((await broken.stats()).count).toBe(0);
-  });
-
-  test('upsert before load() throws', async () => {
-    const noLoad = new InMemoryJsonStore({
-      filePath: path.join(tmpDir, 'other.vec.json'),
-      dimensions: 4,
-      modelFingerprint: 'test:fp',
-    });
-    await expect(
-      noLoad.upsert('a', Float32Array.from([1, 0, 0, 0]), {}),
-    ).rejects.toThrow(/not loaded/);
-  });
-});
-
-describe('InMemoryJsonStore — substring matches', () => {
-  let tmpDir, store;
-
-  beforeEach(async () => {
-    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-vec-substr-'));
-    store = new InMemoryJsonStore({
-      filePath: path.join(tmpDir, 'store.vec.json'),
-      dimensions: 2,
-      modelFingerprint: 'test:fp',
-    });
-    await store.load();
-  });
-
-  afterEach(async () => {
-    await fsp.rm(tmpDir, { recursive: true, force: true });
-  });
-
-  test('hybridQuery surfaces a row that matches ONLY on substring', async () => {
-    await store.upsertBatch([
-      { id: 'a', vector: Float32Array.from([1, 0]), metadata: { text: 'banana' } },
-      { id: 'b', vector: Float32Array.from([0, 1]), metadata: { text: 'apple pie' } },
-    ]);
-    // Query vector orthogonal to 'a' (cosine=0) but text matches 'apple'.
-    const hits = await store.hybridQuery(Float32Array.from([1, 0]), 'apple');
-    expect(hits.find(h => h.id === 'b')).toBeDefined();
-  });
-
-  test('case-insensitive substring', async () => {
-    await store.upsert('a', Float32Array.from([1, 0]), { text: 'Telegram Errors' });
-    const hits = await store.hybridQuery(Float32Array.from([0, 0]), 'telegram');
-    expect(hits.find(h => h.id === 'a')).toBeDefined();
-  });
-
-  test('rows missing metadata.text are skipped on substring side', async () => {
-    await store.upsertBatch([
-      { id: 'a', vector: Float32Array.from([1, 0]), metadata: {} },
-      { id: 'b', vector: Float32Array.from([0, 1]), metadata: { text: null } },
-      { id: 'c', vector: Float32Array.from([0.5, 0.5]), metadata: { text: 'matches' } },
-    ]);
-    const hits = await store.hybridQuery(Float32Array.from([0, 0]), 'matches');
-    // Only 'c' should appear from the substring side — the semantic side
-    // adds 'a' and 'b' but their substring rank is empty.
-    expect(hits.map(h => h.id)).toContain('c');
-  });
-});
-
-describe('InMemoryJsonStore — drift recovery', () => {
-  let tmpDir, filePath;
-
-  beforeEach(async () => {
-    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-vec-drift-'));
-    filePath = path.join(tmpDir, 'store.vec.json');
-    const initial = new InMemoryJsonStore({
-      filePath, dimensions: 4, modelFingerprint: 'azure:text-embedding-3-small',
-    });
-    await initial.load();
-    await initial.upsert('a', Float32Array.from([1, 0, 0, 0]), { text: 'kept?' });
-  });
-
-  afterEach(async () => {
-    await fsp.rm(tmpDir, { recursive: true, force: true });
-  });
-
-  test('switching provider (fingerprint mismatch) presents empty corpus', async () => {
-    const swapped = new InMemoryJsonStore({
-      filePath, dimensions: 4, modelFingerprint: 'local:Xenova/nomic-embed-text-v1.5',
-    });
-    await swapped.load();
-    expect((await swapped.stats()).count).toBe(0);
-  });
-
-  test('after re-embed, new fingerprint persists', async () => {
-    const swapped = new InMemoryJsonStore({
-      filePath, dimensions: 768, modelFingerprint: 'local:Xenova/nomic-embed-text-v1.5',
-    });
-    await swapped.load();
-    await swapped.upsert('a', Float32Array.from(Array(768).fill(0.01)), { text: 're-embedded' });
-    const re = new InMemoryJsonStore({
-      filePath, dimensions: 768, modelFingerprint: 'local:Xenova/nomic-embed-text-v1.5',
-    });
-    await re.load();
-    expect((await re.stats()).count).toBe(1);
-    expect((await re.stats()).modelFingerprint).toBe('local:Xenova/nomic-embed-text-v1.5');
-  });
-});
+/**
+ * Conformance tests for InMemoryJsonStore. Most of the test surface lives
+ * in `runStoreContractTests`, which any future VectorStore implementation
+ * also has to pass (e.g. sqliteVecStore later). This file adds a few
+ * implementation-specific tests on top: atomic-write behavior, the
+ * substring path, and the RRF fusion helper directly.
+ */
+
+import { describe, test, expect, beforeEach, afterEach } from '@jest/globals';
+import os from 'os';
+import fsp from 'fs/promises';
+import path from 'path';
+import { InMemoryJsonStore } from '../vectorStore/inMemoryJsonStore.js';
+import { reciprocalRankFusion } from '../vectorStore/storeInterface.js';
+import { runStoreContractTests } from '../vectorStore/storeContractTests.js';
+
+// Shared conformance contract — every VectorStore impl runs through these.
+runStoreContractTests((opts) => new InMemoryJsonStore(opts));
+
+describe('reciprocalRankFusion', () => {
+  test('items present in both lists rank higher than items in only one', () => {
+    const out = reciprocalRankFusion([
+      [{ id: 'both' }, { id: 'only-semantic' }],
+      [{ id: 'both' }, { id: 'only-substring' }],
+    ]);
+    expect(out[0].id).toBe('both');
+  });
+
+  test('preserves input ranking when only one list has results', () => {
+    const out = reciprocalRankFusion([
+      [{ id: 'a' }, { id: 'b' }, { id: 'c' }],
+      [],
+    ]);
+    expect(out.map(x => x.id)).toEqual(['a', 'b', 'c']);
+  });
+
+  test('topK trims output', () => {
+    const out = reciprocalRankFusion(
+      [[{ id: 'a' }, { id: 'b' }, { id: 'c' }]],
+      { topK: 2 },
+    );
+    expect(out).toHaveLength(2);
+  });
+
+  test('empty input lists produce empty output', () => {
+    expect(reciprocalRankFusion([])).toEqual([]);
+    expect(reciprocalRankFusion([[], []])).toEqual([]);
+  });
+
+  test('ignores non-array entries gracefully', () => {
+    expect(() => reciprocalRankFusion([null, undefined, [{ id: 'x' }]])).not.toThrow();
+    const out = reciprocalRankFusion([null, undefined, [{ id: 'x' }]]);
+    expect(out.map(o => o.id)).toEqual(['x']);
+  });
+});
+
+describe('InMemoryJsonStore — atomic persistence', () => {
+  let tmpDir, filePath, store;
+
+  beforeEach(async () => {
+    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-vec-atomic-'));
+    filePath = path.join(tmpDir, 'store.vec.json');
+    store = new InMemoryJsonStore({
+      filePath,
+      dimensions: 4,
+      modelFingerprint: 'test:fp',
+    });
+    await store.load();
+  });
+
+  afterEach(async () => {
+    await fsp.rm(tmpDir, { recursive: true, force: true });
+  });
+
+  test('writes via .tmp + rename (no partial file on disk after success)', async () => {
+    await store.upsert('a', Float32Array.from([1, 0, 0, 0]), { text: 'a' });
+    const entries = await fsp.readdir(tmpDir);
+    expect(entries).toContain('store.vec.json');
+    expect(entries.filter(e => e.endsWith('.tmp'))).toEqual([]);
+  });
+
+  test('drop() removes the file from disk', async () => {
+    await store.upsert('a', Float32Array.from([1, 0, 0, 0]), {});
+    await store.drop();
+    await expect(fsp.access(filePath)).rejects.toMatchObject({ code: 'ENOENT' });
+  });
+
+  test('writes parent directory if missing', async () => {
+    const nested = path.join(tmpDir, 'nested', 'dirs', 'store.vec.json');
+    const nestedStore = new InMemoryJsonStore({
+      filePath: nested,
+      dimensions: 4,
+      modelFingerprint: 'test:fp',
+    });
+    await nestedStore.load();
+    await nestedStore.upsert('a', Float32Array.from([1, 0, 0, 0]), {});
+    await expect(fsp.access(nested)).resolves.toBeUndefined();
+  });
+
+  test('corrupt JSON on load → starts empty, no throw', async () => {
+    await fsp.writeFile(filePath, 'not json{{');
+    const broken = new InMemoryJsonStore({
+      filePath,
+      dimensions: 4,
+      modelFingerprint: 'test:fp',
+    });
+    await expect(broken.load()).resolves.toBeUndefined();
+    expect((await broken.stats()).count).toBe(0);
+  });
+
+  test('upsert before load() throws', async () => {
+    const noLoad = new InMemoryJsonStore({
+      filePath: path.join(tmpDir, 'other.vec.json'),
+      dimensions: 4,
+      modelFingerprint: 'test:fp',
+    });
+    await expect(
+      noLoad.upsert('a', Float32Array.from([1, 0, 0, 0]), {}),
+    ).rejects.toThrow(/not loaded/);
+  });
+});
+
+describe('InMemoryJsonStore — substring matches', () => {
+  let tmpDir, store;
+
+  beforeEach(async () => {
+    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-vec-substr-'));
+    store = new InMemoryJsonStore({
+      filePath: path.join(tmpDir, 'store.vec.json'),
+      dimensions: 2,
+      modelFingerprint: 'test:fp',
+    });
+    await store.load();
+  });
+
+  afterEach(async () => {
+    await fsp.rm(tmpDir, { recursive: true, force: true });
+  });
+
+  test('hybridQuery surfaces a row that matches ONLY on substring', async () => {
+    await store.upsertBatch([
+      { id: 'a', vector: Float32Array.from([1, 0]), metadata: { text: 'banana' } },
+      { id: 'b', vector: Float32Array.from([0, 1]), metadata: { text: 'apple pie' } },
+    ]);
+    // Query vector orthogonal to 'a' (cosine=0) but text matches 'apple'.
+    const hits = await store.hybridQuery(Float32Array.from([1, 0]), 'apple');
+    expect(hits.find(h => h.id === 'b')).toBeDefined();
+  });
+
+  test('case-insensitive substring', async () => {
+    await store.upsert('a', Float32Array.from([1, 0]), { text: 'Telegram Errors' });
+    const hits = await store.hybridQuery(Float32Array.from([0, 0]), 'telegram');
+    expect(hits.find(h => h.id === 'a')).toBeDefined();
+  });
+
+  test('rows missing metadata.text are skipped on substring side', async () => {
+    await store.upsertBatch([
+      { id: 'a', vector: Float32Array.from([1, 0]), metadata: {} },
+      { id: 'b', vector: Float32Array.from([0, 1]), metadata: { text: null } },
+      { id: 'c', vector: Float32Array.from([0.5, 0.5]), metadata: { text: 'matches' } },
+    ]);
+    const hits = await store.hybridQuery(Float32Array.from([0, 0]), 'matches');
+    // Only 'c' should appear from the substring side — the semantic side
+    // adds 'a' and 'b' but their substring rank is empty.
+    expect(hits.map(h => h.id)).toContain('c');
+  });
+});
+
+describe('InMemoryJsonStore — drift recovery', () => {
+  let tmpDir, filePath;
+
+  beforeEach(async () => {
+    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'loxia-vec-drift-'));
+    filePath = path.join(tmpDir, 'store.vec.json');
+    const initial = new InMemoryJsonStore({
+      filePath, dimensions: 4, modelFingerprint: 'azure:text-embedding-3-small',
+    });
+    await initial.load();
+    await initial.upsert('a', Float32Array.from([1, 0, 0, 0]), { text: 'kept?' });
+  });
+
+  afterEach(async () => {
+    await fsp.rm(tmpDir, { recursive: true, force: true });
+  });
+
+  test('switching provider (fingerprint mismatch) presents empty corpus', async () => {
+    const swapped = new InMemoryJsonStore({
+      filePath, dimensions: 4, modelFingerprint: 'local:nomic-ai/nomic-embed-text-v1',
+    });
+    await swapped.load();
+    expect((await swapped.stats()).count).toBe(0);
+  });
+
+  test('after re-embed, new fingerprint persists', async () => {
+    const swapped = new InMemoryJsonStore({
+      filePath, dimensions: 768, modelFingerprint: 'local:nomic-ai/nomic-embed-text-v1',
+    });
+    await swapped.load();
+    await swapped.upsert('a', Float32Array.from(Array(768).fill(0.01)), { text: 're-embedded' });
+    const re = new InMemoryJsonStore({
+      filePath, dimensions: 768, modelFingerprint: 'local:nomic-ai/nomic-embed-text-v1',
+    });
+    await re.load();
+    expect((await re.stats()).count).toBe(1);
+    expect((await re.stats()).modelFingerprint).toBe('local:nomic-ai/nomic-embed-text-v1');
+  });
+});

package/src/services/embeddings/__tests__/localProvider.test.js

@@ -53,7 +53,7 @@ function makeStubPipeline({ dimensions, returnShape = '2d' } = {}) {
 }
 
 describe('LocalEmbeddingProvider — defaults + info', () => {
-  test('default model is nomic-embed-text-v1.5', () => {
+  test('default model is nomic-embed-text-v1 (anonymously-fetchable namespace)', () => {
     const p = new LocalEmbeddingProvider();
     const info = p.getInfo();
     expect(info.name).toBe('local');
@@ -80,14 +80,14 @@ describe('LocalEmbeddingProvider — defaults + info', () => {
     expect(Object.keys(LOCAL_MODELS).sort()).toEqual([
       'Xenova/all-MiniLM-L6-v2',
       'Xenova/bge-small-en-v1.5',
-      'Xenova/nomic-embed-text-v1.5',
-    ]);
+      'nomic-ai/nomic-embed-text-v1',
+    ].sort());
   });
 
   test('getModelFingerprint includes modelId', () => {
     const a = new LocalEmbeddingProvider();
     const b = new LocalEmbeddingProvider({ modelId: 'Xenova/bge-small-en-v1.5' });
-    expect(a.getModelFingerprint()).toBe('local:Xenova/nomic-embed-text-v1.5');
+    expect(a.getModelFingerprint()).toBe('local:nomic-ai/nomic-embed-text-v1');
     expect(b.getModelFingerprint()).toBe('local:Xenova/bge-small-en-v1.5');
     expect(a.getModelFingerprint()).not.toBe(b.getModelFingerprint());
   });

package/src/services/embeddings/embeddingsConfig.js

@@ -85,9 +85,29 @@ export const DEFAULT_CONFIG = Object.freeze({
   }),
 });
 
+/**
+ * Migration table for deprecated local-model ids. Any saved config that
+ * references a key here is silently rewritten to its replacement on
+ * load. This is the auto-upgrade path for users who picked a model
+ * before we changed the default — without it, their old `local.modelId`
+ * keeps winning the merge and we hit the same warmup error they had
+ * before the upgrade.
+ *
+ * 2026-06-07: `Xenova/nomic-embed-text-v1.5` started returning 401 on
+ * the Hugging Face Hub for anonymous fetches. Replaced with
+ * `nomic-ai/nomic-embed-text-v1` (same prefix conventions, 768d, ONNX
+ * available). See providers/localProvider.js for the wider context.
+ */
+const DEPRECATED_LOCAL_MODELS = Object.freeze({
+  'Xenova/nomic-embed-text-v1.5': 'nomic-ai/nomic-embed-text-v1',
+});
+
 /**
  * Deep-merge a user config object into DEFAULT_CONFIG. Frozen defaults
- * stay intact; user overrides win on a per-leaf basis.
+ * stay intact; user overrides win on a per-leaf basis. Also migrates
+ * any deprecated local.modelId (see DEPRECATED_LOCAL_MODELS) so users
+ * who saved a now-broken model id get auto-healed instead of stuck on
+ * a warmup-fails error.
  *
  * @param {object|null|undefined} userConfig
  * @returns {object} a fresh, mutable config
@@ -113,6 +133,13 @@ export function mergeConfig(userConfig) {
       }
     }
   }
+  // Auto-migrate deprecated local-model ids. Runs after the merge so it
+  // catches both the user-supplied id and any future case where the
+  // default itself moved on while the user's saved config still pins
+  // the old one.
+  if (typeof out.local?.modelId === 'string' && DEPRECATED_LOCAL_MODELS[out.local.modelId]) {
+    out.local.modelId = DEPRECATED_LOCAL_MODELS[out.local.modelId];
+  }
   return out;
 }