@namzu/sdk 2.0.0 → 3.1.0

package/src/rag/__tests__/retrieval-math.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from 'vitest'
+
+import { TextChunker, splitKeepingSeparator } from '../chunking.js'
+
+/**
+ * Retrieval arithmetic, pinned by the counterexample that proves each
+ * defect. Every test here fails against the previous formula.
+ */
+
+describe('splitKeepingSeparator', () => {
+	it('does not delete the separator it split on', () => {
+		// `'A. B. C.'.split('. ')` returns ['A','B','C.'] — the sentence
+		// terminators are gone, so the retrieved chunk no longer says what
+		// the document said.
+		expect(splitKeepingSeparator('A. B. C.', '. ')).toEqual(['A. ', 'B. ', 'C.'])
+	})
+
+	it('reconstructs the input exactly', () => {
+		const text = 'First para.\n\nSecond para.\n\nThird.'
+		expect(splitKeepingSeparator(text, '\n\n').join('')).toBe(text)
+	})
+
+	it('handles a trailing separator without emitting an empty tail', () => {
+		expect(splitKeepingSeparator('a\n\nb\n\n', '\n\n')).toEqual(['a\n\n', 'b\n\n'])
+	})
+
+	it('returns the whole text when the separator is absent', () => {
+		expect(splitKeepingSeparator('no breaks here', '\n\n')).toEqual(['no breaks here'])
+	})
+
+	it('handles consecutive separators', () => {
+		const text = 'a\n\n\n\nb'
+		expect(splitKeepingSeparator(text, '\n\n').join('')).toBe(text)
+	})
+})
+
+describe('TextChunker preserves the source text', () => {
+	it('keeps sentence terminators in recursive chunks', () => {
+		// The seam, not the helper: the chunker used `text.split(sep)`, so a
+		// recursive split on '. ' returned chunks with the periods removed.
+		const text = `${'Alpha beta gamma. '.repeat(20)}Delta epsilon zeta.`
+		const chunks = new TextChunker().chunk(text, {
+			strategy: 'recursive',
+			chunkSize: 100,
+			chunkOverlap: 0,
+		})
+
+		expect(chunks.length).toBeGreaterThan(1)
+		// Every sentence in a chunk still ends the way the document did.
+		const joined = chunks.map((c) => c.content).join(' ')
+		expect(joined).toContain('Alpha beta gamma.')
+		expect(joined).not.toContain('Alpha beta gamma Alpha')
+	})
+})

package/src/rag/chunking.ts

@@ -63,7 +63,12 @@ export class TextChunker implements Chunker {
 		}
 
 		for (const sep of separators) {
-			const parts = sep === '' ? [...text] : text.split(sep).filter((p) => p.trim().length > 0)
+			// Keep the separator on the piece it terminated. `text.split(sep)`
+			// DELETES it, so splitting on '. ' stripped the sentence-ending
+			// period and splitting on '\n\n' stripped the paragraph break —
+			// the retrieved chunk then no longer said what the document said.
+			const parts =
+				sep === '' ? [...text] : splitKeepingSeparator(text, sep).filter((p) => p.trim().length > 0)
 			if (parts.length <= 1) continue
 
 			const remaining = separators.slice(separators.indexOf(sep) + 1)
@@ -133,3 +138,29 @@ export class TextChunker implements Chunker {
 }
 
 export { DEFAULT_CHUNKING_CONFIG }
+
+/**
+ * Split on `sep` but keep each separator attached to the piece it ended.
+ *
+ * `String.prototype.split` discards the separator, which silently rewrites
+ * the text: a document split on '. ' comes back with its sentence-ending
+ * periods gone, and one split on '\n\n' loses every paragraph break. For a
+ * retrieval chunk that is a correctness problem, not a cosmetic one — the
+ * chunk shown to the model is no longer what the source said.
+ *
+ * Concatenating the result reproduces the input exactly.
+ */
+export function splitKeepingSeparator(text: string, sep: string): string[] {
+	if (sep === '') return [text]
+
+	const parts: string[] = []
+	let start = 0
+	for (;;) {
+		const at = text.indexOf(sep, start)
+		if (at === -1) break
+		parts.push(text.slice(start, at + sep.length))
+		start = at + sep.length
+	}
+	if (start < text.length) parts.push(text.slice(start))
+	return parts
+}

package/src/rag/embedding.test.ts

@@ -1,7 +1,7 @@
 /**
  * Current-code invariants asserted (2026-04-21, ses_006 Phase 4):
  *
- *   - `OpenRouterEmbeddingProvider`:
+ *   - `HttpEmbeddingProvider`:
  *     - Defaults: dimensions = 1536; baseUrl = openrouter.ai/api/v1;
  *       batchSize = 64.
  *     - `embed(texts)` batches into `batchSize` slices and concatenates
@@ -18,9 +18,9 @@
 
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
-import { OpenRouterEmbeddingProvider } from './embedding.js'
+import { HttpEmbeddingProvider } from './embedding.js'
 
-describe('OpenRouterEmbeddingProvider', () => {
+describe('HttpEmbeddingProvider', () => {
 	let fetchMock: ReturnType<typeof vi.fn>
 
 	beforeEach(() => {
@@ -33,7 +33,11 @@ describe('OpenRouterEmbeddingProvider', () => {
 	})
 
 	it('carries model / dimensions defaults + batchSize', () => {
-		const p = new OpenRouterEmbeddingProvider({ apiKey: 'k', model: 'm' })
+		const p = new HttpEmbeddingProvider({
+			apiKey: 'k',
+			model: 'm',
+			baseUrl: 'https://embeddings.test/v1',
+		})
 		expect(p.model).toBe('m')
 		expect(p.dimensions).toBe(1536)
 	})
@@ -43,7 +47,7 @@ describe('OpenRouterEmbeddingProvider', () => {
 			ok: true,
 			json: async () => ({ data: [{ index: 0, embedding: [1, 2, 3] }] }),
 		})
-		const p = new OpenRouterEmbeddingProvider({
+		const p = new HttpEmbeddingProvider({
 			apiKey: 'k',
 			model: 'm',
 			dimensions: 256,
@@ -76,9 +80,10 @@ describe('OpenRouterEmbeddingProvider', () => {
 				],
 			}),
 		})
-		const p = new OpenRouterEmbeddingProvider({
+		const p = new HttpEmbeddingProvider({
 			apiKey: 'k',
 			model: 'm',
+			baseUrl: 'https://embeddings.test/v1',
 			batchSize: 2,
 		})
 		await p.embed(['a', 'b', 'c', 'd'])
@@ -96,7 +101,11 @@ describe('OpenRouterEmbeddingProvider', () => {
 				],
 			}),
 		})
-		const p = new OpenRouterEmbeddingProvider({ apiKey: 'k', model: 'm' })
+		const p = new HttpEmbeddingProvider({
+			apiKey: 'k',
+			model: 'm',
+			baseUrl: 'https://embeddings.test/v1',
+		})
 		expect(await p.embed(['a', 'b', 'c'])).toEqual([[1], [2], [3]])
 	})
 
@@ -105,7 +114,11 @@ describe('OpenRouterEmbeddingProvider', () => {
 			ok: true,
 			json: async () => ({ data: [{ index: 0, embedding: [9, 9] }] }),
 		})
-		const p = new OpenRouterEmbeddingProvider({ apiKey: 'k', model: 'm' })
+		const p = new HttpEmbeddingProvider({
+			apiKey: 'k',
+			model: 'm',
+			baseUrl: 'https://embeddings.test/v1',
+		})
 		expect(await p.embedQuery('hi')).toEqual([9, 9])
 	})
 
@@ -114,7 +127,11 @@ describe('OpenRouterEmbeddingProvider', () => {
 			ok: true,
 			json: async () => ({ data: [] }),
 		})
-		const p = new OpenRouterEmbeddingProvider({ apiKey: 'k', model: 'm' })
+		const p = new HttpEmbeddingProvider({
+			apiKey: 'k',
+			model: 'm',
+			baseUrl: 'https://embeddings.test/v1',
+		})
 		await expect(p.embedQuery('hi')).rejects.toThrow(/no results/)
 	})
 
@@ -124,7 +141,45 @@ describe('OpenRouterEmbeddingProvider', () => {
 			status: 503,
 			text: async () => 'service unavailable',
 		})
-		const p = new OpenRouterEmbeddingProvider({ apiKey: 'k', model: 'm' })
+		const p = new HttpEmbeddingProvider({
+			apiKey: 'k',
+			model: 'm',
+			baseUrl: 'https://embeddings.test/v1',
+		})
 		await expect(p.embed(['hi'])).rejects.toThrow(/503.*service unavailable/)
 	})
 })
+
+describe('the endpoint is the caller\u2019s decision', () => {
+	it('appends the path to whatever root the caller named', async () => {
+		const fetchMock = vi.fn().mockResolvedValue({
+			ok: true,
+			json: async () => ({ data: [{ index: 0, embedding: [1] }] }),
+		})
+		global.fetch = fetchMock as unknown as typeof fetch
+
+		await new HttpEmbeddingProvider({
+			apiKey: 'k',
+			model: 'm',
+			baseUrl: 'https://vectors.internal/v2',
+		}).embed(['x'])
+
+		expect(fetchMock.mock.calls[0]?.[0]).toBe('https://vectors.internal/v2/embeddings')
+	})
+
+	it('tolerates a trailing slash instead of building a double one', async () => {
+		const fetchMock = vi.fn().mockResolvedValue({
+			ok: true,
+			json: async () => ({ data: [{ index: 0, embedding: [1] }] }),
+		})
+		global.fetch = fetchMock as unknown as typeof fetch
+
+		await new HttpEmbeddingProvider({
+			apiKey: 'k',
+			model: 'm',
+			baseUrl: 'https://vectors.internal/v2/',
+		}).embed(['x'])
+
+		expect(fetchMock.mock.calls[0]?.[0]).toBe('https://vectors.internal/v2/embeddings')
+	})
+})

package/src/rag/embedding.ts

@@ -1,7 +1,28 @@
-import type { EmbeddingProvider, OpenRouterEmbeddingConfig } from '../types/rag/index.js'
+import type { EmbeddingProvider, HttpEmbeddingConfig } from '../types/rag/index.js'
 
-export class OpenRouterEmbeddingProvider implements EmbeddingProvider {
-	readonly id = 'openrouter-embedding'
+/**
+ * Embeddings over the common HTTP shape: `POST {baseUrl}/embeddings` with
+ * a bearer key, `{ model, input, dimensions }` in, `{ data: [{ index,
+ * embedding }] }` out. Every hosted embeddings service worth pointing at
+ * speaks it, so the driver is the shape rather than any one host.
+ */
+/**
+ * Trim trailing slashes without a regex.
+ *
+ * `/\/+$/` backtracks quadratically on a long run of slashes, and this
+ * value crosses a trust boundary — a host-supplied endpoint on a shared
+ * event loop. The scan is linear and says the same thing.
+ */
+function stripTrailingSlashes(value: string): string {
+	let end = value.length
+	while (end > 0 && value[end - 1] === '/') {
+		end--
+	}
+	return value.slice(0, end)
+}
+
+export class HttpEmbeddingProvider implements EmbeddingProvider {
+	readonly id = 'http-embedding'
 	readonly model: string
 	readonly dimensions: number
 
@@ -9,11 +30,11 @@ export class OpenRouterEmbeddingProvider implements EmbeddingProvider {
 	private readonly baseUrl: string
 	private readonly batchSize: number
 
-	constructor(config: OpenRouterEmbeddingConfig) {
+	constructor(config: HttpEmbeddingConfig) {
 		this.model = config.model
 		this.dimensions = config.dimensions ?? 1536
 		this.apiKey = config.apiKey
-		this.baseUrl = config.baseUrl ?? 'https://openrouter.ai/api/v1'
+		this.baseUrl = stripTrailingSlashes(config.baseUrl)
 		this.batchSize = config.batchSize ?? 64
 	}
 

package/src/rag/index.ts

@@ -28,7 +28,7 @@ export type {
 
 export { TextChunker, DEFAULT_CHUNKING_CONFIG } from './chunking.js'
 
-export { OpenRouterEmbeddingProvider } from './embedding.js'
+export { HttpEmbeddingProvider } from './embedding.js'
 
 export { InMemoryVectorStore, cosineSimilarity } from './vector-store.js'
 

package/src/rag/retriever.ts

@@ -97,10 +97,19 @@ export class DefaultRetriever implements Retriever {
 		})
 
 		const queryTerms = this.tokenize(query)
+
+		// BM25 needs corpus statistics, so tokenize the whole candidate set
+		// first: document frequency per term for IDF, and the real average
+		// document length for the length normalization. Both used to be
+		// missing — IDF entirely, and `avgDl` as a hardcoded 256 that had
+		// nothing to do with these documents.
+		const tokenized = vectorResults.map((result) => this.tokenize(result.chunk.content))
+		const stats = buildCorpusStats(tokenized)
+
 		return vectorResults
-			.map((result) => ({
+			.map((result, i) => ({
 				...result,
-				score: this.bm25Score(queryTerms, this.tokenize(result.chunk.content)),
+				score: bm25Score(queryTerms, tokenized[i] as string[], stats),
 			}))
 			.filter((r) => r.score > 0)
 			.sort((a, b) => b.score - a.score)
@@ -122,14 +131,17 @@ export class DefaultRetriever implements Retriever {
 
 		const scoreMap = new Map<string, { chunk: VectorSearchResult['chunk']; score: number }>()
 
-		for (const result of vectorResults) {
+		// Put both rankings on the same scale before blending. Cosine is
+		// bounded, BM25 is not, so the raw mix let the larger scale win
+		// regardless of `alpha`.
+		for (const result of normalizeByMax(vectorResults)) {
 			scoreMap.set(result.chunk.id, {
 				chunk: result.chunk,
 				score: alpha * result.score,
 			})
 		}
 
-		for (const result of keywordResults) {
+		for (const result of normalizeByMax(keywordResults)) {
 			const existing = scoreMap.get(result.chunk.id)
 			if (existing) {
 				existing.score += (1 - alpha) * result.score
@@ -151,27 +163,86 @@ export class DefaultRetriever implements Retriever {
 			.split(/\s+/)
 			.filter((t) => t.length > 1)
 	}
+}
+
+interface CorpusStats {
+	/** Number of documents in the candidate set. */
+	readonly n: number
+	/** Mean document length, in tokens. */
+	readonly avgDl: number
+	/** How many documents contain each term. */
+	readonly df: ReadonlyMap<string, number>
+}
 
-	private bm25Score(queryTerms: string[], docTerms: string[]): number {
-		const k1 = 1.2
-		const b = 0.75
-		const avgDl = 256
-		const dl = docTerms.length
+function buildCorpusStats(docs: readonly string[][]): CorpusStats {
+	const df = new Map<string, number>()
+	let totalLength = 0
 
-		const termFreq = new Map<string, number>()
-		for (const term of docTerms) {
-			termFreq.set(term, (termFreq.get(term) ?? 0) + 1)
+	for (const doc of docs) {
+		totalLength += doc.length
+		for (const term of new Set(doc)) {
+			df.set(term, (df.get(term) ?? 0) + 1)
 		}
+	}
 
-		let score = 0
-		for (const term of queryTerms) {
-			const tf = termFreq.get(term) ?? 0
-			if (tf === 0) continue
-			const numerator = tf * (k1 + 1)
-			const denominator = tf + k1 * (1 - b + b * (dl / avgDl))
-			score += numerator / denominator
-		}
+	return { n: docs.length, avgDl: docs.length === 0 ? 1 : totalLength / docs.length, df }
+}
 
-		return score
+/**
+ * Okapi BM25.
+ *
+ * The previous implementation had the term-frequency saturation half and no
+ * IDF at all, which is the half that does the discriminating: without it
+ * every matched term is worth the same, so a chunk matching three common
+ * words outscores the one chunk containing the rare term the query was
+ * actually about. It also normalized document length against a hardcoded
+ * `avgDl = 256` rather than the corpus in front of it, so the `b` term was
+ * measuring against a number with no relationship to these documents.
+ *
+ * IDF uses the standard Robertson/Sparck-Jones form with 0.5 smoothing,
+ * wrapped in `ln(1 + x)` so a term appearing in EVERY document scores a
+ * small positive weight rather than the negative one the unwrapped form
+ * produces when `df > n/2`.
+ */
+function bm25Score(queryTerms: string[], docTerms: string[], stats: CorpusStats): number {
+	const k1 = 1.2
+	const b = 0.75
+	const dl = docTerms.length
+
+	const termFreq = new Map<string, number>()
+	for (const term of docTerms) {
+		termFreq.set(term, (termFreq.get(term) ?? 0) + 1)
 	}
+
+	let score = 0
+	for (const term of queryTerms) {
+		const tf = termFreq.get(term) ?? 0
+		if (tf === 0) continue
+
+		const df = stats.df.get(term) ?? 0
+		const idf = Math.log(1 + (stats.n - df + 0.5) / (df + 0.5))
+
+		const numerator = tf * (k1 + 1)
+		const denominator = tf + k1 * (1 - b + b * (dl / stats.avgDl))
+		score += idf * (numerator / denominator)
+	}
+
+	return score
+}
+
+/**
+ * Rescale scores into [0, 1] by their maximum.
+ *
+ * Cosine similarity is bounded and BM25 is not, so blending them with a
+ * fixed `alpha` compared quantities on different scales — a BM25 of 7.4
+ * against a cosine of 0.83 meant `alpha` did not weight the two halves, it
+ * just let whichever scale happened to be larger win. Normalizing each
+ * ranking to its own maximum makes `alpha` mean what its name says.
+ */
+function normalizeByMax(
+	results: readonly VectorSearchResult[],
+): { chunk: VectorSearchResult['chunk']; score: number }[] {
+	const max = results.reduce((m, r) => Math.max(m, r.score), 0)
+	if (max <= 0) return results.map((r) => ({ chunk: r.chunk, score: 0 }))
+	return results.map((r) => ({ chunk: r.chunk, score: r.score / max }))
 }

package/src/rag/vector-store.test.ts

@@ -27,7 +27,7 @@
  *   - `deleteByDocument(documentId)`: NOT tenant-scoped in the
  *     current API — deletes every chunk with matching `documentId`
  *     ACROSS tenants. This is an asymmetry with `search` /
- *     `deleteByKnowledgeBase` and is intentional to pin (Codex #9 flag).
+ *     `deleteByKnowledgeBase` and is intentional to pin.
  *   - `deleteByKnowledgeBase(kbId, tenantId)`: tenant-scoped deletion
  *     by knowledge base.
  *   - `cosineSimilarity(a, b)`: returns 0 when arrays differ in length
@@ -151,7 +151,7 @@ describe('InMemoryVectorStore — delete', () => {
 		expect((await s.search(baseQuery)).map((r) => r.chunk.id)).toEqual(['b'])
 	})
 
-	it('deleteByDocument removes every chunk of the document ACROSS tenants (asymmetry per Codex #9)', async () => {
+	it('deleteByDocument removes every chunk of the document ACROSS tenants (asymmetry)', async () => {
 		const s = new InMemoryVectorStore()
 		await s.upsert([
 			chunk('a', { documentId: D1, tenantId: T1 }),

package/src/registry/ManagedRegistry.test.ts

@@ -13,7 +13,7 @@
  *     throws and aborts the batch (no partial-success semantics).
  *   - `getOrThrow(id)`: returns the item; throws
  *     `new Error("Not found: <id>. Available: <csv of known ids>")`
- *     — a plain `Error`, NOT a typed `XYZNotFoundError` (Codex #1).
+ *     — a plain `Error`, NOT a typed `XYZNotFoundError`.
  *   - No start / stop lifecycle exists (design §2.3 claim was
  *     fictional).
  */

package/src/registry/Registry.test.ts

@@ -2,7 +2,7 @@
  * Current-code invariants asserted (2026-04-21, ses_006 Phase 3):
  *
  *   - `register(id, item)` silently overwrites an existing entry —
- *     NO duplicate-error is thrown (Codex #1 verified the design
+ *     NO duplicate-error is thrown (review verified the design
  *     §2.3 "typed DuplicateRegistrationError" claim was fiction).
  *   - `get(id)` returns `undefined` for an unknown id (NOT a typed
  *     error). `getAll`, `listIds`, `has`, `size` reflect the current

package/src/registry/connector/definitions.test.ts

@@ -7,7 +7,7 @@
  *   - `listByType(connectionType)` filters by `connectionType`.
  *   - As a global (non-tenant-scoped) registry, ConnectorRegistry is
  *     shared across tenants; tenant isolation lives in
- *     `TenantConnectorManager` per Codex #5.
+ *     `TenantConnectorManager`.
  */
 
 import { describe, expect, it } from 'vitest'