@quaesitor-textus/mongo 0.2.0

package/src/sse.test.ts

@@ -0,0 +1,11 @@
+import { describe, it, expect } from 'vitest'
+import { formatSse, sseComment } from './sse'
+
+describe('sse', () => {
+  it('formats a data event with trailing blank line', () => {
+    expect(formatSse({ type: 'match', item: { _id: 'x' } })).toBe('data: {"type":"match","item":{"_id":"x"}}\n\n')
+  })
+  it('formats a heartbeat comment', () => {
+    expect(sseComment()).toBe(': ping\n\n')
+  })
+})

package/src/sse.ts

@@ -0,0 +1,7 @@
+// Framework-agnostic SSE wire helpers.
+export function formatSse(event: unknown): string {
+  return `data: ${JSON.stringify(event)}\n\n`
+}
+export function sseComment(text = 'ping'): string {
+  return `: ${text}\n\n`
+}

package/src/startSearchSync.test.ts

@@ -0,0 +1,42 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest'
+import { MongoClient } from 'mongodb'
+import { startSearchSync } from './startSearchSync'
+import type { MongoSearchConfig } from './config'
+
+const URL = process.env.MONGO_URL ?? 'mongodb://localhost:27018/?directConnection=true'
+const config: MongoSearchConfig = { targets: { name: { fields: ['name'] } } }
+let client: MongoClient
+let available = true
+
+beforeAll(async () => {
+  try { client = await MongoClient.connect(URL, { serverSelectionTimeoutMS: 1500 }) }
+  catch { available = false }
+})
+afterAll(async () => { await client?.close() })
+
+describe('startSearchSync backfill', () => {
+  it('derives a pre-existing doc written before the watcher starts', async () => {
+    if (!available) return
+    const col = client.db('qt_backfill_test').collection('docs')
+    await col.deleteMany({})
+    await col.insertOne({ _id: 'pre', name: 'Émile Zola' } as never) // raw, before watcher
+    const sync = startSearchSync(col, config, { backfill: true })
+    await new Promise(r => setTimeout(r, 1200))
+    const doc = await col.findOne({ _id: 'pre' as never }) as any
+    expect(doc?._qt?.name?.norm).toBe('emile zola')
+    await sync.stop()
+  })
+
+  it('re-derives a doc whose stored version is stale', async () => {
+    if (!available) return
+    const col = client.db('qt_backfill_test').collection('docs')
+    await col.deleteMany({})
+    // Doc that already has derived fields but stamped with an obsolete version.
+    await col.insertOne({ _id: 'stale', name: 'Wisława', _qt: { name: { norm: 'WRONG', ngrams: [] }, _v: 'old:0' } } as never)
+    const sync = startSearchSync(col, config, { backfill: true })
+    await new Promise(r => setTimeout(r, 1200))
+    const doc = await col.findOne({ _id: 'stale' as never }) as any
+    expect(doc?._qt?.name?.norm).toBe('wislawa') // re-derived with current folding
+    await sync.stop()
+  })
+})

package/src/startSearchSync.ts

@@ -0,0 +1,91 @@
+import type { ChangeStream, Collection } from 'mongodb'
+import type { MongoSearchConfig } from './config'
+import { DEFAULT_NAMESPACE } from './config'
+import { computeSearchFields } from './computeSearchFields'
+import { searchFieldsVersion } from './version'
+
+export type SearchSyncEvent =
+  | { type: 'indexing-started' }
+  | { type: 'indexing-finished'; count: number; durationMs: number }
+  | { type: 'indexed'; id: unknown }
+export type SearchSyncListener = (event: SearchSyncEvent) => void
+export interface SearchSync {
+  on(listener: SearchSyncListener): void
+  off(listener: SearchSyncListener): void
+  stop(): Promise<void>
+}
+export interface StartSearchSyncOptions { idleMs?: number; backfill?: boolean }
+
+// Tails the collection change stream, derives search fields, and notifies
+// listeners. Requires a replica set. Emits indexing-started / indexing-finished
+// (debounced burst, for logging) and a per-doc `indexed` event AFTER the derive
+// write resolves (so filters on the derived fields will match). With
+// `backfill: true`, derives any pre-existing documents missing the namespace on
+// start (change streams are forward-only, so this catches docs written before
+// the watcher ran or during downtime — e.g. an external Python writer).
+export function startSearchSync(
+  collection: Collection,
+  config: MongoSearchConfig,
+  options: StartSearchSyncOptions = {},
+): SearchSync {
+  const ns = config.namespace ?? DEFAULT_NAMESPACE
+  const { idleMs = 750, backfill = false } = options
+  const stream: ChangeStream = collection.watch([], { fullDocument: 'updateLookup' })
+  const listeners = new Set<SearchSyncListener>()
+  const emit = (e: SearchSyncEvent) => { for (const l of listeners) l(e) }
+
+  let active = false
+  let count = 0
+  let startedAt = 0
+  let idleTimer: ReturnType<typeof setTimeout> | undefined
+
+  stream.on('change', (change: any) => {
+    if (!['insert', 'update', 'replace'].includes(change.operationType)) return
+    const doc = change.fullDocument
+    if (!doc) return
+    const derived = computeSearchFields(doc, config) as Record<string, unknown>
+    // Loop guard: our own echo writes already match -> skip (and don't count).
+    if (JSON.stringify(doc[ns]) === JSON.stringify(derived[ns])) return
+
+    if (!active) { active = true; count = 0; startedAt = Date.now(); emit({ type: 'indexing-started' }) }
+    count += 1
+    // Emit `indexed` only AFTER the derive write lands, so live match-tests see
+    // the derived fields.
+    void collection.updateOne({ _id: doc._id }, { $set: { [ns]: derived[ns] } })
+      .then(() => emit({ type: 'indexed', id: doc._id }))
+      .catch(() => { /* ignore individual write failures */ })
+
+    if (idleTimer) clearTimeout(idleTimer)
+    idleTimer = setTimeout(() => {
+      active = false
+      emit({ type: 'indexing-finished', count, durationMs: Date.now() - startedAt })
+    }, idleMs)
+  })
+
+  // Optional one-time backfill. The stream is already open, so writes arriving
+  // during the sweep are handled normally; the loop-guard dedups the overlap.
+  if (backfill) void runBackfill()
+  async function runBackfill() {
+    const startedAt = Date.now()
+    let n = 0
+    emit({ type: 'indexing-started' })
+    // Re-derive documents whose search fields are missing OR were derived under a
+    // different version (library upgrade or config change).
+    const version = searchFieldsVersion(config)
+    const cursor = collection.find({
+      $or: [{ [ns]: { $exists: false } }, { [`${ns}._v`]: { $ne: version } }],
+    })
+    for await (const doc of cursor) {
+      const derived = computeSearchFields(doc, config) as Record<string, unknown>
+      await collection.updateOne({ _id: doc._id }, { $set: { [ns]: derived[ns] } }).catch(() => {})
+      n += 1
+    }
+    emit({ type: 'indexing-finished', count: n, durationMs: Date.now() - startedAt })
+  }
+
+  return {
+    on: (l) => { listeners.add(l) },
+    off: (l) => { listeners.delete(l) },
+    stop: async () => { if (idleTimer) clearTimeout(idleTimer); listeners.clear(); await stream.close() },
+  }
+}

package/src/version.test.ts

@@ -0,0 +1,40 @@
+import { describe, it, expect } from 'vitest'
+import { SEARCH_FIELDS_VERSION, searchFieldsVersion } from './version'
+import { computeSearchFields } from './computeSearchFields'
+import type { MongoSearchConfig } from './config'
+
+const cfg = (extra?: Partial<MongoSearchConfig>): MongoSearchConfig => ({
+  targets: { author: { fields: ['author'] } },
+  ...extra,
+})
+
+describe('searchFieldsVersion', () => {
+  it('starts with the code version prefix', () => {
+    expect(searchFieldsVersion(cfg()).startsWith(`${SEARCH_FIELDS_VERSION}:`)).toBe(true)
+  })
+
+  it('is stable for the same config', () => {
+    expect(searchFieldsVersion(cfg())).toBe(searchFieldsVersion(cfg()))
+  })
+
+  it('changes when the config changes', () => {
+    const a = searchFieldsVersion(cfg())
+    const b = searchFieldsVersion(cfg({ ngramSizes: [2, 3, 4] }))
+    const c = searchFieldsVersion({ targets: { author: { fields: ['author', 'title'] } } })
+    expect(a).not.toBe(b)
+    expect(a).not.toBe(c)
+  })
+
+  it('is order-independent over target keys', () => {
+    const x: MongoSearchConfig = { targets: { author: { fields: ['author'] }, title: { fields: ['title'] } } }
+    const y: MongoSearchConfig = { targets: { title: { fields: ['title'] }, author: { fields: ['author'] } } }
+    expect(searchFieldsVersion(x)).toBe(searchFieldsVersion(y))
+  })
+})
+
+describe('computeSearchFields version stamp', () => {
+  it('stamps _v on the derived namespace block', () => {
+    const out = computeSearchFields({ author: 'x' }, cfg()) as any
+    expect(out._qt._v).toBe(searchFieldsVersion(cfg()))
+  })
+})

package/src/version.ts

@@ -0,0 +1,41 @@
+import type { MongoSearchConfig } from './config'
+import { DEFAULT_NAMESPACE, DEFAULT_NGRAM_SIZES } from './config'
+
+// Bump whenever the DERIVED OUTPUT changes for the same input — i.e. any change
+// to normalizeText, toNgrams, buildCorpus, or computeSearchFields's shape.
+// History: 1 = initial; 2 = precomposed-letter folding (ł, ø, ß→ss, …).
+export const SEARCH_FIELDS_VERSION = 2
+
+// Order-independent JSON so two equal configs hash the same regardless of key order.
+function stableStringify(v: unknown): string {
+  if (v === null || typeof v !== 'object') return JSON.stringify(v) ?? 'null'
+  if (Array.isArray(v)) return '[' + v.map(stableStringify).join(',') + ']'
+  const obj = v as Record<string, unknown>
+  return '{' + Object.keys(obj).sort().map((k) => JSON.stringify(k) + ':' + stableStringify(obj[k])).join(',') + '}'
+}
+
+// Compact, dependency-free non-cryptographic hash (cyrb53).
+function cyrb53(str: string): string {
+  let h1 = 0xdeadbeef
+  let h2 = 0x41c6ce57
+  for (let i = 0; i < str.length; i++) {
+    const ch = str.charCodeAt(i)
+    h1 = Math.imul(h1 ^ ch, 2654435761)
+    h2 = Math.imul(h2 ^ ch, 1597334677)
+  }
+  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909)
+  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909)
+  return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(36)
+}
+
+// Effective version stamped on each document's derived block: code version plus
+// a fingerprint of the derivation-affecting config (namespace, n-gram sizes,
+// targets). A library upgrade (code version) OR a config change re-derives.
+export function searchFieldsVersion(config: MongoSearchConfig): string {
+  const sig = stableStringify({
+    ns: config.namespace ?? DEFAULT_NAMESPACE,
+    sizes: config.ngramSizes ?? DEFAULT_NGRAM_SIZES,
+    targets: config.targets,
+  })
+  return `${SEARCH_FIELDS_VERSION}:${cyrb53(sig)}`
+}