@strav/search 0.4.31 → 1.0.0-alpha.31

package/src/drivers/embedded/embedded_driver.ts

@@ -1,136 +0,0 @@
-import type { SearchEngine } from '../../search_engine.ts'
-import type {
-  SearchDocument,
-  SearchOptions,
-  SearchResult,
-  IndexSettings,
-  DriverConfig,
-} from '../../types.ts'
-import { SqliteEngine } from './engine/sqlite_engine.ts'
-import { resolveTypoTolerance } from './engine/typo_expander.ts'
-import { resolveIndexPath, MEMORY_PATH } from './storage/paths.ts'
-import type { EmbeddedConfig, ResolvedTypoTolerance } from './types.ts'
-
-/**
- * In-process full-text search driver backed by SQLite FTS5.
- *
- * Each index lives in its own SQLite file (or `:memory:` for tests). The
- * driver maintains a `Map<indexName, SqliteEngine>` and creates engines
- * lazily on first reference. This means a fresh `upsert()` against a
- * never-created index will auto-create a default schema (single `_text`
- * column). Callers that want per-field weights call `createIndex()` first
- * with their `IndexSettings`.
- */
-export class EmbeddedDriver implements SearchEngine {
-  readonly name = 'embedded'
-
-  private readonly config: EmbeddedConfig
-  private readonly synchronous: 'OFF' | 'NORMAL' | 'FULL'
-  private readonly typo: ResolvedTypoTolerance
-  private readonly engines = new Map<string, SqliteEngine>()
-  /** Pending settings for indexes that haven't been opened yet. */
-  private readonly pendingSettings = new Map<string, IndexSettings>()
-
-  constructor(config: DriverConfig) {
-    this.config = (config ?? {}) as EmbeddedConfig
-    this.synchronous = this.config.synchronous ?? 'NORMAL'
-    this.typo = resolveTypoTolerance(this.config.typoTolerance)
-  }
-
-  // ── Document operations ──────────────────────────────────────────────────
-
-  async upsert(
-    index: string,
-    id: string | number,
-    document: Record<string, unknown>
-  ): Promise<void> {
-    this.engineFor(index).upsert(id, document)
-  }
-
-  async upsertMany(index: string, documents: SearchDocument[]): Promise<void> {
-    this.engineFor(index).upsertMany(documents)
-  }
-
-  async delete(index: string, id: string | number): Promise<void> {
-    this.engineFor(index).delete(id)
-  }
-
-  async deleteMany(index: string, ids: Array<string | number>): Promise<void> {
-    this.engineFor(index).deleteMany(ids)
-  }
-
-  // ── Index operations ─────────────────────────────────────────────────────
-
-  async flush(index: string): Promise<void> {
-    this.engineFor(index).flush()
-  }
-
-  async deleteIndex(index: string): Promise<void> {
-    const engine = this.engines.get(index)
-    if (engine) {
-      engine.close()
-      this.engines.delete(index)
-    }
-    this.pendingSettings.delete(index)
-
-    const path = resolveIndexPath(this.config, index)
-    if (path === MEMORY_PATH) return
-
-    const fs = await import('node:fs/promises')
-    for (const suffix of ['', '-wal', '-shm']) {
-      try {
-        await fs.unlink(`${path}${suffix}`)
-      } catch (err: any) {
-        if (err?.code !== 'ENOENT') throw err
-      }
-    }
-  }
-
-  async createIndex(index: string, options?: IndexSettings): Promise<void> {
-    if (options) this.pendingSettings.set(index, options)
-    // Force engine instantiation so the schema exists on disk.
-    this.engineFor(index)
-  }
-
-  // ── Search ───────────────────────────────────────────────────────────────
-
-  async search(index: string, query: string, options?: SearchOptions): Promise<SearchResult> {
-    return this.engineFor(index).search(query, options)
-  }
-
-  // ── Lifecycle ────────────────────────────────────────────────────────────
-
-  /** Close all open engines. Call from app shutdown. */
-  close(): void {
-    for (const engine of this.engines.values()) engine.close()
-    this.engines.clear()
-  }
-
-  /** Run FTS5 segment merge on every open index. Use from CLI for periodic ops. */
-  optimize(index?: string): void {
-    if (index) {
-      this.engineFor(index).optimize()
-      return
-    }
-    for (const engine of this.engines.values()) engine.optimize()
-  }
-
-  // ── Internals ────────────────────────────────────────────────────────────
-
-  private engineFor(index: string): SqliteEngine {
-    let engine = this.engines.get(index)
-    if (engine) return engine
-
-    const settings = this.pendingSettings.get(index)
-    engine = new SqliteEngine({
-      path: resolveIndexPath(this.config, index),
-      synchronous: this.synchronous,
-      typoTolerance: this.typo,
-      indexName: index,
-      settings,
-    })
-    this.engines.set(index, engine)
-    this.pendingSettings.delete(index)
-    return engine
-  }
-}

package/src/drivers/embedded/engine/field_registry.ts

@@ -1,97 +0,0 @@
-import type { IndexSettings } from '../../../types.ts'
-
-/** The default searchable column name used when no `searchableAttributes` are configured. */
-export const DEFAULT_TEXT_COLUMN = '_text'
-
-/**
- * The schema layout for one index: which document attributes feed which FTS5
- * column and which typed `documents` columns exist for filtering / sorting.
- *
- * When a caller doesn't declare `searchableAttributes`, we fall back to a
- * single `_text` column that concatenates every string-valued field at
- * indexing time. Users who want per-field weights opt in by passing
- * `IndexSettings`.
- */
-export class FieldRegistry {
-  /** FTS5 columns in declaration order — also the order BM25 weights apply in. */
-  readonly searchable: string[]
-  /** Filterable attributes — materialized as typed columns on `documents`. */
-  readonly filterable: string[]
-  /** Sortable attributes — materialized as typed columns on `documents`. */
-  readonly sortable: string[]
-  /** Union of filterable + sortable, deduplicated. */
-  readonly typedColumns: string[]
-  /** Primary key field name — defaults to 'id'. */
-  readonly primaryKey: string
-
-  constructor(settings?: IndexSettings) {
-    this.primaryKey = settings?.primaryKey ?? 'id'
-    this.searchable =
-      settings?.searchableAttributes && settings.searchableAttributes.length > 0
-        ? [...settings.searchableAttributes]
-        : [DEFAULT_TEXT_COLUMN]
-    this.filterable = settings?.filterableAttributes ?? []
-    this.sortable = settings?.sortableAttributes ?? []
-    this.typedColumns = Array.from(new Set([...this.filterable, ...this.sortable]))
-  }
-
-  /** Whether this registry uses the synthesised `_text` column. */
-  get usesDefaultTextColumn(): boolean {
-    return this.searchable.length === 1 && this.searchable[0] === DEFAULT_TEXT_COLUMN
-  }
-
-  /**
-   * Project a document into the values that go into the FTS5 row.
-   * For default mode, concatenate every string-valued field.
-   * For declared mode, pick each named attribute (coerced to string).
-   */
-  projectFtsValues(document: Record<string, unknown>): string[] {
-    if (this.usesDefaultTextColumn) {
-      const parts: string[] = []
-      for (const value of Object.values(document)) {
-        if (typeof value === 'string' && value.length > 0) parts.push(value)
-        else if (Array.isArray(value)) {
-          for (const item of value) {
-            if (typeof item === 'string' && item.length > 0) parts.push(item)
-          }
-        }
-      }
-      return [parts.join(' ')]
-    }
-
-    return this.searchable.map(attr => coerceText(document[attr]))
-  }
-
-  /**
-   * Project a document into the typed-column values stored on `documents`.
-   * Returned in the same order as `typedColumns`.
-   */
-  projectTypedValues(document: Record<string, unknown>): unknown[] {
-    return this.typedColumns.map(attr => coerceTyped(document[attr]))
-  }
-
-  /**
-   * Concatenate every searchable attribute into one long string suitable for
-   * tokenization (used for terms-dictionary maintenance).
-   */
-  concatSearchableText(document: Record<string, unknown>): string {
-    return this.projectFtsValues(document).join(' ')
-  }
-}
-
-function coerceText(value: unknown): string {
-  if (value === null || value === undefined) return ''
-  if (typeof value === 'string') return value
-  if (Array.isArray(value)) {
-    return value.map(v => coerceText(v)).filter(Boolean).join(' ')
-  }
-  if (typeof value === 'number' || typeof value === 'boolean') return String(value)
-  return ''
-}
-
-function coerceTyped(value: unknown): unknown {
-  if (value === null || value === undefined) return null
-  if (Array.isArray(value)) return JSON.stringify(value)
-  if (typeof value === 'object') return JSON.stringify(value)
-  return value
-}

package/src/drivers/embedded/engine/fts_query_builder.ts

@@ -1,184 +0,0 @@
-/**
- * Translate a user-facing query string into a sanitized FTS5 MATCH expression.
- *
- * Supported syntax (subset of Google-style search):
- * - `"foo bar"`   — exact phrase
- * - `-foo`        — exclude documents containing this token
- * - `+foo`        — required (default for all positive tokens — accepted for symmetry)
- * - `foo*`        — prefix match
- *
- * Everything else is treated as a positive ANDed token.
- *
- * Defends against FTS5 syntax injection by stripping or escaping any FTS5
- * operator characters from raw user tokens. The user never gets to write a
- * raw MATCH expression.
- */
-export interface FtsExpression {
-  /** Final MATCH expression, ready to bind into a query. */
-  match: string
-  /** The positive tokens (no quotes, no operators) — useful for typo expansion. */
-  positiveTokens: string[]
-  /** Whether the expression is empty (caller may short-circuit to "match all"). */
-  isEmpty: boolean
-}
-
-interface ParsedToken {
-  text: string
-  negate: boolean
-  phrase: boolean
-  prefix: boolean
-}
-
-const FTS5_RESERVED = /["()*:^]/g
-const PHRASE_RE = /"([^"]*)"/g
-
-export function compileQuery(input: string): FtsExpression {
-  const trimmed = input.trim()
-  if (!trimmed) return { match: '', positiveTokens: [], isEmpty: true }
-
-  const tokens = parseTokens(trimmed)
-  if (tokens.length === 0) return { match: '', positiveTokens: [], isEmpty: true }
-
-  const positives: string[] = []
-  const negatives: string[] = []
-  const positiveTokens: string[] = []
-
-  for (const tok of tokens) {
-    const rendered = renderToken(tok)
-    if (!rendered) continue
-
-    if (tok.negate) {
-      negatives.push(rendered)
-    } else {
-      positives.push(rendered)
-      if (!tok.phrase && !tok.prefix) positiveTokens.push(tok.text.toLowerCase())
-    }
-  }
-
-  if (positives.length === 0 && negatives.length === 0) {
-    return { match: '', positiveTokens: [], isEmpty: true }
-  }
-
-  // Pure-negative queries can't be expressed in FTS5 — fall back to no-match.
-  if (positives.length === 0) {
-    return { match: '', positiveTokens: [], isEmpty: true }
-  }
-
-  let expr = positives.join(' AND ')
-  if (negatives.length > 0) {
-    expr = `${expr} NOT (${negatives.join(' OR ')})`
-  }
-
-  return { match: expr, positiveTokens, isEmpty: false }
-}
-
-/**
- * Re-render a previously parsed query but with extra OR-candidates injected
- * for each positive token. Used by the typo expander.
- */
-export function compileQueryWithExpansions(
-  input: string,
-  expansions: Map<string, string[]>
-): FtsExpression {
-  const trimmed = input.trim()
-  if (!trimmed) return { match: '', positiveTokens: [], isEmpty: true }
-
-  const tokens = parseTokens(trimmed)
-  const positives: string[] = []
-  const negatives: string[] = []
-  const positiveTokens: string[] = []
-
-  for (const tok of tokens) {
-    if (tok.negate) {
-      const r = renderToken(tok)
-      if (r) negatives.push(r)
-      continue
-    }
-
-    if (tok.phrase || tok.prefix) {
-      const r = renderToken(tok)
-      if (r) positives.push(r)
-      continue
-    }
-
-    const sanitized = sanitizeBareToken(tok.text)
-    if (!sanitized) continue
-    positiveTokens.push(sanitized.toLowerCase())
-
-    const cands = expansions.get(sanitized.toLowerCase()) ?? []
-    if (cands.length === 0) {
-      positives.push(sanitized)
-    } else {
-      const all = [sanitized, ...cands].map(t => sanitizeBareToken(t)).filter(Boolean) as string[]
-      const unique = Array.from(new Set(all))
-      positives.push(`(${unique.join(' OR ')})`)
-    }
-  }
-
-  if (positives.length === 0) return { match: '', positiveTokens: [], isEmpty: true }
-
-  let expr = positives.join(' AND ')
-  if (negatives.length > 0) {
-    expr = `${expr} NOT (${negatives.join(' OR ')})`
-  }
-  return { match: expr, positiveTokens, isEmpty: false }
-}
-
-function parseTokens(input: string): ParsedToken[] {
-  const tokens: ParsedToken[] = []
-  let cursor = 0
-  let working = input
-
-  // Pull out phrase tokens first to avoid splitting on inner whitespace.
-  working = working.replace(PHRASE_RE, (_, phrase, offset) => {
-    const negate = offset > 0 && input[offset - 1] === '-'
-    tokens.push({ text: phrase, negate, phrase: true, prefix: false })
-    return ' '.repeat(_.length + (negate ? 1 : 0))
-  })
-
-  for (const raw of working.split(/\s+/)) {
-    if (!raw) continue
-    let text = raw
-    let negate = false
-    let prefix = false
-
-    if (text.startsWith('-')) {
-      negate = true
-      text = text.slice(1)
-    } else if (text.startsWith('+')) {
-      text = text.slice(1)
-    }
-    if (text.endsWith('*')) {
-      prefix = true
-      text = text.slice(0, -1)
-    }
-    if (!text) continue
-
-    tokens.push({ text, negate, phrase: false, prefix })
-  }
-
-  void cursor
-  return tokens
-}
-
-function renderToken(tok: ParsedToken): string | null {
-  if (tok.phrase) {
-    const cleaned = tok.text.replace(/"/g, '').trim()
-    if (!cleaned) return null
-    return `"${cleaned}"`
-  }
-  const sanitized = sanitizeBareToken(tok.text)
-  if (!sanitized) return null
-  return tok.prefix ? `${sanitized}*` : sanitized
-}
-
-function sanitizeBareToken(token: string): string {
-  // Replace any FTS5 operator characters with a space, then collapse to one
-  // word. If only one word survives we use it bare; otherwise wrap in quotes
-  // so FTS5 treats it as a phrase rather than two ANDed tokens.
-  const cleaned = token.replace(FTS5_RESERVED, ' ').trim()
-  if (!cleaned) return ''
-  const parts = cleaned.split(/\s+/).filter(Boolean)
-  if (parts.length === 1) return parts[0]!
-  return `"${parts.join(' ')}"`
-}

package/src/drivers/embedded/engine/query_compiler.ts

@@ -1,134 +0,0 @@
-import type { SearchOptions } from '../../../types.ts'
-import type { FieldRegistry } from './field_registry.ts'
-import type { FtsExpression } from './fts_query_builder.ts'
-import { compileFilter } from '../filters/filter_compiler.ts'
-import { quoteIdent } from './schema.ts'
-import { OPEN_SENTINEL, CLOSE_SENTINEL } from './snippet_formatter.ts'
-
-export interface CompiledSearch {
-  /** Main SELECT returning hits + score + snippets. */
-  sql: string
-  /** Bound parameters for the main SELECT. */
-  params: unknown[]
-  /** COUNT(*) variant for totalHits. */
-  countSql: string
-  /** Bound parameters for the count query (subset of `params`). */
-  countParams: unknown[]
-  /** Names of columns we asked SQLite to return for snippets, in order. */
-  snippetColumns: string[]
-}
-
-const DEFAULT_SNIPPET_TOKENS = 24
-
-export interface QueryCompilerOptions {
-  registry: FieldRegistry
-  expression: FtsExpression
-  search: SearchOptions
-  /** Per-column BM25 weights, matching `registry.searchable` order. Defaults to all-1. */
-  weights?: number[]
-}
-
-export function compileSearch(opts: QueryCompilerOptions): CompiledSearch {
-  const { registry, expression, search, weights } = opts
-  const filterableSet = new Set(registry.filterable)
-  const sortableSet = new Set(registry.sortable)
-
-  const filter = compileFilter(search.filter, filterableSet)
-
-  const whereParts: string[] = []
-  const matchParams: unknown[] = []
-
-  if (!expression.isEmpty) {
-    whereParts.push('fts.fts MATCH ?')
-    matchParams.push(expression.match)
-  }
-  if (filter.sql) whereParts.push(filter.sql)
-
-  const where = whereParts.length > 0 ? `WHERE ${whereParts.join(' AND ')}` : ''
-
-  // BM25 score (negative = better). Defaults to weight 1.0 for every column.
-  const ws = (weights ?? registry.searchable.map(() => 1)).map(w => Number(w) || 1)
-  const bm25Args = ws.length > 0 ? `, ${ws.join(', ')}` : ''
-
-  const orderBy = compileOrder(search.sort, sortableSet, expression.isEmpty, bm25Args)
-
-  // Build snippet expressions for each field the caller wants highlighted.
-  const wantedHighlights = pickHighlightFields(search.attributesToHighlight, registry)
-  const snippetSelect = wantedHighlights
-    .map(field => {
-      const idx = registry.searchable.indexOf(field)
-      return `snippet(fts.fts, ${idx}, '${OPEN_SENTINEL}', '${CLOSE_SENTINEL}', ' … ', ${DEFAULT_SNIPPET_TOKENS}) AS ${quoteIdent(`__snip_${field}`)}`
-    })
-    .join(', ')
-
-  const perPage = Math.max(1, search.perPage ?? 20)
-  const page = Math.max(1, search.page ?? 1)
-  const offset = (page - 1) * perPage
-
-  const selectCols = [
-    'documents.id AS id',
-    'documents.doc AS doc',
-    expression.isEmpty ? '0 AS score' : `bm25(fts.fts${bm25Args}) AS score`,
-  ]
-  if (snippetSelect) selectCols.push(snippetSelect)
-
-  const sql = `
-    SELECT ${selectCols.join(', ')}
-    FROM documents
-    ${expression.isEmpty ? '' : 'JOIN fts ON fts.rowid = documents.rowid'}
-    ${where}
-    ${orderBy}
-    LIMIT ? OFFSET ?
-  `.trim()
-
-  const countSql = `
-    SELECT COUNT(*) AS n
-    FROM documents
-    ${expression.isEmpty ? '' : 'JOIN fts ON fts.rowid = documents.rowid'}
-    ${where}
-  `.trim()
-
-  const allParams = [...matchParams, ...filter.params]
-  const params = [...allParams, perPage, offset]
-
-  return {
-    sql,
-    params,
-    countSql,
-    countParams: allParams,
-    snippetColumns: wantedHighlights,
-  }
-}
-
-function compileOrder(
-  sort: string[] | undefined,
-  sortableSet: ReadonlySet<string>,
-  matchAll: boolean,
-  bm25Args: string
-): string {
-  if (sort && sort.length > 0) {
-    const parts: string[] = []
-    for (const spec of sort) {
-      const [field, dirRaw] = spec.split(':') as [string, string | undefined]
-      if (!field || !sortableSet.has(field)) {
-        throw new Error(
-          `Field "${field}" is not in sortableAttributes. Add it to the index settings before sorting on it.`
-        )
-      }
-      const dir = dirRaw?.toLowerCase() === 'desc' ? 'DESC' : 'ASC'
-      parts.push(`${quoteIdent(field)} ${dir}`)
-    }
-    return `ORDER BY ${parts.join(', ')}`
-  }
-  if (matchAll) return 'ORDER BY documents.rowid ASC'
-  return `ORDER BY bm25(fts.fts${bm25Args}) ASC`
-}
-
-function pickHighlightFields(
-  requested: string[] | undefined,
-  registry: FieldRegistry
-): string[] {
-  if (registry.usesDefaultTextColumn) return []
-  if (!requested || requested.length === 0) return []
-  return requested.filter(f => registry.searchable.includes(f))
-}

package/src/drivers/embedded/engine/schema.ts

@@ -1,99 +0,0 @@
-import type { Database } from 'bun:sqlite'
-import type { FieldRegistry } from './field_registry.ts'
-
-const SCHEMA_VERSION = 1
-
-/** Quote a SQLite identifier (column or table). Throws on identifiers that contain a NUL byte. */
-export function quoteIdent(name: string): string {
-  if (name.includes('\0')) throw new Error(`Invalid identifier: ${name}`)
-  return `"${name.replace(/"/g, '""')}"`
-}
-
-export function applyConnectionPragmas(
-  db: Database,
-  synchronous: 'OFF' | 'NORMAL' | 'FULL'
-): void {
-  db.exec('PRAGMA journal_mode = WAL')
-  db.exec(`PRAGMA synchronous = ${synchronous}`)
-  db.exec('PRAGMA temp_store = MEMORY')
-  db.exec('PRAGMA foreign_keys = ON')
-}
-
-/**
- * Create the documents + FTS5 + terms_dict tables for a fresh index.
- * Idempotent: skips creation if `_meta` already exists.
- */
-export function createSchema(db: Database, registry: FieldRegistry): void {
-  if (schemaExists(db)) return
-
-  db.exec(`
-    CREATE TABLE _meta (
-      key TEXT PRIMARY KEY,
-      value TEXT NOT NULL
-    )
-  `)
-
-  const typedColumns = registry.typedColumns
-    .map(c => `${quoteIdent(c)} BLOB`)
-    .join(', ')
-  const typedColumnsClause = typedColumns ? `, ${typedColumns}` : ''
-
-  db.exec(`
-    CREATE TABLE documents (
-      rowid INTEGER PRIMARY KEY AUTOINCREMENT,
-      id TEXT NOT NULL UNIQUE,
-      doc TEXT NOT NULL${typedColumnsClause}
-    )
-  `)
-  db.exec('CREATE UNIQUE INDEX documents_id_idx ON documents(id)')
-
-  // One index per filterable column so WHERE clauses can use it.
-  for (const col of registry.filterable) {
-    db.exec(`CREATE INDEX ${quoteIdent(`documents_${col}_idx`)} ON documents(${quoteIdent(col)})`)
-  }
-
-  const ftsCols = registry.searchable.map(quoteIdent).join(', ')
-  // FTS5 default mode (no `content` option): the original text is stored in
-  // FTS5 itself so `snippet()` can echo it back highlighted. The Porter
-  // tokenizer applies English stemming; unicode61 normalises and folds
-  // diacritics so accented input matches its ASCII form.
-  db.exec(`
-    CREATE VIRTUAL TABLE fts USING fts5(
-      ${ftsCols},
-      tokenize = 'porter unicode61 remove_diacritics 2'
-    )
-  `)
-
-  // Maintain a terms dictionary for typo expansion.
-  db.exec(`
-    CREATE TABLE terms_dict (
-      term TEXT PRIMARY KEY,
-      doc_freq INTEGER NOT NULL DEFAULT 0
-    )
-  `)
-  db.exec('CREATE INDEX terms_dict_len_idx ON terms_dict(length(term))')
-
-  // Persist registry layout so we can detect mismatches on reopen.
-  const stmt = db.prepare('INSERT INTO _meta (key, value) VALUES (?, ?)')
-  stmt.run('schema_version', String(SCHEMA_VERSION))
-  stmt.run('searchable', JSON.stringify(registry.searchable))
-  stmt.run('filterable', JSON.stringify(registry.filterable))
-  stmt.run('sortable', JSON.stringify(registry.sortable))
-  stmt.run('primary_key', registry.primaryKey)
-}
-
-function schemaExists(db: Database): boolean {
-  const row = db
-    .query<{ name: string }, []>(
-      "SELECT name FROM sqlite_master WHERE type='table' AND name='_meta'"
-    )
-    .get()
-  return row !== null
-}
-
-export function readMeta(db: Database, key: string): string | null {
-  const row = db
-    .query<{ value: string }, [string]>('SELECT value FROM _meta WHERE key = ?')
-    .get(key)
-  return row ? row.value : null
-}

package/src/drivers/embedded/engine/snippet_formatter.ts

@@ -1,29 +0,0 @@
-/**
- * SQLite's snippet() returns text with the requested marker tokens around hits.
- * We use sentinel markers instead of `<mark>` directly so we can safely escape
- * any HTML in the source text first, then swap sentinels for the real tags.
- */
-export const OPEN_SENTINEL = '
STRAV_OPEN
'
-export const CLOSE_SENTINEL = '
STRAV_CLOSE
'
-
-export const OPEN_TAG = '<mark>'
-export const CLOSE_TAG = '</mark>'
-
-/**
- * Convert SQLite-snippet output (already wrapped in sentinels) into the
- * caller-facing string with `<mark>...</mark>` around hits and HTML-escaped
- * surrounding text.
- */
-export function formatSnippet(snippet: string | null | undefined): string {
-  if (!snippet) return ''
-  return escapeHtml(snippet).replaceAll(OPEN_SENTINEL, OPEN_TAG).replaceAll(CLOSE_SENTINEL, CLOSE_TAG)
-}
-
-function escapeHtml(input: string): string {
-  return input
-    .replace(/&/g, '&amp;')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;')
-    .replace(/'/g, '&#39;')
-}