@thyn-ai/fuse-mojo-core 0.1.7

package/src/native.cjs

@@ -0,0 +1,374 @@
+'use strict'
+
+/**
+ * koffi loader for the fusemojo native kernel, with an ABI-version handshake.
+ *
+ * Resolution order:
+ *
+ *   1. `$FUSE_MOJO_NATIVE_LIB` (explicit path override, for development; the
+ *      pthread shim is expected next to it)
+ *   2. the platform package (`@thyn-ai/fuse-mojo-<platform>-<arch>`) installed as an
+ *      optional dependency of `@thyn-ai/fuse-mojo-core`
+ *   3. the repository development build output `kernels/fuse/build/`
+ *
+ * The pthread shim (`libfusemojoshim`) is loaded from the same directory as
+ * the kernel; without it, queries run sequentially on the calling thread
+ * (same results, less speed).
+ *
+ * If the kernel cannot be found, fails to load, or reports an ABI version
+ * this package does not understand, `NativeUnavailable` is thrown and the
+ * caller falls back to the vendored Fuse.js implementation. Set
+ * `FUSE_MOJO_DISABLE_NATIVE=1` to force that fallback (used by the
+ * differential test suite and on platforms without a prebuilt library).
+ * `FUSE_MOJO_THREADS=N` caps the native thread count (1 = sequential).
+ *
+ * Stable C ABI (v1)::
+ *
+ *   int32_t  fusemojo_abi_version(void)
+ *   void*    fusemojo_index_create(const uint16_t* chars, const int32_t* offsets,
+ *                                  int32_t n_texts, int32_t location, double distance,
+ *                                  double threshold, int32_t min_match_char_length,
+ *                                  int32_t find_all_matches, int32_t ignore_location,
+ *                                  int32_t compute_matches, int32_t include_matches)
+ *   void*    fusemojo_search_begin(void* handle, const uint16_t* pattern,
+ *                                  int32_t pattern_len, int32_t location_offset,
+ *                                  int32_t exact_check, double* out_scores,
+ *                                  int32_t* out_is_match, int32_t* out_idx_offsets,
+ *                                  int32_t n_jobs)
+ *   int32_t  fusemojo_search_range(void* ctx, int32_t job_id, int32_t start, int32_t end)
+ *   int32_t  fusemojo_search_end(void* ctx)
+ *   void     fusemojo_copy_indices(void* handle, int32_t* out_pairs)
+ *   void     fusemojo_index_destroy(void* handle)
+ *   int32_t  fusemojo_search_parallel(void* ctx, int32_t n_texts, int32_t max_threads)  [shim]
+ */
+
+const fs = require('node:fs')
+const os = require('node:os')
+const path = require('node:path')
+
+// Must equal ABI_VERSION in kernels/fuse/src/fusemojo.mojo. A mismatch means
+// the installed package and the resolved shared library disagree; fall back.
+const ABI_VERSION = 1
+
+const ENV_LIB = 'FUSE_MOJO_NATIVE_LIB'
+const ENV_DISABLE = 'FUSE_MOJO_DISABLE_NATIVE'
+const ENV_THREADS = 'FUSE_MOJO_THREADS'
+const ENV_MIN_CHUNK = 'FUSE_MOJO_MIN_CHUNK'
+
+class NativeUnavailable extends Error {
+  constructor(message) {
+    super(message)
+    this.name = 'NativeUnavailable'
+    this.code = 'FUSE_MOJO_NATIVE_UNAVAILABLE'
+  }
+}
+
+function libBasename() {
+  if (process.platform === 'darwin') return 'libfusemojo.dylib'
+  if (process.platform === 'linux') return 'libfusemojo.so'
+  if (process.platform === 'win32') return 'fusemojo.dll' // no Mojo toolchain builds this today
+  return 'libfusemojo.so'
+}
+
+function shimBasename() {
+  if (process.platform === 'darwin') return 'libfusemojoshim.dylib'
+  if (process.platform === 'linux') return 'libfusemojoshim.so'
+  if (process.platform === 'win32') return 'fusemojoshim.dll'
+  return 'libfusemojoshim.so'
+}
+
+function candidatePaths() {
+  /** [sourceLabel, absolutePath] candidates, in resolver order. */
+  const out = []
+  const override = process.env[ENV_LIB]
+  if (override) {
+    out.push([`env ${ENV_LIB}`, override])
+  }
+  // Platform package, resolved relative to this package so npm's nested or
+  // hoisted layouts both work.
+  const platformPkg = `@thyn-ai/fuse-mojo-${process.platform}-${process.arch}`
+  try {
+    const pkgJson = require.resolve(`${platformPkg}/package.json`, { paths: [__dirname] })
+    out.push([`platform package ${platformPkg}`, path.join(path.dirname(pkgJson), 'lib', libBasename())])
+  } catch {
+    // optional dependency not installed (unsupported platform or pruned)
+  }
+  out.push([
+    'repo-dev build output',
+    path.resolve(__dirname, '..', '..', '..', '..', '..', 'kernels', 'fuse', 'build', libBasename()),
+  ])
+  return out
+}
+
+function bindAbi(lib) {
+  // koffi parses C prototypes; TypedArrays passed to pointer parameters are
+  // pinned for the duration of the call.
+  return {
+    abi_version: lib.func('int32_t fusemojo_abi_version(void)'),
+    index_create: lib.func(`void* fusemojo_index_create(
+      const uint16_t* chars, const int32_t* offsets, int32_t n_texts,
+      int32_t location, double distance, double threshold,
+      int32_t min_match_char_length, int32_t find_all_matches,
+      int32_t ignore_location, int32_t compute_matches, int32_t include_matches)`),
+    search_begin: lib.func(`void* fusemojo_search_begin(
+      void* handle, const uint16_t* pattern, int32_t pattern_len,
+      int32_t location_offset, int32_t exact_check,
+      double* out_scores, int32_t* out_is_match, int32_t* out_idx_offsets,
+      int32_t n_jobs)`),
+    search_range: lib.func(
+      'int32_t fusemojo_search_range(void* ctx, int32_t job_id, int32_t start, int32_t end)'
+    ),
+    search_end: lib.func('int32_t fusemojo_search_end(void* ctx)'),
+    copy_indices: lib.func('void fusemojo_copy_indices(void* handle, int32_t* out_pairs)'),
+    index_destroy: lib.func('void fusemojo_index_destroy(void* handle)'),
+  }
+}
+
+let _koffi = null
+let _lib = null
+let _shim = null // null = not tried; false = unavailable; function = parallel runner
+let _libSource = null
+let _loadError = null
+
+function loadKoffi() {
+  if (_koffi) return _koffi
+  try {
+    _koffi = require('koffi')
+  } catch (err) {
+    throw new NativeUnavailable(`koffi is not available: ${err.message}`)
+  }
+  return _koffi
+}
+
+function threadCap() {
+  const raw = process.env[ENV_THREADS]
+  if (raw !== undefined) {
+    const n = Number.parseInt(raw, 10)
+    if (Number.isInteger(n) && n > 0) return n
+  }
+  const cpus = os.cpus().length || 1
+  return Math.min(cpus, 64)
+}
+
+function minChunk() {
+  const raw = process.env[ENV_MIN_CHUNK]
+  if (raw !== undefined) {
+    const n = Number.parseInt(raw, 10)
+    if (Number.isInteger(n) && n > 0) return n
+  }
+  return 2048
+}
+
+function load() {
+  /** Resolve, dlopen, and ABI-handshake the native kernel. Never caches failure. */
+  if (process.env[ENV_DISABLE] === '1') {
+    throw new NativeUnavailable(`native kernel disabled by ${ENV_DISABLE}=1`)
+  }
+  if (_lib !== null) {
+    return _lib
+  }
+  const koffi = loadKoffi()
+  const errors = []
+  for (const [label, candidate] of candidatePaths()) {
+    let resolved
+    try {
+      if (!candidate || !fs.existsSync(candidate)) {
+        continue
+      }
+      try {
+        resolved = koffi.load(candidate)
+      } catch (err) {
+        errors.push(`${label} (${candidate}): ${err.message}`)
+        continue
+      }
+      let abi
+      try {
+        const bound = bindAbi(resolved)
+        abi = bound.abi_version()
+        if (abi !== ABI_VERSION) {
+          errors.push(`${label} (${candidate}): native ABI v${abi} != wrapper ABI v${ABI_VERSION}`)
+          continue
+        }
+        // The pthread shim is optional; without it queries run sequentially.
+        _shim = false
+        const shimPath = path.join(path.dirname(candidate), shimBasename())
+        try {
+          if (fs.existsSync(shimPath)) {
+            const shimLib = koffi.load(shimPath)
+            _shim = shimLib.func(
+              'int32_t fusemojo_search_parallel(void* ctx, int32_t n_texts, int32_t max_threads, int32_t min_chunk)'
+            )
+          }
+        } catch {
+          _shim = false
+        }
+        _lib = bound
+        _libSource = `${label} (${candidate})${_shim ? ' +pthread shim' : ''}`
+        _loadError = null
+        return bound
+      } catch (err) {
+        errors.push(`${label} (${candidate}): ABI not recognized: ${err.message}`)
+        continue
+      }
+    } catch (err) {
+      errors.push(`${label}: ${err.message}`)
+    }
+  }
+  _loadError = errors.join('; ') || 'no native kernel found on any resolver path'
+  throw new NativeUnavailable(_loadError)
+}
+
+function nativeAvailable() {
+  /** True if the native kernel can score right now. Never throws. */
+  try {
+    load()
+    return true
+  } catch {
+    return false
+  }
+}
+
+function backendInfo() {
+  /** Diagnostics for the active backend. Never throws. */
+  const info = {
+    native_available: false,
+    native_source: null,
+    abi_version_expected: ABI_VERSION,
+    abi_version_native: null,
+    disabled_by_env: process.env[ENV_DISABLE] === '1',
+    threads: 0,
+    platform: process.platform,
+    arch: process.arch,
+    error: null,
+  }
+  try {
+    const lib = load()
+    info.native_available = true
+    info.native_source = _libSource
+    info.abi_version_native = lib.abi_version()
+    info.threads = threadCap()
+  } catch (err) {
+    info.error = err.message
+  }
+  return info
+}
+
+const registry = new FinalizationRegistry((handle) => {
+  try {
+    if (_lib) _lib.index_destroy(handle)
+  } catch {
+    // best-effort cleanup during GC; never throw
+  }
+})
+
+class NativeIndex {
+  /** Owned handle to a native text-collection index. */
+  constructor(chars, offsets, options) {
+    const lib = load() // throws NativeUnavailable
+    const nTexts = offsets.length - 1
+    const handle = lib.index_create(
+      chars,
+      offsets,
+      nTexts,
+      options.location,
+      options.distance,
+      options.threshold,
+      options.minMatchCharLength,
+      options.findAllMatches ? 1 : 0,
+      options.ignoreLocation ? 1 : 0,
+      options.computeMatches ? 1 : 0,
+      options.includeMatches ? 1 : 0
+    )
+    if (handle === null || handle === undefined) {
+      throw new NativeUnavailable(
+        'native kernel rejected the index (invalid sizes); falling back to the vendored Fuse.js'
+      )
+    }
+    // The kernel copies every buffer; the JS arrays may be GC'd.
+    this._lib = lib
+    this._handle = handle
+    this.nTexts = nTexts
+    this._threads = threadCap()
+    this._minChunk = minChunk()
+    // Reusable per-query buffers (single-threaded per instance).
+    this._scores = new Float64Array(nTexts)
+    this._isMatch = new Int32Array(nTexts)
+    this._idxOffsets = new Int32Array(nTexts + 1)
+    registry.register(this, handle, this)
+  }
+
+  /**
+   * Score one pattern chunk (Uint16Array of length 1..32) against every text.
+   * Returns { totalPairs, scores, isMatch, idxOffsets } views over the
+   * instance's shared buffers (valid until the next searchChunk call).
+   */
+  searchChunk(patternU16, locationOffset, exactCheck) {
+    if (this._handle === null) {
+      throw new NativeUnavailable('native index is closed')
+    }
+    const ctx = this._lib.search_begin(
+      this._handle,
+      patternU16,
+      patternU16.length,
+      locationOffset,
+      exactCheck ? 1 : 0,
+      this._scores,
+      this._isMatch,
+      this._idxOffsets,
+      this._threads
+    )
+    if (ctx === null || ctx === undefined) {
+      throw new NativeUnavailable('native kernel rejected the pattern chunk')
+    }
+    let rc
+    if (_shim) {
+      rc = _shim(ctx, this.nTexts, this._threads, this._minChunk)
+    } else {
+      rc = this._lib.search_range(ctx, 0, 0, this.nTexts)
+    }
+    if (rc !== 0) {
+      // Drain the context so buffers/stash stay consistent, then fail.
+      this._lib.search_end(ctx)
+      throw new NativeUnavailable(`native scoring failed with status ${rc}`)
+    }
+    const totalPairs = this._lib.search_end(ctx)
+    if (totalPairs < 0) {
+      throw new NativeUnavailable(`native scoring finalize failed with status ${totalPairs}`)
+    }
+    return {
+      totalPairs,
+      scores: this._scores,
+      isMatch: this._isMatch,
+      idxOffsets: this._idxOffsets,
+    }
+  }
+
+  copyIndices(totalPairs) {
+    const out = new Int32Array(totalPairs * 2)
+    if (totalPairs > 0) {
+      this._lib.copy_indices(this._handle, out)
+    }
+    return out
+  }
+
+  destroy() {
+    const handle = this._handle
+    this._handle = null
+    if (handle) {
+      registry.unregister(this)
+      this._lib.index_destroy(handle)
+    }
+  }
+}
+
+module.exports = {
+  NativeIndex,
+  NativeUnavailable,
+  nativeAvailable,
+  backendInfo,
+  ABI_VERSION,
+  ENV_LIB,
+  ENV_DISABLE,
+  ENV_THREADS,
+  ENV_MIN_CHUNK,
+}

package/src/options.cjs

@@ -0,0 +1,242 @@
+'use strict'
+
+/**
+ * Option handling for @thyn-ai/fuse-mojo-core: defaults, validation, key store,
+ * property-path getter, and the field-length norm.
+ *
+ * These functions replicate the *observable* behavior of the corresponding
+ * Fuse.js v7 configuration code paths (default values, key weight
+ * normalization, dotted-path value extraction, `1/sqrt(tokens)` field norm
+ * rounded to 3 decimals) so that results are identical on both backends.
+ * Written clean-room against the published documentation and observed
+ * behavior of the reference package.
+ */
+
+const SPACE_RUN = /[^ ]+/g
+
+/** Default options, identical to Fuse.js v7. */
+const DEFAULT_CONFIG = {
+  isCaseSensitive: false,
+  includeMatches: false,
+  findAllMatches: false,
+  minMatchCharLength: 1,
+  includeScore: false,
+  keys: [],
+  shouldSort: true,
+  sortFn: defaultSortFn,
+  location: 0,
+  threshold: 0.6,
+  distance: 100,
+  useExtendedSearch: false,
+  getFn: fuseGet,
+  ignoreLocation: false,
+  ignoreFieldNorm: false,
+  fieldNormWeight: 1,
+  ignoreDiacritics: false,
+}
+
+function defaultSortFn(a, b) {
+  return a.score === b.score ? (a.idx < b.idx ? -1 : 1) : a.score < b.score ? -1 : 1
+}
+
+const SUPPORTED_OPTIONS = [
+  'keys',
+  'threshold',
+  'location',
+  'distance',
+  'minMatchCharLength',
+  'includeScore',
+  'includeMatches',
+  'shouldSort',
+  'ignoreLocation',
+  'isCaseSensitive',
+  'findAllMatches',
+  'ignoreFieldNorm',
+  'fieldNormWeight',
+]
+
+class UnsupportedOptionError extends Error {
+  constructor(what) {
+    super(
+      `fuse-mojo: unsupported option or input: ${what}. ` +
+        `Supported options are: ${SUPPORTED_OPTIONS.join(', ')} ` +
+        '(plus the `limit` search parameter). Extended search ' +
+        '(`useExtendedSearch`, logical `$and`/`$or` queries), ' +
+        '`ignoreDiacritics`, custom `getFn`, custom `sortFn`, and external ' +
+        'indices (`Fuse.createIndex`/`parseIndex`) are not supported.'
+    )
+    this.name = 'UnsupportedOptionError'
+    this.code = 'FUSE_MOJO_UNSUPPORTED_OPTION'
+  }
+}
+
+function validateOptions(options) {
+  if (options.useExtendedSearch) {
+    throw new UnsupportedOptionError('useExtendedSearch')
+  }
+  if (options.ignoreDiacritics) {
+    throw new UnsupportedOptionError('ignoreDiacritics')
+  }
+  if (options.getFn !== undefined && options.getFn !== fuseGet) {
+    throw new UnsupportedOptionError('custom getFn')
+  }
+  if (options.sortFn !== undefined && options.sortFn !== defaultSortFn) {
+    throw new UnsupportedOptionError('custom sortFn')
+  }
+  if (!Array.isArray(options.keys)) {
+    throw new UnsupportedOptionError('keys must be an array of strings or {name, weight} objects')
+  }
+  if (!Number.isFinite(options.threshold)) {
+    throw new UnsupportedOptionError('non-finite threshold')
+  }
+  if (typeof options.distance !== 'number' || Number.isNaN(options.distance)) {
+    throw new UnsupportedOptionError('non-numeric distance')
+  }
+  if (!Number.isInteger(options.location) || Math.abs(options.location) > 0x7fffffff) {
+    throw new UnsupportedOptionError('non-integer or out-of-range location')
+  }
+  if (
+    !Number.isInteger(options.minMatchCharLength) ||
+    options.minMatchCharLength > 0x7fffffff
+  ) {
+    throw new UnsupportedOptionError('non-integer or out-of-range minMatchCharLength')
+  }
+  for (const key of options.keys) {
+    if (typeof key === 'object' && key !== null && !Array.isArray(key) && key.getFn) {
+      throw new UnsupportedOptionError('key getFn')
+    }
+  }
+}
+
+/**
+ * Key store for standard (non-logical) search. Weights are the RAW weights
+ * from the key specifications: the reference only normalizes weights (to sum
+ * to 1) inside the KeyStore used by logical $and/$or queries; the regular
+ * object-list search path reads the un-normalized weights from the index's
+ * key entries. Logical queries are unsupported, so no normalization happens
+ * here either.
+ */
+function createKeyStore(keys) {
+  const store = []
+  for (const key of keys) {
+    let path
+    let id
+    let src
+    let weight = 1
+    if (typeof key === 'string' || Array.isArray(key)) {
+      src = key
+      path = Array.isArray(key) ? key : key.split('.')
+      id = Array.isArray(key) ? key.join('.') : key
+    } else if (typeof key === 'object' && key !== null) {
+      if (!Object.prototype.hasOwnProperty.call(key, 'name')) {
+        throw new Error('Missing "name" property in key object')
+      }
+      const name = key.name
+      src = name
+      if (Object.prototype.hasOwnProperty.call(key, 'weight')) {
+        weight = key.weight
+        if (weight <= 0) {
+          throw new Error(`Invalid "weight" property in key object: ${name}`)
+        }
+      }
+      path = typeof name === 'string' ? name.split('.') : name
+      id = Array.isArray(name) ? name.join('.') : name
+    } else {
+      throw new UnsupportedOptionError(`key of type ${typeof key}`)
+    }
+    store.push({ path, id, src, weight })
+  }
+  return store
+}
+
+const INFINITY = 1 / 0
+
+function baseToString(value) {
+  if (typeof value === 'string') {
+    return value
+  }
+  const result = value + ''
+  return result === '0' && 1 / value === -INFINITY ? '-0' : result
+}
+
+function toString(value) {
+  return value == null ? '' : baseToString(value)
+}
+
+/**
+ * Dotted-path property getter with array descent. Behavioral replica of the
+ * reference default getFn: string/number/boolean leaves are stringified,
+ * arrays are flattened depth-first in ascending index order, and the result
+ * is the single value, or the list of values when any array was traversed.
+ */
+function fuseGet(obj, path) {
+  const list = []
+  let arr = false
+
+  const deepGet = (obj, path, index) => {
+    if (obj === undefined || obj === null) {
+      return
+    }
+    if (!path[index]) {
+      // No path left (or an empty path segment): the object itself.
+      list.push(obj)
+      return
+    }
+    const key = path[index]
+    const value = obj[key]
+    if (value === undefined || value === null) {
+      return
+    }
+    if (
+      index === path.length - 1 &&
+      (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean')
+    ) {
+      list.push(toString(value))
+    } else if (Array.isArray(value)) {
+      arr = true
+      for (let i = 0, len = value.length; i < len; i += 1) {
+        deepGet(value[i], path, index + 1)
+      }
+    } else if (path.length) {
+      deepGet(value, path, index + 1)
+    }
+  }
+
+  deepGet(obj, typeof path === 'string' ? path.split('.') : path, 0)
+  return arr ? list : list[0]
+}
+
+function isBlank(value) {
+  return !value.trim().length
+}
+
+/**
+ * Field-length norm: 1 / numTokens^(0.5 * weight), rounded to 3 decimals,
+ * cached by token count — identical to the reference norm generator.
+ */
+function createNormGetter(weight, mantissa = 3) {
+  const cache = new Map()
+  const m = Math.pow(10, mantissa)
+  return (value) => {
+    const numTokens = value.match(SPACE_RUN).length
+    if (cache.has(numTokens)) {
+      return cache.get(numTokens)
+    }
+    const norm = 1 / Math.pow(numTokens, 0.5 * weight)
+    const n = parseFloat(Math.round(norm * m) / m)
+    cache.set(numTokens, n)
+    return n
+  }
+}
+
+module.exports = {
+  DEFAULT_CONFIG,
+  SUPPORTED_OPTIONS,
+  UnsupportedOptionError,
+  validateOptions,
+  createKeyStore,
+  fuseGet,
+  isBlank,
+  createNormGetter,
+  defaultSortFn,
+}