@delali/narsil-react-native 0.0.0-stage → 0.3.1-next.146

package/src/engine/stand-ins/structured-clone.ts

@@ -0,0 +1,118 @@
+type TypedArrayConstructor = new (buffer: ArrayBuffer, byteOffset: number, length: number) => ArrayBufferView
+
+const ERROR_CONSTRUCTORS: Record<string, ErrorConstructor> = {
+  Error,
+  EvalError,
+  RangeError,
+  ReferenceError,
+  SyntaxError,
+  TypeError,
+  URIError,
+}
+
+function dataCloneError(message: string): Error {
+  const DomException = (globalThis as { DOMException?: new (message: string, name: string) => Error }).DOMException
+  if (DomException !== undefined) return new DomException(message, 'DataCloneError')
+  const error = new Error(message)
+  error.name = 'DataCloneError'
+  return error
+}
+
+function typedArrayConstructorOf(view: ArrayBufferView): TypedArrayConstructor | null {
+  const viewType = (view as { constructor?: unknown }).constructor
+  if (typeof viewType !== 'function') return null
+  const scope = globalThis as Record<string, unknown>
+  const name = (viewType as { name?: unknown }).name
+  if (typeof name !== 'string' || scope[name] !== viewType) return null
+  return viewType as TypedArrayConstructor
+}
+
+function cloneValue(value: unknown, memory: Map<object, unknown>): unknown {
+  if (typeof value === 'symbol') throw dataCloneError('A symbol cannot be cloned')
+  if (typeof value === 'function') throw dataCloneError('A function cannot be cloned')
+  if (value === null || typeof value !== 'object') return value
+
+  const remembered = memory.get(value)
+  if (remembered !== undefined || memory.has(value)) return remembered
+
+  if (value instanceof Date) {
+    const copy = new Date(value.getTime())
+    memory.set(value, copy)
+    return copy
+  }
+  if (value instanceof RegExp) {
+    const copy = new RegExp(value.source, value.flags)
+    memory.set(value, copy)
+    return copy
+  }
+  if (value instanceof Boolean || value instanceof Number || value instanceof String) {
+    const copy = Object(value.valueOf())
+    memory.set(value, copy)
+    return copy
+  }
+  if (typeof BigInt !== 'undefined' && Object.prototype.toString.call(value) === '[object BigInt]') {
+    const copy = Object((value as { valueOf(): bigint }).valueOf())
+    memory.set(value, copy)
+    return copy
+  }
+  if (value instanceof ArrayBuffer) {
+    const copy = value.slice(0, value.byteLength)
+    memory.set(value, copy)
+    return copy
+  }
+  if (ArrayBuffer.isView(value)) {
+    const buffer = cloneValue(value.buffer, memory) as ArrayBuffer
+    if (value instanceof DataView) {
+      const copy = new DataView(buffer, value.byteOffset, value.byteLength)
+      memory.set(value, copy)
+      return copy
+    }
+    const ViewType = typedArrayConstructorOf(value)
+    if (ViewType === null) throw dataCloneError('This kind of array buffer view cannot be cloned')
+    const length = (value as unknown as { length: number }).length
+    const copy = new ViewType(buffer, value.byteOffset, length)
+    memory.set(value, copy)
+    return copy
+  }
+  if (value instanceof Map) {
+    const copy = new Map<unknown, unknown>()
+    memory.set(value, copy)
+    for (const [key, entry] of value) copy.set(cloneValue(key, memory), cloneValue(entry, memory))
+    return copy
+  }
+  if (value instanceof Set) {
+    const copy = new Set<unknown>()
+    memory.set(value, copy)
+    for (const entry of value) copy.add(cloneValue(entry, memory))
+    return copy
+  }
+  if (value instanceof Error) {
+    const ErrorType = ERROR_CONSTRUCTORS[value.name] ?? Error
+    const copy = new ErrorType(value.message)
+    memory.set(value, copy)
+    if (typeof value.stack === 'string') copy.stack = value.stack
+    if ('cause' in value) (copy as { cause?: unknown }).cause = cloneValue(value.cause, memory)
+    return copy
+  }
+
+  const copy = (Array.isArray(value) ? new Array<unknown>(value.length) : {}) as Record<string, unknown>
+  memory.set(value, copy)
+  for (const key of Object.keys(value)) {
+    copy[key] = cloneValue((value as Record<string, unknown>)[key], memory)
+  }
+  return copy
+}
+
+function structuredCloneStandIn<T>(value: T): T {
+  return cloneValue(value, new Map()) as T
+}
+
+export function installStructuredClone(): void {
+  if (typeof (globalThis as { structuredClone?: unknown }).structuredClone === 'function') return
+  Object.defineProperty(globalThis, 'structuredClone', {
+    value: structuredCloneStandIn,
+    writable: true,
+    configurable: true,
+    enumerable: false,
+  })
+}

package/src/engine/streamed.ts

@@ -0,0 +1,40 @@
+import type { DurabilityConfig, Narsil, NarsilConfig, ReactNativeHost } from '@delali/narsil'
+import { DEFAULT_DURABILITY_DIRECTORY } from '../constants'
+import type { StreamedMethod } from '../methods'
+import { collectStream, fileChunks, resolveAppPath, writeStreamToFile } from './byte-files'
+
+export function withPhoneDurability<T extends { durability?: DurabilityConfig | null }>(
+  config: T,
+  durabilitySetElsewhere = false,
+): Omit<T, 'durability'> & Pick<NarsilConfig, 'durability'> {
+  const { durability, ...rest } = config
+  if (durability === null || durabilitySetElsewhere) return rest
+  return { ...rest, durability: { directory: DEFAULT_DURABILITY_DIRECTORY, ...durability } }
+}
+
+export function callStreamed(
+  narsil: Narsil,
+  host: ReactNativeHost,
+  method: StreamedMethod,
+  args: readonly unknown[],
+): Promise<unknown> {
+  const indexName = args[0] as string
+  switch (method) {
+    case 'snapshot':
+      return narsil.snapshot(indexName).then(collectStream)
+    case 'exportDocuments':
+      return narsil.exportDocuments(indexName).then(collectStream)
+    case 'snapshotToFile':
+      return narsil
+        .snapshot(indexName)
+        .then(stream => writeStreamToFile(host.files, stream, resolveAppPath(host, args[1] as string)))
+    case 'exportDocumentsToFile':
+      return narsil
+        .exportDocuments(indexName)
+        .then(stream => writeStreamToFile(host.files, stream, resolveAppPath(host, args[1] as string)))
+    case 'restoreFromFile':
+      return narsil.restore(indexName, fileChunks(host.files, resolveAppPath(host, args[1] as string)))
+    case 'importDocumentExportFromFile':
+      return narsil.importDocumentExport(indexName, fileChunks(host.files, resolveAppPath(host, args[1] as string)))
+  }
+}

package/src/engine/transfer.ts

@@ -0,0 +1,72 @@
+import type { TransferredError } from '../transferred'
+
+function isPlainObject(value: object): boolean {
+  const prototype = Object.getPrototypeOf(value)
+  return prototype === Object.prototype || prototype === null
+}
+
+function viewCopy(view: ArrayBufferView): ArrayBufferView {
+  if (view.byteOffset === 0 && view.byteLength === view.buffer.byteLength) return view
+  const copy = new Uint8Array(view.byteLength)
+  copy.set(new Uint8Array(view.buffer, view.byteOffset, view.byteLength))
+  if (view instanceof DataView) return new DataView(copy.buffer)
+  const Constructor = view.constructor as new (buffer: ArrayBuffer) => ArrayBufferView
+  return new Constructor(copy.buffer)
+}
+
+export function transferredError(error: unknown, seen = new Map<object, unknown>()): TransferredError {
+  if (!(error instanceof Error)) {
+    return { transferredError: true, name: 'Error', message: String(error), code: undefined, details: undefined }
+  }
+  const coded = error as Error & { code?: unknown; details?: unknown }
+  return {
+    transferredError: true,
+    name: error.name,
+    message: error.message,
+    code: typeof coded.code === 'string' ? coded.code : undefined,
+    details: coded.details === undefined ? undefined : transferable(coded.details, seen),
+  }
+}
+
+export function transferable(value: unknown, seen = new Map<object, unknown>()): unknown {
+  if (value === null || typeof value !== 'object') return typeof value === 'function' ? undefined : value
+  const known = seen.get(value)
+  if (known !== undefined) return known
+  if (ArrayBuffer.isView(value)) {
+    const copy = viewCopy(value)
+    seen.set(value, copy)
+    return copy
+  }
+  if (value instanceof Error) {
+    const copy = transferredError(value, seen)
+    seen.set(value, copy)
+    return copy
+  }
+  if (Array.isArray(value)) {
+    const copy: unknown[] = new Array(value.length)
+    seen.set(value, copy)
+    for (let index = 0; index < value.length; index++) copy[index] = transferable(value[index], seen)
+    return copy
+  }
+  if (value instanceof Map) {
+    const copy = new Map<unknown, unknown>()
+    seen.set(value, copy)
+    for (const [key, entry] of value) copy.set(transferable(key, seen), transferable(entry, seen))
+    return copy
+  }
+  if (value instanceof Set) {
+    const copy = new Set<unknown>()
+    seen.set(value, copy)
+    for (const entry of value) copy.add(transferable(entry, seen))
+    return copy
+  }
+  if (value instanceof Date || value instanceof RegExp || value instanceof ArrayBuffer) return value
+  if (!isPlainObject(value)) return value
+  const copy: Record<string, unknown> = {}
+  seen.set(value, copy)
+  for (const key of Object.keys(value)) {
+    const entry = transferable((value as Record<string, unknown>)[key], seen)
+    if (entry !== undefined) copy[key] = entry
+  }
+  return copy
+}

package/src/errors.ts

@@ -0,0 +1,73 @@
+import { isTransferredError, type TransferredError } from './transferred'
+
+const NARSIL_ERROR_MARKER = Symbol.for('@delali/narsil.NarsilError')
+const NARSIL_ERROR_NAME = 'NarsilError'
+const NATIVE_DEPENDENCY_MISSING = 'NATIVE_DEPENDENCY_MISSING'
+
+const NATIVE_DEPENDENCY_MISSING_MESSAGE =
+  'This app build does not include the Narsil native module, as happens in Expo Go and on 32-bit Android. Install `@delali/narsil-react-native` and `react-native-nitro-modules`, then make a development build with `npx expo run:ios` or `npx expo run:android`, or, in an app without Expo, run `pod install` in the `ios` folder and rebuild the app for each platform.'
+
+/**
+ * An error that the engine raised, as it reaches the app's JS thread. `code`
+ * holds the engine's error code, such as `INDEX_NOT_FOUND`, and `details`
+ * holds the values behind the failure.
+ *
+ * @public
+ */
+export interface PhoneNarsilError extends Error {
+  readonly code: string
+  readonly details: Record<string, unknown>
+}
+
+/**
+ * Reports whether a value is an error that the Narsil engine raised, whether
+ * it reached the app from the engine's runtime or from this package.
+ *
+ * @param value - Anything that a rejected promise or a `catch` block holds.
+ * @returns True when the value carries a Narsil error code.
+ *
+ * @public
+ */
+export function isNarsilError(value: unknown): value is PhoneNarsilError {
+  return typeof value === 'object' && value !== null && NARSIL_ERROR_MARKER in value
+}
+
+function narsilError(message: string, code: string, details: Record<string, unknown>): PhoneNarsilError {
+  const error = new Error(message)
+  error.name = NARSIL_ERROR_NAME
+  Object.defineProperty(error, 'code', { value: code, enumerable: true })
+  Object.defineProperty(error, 'details', { value: details, enumerable: true })
+  Object.defineProperty(error, NARSIL_ERROR_MARKER, { value: true })
+  return error as PhoneNarsilError
+}
+
+export function nativeDependencyMissing(cause: unknown): PhoneNarsilError {
+  return narsilError(NATIVE_DEPENDENCY_MISSING_MESSAGE, NATIVE_DEPENDENCY_MISSING, {
+    cause: cause instanceof Error ? cause.message : String(cause),
+  })
+}
+
+export function rebuiltError(transferred: TransferredError): Error {
+  const details = rebuiltValue(transferred.details)
+  if (transferred.name === NARSIL_ERROR_NAME && transferred.code !== undefined) {
+    const record = typeof details === 'object' && details !== null ? (details as Record<string, unknown>) : {}
+    return narsilError(transferred.message, transferred.code, record)
+  }
+  const error = new Error(transferred.message)
+  error.name = transferred.name
+  if (transferred.code !== undefined)
+    Object.defineProperty(error, 'code', { value: transferred.code, enumerable: true })
+  if (details !== undefined) Object.defineProperty(error, 'details', { value: details, enumerable: true })
+  return error
+}
+
+export function rebuiltValue(value: unknown): unknown {
+  if (value === null || typeof value !== 'object') return value
+  if (isTransferredError(value)) return rebuiltError(value)
+  const prototype = Object.getPrototypeOf(value)
+  if (!Array.isArray(value) && prototype !== Object.prototype && prototype !== null) return value
+  if (Array.isArray(value)) return value.map(rebuiltValue)
+  const record: Record<string, unknown> = {}
+  for (const key of Object.keys(value)) record[key] = rebuiltValue((value as Record<string, unknown>)[key])
+  return record
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export { createNarsil } from './client'
+export { isNarsilError, type PhoneNarsilError } from './errors'
+export type { PhoneEventHandler, PhoneNarsil, PhoneNarsilConfig } from './types'

package/src/methods.ts

@@ -0,0 +1,52 @@
+export const FORWARDED_METHODS = [
+  'createIndex',
+  'dropIndex',
+  'listIndexes',
+  'getStats',
+  'getPartitionStats',
+  'get',
+  'getMultiple',
+  'has',
+  'countDocuments',
+  'listDocuments',
+  'query',
+  'preflight',
+  'suggest',
+  'rebuildAnalysis',
+  'restore',
+  'importDocumentExport',
+  'checkpoint',
+  'clear',
+  'rebalance',
+  'updatePartitionConfig',
+  'getMemoryStats',
+  'compactVectors',
+  'optimizeVectors',
+  'vectorMaintenanceStatus',
+  'open',
+  'close',
+  'insert',
+  'insertBatch',
+  'remove',
+  'removeBatch',
+  'update',
+  'updateBatch',
+  'waitForWrites',
+  'updateSynonyms',
+  'getSynonyms',
+  'changeAnalysis',
+  'updateBM25',
+] as const
+
+export const STREAMED_METHODS = [
+  'snapshot',
+  'exportDocuments',
+  'snapshotToFile',
+  'exportDocumentsToFile',
+  'restoreFromFile',
+  'importDocumentExportFromFile',
+] as const
+
+export type ForwardedMethod = (typeof FORWARDED_METHODS)[number]
+export type StreamedMethod = (typeof STREAMED_METHODS)[number]
+export type EngineMethod = ForwardedMethod | StreamedMethod | 'shutdown'

package/src/native-objects.ts

@@ -0,0 +1,30 @@
+import type { NativeObjects } from './engine/host'
+import { nativeDependencyMissing } from './errors'
+import type { NarsilCore } from './specs/core.nitro'
+import type { NarsilFiles } from './specs/files.nitro'
+import type { NarsilHost } from './specs/host.nitro'
+
+let loading: Promise<NativeObjects> | null = null
+
+async function createNativeObjects(): Promise<NativeObjects> {
+  try {
+    const { NitroModules } = await import('react-native-nitro-modules')
+    return {
+      host: NitroModules.createHybridObject<NarsilHost>('NarsilHost'),
+      files: NitroModules.createHybridObject<NarsilFiles>('NarsilFiles'),
+      core: NitroModules.createHybridObject<NarsilCore>('NarsilCore'),
+    }
+  } catch (error) {
+    throw nativeDependencyMissing(error)
+  }
+}
+
+export function loadNativeObjects(): Promise<NativeObjects> {
+  if (loading === null) {
+    loading = createNativeObjects()
+    loading.catch(() => {
+      loading = null
+    })
+  }
+  return loading
+}

package/src/specs/core.nitro.ts

@@ -0,0 +1,140 @@
+import type { HybridObject } from 'react-native-nitro-modules'
+
+export interface MemoryView {
+  buffer: ArrayBuffer
+  byteOffset: number
+  byteLength: number
+}
+
+export interface GraphMemory {
+  graphHeader: MemoryView
+  nodeLevels: MemoryView
+  level0: MemoryView
+  upperBase: MemoryView
+  upper: MemoryView
+  locks: MemoryView
+  tombstones: MemoryView
+  incoming: MemoryView
+  nearestLinkDue: MemoryView
+  heldLocks: MemoryView
+}
+
+export interface StoreMemory {
+  storeHeader: MemoryView
+  dimension: number
+  bits: number
+  recordsPerBlock: number
+  vectorsPerBlock: number
+  vectorStrideBytes: number
+  codePresent: MemoryView
+  centroid: MemoryView
+  magnitudes: MemoryView
+  present: MemoryView
+  vectorFile: MemoryView
+  vectorOffset: MemoryView
+  vectorFiles: string[]
+  codeBlocks: MemoryView[]
+  vectorBlocks: MemoryView[]
+}
+
+export interface NativeSearchHits {
+  count: number
+  ordinals: ArrayBuffer
+  distances: ArrayBuffer
+}
+
+export interface NativeScores {
+  status: number
+  distances: ArrayBuffer
+}
+
+export interface NativeConnectOutcome {
+  status: number
+  wakes: ArrayBuffer
+}
+
+export interface NarsilAttachedGraph extends HybridObject<{ ios: 'c++'; android: 'c++' }> {}
+
+export interface NarsilAttachedStore extends HybridObject<{ ios: 'c++'; android: 'c++' }> {
+  detach(): number
+}
+
+export interface NarsilCore extends HybridObject<{ ios: 'c++'; android: 'c++' }> {
+  abiVersion(): number
+  workspaceBytes(): number
+  attachGraph(memory: GraphMemory): NarsilAttachedGraph
+  attachStore(memory: StoreMemory): NarsilAttachedStore
+  search(
+    graph: NarsilAttachedGraph,
+    store: NarsilAttachedStore,
+    query: MemoryView,
+    metric: number,
+    candidateCount: number,
+    threadSlot: number,
+    ordinals: MemoryView,
+    distances: MemoryView,
+    accepted: MemoryView,
+    filtered: boolean,
+  ): number
+  searchAsync(
+    graph: NarsilAttachedGraph,
+    store: NarsilAttachedStore,
+    query: MemoryView,
+    metric: number,
+    candidateCount: number,
+    threadSlot: number,
+    accepted: MemoryView,
+    filtered: boolean,
+  ): Promise<NativeSearchHits>
+  scoreAsync(
+    store: NarsilAttachedStore,
+    query: MemoryView,
+    metric: number,
+    ordinals: MemoryView,
+    count: number,
+    threadSlot: number,
+  ): Promise<NativeScores>
+  score(
+    store: NarsilAttachedStore,
+    query: MemoryView,
+    metric: number,
+    ordinals: MemoryView,
+    count: number,
+    distances: MemoryView,
+  ): number
+  place(
+    graph: NarsilAttachedGraph,
+    store: NarsilAttachedStore,
+    metric: number,
+    ordinal: number,
+    topLayer: number,
+    graphHeldAlone: number,
+    threadSlot: number,
+    wakes: MemoryView,
+  ): number
+  remove(graph: NarsilAttachedGraph, ordinal: number, threadSlot: number, wakes: MemoryView): number
+  compact(
+    graph: NarsilAttachedGraph,
+    store: NarsilAttachedStore,
+    metric: number,
+    threadSlot: number,
+    wakes: MemoryView,
+  ): number
+  connectUnreached(
+    graph: NarsilAttachedGraph,
+    store: NarsilAttachedStore,
+    metric: number,
+    nearby: MemoryView,
+    threadSlot: number,
+    wakes: MemoryView,
+  ): number
+  connectUnreachedAsync(
+    graph: NarsilAttachedGraph,
+    store: NarsilAttachedStore,
+    metric: number,
+    nearby: MemoryView,
+    threadSlot: number,
+  ): Promise<NativeConnectOutcome>
+  measure(store: NarsilAttachedStore, metric: number, ordinals: MemoryView, count: number, totals: MemoryView): number
+  quantise(store: NarsilAttachedStore, metric: number, ordinals: MemoryView, count: number): number
+}

package/src/specs/files.nitro.ts

@@ -0,0 +1,42 @@
+import type { HybridObject } from 'react-native-nitro-modules'
+
+export interface FileStatus {
+  size: number
+  isDirectory: boolean
+  isFile: boolean
+  device: string
+  inode: string
+  modifiedMs: number
+}
+
+export interface DirectoryEntry {
+  name: string
+  isDirectory: boolean
+  isFile: boolean
+}
+
+export interface NarsilFiles extends HybridObject<{ ios: 'c++'; android: 'c++' }> {
+  open(path: string, flags: string): number
+  close(descriptor: number): void
+  read(descriptor: number, target: ArrayBuffer, byteOffset: number, byteLength: number, position: number): number
+  write(descriptor: number, source: ArrayBuffer, byteOffset: number, byteLength: number, position: number): number
+  sync(descriptor: number): void
+  readAsync(descriptor: number, byteLength: number, position: number): Promise<ArrayBuffer>
+  writeAsync(
+    descriptor: number,
+    source: ArrayBuffer,
+    byteOffset: number,
+    byteLength: number,
+    position: number,
+  ): Promise<number>
+  syncAsync(descriptor: number): Promise<void>
+  truncate(descriptor: number, byteLength: number): void
+  descriptorStatus(descriptor: number): FileStatus
+  pathStatus(path: string): FileStatus
+  makeDirectory(path: string): void
+  readDirectory(path: string): DirectoryEntry[]
+  rename(from: string, to: string): void
+  unlink(path: string): void
+  removeDirectory(path: string): void
+  link(existingPath: string, newPath: string): void
+}

package/src/specs/host.nitro.ts

@@ -0,0 +1,30 @@
+import type { HybridObject } from 'react-native-nitro-modules'
+
+export interface ProcessMemory {
+  residentBytes: number
+  availableBytes: number
+}
+
+export interface NarsilHost extends HybridObject<{ ios: 'c++'; android: 'c++' }> {
+  reserveMemory(bytes: number): ArrayBuffer
+  allocateMemory(bytes: number): ArrayBuffer
+  ownsMemory(buffer: ArrayBuffer): boolean
+  atomic(
+    operation: number,
+    elementKind: number,
+    buffer: ArrayBuffer,
+    byteIndex: number,
+    value: number,
+    replacement: number,
+  ): number
+  fillRandom(target: ArrayBuffer, byteOffset: number, byteLength: number): void
+  processId(): number
+  processStartedAtMs(): number
+  processIsAlive(processId: number): boolean
+  processMemory(): ProcessMemory
+  appStorageDirectory(): string
+  deflateRaw(source: ArrayBuffer, byteOffset: number, byteLength: number): ArrayBuffer
+  inflateRaw(source: ArrayBuffer, byteOffset: number, byteLength: number, rawByteLength: number): ArrayBuffer
+  gzip(source: ArrayBuffer, byteOffset: number, byteLength: number): ArrayBuffer
+  gunzip(source: ArrayBuffer, byteOffset: number, byteLength: number, maxByteLength: number): ArrayBuffer
+}

package/src/transferred.ts

@@ -0,0 +1,18 @@
+export interface TransferredError {
+  transferredError: true
+  name: string
+  message: string
+  code: string | undefined
+  details: unknown
+}
+
+export type CallOutcome = { ok: true; value: unknown } | { ok: false; error: TransferredError }
+
+export function isTransferredError(value: unknown): value is TransferredError {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    (value as { transferredError?: unknown }).transferredError === true &&
+    typeof (value as { message?: unknown }).message === 'string'
+  )
+}