@delali/narsil-react-native 0.0.0-stage → 0.3.1-next.151

package/package.json

@@ -1,6 +1,90 @@
 {
   "name": "@delali/narsil-react-native",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.3.1-next.151",
+  "description": "Narsil for React Native: keyword and vector search on the phone, with the index kept in durable files.",
+  "author": "Delali (https://sondelali.com)",
+  "license": "Apache-2.0",
+  "homepage": "https://narsil.sondelali.com",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/assetcorp/narsil.git",
+    "directory": "packages/react-native"
+  },
+  "bugs": {
+    "url": "https://github.com/assetcorp/narsil/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "./lib/index.js",
+  "types": "./lib/index.d.ts",
+  "react-native": "./src/index.ts",
+  "source": "./src/index.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.ts",
+      "react-native": "./src/index.ts",
+      "default": "./lib/index.js"
+    },
+    "./worklets": {
+      "types": "./lib/worklets/index.d.ts",
+      "react-native": "./src/worklets/index.ts",
+      "default": "./lib/worklets/index.js"
+    },
+    "./app.plugin.js": "./app.plugin.js",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "src/",
+    "lib/",
+    "cpp/",
+    "nitrogen/",
+    "android/build.gradle",
+    "android/gradle.properties",
+    "android/CMakeLists.txt",
+    "android/src/",
+    "ios/PrivacyInfo.xcprivacy",
+    "NarsilReactNative.podspec",
+    "nitro.json",
+    "app.plugin.js",
+    "react-native.config.js",
+    "LICENSE",
+    "README.md"
+  ],
+  "sideEffects": false,
+  "peerDependencies": {
+    "@delali/narsil": "0.3.1-next.151",
+    "expo": "*",
+    "react-native": "*",
+    "react-native-nitro-modules": "*",
+    "react-native-worklets": "*"
+  },
+  "peerDependenciesMeta": {
+    "expo": {
+      "optional": true
+    },
+    "react-native-worklets": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@biomejs/biome": "2.4.4",
+    "expo": "58.0.21",
+    "nitrogen": "0.37.1",
+    "react-native": "0.88.1",
+    "react-native-nitro-modules": "0.37.1",
+    "react-native-worklets": "0.13.0",
+    "typescript": "5.9.3",
+    "@delali/narsil": "0.3.1-next.151"
+  },
+  "dependencies": {
+    "web-streams-polyfill": "4.3.0"
+  },
+  "scripts": {
+    "build": "node scripts/copy-core.mjs && tsc -p tsconfig.build.json",
+    "copy-core": "node scripts/copy-core.mjs",
+    "specs": "nitrogen --logLevel=\"warning\"",
+    "lint": "biome check src/ scripts/ app.plugin.js react-native.config.js",
+    "typecheck": "tsc --noEmit"
+  }
 }

package/react-native.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  dependency: {
+    platforms: {
+      ios: {},
+      android: {},
+    },
+  },
+}

package/src/client.ts

@@ -0,0 +1,44 @@
+import { createNarsil as createEngine, type Narsil, type ReactNativeHost } from '@delali/narsil'
+import { installEngineHost, installedHost } from './engine/install'
+import { callStreamed, withPhoneDurability } from './engine/streamed'
+import { FORWARDED_METHODS, STREAMED_METHODS } from './methods'
+import { loadNativeObjects } from './native-objects'
+import type { PhoneNarsil, PhoneNarsilConfig } from './types'
+
+function phoneNarsil(narsil: Narsil, host: ReactNativeHost): PhoneNarsil {
+  const client: Record<string, unknown> = {}
+  for (const method of FORWARDED_METHODS) {
+    const target = narsil[method] as (...values: unknown[]) => unknown
+    client[method] = async (...args: unknown[]) => target.apply(narsil, args)
+  }
+  for (const method of STREAMED_METHODS) {
+    client[method] = async (...args: unknown[]) => callStreamed(narsil, host, method, args)
+  }
+  client.on = narsil.on.bind(narsil)
+  client.off = narsil.off.bind(narsil)
+  client.shutdown = async () => narsil.shutdown()
+  return client as unknown as PhoneNarsil
+}
+
+/**
+ * Starts a Narsil engine on the app's own JS runtime and resolves with a
+ * client whose methods each return a promise. The engine takes the same
+ * settings as on Node.js, function settings such as embedding adapters,
+ * plugins, and an id generator included. File reads, file writes, file
+ * syncs, and vector searches run on native threads, and a long import or
+ * search yields to the app between steps.
+ *
+ * @param config - The engine settings. `durability` defaults to a
+ * write-ahead log in the `narsil` directory of the app's private storage,
+ * and `null` keeps every index in memory alone.
+ * @returns The client, once the engine has started.
+ * @throws A `NarsilError` with `NATIVE_DEPENDENCY_MISSING` when the app's
+ * build lacks the native module, as in Expo Go.
+ *
+ * @public
+ */
+export async function createNarsil(config: PhoneNarsilConfig = {}): Promise<PhoneNarsil> {
+  installEngineHost(await loadNativeObjects(), true)
+  const narsil = await createEngine(withPhoneDurability(config))
+  return phoneNarsil(narsil, installedHost())
+}

package/src/constants.ts

@@ -0,0 +1,9 @@
+export const FILE_CHUNK_BYTES = 1_048_576
+export const DEFAULT_DURABILITY_DIRECTORY = 'narsil'
+export const ENGINE_RUNTIME_NAME = 'narsil-engine'
+export const WORKLET_TRANSFER_TURN_BUDGET_MS = 8
+export const WORKLET_TRANSFER_PART_TARGET_MS = 4
+export const WORKLET_TRANSFER_FIRST_PART_ITEMS = 32
+export const WORKLET_TRANSFER_LARGEST_PART_ITEMS = 4_096
+export const WORKLET_JSON_PART_MAX_DEPTH = 64
+export const APP_TIMER_WAIT_BEFORE_RUN_LOOP_HOP_MS = 100

package/src/engine/byte-files.ts

@@ -0,0 +1,99 @@
+import type { ReactNativeFiles, ReactNativeHost } from '@delali/narsil'
+import { FILE_CHUNK_BYTES } from '../constants'
+
+const CURRENT_POSITION = -1
+
+function invalidPath(message: string): Error {
+  return Object.assign(new TypeError(message), { code: 'EINVAL' })
+}
+
+export function resolveAppPath(host: ReactNativeHost, path: string): string {
+  if (typeof path !== 'string' || path.length === 0) throw invalidPath('A file path must be a non-empty string')
+  if (path.includes('\0')) throw invalidPath('A file path cannot hold a NUL character')
+  if (path.startsWith('/')) return path
+  const directory = host.appStorageDirectory()
+  return directory.endsWith('/') ? `${directory}${path}` : `${directory}/${path}`
+}
+
+async function writeAll(files: ReactNativeFiles, descriptor: number, chunk: Uint8Array): Promise<void> {
+  let written = 0
+  while (written < chunk.byteLength) {
+    const end = Math.min(chunk.byteLength, written + FILE_CHUNK_BYTES)
+    const count = await files.writeAsync(descriptor, chunk.subarray(written, end), CURRENT_POSITION)
+    if (count <= 0) throw Object.assign(new Error('The file accepted no more bytes'), { code: 'EIO' })
+    written += count
+  }
+}
+
+function temporaryNameFor(path: string): string {
+  const suffix = new Uint8Array(8)
+  globalThis.crypto.getRandomValues(suffix)
+  let text = ''
+  for (const byte of suffix) text += byte.toString(16).padStart(2, '0')
+  return `${path}.partial-${text}`
+}
+
+export async function writeStreamToFile(
+  files: ReactNativeFiles,
+  stream: ReadableStream<Uint8Array>,
+  path: string,
+): Promise<number> {
+  const temporary = temporaryNameFor(path)
+  const descriptor = files.open(temporary, 'wx')
+  const reader = stream.getReader()
+  let total = 0
+  try {
+    for (;;) {
+      const { done, value } = await reader.read()
+      if (done) break
+      await writeAll(files, descriptor, value)
+      total += value.byteLength
+    }
+    await files.syncAsync(descriptor)
+  } catch (error) {
+    await reader.cancel(error).catch(() => undefined)
+    files.close(descriptor)
+    try {
+      files.unlink(temporary)
+    } catch {}
+    throw error
+  }
+  files.close(descriptor)
+  files.rename(temporary, path)
+  return total
+}
+
+export async function* fileChunks(files: ReactNativeFiles, path: string): AsyncGenerator<Uint8Array> {
+  const descriptor = files.open(path, 'r')
+  try {
+    let position = 0
+    for (;;) {
+      const chunk = new Uint8Array(FILE_CHUNK_BYTES)
+      const count = await files.readAsync(descriptor, chunk, position)
+      if (count === 0) return
+      position += count
+      yield count === chunk.byteLength ? chunk : chunk.subarray(0, count)
+    }
+  } finally {
+    files.close(descriptor)
+  }
+}
+
+export async function collectStream(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
+  const reader = stream.getReader()
+  const chunks: Uint8Array[] = []
+  let total = 0
+  for (;;) {
+    const { done, value } = await reader.read()
+    if (done) break
+    chunks.push(value)
+    total += value.byteLength
+  }
+  const bytes = new Uint8Array(total)
+  let offset = 0
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset)
+    offset += chunk.byteLength
+  }
+  return bytes
+}

package/src/engine/core.ts

@@ -0,0 +1,162 @@
+import type {
+  NativeCore,
+  NativeGraphHandle,
+  NativeGraphMemory,
+  NativeStoreHandle,
+  NativeStoreMemory,
+} from '@delali/narsil'
+import type {
+  GraphMemory,
+  MemoryView,
+  NarsilAttachedGraph,
+  NarsilAttachedStore,
+  NarsilCore,
+  StoreMemory,
+} from '../specs/core.nitro'
+import { withNativeCode } from './native-error'
+
+const ABSENT_REGION: MemoryView = { buffer: new ArrayBuffer(0), byteOffset: 0, byteLength: 0 }
+
+function viewOf(array: ArrayBufferView): MemoryView {
+  return { buffer: array.buffer as ArrayBuffer, byteOffset: array.byteOffset, byteLength: array.byteLength }
+}
+
+function graphMemoryOf(memory: NativeGraphMemory): GraphMemory {
+  return {
+    graphHeader: viewOf(memory.graphHeader),
+    nodeLevels: viewOf(memory.nodeLevels),
+    level0: viewOf(memory.level0),
+    upperBase: viewOf(memory.upperBase),
+    upper: viewOf(memory.upper),
+    locks: viewOf(memory.locks),
+    tombstones: viewOf(memory.tombstones),
+    incoming: viewOf(memory.incoming),
+    nearestLinkDue: viewOf(memory.nearestLinkDue),
+    heldLocks: viewOf(memory.heldLocks),
+  }
+}
+
+function storeMemoryOf(memory: NativeStoreMemory): StoreMemory {
+  return {
+    storeHeader: viewOf(memory.storeHeader),
+    dimension: memory.dimension,
+    bits: memory.bits,
+    recordsPerBlock: memory.recordsPerBlock,
+    vectorsPerBlock: memory.vectorsPerBlock,
+    vectorStrideBytes: memory.vectorStrideBytes,
+    codePresent: viewOf(memory.codePresent),
+    centroid: viewOf(memory.centroid),
+    magnitudes: viewOf(memory.magnitudes),
+    present: viewOf(memory.present),
+    vectorFile: viewOf(memory.vectorFile),
+    vectorOffset: viewOf(memory.vectorOffset),
+    vectorFiles: memory.vectorFiles,
+    codeBlocks: memory.codeBlocks.map(viewOf),
+    vectorBlocks: memory.vectorBlocks.map(block => (block === null ? ABSENT_REGION : viewOf(block))),
+  }
+}
+
+function graphOf(handle: NativeGraphHandle): NarsilAttachedGraph {
+  return handle as NarsilAttachedGraph
+}
+
+function storeOf(handle: NativeStoreHandle): NarsilAttachedStore {
+  return handle as NarsilAttachedStore
+}
+
+function backgroundCalls(core: NarsilCore): Pick<NativeCore, 'searchAsync' | 'scoreAsync' | 'connectUnreachedAsync'> {
+  return {
+    async connectUnreachedAsync(graph, store, metric, nearby, threadSlot) {
+      try {
+        const outcome = await core.connectUnreachedAsync(
+          graphOf(graph),
+          storeOf(store),
+          metric,
+          viewOf(nearby),
+          threadSlot,
+        )
+        return { status: outcome.status, wakes: new Int32Array(outcome.wakes) }
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    async searchAsync(graph, store, query, metric, candidateCount, threadSlot, accepted) {
+      try {
+        const hits = await core.searchAsync(
+          graphOf(graph),
+          storeOf(store),
+          viewOf(query),
+          metric,
+          candidateCount,
+          threadSlot,
+          accepted === null ? ABSENT_REGION : viewOf(accepted),
+          accepted !== null,
+        )
+        return {
+          count: hits.count,
+          ordinals: new Int32Array(hits.ordinals),
+          distances: new Float64Array(hits.distances),
+        }
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    async scoreAsync(store, query, metric, ordinals, count, threadSlot) {
+      try {
+        const scores = await core.scoreAsync(storeOf(store), viewOf(query), metric, viewOf(ordinals), count, threadSlot)
+        return { status: scores.status, distances: new Float64Array(scores.distances) }
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+  }
+}
+
+export function adaptCore(core: NarsilCore, nativeThreads: boolean): NativeCore {
+  return {
+    abiVersion: () => core.abiVersion(),
+    workspaceBytes: () => core.workspaceBytes(),
+    attachGraph(memory) {
+      try {
+        return core.attachGraph(graphMemoryOf(memory))
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    attachStore(memory) {
+      try {
+        return core.attachStore(storeMemoryOf(memory))
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    detachStore: store => storeOf(store).detach(),
+    search(graph, store, query, metric, candidateCount, threadSlot, ordinals, distances, accepted) {
+      return core.search(
+        graphOf(graph),
+        storeOf(store),
+        viewOf(query),
+        metric,
+        candidateCount,
+        threadSlot,
+        viewOf(ordinals),
+        viewOf(distances),
+        accepted === null ? ABSENT_REGION : viewOf(accepted),
+        accepted !== null,
+      )
+    },
+    score: (store, query, metric, ordinals, count, distances) =>
+      core.score(storeOf(store), viewOf(query), metric, viewOf(ordinals), count, viewOf(distances)),
+    ...(nativeThreads ? backgroundCalls(core) : {}),
+    place: (graph, store, metric, ordinal, topLayer, graphHeldAlone, threadSlot, wakes) =>
+      core.place(graphOf(graph), storeOf(store), metric, ordinal, topLayer, graphHeldAlone, threadSlot, viewOf(wakes)),
+    remove: (graph, ordinal, threadSlot, wakes) => core.remove(graphOf(graph), ordinal, threadSlot, viewOf(wakes)),
+    compact: (graph, store, metric, threadSlot, wakes) =>
+      core.compact(graphOf(graph), storeOf(store), metric, threadSlot, viewOf(wakes)),
+    connectUnreached: (graph, store, metric, nearby, threadSlot, wakes) =>
+      core.connectUnreached(graphOf(graph), storeOf(store), metric, viewOf(nearby), threadSlot, viewOf(wakes)),
+    measure: (store, metric, ordinals, count, totals) =>
+      core.measure(storeOf(store), metric, viewOf(ordinals), count, viewOf(totals)),
+    quantise: (store, metric, ordinals, count) => core.quantise(storeOf(store), metric, viewOf(ordinals), count),
+  }
+}

package/src/engine/files.ts

@@ -0,0 +1,186 @@
+import type { ReactNativeDirectoryEntry, ReactNativeFileStatus, ReactNativeFiles } from '@delali/narsil'
+import type { NarsilFiles } from '../specs/files.nitro'
+import { withNativeCode } from './native-error'
+
+type BlockingFiles = Omit<ReactNativeFiles, 'readAsync' | 'writeAsync' | 'syncAsync'>
+
+function rejectedWithNativeCode(error: unknown): Promise<never> {
+  return Promise.reject(withNativeCode(error))
+}
+
+function blockingFiles(files: NarsilFiles): BlockingFiles {
+  return {
+    open(path: string, flags: string): number {
+      try {
+        return files.open(path, flags)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    close(descriptor: number): void {
+      try {
+        files.close(descriptor)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    read(descriptor: number, target: Uint8Array, position: number): number {
+      try {
+        return files.read(descriptor, target.buffer as ArrayBuffer, target.byteOffset, target.byteLength, position)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    write(descriptor: number, source: Uint8Array, position: number): number {
+      try {
+        return files.write(descriptor, source.buffer as ArrayBuffer, source.byteOffset, source.byteLength, position)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    sync(descriptor: number): void {
+      try {
+        files.sync(descriptor)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    truncate(descriptor: number, byteLength: number): void {
+      try {
+        files.truncate(descriptor, byteLength)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    descriptorStatus(descriptor: number): ReactNativeFileStatus {
+      try {
+        return files.descriptorStatus(descriptor)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    pathStatus(path: string): ReactNativeFileStatus {
+      try {
+        return files.pathStatus(path)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    makeDirectory(path: string): void {
+      try {
+        files.makeDirectory(path)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    readDirectory(path: string): ReactNativeDirectoryEntry[] {
+      try {
+        return files.readDirectory(path)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    rename(from: string, to: string): void {
+      try {
+        files.rename(from, to)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    unlink(path: string): void {
+      try {
+        files.unlink(path)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    removeDirectory(path: string): void {
+      try {
+        files.removeDirectory(path)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+    link(existingPath: string, newPath: string): void {
+      try {
+        files.link(existingPath, newPath)
+      } catch (error) {
+        throw withNativeCode(error)
+      }
+    },
+  }
+}
+
+function onTheCallingThread(blocking: BlockingFiles): ReactNativeFiles {
+  return {
+    ...blocking,
+    readAsync: async (descriptor, target, position) => blocking.read(descriptor, target, position),
+    writeAsync: async (descriptor, source, position) => blocking.write(descriptor, source, position),
+    syncAsync: async descriptor => blocking.sync(descriptor),
+  }
+}
+
+function onNativeThreads(files: NarsilFiles, blocking: BlockingFiles): ReactNativeFiles {
+  const pendingByDescriptor = new Map<number, Promise<unknown>>()
+
+  function inOrder<T>(descriptor: number, run: () => Promise<T>): Promise<T> {
+    const previous = pendingByDescriptor.get(descriptor)
+    let started: Promise<T>
+    if (previous !== undefined) {
+      started = previous.then(run, run)
+    } else {
+      try {
+        started = run()
+      } catch (error) {
+        started = Promise.reject(error)
+      }
+    }
+    const settled = started.then(
+      () => {},
+      () => {},
+    )
+    pendingByDescriptor.set(descriptor, settled)
+    settled.then(() => {
+      if (pendingByDescriptor.get(descriptor) === settled) pendingByDescriptor.delete(descriptor)
+    })
+    return started.catch(rejectedWithNativeCode)
+  }
+
+  return {
+    ...blocking,
+    close(descriptor: number): void {
+      const pending = pendingByDescriptor.get(descriptor)
+      if (pending === undefined) {
+        blocking.close(descriptor)
+        return
+      }
+      pendingByDescriptor.delete(descriptor)
+      pending.then(() => {
+        try {
+          files.close(descriptor)
+        } catch {}
+      })
+    },
+    readAsync(descriptor: number, target: Uint8Array, position: number): Promise<number> {
+      return inOrder(descriptor, async () => {
+        const bytes = await files.readAsync(descriptor, target.byteLength, position)
+        target.set(new Uint8Array(bytes))
+        return bytes.byteLength
+      })
+    },
+    writeAsync(descriptor: number, source: Uint8Array, position: number): Promise<number> {
+      const bytes = pendingByDescriptor.has(descriptor) ? source.slice() : source
+      return inOrder(descriptor, () =>
+        files.writeAsync(descriptor, bytes.buffer as ArrayBuffer, bytes.byteOffset, bytes.byteLength, position),
+      )
+    },
+    syncAsync(descriptor: number): Promise<void> {
+      return inOrder(descriptor, () => files.syncAsync(descriptor))
+    },
+  }
+}
+
+export function adaptFiles(files: NarsilFiles, nativeThreads: boolean): ReactNativeFiles {
+  const blocking = blockingFiles(files)
+  return nativeThreads ? onNativeThreads(files, blocking) : onTheCallingThread(blocking)
+}