@bridgerust/embex 0.1.16 → 0.1.17

package/CHANGELOG.md

@@ -0,0 +1,32 @@
+# Changelog
+
+All notable changes to the Embex Node.js bindings will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.16] - 2026-01-05
+
+### Added
+
+#### Performance
+
+- **SIMD Optimizations**: Added SIMD-accelerated vector operations with scalar fallback.
+- **Connection Pooling**: Implemented internal pooling for Qdrant and Chroma adapters.
+
+#### Features
+
+- **API Parity**: Added new methods to match Python SDK (`updateMetadata`, `buildQuery`, `aggregation`).
+- **Query Builder**: Added support for filter-only queries and aggregations.
+- **Observability**: Added metrics collection and structured logging instrumentation.
+
+### Changed
+
+- **Architecture**: Refactored internal architecture for better separation of concerns.
+- **Error Handling**: Introduced more specific error types and better exception messages.
+
+### Fixed
+
+- **Output Filename**: Resolved filename collision with `embex-bridge` crate.

package/README.md

@@ -106,6 +106,27 @@ const client = new EmbexClient(
 );
 ```
 
+## 🔄 Data Migration
+
+Move data between providers effortlessly using the built-in `DataMigrator`.
+
+```typescript
+import { EmbexClient, DataMigrator } from "@bridgerust/embex";
+
+// 1. Setup clients
+const source = await EmbexClient.newAsync("lancedb", "./local_data");
+const dest = new EmbexClient("qdrant", "http://prod-db:6333");
+
+// 2. Migrate
+const migrator = new DataMigrator(source, dest);
+const result = await migrator.migrateSimple(
+  "products", // source
+  "products_v2" // destination
+);
+
+console.log(`Migrated ${result.pointsMigrated} points!`);
+```
+
 ## 🔗 Resources
 
 - **Full Documentation**: [bridgerust.dev/embex](https://bridgerust.dev/embex/introduction)

package/dist/native.d.ts

@@ -35,6 +35,14 @@ export declare class Collection {
    */
   createAuto(dimension?: number | undefined | null, distance?: string | undefined | null): Promise<void>
   insertBatch(points: Array<Point>, batchSize?: number | undefined | null, parallel?: number | undefined | null): Promise<void>
+  /** Scroll through points in the collection (paginated export). */
+  scroll(offset?: string | undefined | null, limit?: number | undefined | null): Promise<ScrollResponse>
+}
+
+export declare class DataMigrator {
+  constructor(source: EmbexClient, dest: EmbexClient)
+  migrate(sourceCollection: string, destCollection: string, dimension: number, batchSize?: number | undefined | null, distance?: string | undefined | null): Promise<MigrationResult>
+  migrateSimple(sourceCollection: string, destCollection: string, batchSize?: number | undefined | null): Promise<MigrationResult>
 }
 
 /** Main client for the Embex vector database. */
@@ -105,6 +113,11 @@ export interface MigrationInput {
   downOperations?: any
 }
 
+export interface MigrationResult {
+  pointsMigrated: number
+  elapsedMs: number
+}
+
 /** A point in the vector database. */
 export interface Point {
   id: string
@@ -112,6 +125,11 @@ export interface Point {
   metadata?: Record<string, any>
 }
 
+export interface ScrollResponse {
+  points: Array<Point>
+  nextOffset?: string
+}
+
 export interface SearchOptions {
   /** Number of results to return. */
   limit?: number

package/dist/native.js

@@ -77,8 +77,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-android-arm64')
         const bindingPackageVersion = require('@bridgerust/embex-android-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -93,8 +93,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-android-arm-eabi')
         const bindingPackageVersion = require('@bridgerust/embex-android-arm-eabi/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -114,8 +114,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-win32-x64-gnu')
         const bindingPackageVersion = require('@bridgerust/embex-win32-x64-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -130,8 +130,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-win32-x64-msvc')
         const bindingPackageVersion = require('@bridgerust/embex-win32-x64-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -147,8 +147,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-win32-ia32-msvc')
         const bindingPackageVersion = require('@bridgerust/embex-win32-ia32-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -163,8 +163,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-win32-arm64-msvc')
         const bindingPackageVersion = require('@bridgerust/embex-win32-arm64-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -182,8 +182,8 @@ function requireNative() {
     try {
       const binding = require('@bridgerust/embex-darwin-universal')
       const bindingPackageVersion = require('@bridgerust/embex-darwin-universal/package.json').version
-      if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-        throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+      if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+        throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
       }
       return binding
     } catch (e) {
@@ -198,8 +198,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-darwin-x64')
         const bindingPackageVersion = require('@bridgerust/embex-darwin-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -214,8 +214,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-darwin-arm64')
         const bindingPackageVersion = require('@bridgerust/embex-darwin-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -234,8 +234,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-freebsd-x64')
         const bindingPackageVersion = require('@bridgerust/embex-freebsd-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -250,8 +250,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-freebsd-arm64')
         const bindingPackageVersion = require('@bridgerust/embex-freebsd-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -271,8 +271,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-x64-musl')
           const bindingPackageVersion = require('@bridgerust/embex-linux-x64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -287,8 +287,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-x64-gnu')
           const bindingPackageVersion = require('@bridgerust/embex-linux-x64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -305,8 +305,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-arm64-musl')
           const bindingPackageVersion = require('@bridgerust/embex-linux-arm64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -321,8 +321,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-arm64-gnu')
           const bindingPackageVersion = require('@bridgerust/embex-linux-arm64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -339,8 +339,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-arm-musleabihf')
           const bindingPackageVersion = require('@bridgerust/embex-linux-arm-musleabihf/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -355,8 +355,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-arm-gnueabihf')
           const bindingPackageVersion = require('@bridgerust/embex-linux-arm-gnueabihf/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -373,8 +373,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-loong64-musl')
           const bindingPackageVersion = require('@bridgerust/embex-linux-loong64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -389,8 +389,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-loong64-gnu')
           const bindingPackageVersion = require('@bridgerust/embex-linux-loong64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -407,8 +407,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-riscv64-musl')
           const bindingPackageVersion = require('@bridgerust/embex-linux-riscv64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -423,8 +423,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-riscv64-gnu')
           const bindingPackageVersion = require('@bridgerust/embex-linux-riscv64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -440,8 +440,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-linux-ppc64-gnu')
         const bindingPackageVersion = require('@bridgerust/embex-linux-ppc64-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -456,8 +456,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-linux-s390x-gnu')
         const bindingPackageVersion = require('@bridgerust/embex-linux-s390x-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -476,8 +476,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-openharmony-arm64')
         const bindingPackageVersion = require('@bridgerust/embex-openharmony-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -492,8 +492,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-openharmony-x64')
         const bindingPackageVersion = require('@bridgerust/embex-openharmony-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -508,8 +508,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-openharmony-arm')
         const bindingPackageVersion = require('@bridgerust/embex-openharmony-arm/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -536,13 +536,17 @@ if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
       wasiBindingError = err
     }
   }
-  if (!nativeBinding) {
+  if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
     try {
       wasiBinding = require('@bridgerust/embex-wasm32-wasi')
       nativeBinding = wasiBinding
     } catch (err) {
       if (process.env.NAPI_RS_FORCE_WASI) {
-        wasiBindingError.cause = err
+        if (!wasiBindingError) {
+          wasiBindingError = err
+        } else {
+          wasiBindingError.cause = err
+        }
         loadErrors.push(err)
       }
     }
@@ -573,6 +577,7 @@ if (!nativeBinding) {
 
 module.exports = nativeBinding
 module.exports.Collection = nativeBinding.Collection
+module.exports.DataMigrator = nativeBinding.DataMigrator
 module.exports.EmbexClient = nativeBinding.EmbexClient
 module.exports.QueryBuilder = nativeBinding.QueryBuilder
 module.exports.SearchBuilder = nativeBinding.SearchBuilder

package/dist/src/index.d.ts

@@ -1,8 +1,8 @@
-import { Collection, EmbexClient, QueryBuilder, SearchBuilder, cli, Point } from "../native";
+import { Collection, EmbexClient, QueryBuilder, SearchBuilder, DataMigrator, ScrollResponse, MigrationResult, cli, Point } from "../native";
 declare module "../native" {
     interface Collection {
         insertStream(points: AsyncIterable<Point>, parallel?: number): Promise<void>;
     }
 }
-export { Collection, EmbexClient, QueryBuilder, SearchBuilder, cli, Point };
+export { Collection, EmbexClient, QueryBuilder, SearchBuilder, DataMigrator, ScrollResponse, MigrationResult, cli, Point, };
 //# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,WAAW,EACX,YAAY,EACZ,aAAa,EAEb,GAAG,EACH,KAAK,EACN,MAAM,WAAW,CAAC;AAEnB,OAAO,QAAQ,WAAW,CAAC;IACzB,UAAU,UAAU;QAClB,YAAY,CACV,MAAM,EAAE,aAAa,CAAC,KAAK,CAAC,EAC5B,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;KAClB;CACF;AAwBD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,WAAW,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,cAAc,EACd,eAAe,EAEf,GAAG,EACH,KAAK,EACN,MAAM,WAAW,CAAC;AAEnB,OAAO,QAAQ,WAAW,CAAC;IACzB,UAAU,UAAU;QAClB,YAAY,CACV,MAAM,EAAE,aAAa,CAAC,KAAK,CAAC,EAC5B,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;KAClB;CACF;AAwBD,OAAO,EACL,UAAU,EACV,WAAW,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,cAAc,EACd,eAAe,EACf,GAAG,EACH,KAAK,GACN,CAAC"}

package/dist/src/index.js

@@ -1,11 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.cli = exports.SearchBuilder = exports.QueryBuilder = exports.EmbexClient = exports.Collection = void 0;
+exports.cli = exports.DataMigrator = exports.SearchBuilder = exports.QueryBuilder = exports.EmbexClient = exports.Collection = void 0;
 const native_1 = require("../native");
 Object.defineProperty(exports, "Collection", { enumerable: true, get: function () { return native_1.Collection; } });
 Object.defineProperty(exports, "EmbexClient", { enumerable: true, get: function () { return native_1.EmbexClient; } });
 Object.defineProperty(exports, "QueryBuilder", { enumerable: true, get: function () { return native_1.QueryBuilder; } });
 Object.defineProperty(exports, "SearchBuilder", { enumerable: true, get: function () { return native_1.SearchBuilder; } });
+Object.defineProperty(exports, "DataMigrator", { enumerable: true, get: function () { return native_1.DataMigrator; } });
 Object.defineProperty(exports, "cli", { enumerable: true, get: function () { return 
     // @ts-ignore
     native_1.cli; } });

package/dist/src/tests/integration/data_migration.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=data_migration.test.d.ts.map

package/dist/src/tests/integration/data_migration.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data_migration.test.d.ts","sourceRoot":"","sources":["../../../../src/tests/integration/data_migration.test.ts"],"names":[],"mappings":""}

package/dist/src/tests/integration/data_migration.test.js

@@ -0,0 +1,114 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const index_1 = require("../../index"); // Assuming index exports DataMigrator
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const os = __importStar(require("os"));
+// Helper to create temp dir
+const createTempDir = () => {
+    return fs.mkdtempSync(path.join(os.tmpdir(), "embex-test-"));
+};
+(0, vitest_1.describe)("Data Migrator and Scroll", () => {
+    let sourceDbPath;
+    let destDbPath;
+    let sourceClient;
+    let destClient;
+    (0, vitest_1.beforeAll)(async () => {
+        sourceDbPath = createTempDir();
+        destDbPath = createTempDir();
+        // Initialize source and destination (LanceDB)
+        sourceClient = await index_1.EmbexClient.newAsync("lancedb", sourceDbPath);
+        destClient = await index_1.EmbexClient.newAsync("lancedb", destDbPath);
+    });
+    (0, vitest_1.afterAll)(async () => {
+        // Cleanup
+        try {
+            fs.rmSync(sourceDbPath, { recursive: true, force: true });
+            fs.rmSync(destDbPath, { recursive: true, force: true });
+        }
+        catch (e) {
+            console.error("Cleanup failed", e);
+        }
+    });
+    (0, vitest_1.it)("should migrate data between databases", async () => {
+        const srcColName = "products_src";
+        const destColName = "products_dest";
+        const srcCol = sourceClient.collection(srcColName);
+        // Insert data into source
+        // Note: LanceDB creates auto-schema on first insert if not exists
+        const points = [
+            {
+                id: "p1",
+                vector: [0.1, 0.1, 0.1, 0.1],
+                metadata: { type: "electronics" },
+            },
+            {
+                id: "p2",
+                vector: [0.2, 0.2, 0.2, 0.2],
+                metadata: { type: "clothing" },
+            },
+            {
+                id: "p3",
+                vector: [0.3, 0.3, 0.3, 0.3],
+                metadata: { type: "electronics" },
+            },
+        ];
+        await srcCol.createAuto(4, "cosine");
+        await srcCol.insert(points);
+        // Verify source
+        const srcRes = await srcCol.search([0.1, 0.1, 0.1, 0.1], 10);
+        (0, vitest_1.expect)(srcRes.results.length).toBeGreaterThanOrEqual(3);
+        // Test scroll explicitly
+        const scrollRes = await srcCol.scroll(undefined, 2); // limit 2
+        (0, vitest_1.expect)(scrollRes.points.length).toBe(2);
+        (0, vitest_1.expect)(scrollRes.nextOffset).toBeDefined(); // Should have next offset
+        // Migrate
+        const migrator = new index_1.DataMigrator(sourceClient, destClient);
+        // Simple migration
+        const result = await migrator.migrateSimple(srcColName, destColName, 10);
+        console.log("Migration result:", result);
+        (0, vitest_1.expect)(result.pointsMigrated).toBe(3);
+        // Verify destination
+        const destCol = destClient.collection(destColName);
+        const destRes = await destCol.search([0.1, 0.1, 0.1, 0.1], 10);
+        (0, vitest_1.expect)(destRes.results.length).toBe(3);
+        // Verify metadata
+        const p1 = destRes.results.find((r) => r.id === "p1");
+        (0, vitest_1.expect)(p1).toBeDefined();
+        (0, vitest_1.expect)(p1?.metadata?.type).toBe("electronics");
+    });
+});

package/native.d.ts

@@ -35,6 +35,14 @@ export declare class Collection {
    */
   createAuto(dimension?: number | undefined | null, distance?: string | undefined | null): Promise<void>
   insertBatch(points: Array<Point>, batchSize?: number | undefined | null, parallel?: number | undefined | null): Promise<void>
+  /** Scroll through points in the collection (paginated export). */
+  scroll(offset?: string | undefined | null, limit?: number | undefined | null): Promise<ScrollResponse>
+}
+
+export declare class DataMigrator {
+  constructor(source: EmbexClient, dest: EmbexClient)
+  migrate(sourceCollection: string, destCollection: string, dimension: number, batchSize?: number | undefined | null, distance?: string | undefined | null): Promise<MigrationResult>
+  migrateSimple(sourceCollection: string, destCollection: string, batchSize?: number | undefined | null): Promise<MigrationResult>
 }
 
 /** Main client for the Embex vector database. */
@@ -105,6 +113,11 @@ export interface MigrationInput {
   downOperations?: any
 }
 
+export interface MigrationResult {
+  pointsMigrated: number
+  elapsedMs: number
+}
+
 /** A point in the vector database. */
 export interface Point {
   id: string
@@ -112,6 +125,11 @@ export interface Point {
   metadata?: Record<string, any>
 }
 
+export interface ScrollResponse {
+  points: Array<Point>
+  nextOffset?: string
+}
+
 export interface SearchOptions {
   /** Number of results to return. */
   limit?: number

package/native.js

@@ -77,8 +77,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-android-arm64')
         const bindingPackageVersion = require('@bridgerust/embex-android-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -93,8 +93,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-android-arm-eabi')
         const bindingPackageVersion = require('@bridgerust/embex-android-arm-eabi/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -114,8 +114,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-win32-x64-gnu')
         const bindingPackageVersion = require('@bridgerust/embex-win32-x64-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -130,8 +130,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-win32-x64-msvc')
         const bindingPackageVersion = require('@bridgerust/embex-win32-x64-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -147,8 +147,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-win32-ia32-msvc')
         const bindingPackageVersion = require('@bridgerust/embex-win32-ia32-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -163,8 +163,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-win32-arm64-msvc')
         const bindingPackageVersion = require('@bridgerust/embex-win32-arm64-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -182,8 +182,8 @@ function requireNative() {
     try {
       const binding = require('@bridgerust/embex-darwin-universal')
       const bindingPackageVersion = require('@bridgerust/embex-darwin-universal/package.json').version
-      if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-        throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+      if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+        throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
       }
       return binding
     } catch (e) {
@@ -198,8 +198,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-darwin-x64')
         const bindingPackageVersion = require('@bridgerust/embex-darwin-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -214,8 +214,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-darwin-arm64')
         const bindingPackageVersion = require('@bridgerust/embex-darwin-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -234,8 +234,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-freebsd-x64')
         const bindingPackageVersion = require('@bridgerust/embex-freebsd-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -250,8 +250,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-freebsd-arm64')
         const bindingPackageVersion = require('@bridgerust/embex-freebsd-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -271,8 +271,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-x64-musl')
           const bindingPackageVersion = require('@bridgerust/embex-linux-x64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -287,8 +287,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-x64-gnu')
           const bindingPackageVersion = require('@bridgerust/embex-linux-x64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -305,8 +305,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-arm64-musl')
           const bindingPackageVersion = require('@bridgerust/embex-linux-arm64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -321,8 +321,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-arm64-gnu')
           const bindingPackageVersion = require('@bridgerust/embex-linux-arm64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -339,8 +339,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-arm-musleabihf')
           const bindingPackageVersion = require('@bridgerust/embex-linux-arm-musleabihf/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -355,8 +355,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-arm-gnueabihf')
           const bindingPackageVersion = require('@bridgerust/embex-linux-arm-gnueabihf/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -373,8 +373,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-loong64-musl')
           const bindingPackageVersion = require('@bridgerust/embex-linux-loong64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -389,8 +389,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-loong64-gnu')
           const bindingPackageVersion = require('@bridgerust/embex-linux-loong64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -407,8 +407,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-riscv64-musl')
           const bindingPackageVersion = require('@bridgerust/embex-linux-riscv64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -423,8 +423,8 @@ function requireNative() {
         try {
           const binding = require('@bridgerust/embex-linux-riscv64-gnu')
           const bindingPackageVersion = require('@bridgerust/embex-linux-riscv64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -440,8 +440,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-linux-ppc64-gnu')
         const bindingPackageVersion = require('@bridgerust/embex-linux-ppc64-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -456,8 +456,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-linux-s390x-gnu')
         const bindingPackageVersion = require('@bridgerust/embex-linux-s390x-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -476,8 +476,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-openharmony-arm64')
         const bindingPackageVersion = require('@bridgerust/embex-openharmony-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -492,8 +492,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-openharmony-x64')
         const bindingPackageVersion = require('@bridgerust/embex-openharmony-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -508,8 +508,8 @@ function requireNative() {
       try {
         const binding = require('@bridgerust/embex-openharmony-arm')
         const bindingPackageVersion = require('@bridgerust/embex-openharmony-arm/package.json').version
-        if (bindingPackageVersion !== '0.1.8' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.8 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.1.17' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.17 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -536,13 +536,17 @@ if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
       wasiBindingError = err
     }
   }
-  if (!nativeBinding) {
+  if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
     try {
       wasiBinding = require('@bridgerust/embex-wasm32-wasi')
       nativeBinding = wasiBinding
     } catch (err) {
       if (process.env.NAPI_RS_FORCE_WASI) {
-        wasiBindingError.cause = err
+        if (!wasiBindingError) {
+          wasiBindingError = err
+        } else {
+          wasiBindingError.cause = err
+        }
         loadErrors.push(err)
       }
     }
@@ -573,6 +577,7 @@ if (!nativeBinding) {
 
 module.exports = nativeBinding
 module.exports.Collection = nativeBinding.Collection
+module.exports.DataMigrator = nativeBinding.DataMigrator
 module.exports.EmbexClient = nativeBinding.EmbexClient
 module.exports.QueryBuilder = nativeBinding.QueryBuilder
 module.exports.SearchBuilder = nativeBinding.SearchBuilder

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bridgerust/embex",
-  "version": "0.1.16",
+  "version": "0.1.17",
   "description": "Embex Vector Database ORM Node.js Bindings",
   "homepage": "https://github.com/bridgerust/bridgerust/tree/main/crates/embex",
   "authors": [

package/src/index.ts

@@ -3,6 +3,9 @@ import {
   EmbexClient,
   QueryBuilder,
   SearchBuilder,
+  DataMigrator,
+  ScrollResponse,
+  MigrationResult,
   // @ts-ignore
   cli,
   Point,
@@ -39,4 +42,14 @@ declare module "../native" {
   }
 };
 
-export { Collection, EmbexClient, QueryBuilder, SearchBuilder, cli, Point };
+export {
+  Collection,
+  EmbexClient,
+  QueryBuilder,
+  SearchBuilder,
+  DataMigrator,
+  ScrollResponse,
+  MigrationResult,
+  cli,
+  Point,
+};

package/src/lib.rs

@@ -5,8 +5,10 @@ use std::sync::Arc;
 use std::sync::Mutex;
 
 use bridge_embex::{
-    config::EmbexConfig, types::Point as RustPoint, EmbexClient as RustClient, Migration,
-    MigrationManager, QueryBuilder as RustQueryBuilder, VectorDatabase,
+    config::EmbexConfig,
+    types::{CollectionSchema, DistanceMetric, Point as RustPoint},
+    DataMigrator as RustDataMigrator, EmbexClient as RustClient, Migration, MigrationManager,
+    MigrationProgress as RustMigrationProgress, QueryBuilder as RustQueryBuilder, VectorDatabase,
 };
 
 use napi_derive::napi;
@@ -227,6 +229,98 @@ pub struct MetadataUpdate {
     pub updates: HashMap<String, serde_json::Value>,
 }
 
+#[napi(object)]
+pub struct ScrollResponse {
+    pub points: Vec<Point>,
+    pub next_offset: Option<String>,
+}
+
+#[napi(object)]
+pub struct MigrationResult {
+    pub points_migrated: u32,
+    pub elapsed_ms: f64,
+}
+
+#[napi]
+pub struct DataMigrator {
+    source_db: Arc<dyn VectorDatabase>,
+    dest_db: Arc<dyn VectorDatabase>,
+}
+
+#[napi]
+impl DataMigrator {
+    #[napi(constructor)]
+    pub fn new(source: &EmbexClient, dest: &EmbexClient) -> Self {
+        Self {
+            source_db: source.inner.db(),
+            dest_db: dest.inner.db(),
+        }
+    }
+
+    #[napi]
+    pub async fn migrate(
+        &self,
+        source_collection: String,
+        dest_collection: String,
+        dimension: u32,
+        batch_size: Option<u32>,
+        distance: Option<String>,
+    ) -> Result<MigrationResult> {
+        let migrator = RustDataMigrator::new(self.source_db.clone(), self.dest_db.clone());
+
+        let batch_size = batch_size.unwrap_or(1000) as usize;
+        let metric = match distance.as_deref().unwrap_or("cosine") {
+            "cosine" => DistanceMetric::Cosine,
+            "euclidean" => DistanceMetric::Euclidean,
+            "dot" => DistanceMetric::Dot,
+            _ => return Err(Error::from_reason("Invalid distance metric")),
+        };
+
+        let schema = CollectionSchema {
+            name: dest_collection.clone(),
+            dimension: dimension as usize,
+            metric,
+        };
+
+        let result = migrator
+            .migrate::<fn(RustMigrationProgress)>(
+                &source_collection,
+                &dest_collection,
+                schema,
+                batch_size,
+                None,
+            )
+            .await
+            .map_err(to_napi_err)?;
+
+        Ok(MigrationResult {
+            points_migrated: result.points_migrated as u32,
+            elapsed_ms: result.elapsed_ms as f64,
+        })
+    }
+
+    #[napi]
+    pub async fn migrate_simple(
+        &self,
+        source_collection: String,
+        dest_collection: String,
+        batch_size: Option<u32>,
+    ) -> Result<MigrationResult> {
+        let migrator = RustDataMigrator::new(self.source_db.clone(), self.dest_db.clone());
+        let batch_size = batch_size.unwrap_or(1000) as usize;
+
+        let result = migrator
+            .migrate_simple(&source_collection, &dest_collection, batch_size)
+            .await
+            .map_err(to_napi_err)?;
+
+        Ok(MigrationResult {
+            points_migrated: result.points_migrated as u32,
+            elapsed_ms: result.elapsed_ms as f64,
+        })
+    }
+}
+
 #[napi]
 impl Collection {
     /// Insert points into the collection.
@@ -501,6 +595,34 @@ impl Collection {
     // Note: insert_stream is not yet implemented for Node.js.
     // The Python version uses async iterables which require complex napi-rs interop.
     // For now, use insert_batch for bulk operations.
+
+    /// Scroll through points in the collection (paginated export).
+    #[napi]
+    pub async fn scroll(
+        &self,
+        offset: Option<String>,
+        limit: Option<u32>,
+    ) -> Result<ScrollResponse> {
+        let inner = self.inner.clone();
+        let limit = limit.unwrap_or(100) as usize;
+
+        let res = inner.scroll(offset, limit).await.map_err(to_napi_err)?;
+
+        let points = res
+            .points
+            .into_iter()
+            .map(|p| Point {
+                id: p.id,
+                vector: p.vector.into_iter().map(|v| v as f64).collect(),
+                metadata: p.metadata,
+            })
+            .collect();
+
+        Ok(ScrollResponse {
+            points,
+            next_offset: res.next_offset,
+        })
+    }
 }
 
 #[napi]

package/src/tests/integration/data_migration.test.ts

@@ -0,0 +1,94 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { EmbexClient, DataMigrator, Point } from "../../index"; // Assuming index exports DataMigrator
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+
+// Helper to create temp dir
+const createTempDir = () => {
+  return fs.mkdtempSync(path.join(os.tmpdir(), "embex-test-"));
+};
+
+describe("Data Migrator and Scroll", () => {
+  let sourceDbPath: string;
+  let destDbPath: string;
+  let sourceClient: EmbexClient;
+  let destClient: EmbexClient;
+
+  beforeAll(async () => {
+    sourceDbPath = createTempDir();
+    destDbPath = createTempDir();
+
+    // Initialize source and destination (LanceDB)
+    sourceClient = await EmbexClient.newAsync("lancedb", sourceDbPath);
+    destClient = await EmbexClient.newAsync("lancedb", destDbPath);
+  });
+
+  afterAll(async () => {
+    // Cleanup
+    try {
+      fs.rmSync(sourceDbPath, { recursive: true, force: true });
+      fs.rmSync(destDbPath, { recursive: true, force: true });
+    } catch (e) {
+      console.error("Cleanup failed", e);
+    }
+  });
+
+  it("should migrate data between databases", async () => {
+    const srcColName = "products_src";
+    const destColName = "products_dest";
+
+    const srcCol = sourceClient.collection(srcColName);
+
+    // Insert data into source
+    // Note: LanceDB creates auto-schema on first insert if not exists
+    const points: Point[] = [
+      {
+        id: "p1",
+        vector: [0.1, 0.1, 0.1, 0.1],
+        metadata: { type: "electronics" },
+      },
+      {
+        id: "p2",
+        vector: [0.2, 0.2, 0.2, 0.2],
+        metadata: { type: "clothing" },
+      },
+      {
+        id: "p3",
+        vector: [0.3, 0.3, 0.3, 0.3],
+        metadata: { type: "electronics" },
+      },
+    ];
+
+    await srcCol.createAuto(4, "cosine");
+    await srcCol.insert(points);
+
+    // Verify source
+    const srcRes = await srcCol.search([0.1, 0.1, 0.1, 0.1], 10);
+    expect(srcRes.results.length).toBeGreaterThanOrEqual(3);
+
+    // Test scroll explicitly
+    const scrollRes = await srcCol.scroll(undefined, 2); // limit 2
+    expect(scrollRes.points.length).toBe(2);
+    expect(scrollRes.nextOffset).toBeDefined(); // Should have next offset
+
+    // Migrate
+    const migrator = new DataMigrator(sourceClient, destClient);
+
+    // Simple migration
+    const result = await migrator.migrateSimple(srcColName, destColName, 10);
+
+    console.log("Migration result:", result);
+    expect(result.pointsMigrated).toBe(3);
+
+    // Verify destination
+    const destCol = destClient.collection(destColName);
+    const destRes = await destCol.search([0.1, 0.1, 0.1, 0.1], 10);
+    expect(destRes.results.length).toBe(3);
+
+    // Verify metadata
+    const p1 = destRes.results.find((r) => r.id === "p1");
+    expect(p1).toBeDefined();
+    expect(p1?.metadata?.type).toBe("electronics");
+  });
+});