@h-ai/vecdb 1.0.0-beta.25 → 1.0.0-beta.27

package/README.en.md

@@ -79,6 +79,19 @@ Insert/upsert take collection/documents, delete IDs, yielding void; search takes
 | `vecdb.vector.delete` | collection, ids             | `void`                 | `await vecdb.vector.delete('documents', ['doc-1'])`                |
 | `vecdb.vector.search` | collection, vector, options | `VectorSearchResult[]` | `await vecdb.vector.search('documents', queryVector, { topK: 5 })` |
 | `vecdb.vector.count`  | collection                  | `number`               | `await vecdb.vector.count('documents')`                            |
+| `vecdb.vector.scan`   | collection, options         | `VectorScanPage`       | `await vecdb.vector.scan('documents', { limit: 32 })`              |
+
+Use `scan` to page through document IDs for reconciliation or cleanup. `collection` is the collection name. `options.limit` defaults to 64 and accepts 1 through 128. Omit `options.cursor` for the first page. Each page contains `{ id }` items. If `nextCursor` is present, continue with that cursor on the same collection and backend even when `items` is empty. Concurrent writes or deletions can affect pagination, so reconciliation jobs should run repeated sweeps. Qdrant returns only points that map back to framework document IDs.
+
+```ts
+// Input: collection name and page size. Output: HaiResult<VectorScanPage>.
+const page = await vecdb.vector.scan('documents', { limit: 32 })
+// On success, pass the cursor and page size to the same collection when another page exists.
+if (page.success && page.data.nextCursor !== undefined) {
+  const nextPage = await vecdb.vector.scan('documents', { cursor: page.data.nextCursor, limit: 32 })
+  // On success, nextPage.data.items contains the next IDs. An absent nextCursor marks the last page.
+}
+```
 
 ### Production shared state
 

package/README.ja.md

@@ -79,6 +79,19 @@ insert/upsert は集合と文書配列、delete は ID 配列から void。searc
 | `vecdb.vector.delete` | collection, ids             | `void`                 | `await vecdb.vector.delete('documents', ['doc-1'])`                |
 | `vecdb.vector.search` | collection, vector, options | `VectorSearchResult[]` | `await vecdb.vector.search('documents', queryVector, { topK: 5 })` |
 | `vecdb.vector.count`  | collection                  | `number`               | `await vecdb.vector.count('documents')`                            |
+| `vecdb.vector.scan`   | collection, options         | `VectorScanPage`       | `await vecdb.vector.scan('documents', { limit: 32 })`              |
+
+`scan` は照合やクリーンアップのために文書 ID をページ単位で取得します。`collection` はコレクション名です。`options.limit` の既定値は 64、範囲は 1〜128 です。最初のページでは `options.cursor` を省略します。各ページの `items` には `{ id }` が入ります。`items` が空でも `nextCursor` があれば、同じバックエンドとコレクションにそのカーソルを渡して続けます。同時書き込みや削除でページの内容が変わるため、照合処理はスキャンを繰り返します。Qdrant はフレームワークの文書 ID に対応するポイントだけを返します。
+
+```ts
+// 入力はコレクション名とページサイズ、出力は HaiResult<VectorScanPage>。
+const page = await vecdb.vector.scan('documents', { limit: 32 })
+// 成功して次のページがある場合は、同じコレクションにカーソルとページサイズを渡す。
+if (page.success && page.data.nextCursor !== undefined) {
+  const nextPage = await vecdb.vector.scan('documents', { cursor: page.data.nextCursor, limit: 32 })
+  // 成功時の nextPage.data.items は次ページの ID。nextCursor のないページが最後。
+}
+```
 
 ### 本番の共有状態
 

package/README.md

@@ -79,6 +79,19 @@ if (!closeResult.success) {
 | `vecdb.vector.delete` | collection, ids             | `void`                 | `await vecdb.vector.delete('documents', ['doc-1'])`                |
 | `vecdb.vector.search` | collection, vector, options | `VectorSearchResult[]` | `await vecdb.vector.search('documents', queryVector, { topK: 5 })` |
 | `vecdb.vector.count`  | collection                  | `number`               | `await vecdb.vector.count('documents')`                            |
+| `vecdb.vector.scan`   | collection, options         | `VectorScanPage`       | `await vecdb.vector.scan('documents', { limit: 32 })`              |
+
+`scan` 分页返回文档 ID，可用于核对或批量清理集合。`collection` 是集合名，`options.limit` 默认为 64，取值为 1 到 128，`options.cursor` 在首页省略。每页的 `items` 只包含 `{ id }`。有 `nextCursor` 时，即使本页为空也继续读取，并将游标传给同一后端的同一集合。并发写入或删除可能改变分页结果，清理任务应重复扫描。Qdrant 只返回能映射回框架文档的记录。
+
+```ts
+// 输入集合名与页大小，输出 HaiResult<VectorScanPage>。
+const page = await vecdb.vector.scan('documents', { limit: 32 })
+// 成功且存在下一页时，将上一页游标与页大小传入同一集合。
+if (page.success && page.data.nextCursor !== undefined) {
+  const nextPage = await vecdb.vector.scan('documents', { cursor: page.data.nextCursor, limit: 32 })
+  // nextPage 成功时 data.items 是下一页文档 ID，data.nextCursor 缺失时已到末页。
+}
+```
 
 ### 生产共享状态
 

package/dist/index.d.ts

@@ -631,6 +631,45 @@ interface VectorOperations {
      * Qdrant 使用精确 count，不依赖集合信息的近似统计。
      */
     count: (collection: string) => Promise<HaiResult<number>>;
+    /**
+     * 分页读取文档 ID，仅返回可以传给本模块删除操作的 ID。
+     *
+     * @param collection - 集合名称
+     * @param options - 扫描游标与每页条数
+     * @returns 本页 ID 和后续游标。扫描不提供跨页快照。
+     * @example await vecdb.vector.scan('docs', { limit: 32 })
+     */
+    scan?: (collection: string, options?: VectorScanOptions) => Promise<HaiResult<VectorScanPage>>;
+}
+/**
+ * 文档 ID 扫描选项。
+ * @example
+ * ```ts
+ * import type { VectorScanOptions } from '@h-ai/vecdb'
+ * const options: VectorScanOptions = { limit: 32 }
+ * ```
+ */
+interface VectorScanOptions {
+    /** 上一页返回的游标，首次扫描省略。游标仅用于同一后端的同一个集合。 */
+    cursor?: string;
+    /** 每页最多返回的文档数量，默认 64，允许 1 到 128。 */
+    limit?: number;
+}
+/**
+ * 一页文档 ID。
+ * @example
+ * ```ts
+ * import type { VectorScanPage } from '@h-ai/vecdb'
+ * const page: VectorScanPage = { items: [{ id: 'doc-1' }] }
+ * ```
+ */
+interface VectorScanPage {
+    /** 文档 ID。Qdrant 只包含可映射回框架文档的记录。 */
+    items: Array<{
+        id: string;
+    }>;
+    /** 下一页游标。即使当前页为空，只要存在游标就应继续扫描。 */
+    nextCursor?: string;
 }
 /**
  * 向量数据库服务接口（通过 `vecdb` 对象访问）
@@ -706,6 +745,27 @@ interface VecdbConnection {
     path?: string;
 }
 
+/**
+ * @h-ai/vecdb — Provider 共享基础层
+ *
+ * 将各 Provider 的共性逻辑抽取为统一实现：连接守卫、运行时异常捕获、HaiResult 包装。
+ *
+ * 各 Provider 只需提供 CollectionDriver / VectorDriver（原始操作适配器），
+ * base 层统一处理初始化检查与 catch-all 安全网。
+ * @module vecdb-provider-base
+ */
+
+/**
+ * 内置后端全部支持文档 ID 扫描。
+ * @example 模块内部上下文
+ * ```ts
+ * function firstPage(vector: BuiltinVectorOperations, name: string) {
+ *   return vector.scan(name)
+ * }
+ * ```
+ */
+type BuiltinVectorOperations = VectorOperations & Required<Pick<VectorOperations, 'scan'>>;
+
 /**
  * @h-ai/vecdb — 向量数据库服务主入口
  *
@@ -739,7 +799,9 @@ interface VecdbConnection {
  * }
  * ```
  */
-declare const vecdb: VecdbFunctions;
+declare const vecdb: VecdbFunctions & {
+    readonly vector: BuiltinVectorOperations;
+};
 
 export { DistanceMetricSchema, HaiVecdbError, VecdbConfigSchema, VecdbOperationLogConfigSchema, vecdb };
-export type { ChromaConfig, CollectionCreateOptions, CollectionInfo, CollectionOperations, DistanceMetric, LancedbConfig, PgvectorConfig, QdrantConfig, VecdbConfig, VecdbConfigInput, VecdbConnection, VecdbFunctions, VecdbOperationLogConfig, VecdbType, VectorDocument, VectorOperations, VectorSearchOptions, VectorSearchResult };
+export type { ChromaConfig, CollectionCreateOptions, CollectionInfo, CollectionOperations, DistanceMetric, LancedbConfig, PgvectorConfig, QdrantConfig, VecdbConfig, VecdbConfigInput, VecdbConnection, VecdbFunctions, VecdbOperationLogConfig, VecdbType, VectorDocument, VectorOperations, VectorScanOptions, VectorScanPage, VectorSearchOptions, VectorSearchResult };

package/dist/index.js

@@ -34,6 +34,8 @@ var en_US_default = {
   vecdb_indexBuildFailed: "Failed to build index: {error}",
   vecdb_serializationFailed: "Serialization/deserialization failed: {error}",
   vecdb_invalidDimension: "Invalid vector dimension: {dimension}, must be a positive integer",
+  vecdb_invalidScanLimit: "The vector scan page size must be an integer between 1 and 128",
+  vecdb_invalidScanCursor: "Invalid vector scan cursor. Use the cursor returned by the previous page",
   vecdb_configPathRequired: "Database storage path is required",
   vecdb_configDatabaseRequired: "Database name is required",
   vecdb_sharedBackendRequired: "Production mode requires a shared backend; local files and memory backends are for development or explicit end-to-end tests.",
@@ -68,6 +70,8 @@ var ja_JP_default = {
   vecdb_indexBuildFailed: "\u30A4\u30F3\u30C7\u30C3\u30AF\u30B9\u306E\u69CB\u7BC9\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
   vecdb_serializationFailed: "\u30B7\u30EA\u30A2\u30E9\u30A4\u30BA\u307E\u305F\u306F\u30C7\u30B7\u30EA\u30A2\u30E9\u30A4\u30BA\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
   vecdb_invalidDimension: "\u30D9\u30AF\u30C8\u30EB\u306E\u6B21\u5143\u304C\u7121\u52B9\u3067\u3059: {dimension}\u3002\u6B63\u306E\u6574\u6570\u3092\u6307\u5B9A\u3057\u3066\u304F\u3060\u3055\u3044",
+  vecdb_invalidScanLimit: "\u30D9\u30AF\u30C8\u30EB\u30B9\u30AD\u30E3\u30F3\u306E\u30DA\u30FC\u30B8\u30B5\u30A4\u30BA\u306B\u306F 1 \u304B\u3089 128 \u306E\u6574\u6570\u3092\u6307\u5B9A\u3057\u3066\u304F\u3060\u3055\u3044",
+  vecdb_invalidScanCursor: "\u30D9\u30AF\u30C8\u30EB\u30B9\u30AD\u30E3\u30F3\u306E\u30AB\u30FC\u30BD\u30EB\u304C\u7121\u52B9\u3067\u3059\u3002\u524D\u306E\u30DA\u30FC\u30B8\u304B\u3089\u8FD4\u3055\u308C\u305F\u30AB\u30FC\u30BD\u30EB\u3092\u6307\u5B9A\u3057\u3066\u304F\u3060\u3055\u3044",
   vecdb_configPathRequired: "\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u306E\u4FDD\u5B58\u30D1\u30B9\u306F\u5FC5\u9808\u3067\u3059",
   vecdb_configDatabaseRequired: "\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u540D\u306F\u5FC5\u9808\u3067\u3059",
   vecdb_sharedBackendRequired: "\u672C\u756A\u74B0\u5883\u306B\u306F\u5171\u6709\u30D0\u30C3\u30AF\u30A8\u30F3\u30C9\u304C\u5FC5\u8981\u3067\u3059\u3002\u30ED\u30FC\u30AB\u30EB\u30D5\u30A1\u30A4\u30EB\u3068\u30E1\u30E2\u30EA\u30FC\u30D0\u30C3\u30AF\u30A8\u30F3\u30C9\u306F\u958B\u767A\u74B0\u5883\u307E\u305F\u306F\u660E\u793A\u7684\u306A E2E \u30C6\u30B9\u30C8\u3067\u306E\u307F\u4F7F\u7528\u3067\u304D\u307E\u3059\u3002",
@@ -102,6 +106,8 @@ var zh_CN_default = {
   vecdb_indexBuildFailed: "\u6784\u5EFA\u7D22\u5F15\u5931\u8D25: {error}",
   vecdb_serializationFailed: "\u5E8F\u5217\u5316/\u53CD\u5E8F\u5217\u5316\u5931\u8D25: {error}",
   vecdb_invalidDimension: "\u65E0\u6548\u7684\u5411\u91CF\u7EF4\u5EA6: {dimension}\uFF0C\u5FC5\u987B\u4E3A\u6B63\u6574\u6570",
+  vecdb_invalidScanLimit: "\u5411\u91CF\u626B\u63CF\u6BCF\u9875\u6761\u6570\u5FC5\u987B\u4E3A 1 \u5230 128 \u7684\u6574\u6570",
+  vecdb_invalidScanCursor: "\u5411\u91CF\u626B\u63CF\u6E38\u6807\u65E0\u6548\uFF0C\u8BF7\u4F7F\u7528\u4E0A\u4E00\u9875\u8FD4\u56DE\u7684\u6E38\u6807",
   vecdb_configPathRequired: "\u6570\u636E\u5E93\u5B58\u50A8\u8DEF\u5F84\u4E0D\u80FD\u4E3A\u7A7A",
   vecdb_configDatabaseRequired: "\u6570\u636E\u5E93\u540D\u79F0\u4E0D\u80FD\u4E3A\u7A7A",
   vecdb_sharedBackendRequired: "\u751F\u4EA7\u6A21\u5F0F\u9700\u8981\u8DE8\u8282\u70B9\u5171\u4EAB\u540E\u7AEF\uFF1B\u672C\u673A\u6587\u4EF6\u6216\u5185\u5B58\u540E\u7AEF\u4EC5\u7528\u4E8E\u5F00\u53D1\u548C\u663E\u5F0F\u7AEF\u5230\u7AEF\u6D4B\u8BD5\u3002",
@@ -383,6 +389,18 @@ function createBaseVectorOps(ctx, driver) {
         HaiVecdbError.QUERY_FAILED,
         "Failed to count vectors"
       );
+    },
+    scan: (collection, options) => {
+      const limit = options?.limit ?? 64;
+      const cursor = options?.cursor;
+      logVecdbOperation(ctx, "read", "vector.scan", { limit, hasCursor: cursor !== void 0 });
+      return wrapOp(ctx, async () => {
+        if (!Number.isInteger(limit) || limit < 1 || limit > 128)
+          return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidScanLimit"));
+        if (cursor !== void 0 && typeof cursor !== "string")
+          return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidScanCursor"));
+        return driver.scan(collection, limit, cursor);
+      }, HaiVecdbError.QUERY_FAILED, "Failed to scan vector IDs");
     }
   };
 }
@@ -751,6 +769,26 @@ function createChromaProvider() {
       logger.debug("Counting vectors", { collection });
       const target = await getCollection(collection);
       return ok(await target.count());
+    },
+    async scan(collection, limit, cursor) {
+      const offset = cursor === void 0 ? 0 : Number(cursor);
+      if (!Number.isSafeInteger(offset) || offset < 0 || cursor !== void 0 && String(offset) !== cursor)
+        return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidScanCursor"));
+      try {
+        const target = await getCollection(collection);
+        const { ids } = await target.get({ offset, limit: limit + 1, include: [] });
+        return ok({
+          items: ids.slice(0, limit).map((id) => ({ id })),
+          nextCursor: ids.length > limit ? String(offset + limit) : void 0
+        });
+      } catch (error) {
+        if (error instanceof Error && ["ChromaNotFoundError", "InvalidCollectionError"].includes(error.name)) {
+          return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
+            params: { name: collection }
+          }));
+        }
+        throw error;
+      }
     }
   };
   return {
@@ -1113,6 +1151,19 @@ function createLancedbProvider() {
         const count = await table.countRows();
         return ok(count);
       });
+    },
+    async scan(collection, limit, cursor) {
+      return withCollection(collection, async (table) => {
+        const query = table.query().select(["id"]).orderBy({ columnName: "id", ascending: true }).limit(limit + 1);
+        if (cursor !== void 0)
+          query.where(`id > '${escapeLanceSqlValue(cursor)}'`);
+        const rows = await query.toArray();
+        const items = rows.slice(0, limit).map((row) => ({ id: row.id }));
+        return ok({
+          items,
+          nextCursor: rows.length > limit ? items.at(-1)?.id : void 0
+        });
+      });
     }
   };
   return {
@@ -1505,6 +1556,25 @@ function createPgvectorProvider() {
       logger3.debug("Counting vectors", { collection });
       const result = await pool.query(`SELECT COUNT(*) as count FROM ${quoteIdent(table)}`);
       return ok(Number.parseInt(String(result.rows[0].count), 10));
+    },
+    async scan(collection, limit, cursor) {
+      const table = quoteIdent(tableName(collection));
+      const request = cursor === void 0 ? pool.query(`SELECT id FROM ${table} ORDER BY id LIMIT $1`, [limit + 1]) : pool.query(`SELECT id FROM ${table} WHERE id > $1 ORDER BY id LIMIT $2`, [cursor, limit + 1]);
+      const page = await request.catch((error) => {
+        if (typeof error === "object" && error !== null && "code" in error && error.code === "42P01")
+          return void 0;
+        throw error;
+      });
+      if (!page) {
+        return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
+          params: { name: collection }
+        }));
+      }
+      const items = page.rows.slice(0, limit).map((row) => ({ id: row.id }));
+      return ok({
+        items,
+        nextCursor: page.rows.length > limit ? items.at(-1)?.id : void 0
+      });
     }
   };
   return {
@@ -1573,6 +1643,21 @@ function createPgvectorProvider() {
   };
 }
 var MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
+var MAX_QDRANT_POINT_ID = 18446744073709551615n;
+var scrollIdSchema = z.union([
+  z.guid(),
+  z.bigint().min(0n).max(MAX_QDRANT_POINT_ID)
+]);
+var parseWithSource = JSON.parse;
+var rawJSON = JSON.rawJSON;
+function parseScrollJson(text) {
+  return parseWithSource(text, (key, value, context) => {
+    const source = context?.source;
+    if ((key === "id" || key === "next_page_offset") && typeof value === "number" && source && /^\d{1,20}$/.test(source))
+      return BigInt(source);
+    return value;
+  });
+}
 var pointSchema = z.object({
   id: z.union([
     z.string(),
@@ -1606,7 +1691,7 @@ var QdrantRestClient = class {
   }
   apiKey;
   base;
-  async perform(path, method, schema, body) {
+  async perform(path, method, schema, body, parseJson = JSON.parse) {
     const url = new URL(this.base);
     const [pathname, query] = path.split("?");
     url.pathname = `${url.pathname.replace(/\/$/, "")}${pathname}`;
@@ -1652,7 +1737,7 @@ var QdrantRestClient = class {
     }
     let bodyJson;
     try {
-      bodyJson = JSON.parse(new TextDecoder().decode(bytes));
+      bodyJson = parseJson(new TextDecoder().decode(bytes));
     } catch {
       throw new Error("Qdrant returned invalid JSON");
     }
@@ -1664,9 +1749,9 @@ var QdrantRestClient = class {
       throw new Error("Qdrant returned an invalid response");
     return envelope.data.result;
   }
-  async request(path, method, schema, body) {
+  async request(path, method, schema, body, parseJson) {
     try {
-      return await this.perform(path, method, schema, body);
+      return await this.perform(path, method, schema, body, parseJson);
     } catch (error) {
       if (error instanceof QdrantHttpError)
         throw error;
@@ -1709,6 +1794,22 @@ var QdrantRestClient = class {
       }))
     }), options);
   }
+  scroll(name, limit, offset) {
+    return this.request(`/collections/${encodeURIComponent(name)}/points/scroll`, "POST", z.object({
+      points: z.array(z.object({
+        id: scrollIdSchema,
+        payload: z.object({
+          _id: z.unknown().optional()
+        }).nullable().optional()
+      })).max(limit),
+      next_page_offset: scrollIdSchema.nullish()
+    }), {
+      limit,
+      offset: typeof offset === "bigint" ? rawJSON(String(offset)) : offset,
+      with_payload: ["_id"],
+      with_vector: false
+    }, parseScrollJson);
+  }
   count(name, options) {
     return this.request(`/collections/${encodeURIComponent(name)}/points/count`, "POST", z.object({
       count: z.number()
@@ -1962,6 +2063,40 @@ function createQdrantProvider() {
       logger4.debug("Counting vectors", { collection });
       const count = await client.count(collection, { exact: true });
       return ok(count.count);
+    },
+    async scan(collection, limit, cursor) {
+      let offset;
+      if (cursor !== void 0) {
+        if (/^n:(?:0|[1-9]\d{0,19})$/.test(cursor) && BigInt(cursor.slice(2)) <= MAX_QDRANT_POINT_ID)
+          offset = BigInt(cursor.slice(2));
+        else if (/^[\da-f]{8}(?:-[\da-f]{4}){3}-[\da-f]{12}$/i.test(cursor))
+          offset = cursor.toLowerCase();
+        else
+          return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidScanCursor"));
+      }
+      const page = await client.scroll(collection, limit, offset).catch((error) => {
+        if (isQdrantNotFoundError(error))
+          return void 0;
+        throw error;
+      });
+      if (!page) {
+        return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
+          params: { name: collection }
+        }));
+      }
+      const items = page.points.flatMap(({ id, payload }) => {
+        const documentId = payload?._id;
+        return typeof documentId === "string" && hashToUuid(documentId) === id ? [
+          { id: documentId }
+        ] : [];
+      });
+      const next = page.next_page_offset ?? void 0;
+      if (next !== void 0 && next === offset)
+        throw new Error("Qdrant scroll cursor did not advance");
+      return ok({
+        items,
+        nextCursor: typeof next === "bigint" ? `n:${next}` : next
+      });
     }
   };
   return {
@@ -2092,7 +2227,8 @@ var vectorOps = {
   upsert: (collection, documents) => currentProvider && !closingPromise ? currentProvider.vector.upsert(collection, documents) : notInitializedVector.upsert(collection, documents),
   delete: (collection, ids) => currentProvider && !closingPromise ? currentProvider.vector.delete(collection, ids) : notInitializedVector.delete(collection, ids),
   search: (collection, vector, options) => currentProvider && !closingPromise ? currentProvider.vector.search(collection, vector, options) : notInitializedVector.search(collection, vector, options),
-  count: (collection) => currentProvider && !closingPromise ? currentProvider.vector.count(collection) : notInitializedVector.count(collection)
+  count: (collection) => currentProvider && !closingPromise ? currentProvider.vector.count(collection) : notInitializedVector.count(collection),
+  scan: (collection, options) => currentProvider && !closingPromise ? currentProvider.vector.scan(collection, options) : notInitializedVector.scan(collection, options)
 };
 var vecdb = {
   /**