@karmaniverous/jeeves-watcher 0.8.5 → 0.9.0-0

package/dist/cli/jeeves-watcher/index.js

@@ -3373,6 +3373,44 @@ function createRulesUnregisterParamHandler(deps) {
     }, deps.logger, 'RulesUnregister');
 }
 
+/**
+ * @module api/handlers/scan
+ * Fastify route handler for POST /scan. Filter-only point query without vector search.
+ */
+/**
+ * Create handler for POST /scan.
+ *
+ * @param deps - Route dependencies.
+ */
+function createScanHandler(deps) {
+    return wrapHandler(async (request, reply) => {
+        const { filter, limit = 100, cursor, fields, countOnly } = request.body;
+        if (!filter || typeof filter !== 'object') {
+            deps.logger.warn('Scan rejected: missing or invalid filter');
+            void reply
+                .status(400)
+                .send({ error: 'Missing required field: filter (object)' });
+            return;
+        }
+        if (typeof limit !== 'number' || limit < 1 || limit > 1000) {
+            deps.logger.warn({ limit }, 'Scan rejected: limit out of bounds');
+            void reply
+                .status(400)
+                .send({ error: 'limit must be between 1 and 1000' });
+            return;
+        }
+        if (countOnly) {
+            const count = await deps.vectorStore.count(filter);
+            return { count };
+        }
+        const result = await deps.vectorStore.scrollPage(filter, limit, cursor, fields);
+        return {
+            points: result.points,
+            cursor: result.nextCursor ?? null,
+        };
+    }, deps.logger, 'Scan');
+}
+
 /**
  * @module api/handlers/search
  * Fastify route handler for POST /search. Embeds a query and performs vector store similarity search.
@@ -3554,6 +3592,10 @@ function createApiServer(options) {
             textWeight: config.search.hybrid.textWeight,
         }
         : undefined;
+    app.post('/scan', createScanHandler({
+        vectorStore,
+        logger,
+    }));
     app.post('/search', createSearchHandler({
         embeddingProvider,
         vectorStore,
@@ -4811,6 +4853,28 @@ async function getCollectionInfo(client, collectionName) {
     return { pointCount, dimensions, payloadFields };
 }
 
+/**
+ * @module vectorStore/count
+ * Count utility for Qdrant collection points.
+ */
+/**
+ * Count points in a Qdrant collection matching an optional filter.
+ *
+ * Uses exact counting for accurate results.
+ *
+ * @param client - The Qdrant client instance.
+ * @param collectionName - The collection to count.
+ * @param filter - Optional Qdrant filter.
+ * @returns The number of matching points.
+ */
+async function countPoints(client, collectionName, filter) {
+    const result = await client.count(collectionName, {
+        ...(filter ? { filter } : {}),
+        exact: true,
+    });
+    return result.count;
+}
+
 /**
  * @module vectorStore/hybridSearch
  * Hybrid search and text index helpers for Qdrant vector store.
@@ -4910,11 +4974,43 @@ async function hybridSearch(client, collectionName, vector, queryText, limit, te
 
 /**
  * @module vectorStore/scroll
- * Standalone scroll utility for paginating through Qdrant collection points.
+ * Scroll utilities for paginating through Qdrant collection points.
+ */
+/**
+ * Scroll one page of points matching a filter.
+ *
+ * @param client - The Qdrant client instance.
+ * @param collectionName - The collection to scroll.
+ * @param filter - Optional Qdrant filter.
+ * @param limit - Page size.
+ * @param offset - Cursor offset from previous page.
+ * @param fields - Optional payload field projection (array of field names).
+ * @returns Page of points and next cursor.
  */
+async function scrollPage(client, collectionName, filter, limit = 100, offset, fields) {
+    const result = await client.scroll(collectionName, {
+        limit,
+        with_payload: fields ? fields : true,
+        with_vector: false,
+        ...(filter ? { filter } : {}),
+        ...(offset !== undefined ? { offset } : {}),
+    });
+    return {
+        points: result.points.map((p) => ({
+            id: String(p.id),
+            payload: p.payload,
+        })),
+        nextCursor: typeof result.next_page_offset === 'string' ||
+            typeof result.next_page_offset === 'number'
+            ? result.next_page_offset
+            : undefined,
+    };
+}
 /**
  * Scroll through all points in a Qdrant collection matching a filter.
  *
+ * Iterates over pages using {@link scrollPage}.
+ *
  * @param client - The Qdrant client instance.
  * @param collectionName - The collection to scroll.
  * @param filter - Optional Qdrant filter.
@@ -4922,32 +5018,14 @@ async function hybridSearch(client, collectionName, vector, queryText, limit, te
  * @yields Scrolled points.
  */
 async function* scrollCollection(client, collectionName, filter, limit = 100) {
-    let offset = undefined;
-    for (;;) {
-        const result = await client.scroll(collectionName, {
-            limit,
-            with_payload: true,
-            with_vector: false,
-            ...(filter ? { filter } : {}),
-            ...(offset !== undefined ? { offset } : {}),
-        });
-        for (const point of result.points) {
-            yield {
-                id: String(point.id),
-                payload: point.payload,
-            };
-        }
-        const nextOffset = result.next_page_offset;
-        if (nextOffset === null || nextOffset === undefined) {
-            break;
-        }
-        if (typeof nextOffset === 'string' || typeof nextOffset === 'number') {
-            offset = nextOffset;
-        }
-        else {
-            break;
+    let cursor;
+    do {
+        const page = await scrollPage(client, collectionName, filter, limit, cursor);
+        for (const point of page.points) {
+            yield point;
         }
-    }
+        cursor = page.nextCursor;
+    } while (cursor !== undefined);
 }
 
 /**
@@ -4991,6 +5069,15 @@ class VectorStoreClient {
             checkCompatibility: false,
         });
     }
+    /**
+     * Count points matching a filter.
+     *
+     * @param filter - Optional Qdrant filter.
+     * @returns The number of matching points.
+     */
+    async count(filter) {
+        return countPoints(this.client, this.collectionName, filter);
+    }
     /**
      * Ensure the collection exists with correct dimensions and Cosine distance.
      */
@@ -5171,6 +5258,18 @@ class VectorStoreClient {
     async hybridSearch(vector, queryText, limit, textWeight, filter) {
         return hybridSearch(this.client, this.collectionName, vector, queryText, limit, textWeight, filter);
     }
+    /**
+     * Scroll one page of points matching a filter.
+     *
+     * @param filter - Optional Qdrant filter.
+     * @param limit - Page size.
+     * @param offset - Cursor offset from previous page.
+     * @param fields - Optional field projection.
+     * @returns Page of points and next cursor.
+     */
+    async scrollPage(filter, limit = 100, offset, fields) {
+        return scrollPage(this.client, this.collectionName, filter, limit, offset, fields);
+    }
     /**
      * Scroll through all points matching a filter.
      *
@@ -6519,6 +6618,50 @@ function registerReindexCommand(cli) {
     });
 }
 
+/**
+ * @module commands/scan
+ *
+ * CLI command: scan.
+ */
+function registerScanCommand(cli) {
+    const command = cli
+        .command('scan')
+        .description('Scan the vector store (POST /scan)')
+        .option('-f, --filter <filter>', 'Qdrant filter (JSON string)', '{}')
+        .option('-l, --limit <limit>', 'Max results', '100')
+        .option('-c, --cursor <cursor>', 'Cursor from previous response')
+        .option('--fields <fields>', 'Fields to return (comma-separated)')
+        .option('--count-only', 'Return count only');
+    withApiOptions(command).action(async (options) => {
+        let filterObj = {};
+        try {
+            if (options.filter) {
+                filterObj = JSON.parse(options.filter);
+            }
+        }
+        catch (error) {
+            console.error('Invalid filter JSON:', error);
+            process.exit(1);
+        }
+        const fieldsArray = options.fields
+            ? options.fields.split(',').map((f) => f.trim())
+            : undefined;
+        await runApiCommand({
+            host: options.host,
+            port: options.port,
+            method: 'POST',
+            path: '/scan',
+            body: {
+                filter: filterObj,
+                limit: Number(options.limit),
+                cursor: options.cursor,
+                fields: fieldsArray,
+                countOnly: options.countOnly,
+            },
+        });
+    });
+}
+
 /**
  * @module commands/search
  *
@@ -6684,6 +6827,7 @@ registerStatusCommand(cli);
 registerReindexCommand(cli);
 registerRebuildMetadataCommand(cli);
 registerSearchCommand(cli);
+registerScanCommand(cli);
 registerEnrichCommand(cli);
 registerConfigReindexCommand(cli);
 registerServiceCommand(cli);

package/dist/index.d.ts

@@ -676,6 +676,13 @@ interface ScrolledPoint {
     /** The payload metadata. */
     payload: Record<string, unknown>;
 }
+/** Result of a single scroll page. */
+interface ScrollPageResult {
+    /** Matched points. */
+    points: ScrolledPoint[];
+    /** Cursor for next page, or `undefined` when no more pages. */
+    nextCursor?: string | number;
+}
 /** Payload field schema information as reported by Qdrant. */
 interface PayloadFieldSchema {
     /** Qdrant data type for the field (e.g. `keyword`, `text`, `integer`). */
@@ -702,6 +709,13 @@ interface VectorStore {
      * Ensure the collection exists with correct configuration.
      */
     ensureCollection(): Promise<void>;
+    /**
+     * Count points matching a filter.
+     *
+     * @param filter - Optional Qdrant filter.
+     * @returns The number of matching points.
+     */
+    count(filter?: Record<string, unknown>): Promise<number>;
     /**
      * Upsert points into the collection.
      *
@@ -742,6 +756,16 @@ interface VectorStore {
      * @returns An array of search results.
      */
     search(vector: number[], limit: number, filter?: Record<string, unknown>, offset?: number): Promise<SearchResult[]>;
+    /**
+     * Scroll one page of points matching a filter.
+     *
+     * @param filter - Optional Qdrant filter.
+     * @param limit - Page size.
+     * @param offset - Cursor offset from previous page.
+     * @param fields - Optional field projection.
+     * @returns Page of points and next cursor.
+     */
+    scrollPage(filter?: Record<string, unknown>, limit?: number, offset?: string | number, fields?: string[]): Promise<ScrollPageResult>;
     /**
      * Scroll through all points matching a filter.
      *
@@ -798,6 +822,13 @@ declare class VectorStoreClient implements VectorStore {
      * Creating a fresh client for write operations ensures clean TCP connections.
      */
     private createClient;
+    /**
+     * Count points matching a filter.
+     *
+     * @param filter - Optional Qdrant filter.
+     * @returns The number of matching points.
+     */
+    count(filter?: Record<string, unknown>): Promise<number>;
     /**
      * Ensure the collection exists with correct dimensions and Cosine distance.
      */
@@ -877,6 +908,16 @@ declare class VectorStoreClient implements VectorStore {
      * @returns An array of search results.
      */
     hybridSearch(vector: number[], queryText: string, limit: number, textWeight: number, filter?: Record<string, unknown>): Promise<SearchResult[]>;
+    /**
+     * Scroll one page of points matching a filter.
+     *
+     * @param filter - Optional Qdrant filter.
+     * @param limit - Page size.
+     * @param offset - Cursor offset from previous page.
+     * @param fields - Optional field projection.
+     * @returns Page of points and next cursor.
+     */
+    scrollPage(filter?: Record<string, unknown>, limit?: number, offset?: string | number, fields?: string[]): Promise<ScrollPageResult>;
     /**
      * Scroll through all points matching a filter.
      *

package/dist/index.js

@@ -3059,6 +3059,44 @@ function createRulesUnregisterParamHandler(deps) {
     }, deps.logger, 'RulesUnregister');
 }
 
+/**
+ * @module api/handlers/scan
+ * Fastify route handler for POST /scan. Filter-only point query without vector search.
+ */
+/**
+ * Create handler for POST /scan.
+ *
+ * @param deps - Route dependencies.
+ */
+function createScanHandler(deps) {
+    return wrapHandler(async (request, reply) => {
+        const { filter, limit = 100, cursor, fields, countOnly } = request.body;
+        if (!filter || typeof filter !== 'object') {
+            deps.logger.warn('Scan rejected: missing or invalid filter');
+            void reply
+                .status(400)
+                .send({ error: 'Missing required field: filter (object)' });
+            return;
+        }
+        if (typeof limit !== 'number' || limit < 1 || limit > 1000) {
+            deps.logger.warn({ limit }, 'Scan rejected: limit out of bounds');
+            void reply
+                .status(400)
+                .send({ error: 'limit must be between 1 and 1000' });
+            return;
+        }
+        if (countOnly) {
+            const count = await deps.vectorStore.count(filter);
+            return { count };
+        }
+        const result = await deps.vectorStore.scrollPage(filter, limit, cursor, fields);
+        return {
+            points: result.points,
+            cursor: result.nextCursor ?? null,
+        };
+    }, deps.logger, 'Scan');
+}
+
 /**
  * @module api/handlers/search
  * Fastify route handler for POST /search. Embeds a query and performs vector store similarity search.
@@ -3240,6 +3278,10 @@ function createApiServer(options) {
             textWeight: config.search.hybrid.textWeight,
         }
         : undefined;
+    app.post('/scan', createScanHandler({
+        vectorStore,
+        logger,
+    }));
     app.post('/search', createSearchHandler({
         embeddingProvider,
         vectorStore,
@@ -4789,6 +4831,28 @@ async function getCollectionInfo(client, collectionName) {
     return { pointCount, dimensions, payloadFields };
 }
 
+/**
+ * @module vectorStore/count
+ * Count utility for Qdrant collection points.
+ */
+/**
+ * Count points in a Qdrant collection matching an optional filter.
+ *
+ * Uses exact counting for accurate results.
+ *
+ * @param client - The Qdrant client instance.
+ * @param collectionName - The collection to count.
+ * @param filter - Optional Qdrant filter.
+ * @returns The number of matching points.
+ */
+async function countPoints(client, collectionName, filter) {
+    const result = await client.count(collectionName, {
+        ...(filter ? { filter } : {}),
+        exact: true,
+    });
+    return result.count;
+}
+
 /**
  * @module vectorStore/hybridSearch
  * Hybrid search and text index helpers for Qdrant vector store.
@@ -4888,11 +4952,43 @@ async function hybridSearch(client, collectionName, vector, queryText, limit, te
 
 /**
  * @module vectorStore/scroll
- * Standalone scroll utility for paginating through Qdrant collection points.
+ * Scroll utilities for paginating through Qdrant collection points.
+ */
+/**
+ * Scroll one page of points matching a filter.
+ *
+ * @param client - The Qdrant client instance.
+ * @param collectionName - The collection to scroll.
+ * @param filter - Optional Qdrant filter.
+ * @param limit - Page size.
+ * @param offset - Cursor offset from previous page.
+ * @param fields - Optional payload field projection (array of field names).
+ * @returns Page of points and next cursor.
  */
+async function scrollPage(client, collectionName, filter, limit = 100, offset, fields) {
+    const result = await client.scroll(collectionName, {
+        limit,
+        with_payload: fields ? fields : true,
+        with_vector: false,
+        ...(filter ? { filter } : {}),
+        ...(offset !== undefined ? { offset } : {}),
+    });
+    return {
+        points: result.points.map((p) => ({
+            id: String(p.id),
+            payload: p.payload,
+        })),
+        nextCursor: typeof result.next_page_offset === 'string' ||
+            typeof result.next_page_offset === 'number'
+            ? result.next_page_offset
+            : undefined,
+    };
+}
 /**
  * Scroll through all points in a Qdrant collection matching a filter.
  *
+ * Iterates over pages using {@link scrollPage}.
+ *
  * @param client - The Qdrant client instance.
  * @param collectionName - The collection to scroll.
  * @param filter - Optional Qdrant filter.
@@ -4900,32 +4996,14 @@ async function hybridSearch(client, collectionName, vector, queryText, limit, te
  * @yields Scrolled points.
  */
 async function* scrollCollection(client, collectionName, filter, limit = 100) {
-    let offset = undefined;
-    for (;;) {
-        const result = await client.scroll(collectionName, {
-            limit,
-            with_payload: true,
-            with_vector: false,
-            ...(filter ? { filter } : {}),
-            ...(offset !== undefined ? { offset } : {}),
-        });
-        for (const point of result.points) {
-            yield {
-                id: String(point.id),
-                payload: point.payload,
-            };
+    let cursor;
+    do {
+        const page = await scrollPage(client, collectionName, filter, limit, cursor);
+        for (const point of page.points) {
+            yield point;
         }
-        const nextOffset = result.next_page_offset;
-        if (nextOffset === null || nextOffset === undefined) {
-            break;
-        }
-        if (typeof nextOffset === 'string' || typeof nextOffset === 'number') {
-            offset = nextOffset;
-        }
-        else {
-            break;
-        }
-    }
+        cursor = page.nextCursor;
+    } while (cursor !== undefined);
 }
 
 /**
@@ -4969,6 +5047,15 @@ class VectorStoreClient {
             checkCompatibility: false,
         });
     }
+    /**
+     * Count points matching a filter.
+     *
+     * @param filter - Optional Qdrant filter.
+     * @returns The number of matching points.
+     */
+    async count(filter) {
+        return countPoints(this.client, this.collectionName, filter);
+    }
     /**
      * Ensure the collection exists with correct dimensions and Cosine distance.
      */
@@ -5149,6 +5236,18 @@ class VectorStoreClient {
     async hybridSearch(vector, queryText, limit, textWeight, filter) {
         return hybridSearch(this.client, this.collectionName, vector, queryText, limit, textWeight, filter);
     }
+    /**
+     * Scroll one page of points matching a filter.
+     *
+     * @param filter - Optional Qdrant filter.
+     * @param limit - Page size.
+     * @param offset - Cursor offset from previous page.
+     * @param fields - Optional field projection.
+     * @returns Page of points and next cursor.
+     */
+    async scrollPage(filter, limit = 100, offset, fields) {
+        return scrollPage(this.client, this.collectionName, filter, limit, offset, fields);
+    }
     /**
      * Scroll through all points matching a filter.
      *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@karmaniverous/jeeves-watcher",
-  "version": "0.8.5",
+  "version": "0.9.0-0",
   "author": "Jason Williscroft",
   "description": "Filesystem watcher that keeps a Qdrant vector store in sync with document changes",
   "license": "BSD-3-Clause",