@zenera/rag 1.1.8 → 1.1.10

package/dist/schema/store.js

@@ -8,10 +8,13 @@ import { lancePath } from "./files.js";
 // carrying both the embedding and the full-text index, and the handful of
 // enum columns a query filters on.
 //
-// Those enums are the *only* thing that reaches the SQL predicate. Exclusion
-// lists — which arrive from a model, or from a shell — are applied afterwards
-// in JavaScript. Escaping them into `where()` would work right up until it did
-// not, and there is nothing here that a filter string buys.
+// Those closed vocabularies are the *only* thing that reaches the SQL
+// predicate. `source` is one of them: its values are the document names the
+// index itself wrote, so the store is opened with that list and checks against
+// it. Exclusion lists — which arrive from a model, or from a shell — are
+// applied afterwards in JavaScript. Escaping those into `where()` would work
+// right up until it did not, and there is nothing here that a filter string
+// buys.
 // ---------------------------------------------------------------------------
 const TABLE = 'entities';
 /** Below this an IVF index has nothing to train on, and a flat scan is faster. */
@@ -41,14 +44,17 @@ export async function writeStore(dir, rows, vectors) {
 export class EntityStore {
     #db;
     #table;
-    constructor(db, table) {
+    #sources;
+    constructor(db, table, sources = []) {
         this.#db = db;
         this.#table = table;
+        this.#sources = new Set(sources);
     }
-    static async open(dir) {
+    /** `sources` is the document vocabulary a `sources` filter is checked against. */
+    static async open(dir, sources = []) {
         const db = await connect(lancePath(dir));
         try {
-            return new EntityStore(db, await db.openTable(TABLE));
+            return new EntityStore(db, await db.openTable(TABLE), sources);
         }
         catch {
             db.close();
@@ -60,7 +66,7 @@ export class EntityStore {
      * vector, to the nearest-neighbour side, and LanceDB fuses the two.
      */
     async search(text, vector, filter, limit) {
-        const predicate = where(filter);
+        const predicate = where(filter, this.#sources);
         let query = this.#table.query().nearestToText(text).nearestTo(vector).limit(limit);
         if (predicate) {
             query = query.where(predicate);
@@ -78,11 +84,12 @@ export class EntityStore {
 }
 // ---------------------------------------------------------------------------
 /** Closed vocabularies only. Anything else is a bug, and is treated as one. */
-function where(filter) {
+function where(filter, sources) {
     const clauses = [
         clause('kind', filter.kinds, KINDS),
         clause('direction', filter.directions, DIRECTIONS),
         clause('methodType', filter.methodTypes, METHOD_TYPES),
+        clause('source', filter.sources, sources),
     ].filter(Boolean);
     return clauses.join(' AND ');
 }

package/dist/schema/tools.js

@@ -1,9 +1,9 @@
 import { tool } from '@zenera/neo';
 import { loose, matcher, PatternError } from "../common/match.js";
-import { FORMATS, isFormat, present } from "./present.js";
-import { isEmpty, parseQuery, QueryError } from "./query.js";
 import { toTypeScript } from "./hydrate.js";
 import { fields, grepNodes, listNodes, propertyCount } from "./lookup.js";
+import { FORMATS, isFormat, present } from "./present.js";
+import { isEmpty, parseQuery, QueryError } from "./query.js";
 import { sourceTag } from "./render.js";
 import { stitch } from "./subgraph.js";
 import { chainOf, traceNodes } from "./trace.js";
@@ -47,6 +47,7 @@ export function schemaTools(index, options = {}) {
     // line is prompt spent saying the same word; with several it is the only
     // way to tell two revisions of one API apart.
     const source = options.source ?? index.manifest.sources.length > 1;
+    const names = index.manifest.sources.map((s) => s.name);
     const searchApi = tool({
         name: 'search_api',
         group: GROUP,
@@ -81,6 +82,17 @@ export function schemaTools(index, options = {}) {
                 exclude_methods: list('Operation names to leave out.'),
                 exclude_types: list('Schema names to leave out.'),
                 exclude_properties: list('Field names to leave out.'),
+                // Naming the documents is only a choice when there is more than
+                // one, and an enum is what stops a model inventing a third.
+                ...(names.length > 1
+                    ? {
+                        sources: {
+                            type: 'array',
+                            items: { type: 'string', enum: names },
+                            description: 'Search only these documents. Omit it to search all of them.',
+                        },
+                    }
+                    : {}),
                 limit: {
                     type: 'integer',
                     description: `Results per phrase. Default ${DEFAULT_LIMIT}.`,
@@ -111,6 +123,13 @@ export function schemaTools(index, options = {}) {
             if (isEmpty(query)) {
                 return { error: 'nothing was asked for', hint: 'fill at least one search field' };
             }
+            const absent = (query.sources ?? []).filter((name) => !names.includes(name));
+            if (absent.length > 0) {
+                return {
+                    error: `no document called ${absent.join(', ')}`,
+                    hint: `it has: ${names.join(', ')}`,
+                };
+            }
             const result = await index.search({
                 limit: DEFAULT_LIMIT,
                 max_nodes: DEFAULT_MAX_NODES,

package/package.json

@@ -1,11 +1,12 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.8",
-    "description": "Retrieval over API descriptions: openapi/swagger documents as a searchable graph.",
+    "version": "1.1.10",
+    "description": "Retrieval over a corpus: openapi/swagger documents as a searchable graph, markdown and text as searchable passages.",
     "keywords": [
         "agents",
         "ai-agents",
         "llm",
+        "markdown",
         "openapi",
         "rag",
         "vector-search"
@@ -35,6 +36,14 @@
         "./tools": {
             "types": "./dist/schema/tools.d.ts",
             "default": "./dist/schema/tools.js"
+        },
+        "./docs": {
+            "types": "./dist/docs/index.d.ts",
+            "default": "./dist/docs/index.js"
+        },
+        "./docs/tools": {
+            "types": "./dist/docs/tools.d.ts",
+            "default": "./dist/docs/tools.js"
         }
     },
     "files": [
@@ -52,8 +61,12 @@
     "dependencies": {
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
+        "@zenera/cli": "^1.1.10",
+        "@zenera/neo": "^1.1.10",
         "graphology": "^0.26.0",
-        "@zenera/cli": "^1.1.8",
-        "@zenera/neo": "^1.1.8"
+        "remark-frontmatter": "^5.0.0",
+        "remark-gfm": "^4.0.1",
+        "remark-parse": "^11.0.0",
+        "unified": "^11.0.5"
     }
 }