@kuralle-agents/rag 0.5.0 → 0.6.0

package/README.md

@@ -23,6 +23,7 @@ Everything between a raw document and a grounded agent response: chunk documents
 - **Retrievers** — `VectorRetriever`, `HybridRetriever`, `FusionRetriever`, `MultiHopRetriever`, `createLLMRetriever`
 - **Rerankers** — `LLMReranker`, `CohereReranker`
 - **Search** — `BM25Index` (keyword search for hybrid retrieval)
+- **KnowledgeFs** — read-only `FileSystem` over a vector store (`@kuralle-agents/rag/fs`); see [guides/KNOWLEDGEFS.md](./guides/KNOWLEDGEFS.md)
 - **Pipeline** — `RagPipeline`, `RetrievalQualityChecker`
 - **Cache** — `RetrievalCache`, `TurnCache`, `PredictivePreFetcher`
 

package/dist/fs/KnowledgeFs.d.ts

@@ -0,0 +1,58 @@
+import type { CpOptions, FileSystem, FileSystemDirent, FsStat, MkdirOptions, RmOptions } from '@kuralle-agents/core';
+import type { BM25Index } from '../search/BM25Index.js';
+import type { VectorStoreCore } from '../types.js';
+import type { KnowledgeAccessFilter } from './access.js';
+export interface KnowledgeFsOptions {
+    store: VectorStoreCore;
+    indexName: string;
+    bm25?: BM25Index;
+    accessFilter?: KnowledgeAccessFilter;
+    manifestKey?: string;
+}
+export interface KnowledgeSearchHit {
+    slug: string;
+    chunkIndex: number;
+    text: string;
+}
+export declare class KnowledgeFs implements FileSystem {
+    private readonly store;
+    private readonly indexName;
+    private readonly bm25?;
+    private readonly accessFilter?;
+    private readonly manifestKey;
+    private tree;
+    private chunksBySlug;
+    private readonly pageCache;
+    private initialized;
+    constructor(opts: KnowledgeFsOptions);
+    static open(opts: KnowledgeFsOptions): Promise<KnowledgeFs>;
+    init(): Promise<void>;
+    private assertReady;
+    private resolveCanonical;
+    private isFile;
+    private isDirectory;
+    private assertAccessible;
+    search(pattern: string, opts?: {
+        limit?: number;
+        path?: string;
+    }): Promise<KnowledgeSearchHit[]>;
+    readdir(path: string): Promise<string[]>;
+    readdirWithFileTypes(path: string): Promise<FileSystemDirent[]>;
+    exists(path: string): Promise<boolean>;
+    stat(path: string): Promise<FsStat>;
+    lstat(path: string): Promise<FsStat>;
+    readFile(path: string): Promise<string>;
+    readFileBytes(path: string): Promise<Uint8Array>;
+    writeFile(_path: string, _content: string): Promise<void>;
+    writeFileBytes(path: string, _content: Uint8Array): Promise<void>;
+    appendFile(path: string, _content: string | Uint8Array): Promise<void>;
+    mkdir(path: string, _options?: MkdirOptions): Promise<void>;
+    rm(path: string, _options?: RmOptions): Promise<void>;
+    cp(src: string, dest: string, _options?: CpOptions): Promise<void>;
+    mv(src: string, dest: string): Promise<void>;
+    symlink(_target: string, linkPath: string): Promise<void>;
+    readlink(path: string): Promise<string>;
+    resolvePath(base: string, path: string): string;
+    realpath(path: string): Promise<string>;
+    glob(pattern: string): Promise<string[]>;
+}

package/dist/fs/KnowledgeFs.js

@@ -0,0 +1,268 @@
+import { slugAllowed } from './access.js';
+import { PATH_TREE_MANIFEST_ID, allChunkRecords, buildPathTree, chunkRecordsFromEntries, createGlobMatcher, groupChunksBySlug, joinKnowledgePath, normalizeKnowledgePath, parsePathTreeManifest, pathUnderRoot, prunePathTree, resolveKnowledgePath, } from './path-tree.js';
+function hasListEntries(store) {
+    return typeof store.listEntries === 'function';
+}
+function eroFs(op, path) {
+    return Object.assign(new Error(`EROFS: read-only knowledge filesystem, ${op} '${path}'`), { code: 'EROFS' });
+}
+function enoent(op, path) {
+    return Object.assign(new Error(`ENOENT: no such file or directory, ${op} '${path}'`), { code: 'ENOENT' });
+}
+function enotdir(op, path) {
+    return Object.assign(new Error(`ENOTDIR: not a directory, ${op} '${path}'`), {
+        code: 'ENOTDIR',
+    });
+}
+function eisdir(op, path) {
+    return Object.assign(new Error(`EISDIR: illegal operation on a directory, ${op} '${path}'`), { code: 'EISDIR' });
+}
+const utf8 = new TextEncoder();
+export class KnowledgeFs {
+    store;
+    indexName;
+    bm25;
+    accessFilter;
+    manifestKey;
+    tree;
+    chunksBySlug;
+    pageCache = new Map();
+    initialized = false;
+    constructor(opts) {
+        this.store = opts.store;
+        this.indexName = opts.indexName;
+        this.bm25 = opts.bm25;
+        this.accessFilter = opts.accessFilter;
+        this.manifestKey = opts.manifestKey ?? PATH_TREE_MANIFEST_ID;
+    }
+    static async open(opts) {
+        const fs = new KnowledgeFs(opts);
+        await fs.init();
+        return fs;
+    }
+    async init() {
+        if (this.initialized)
+            return;
+        const filter = this.accessFilter?.vectorFilter;
+        const entries = await scanStoreEntries(this.store, this.indexName, filter);
+        const manifestEntry = entries.find((e) => e.id === this.manifestKey);
+        let slugs;
+        let chunkRecords;
+        if (manifestEntry?.document) {
+            slugs = parsePathTreeManifest(manifestEntry.document);
+            chunkRecords = chunkRecordsFromEntries(entries.filter((e) => e.id !== this.manifestKey));
+        }
+        else {
+            chunkRecords = chunkRecordsFromEntries(entries);
+            if (chunkRecords.length === 0 && entries.length > 0) {
+                throw new Error('KnowledgeFs: store entries lack page/chunk_index metadata required for reassembly');
+            }
+            slugs = [...new Set(chunkRecords.map((c) => c.slug))];
+        }
+        const allow = (slug) => slugAllowed(slug, this.accessFilter);
+        slugs = slugs.filter(allow);
+        chunkRecords = chunkRecords.filter((c) => allow(c.slug));
+        this.tree = prunePathTree(buildPathTree(slugs), allow);
+        this.chunksBySlug = groupChunksBySlug(chunkRecords);
+        if (this.bm25) {
+            this.bm25.add(chunkRecords.map((r) => ({
+                id: `${r.slug}#${r.chunkIndex}`,
+                text: r.text,
+            })));
+        }
+        this.initialized = true;
+    }
+    assertReady() {
+        if (!this.initialized) {
+            throw new Error('KnowledgeFs: call KnowledgeFs.open() before using the filesystem');
+        }
+    }
+    resolveCanonical(path) {
+        return normalizeKnowledgePath(path);
+    }
+    isFile(path) {
+        return this.tree.files.has(path);
+    }
+    isDirectory(path) {
+        if (path === '/')
+            return true;
+        return this.tree.dirChildren.has(path);
+    }
+    assertAccessible(path, op) {
+        const canonical = this.resolveCanonical(path);
+        if (canonical.includes('\0'))
+            throw enoent(op, path);
+        if (canonical !== '/' && !this.isFile(canonical) && !this.isDirectory(canonical)) {
+            throw enoent(op, path);
+        }
+        return canonical;
+    }
+    async search(pattern, opts) {
+        this.assertReady();
+        const limit = opts?.limit ?? 50;
+        const root = opts?.path ? normalizeKnowledgePath(opts.path) : '/';
+        let records = allChunkRecords(this.chunksBySlug).filter((r) => pathUnderRoot(r.slug, root));
+        if (this.bm25) {
+            const ranked = this.bm25.search(pattern, limit);
+            const candidateIds = new Set(ranked.map((hit) => hit.id));
+            return records
+                .filter((r) => candidateIds.has(`${r.slug}#${r.chunkIndex}`))
+                .slice(0, limit)
+                .map((r) => ({
+                slug: r.slug,
+                chunkIndex: r.chunkIndex,
+                text: r.text,
+            }));
+        }
+        let re;
+        try {
+            re = new RegExp(pattern, 'i');
+        }
+        catch {
+            throw new Error(`EINVAL: invalid search pattern '${pattern}'`);
+        }
+        const hits = [];
+        for (const record of records) {
+            if (re.test(record.text)) {
+                hits.push({
+                    slug: record.slug,
+                    chunkIndex: record.chunkIndex,
+                    text: record.text,
+                });
+                if (hits.length >= limit)
+                    break;
+            }
+        }
+        return hits;
+    }
+    async readdir(path) {
+        return (await this.readdirWithFileTypes(path)).map((d) => d.name);
+    }
+    async readdirWithFileTypes(path) {
+        this.assertReady();
+        const canonical = this.assertAccessible(path, 'scandir');
+        if (!this.isDirectory(canonical))
+            throw enotdir('scandir', path);
+        const names = this.tree.dirChildren.get(canonical) ?? [];
+        const out = [];
+        for (const name of names) {
+            const child = joinKnowledgePath(canonical, name);
+            const type = this.isDirectory(child) ? 'directory' : 'file';
+            out.push({ name, type });
+        }
+        return out;
+    }
+    async exists(path) {
+        this.assertReady();
+        if (path.includes('\0'))
+            return false;
+        const canonical = this.resolveCanonical(path);
+        if (canonical === '/')
+            return true;
+        return this.isFile(canonical) || this.isDirectory(canonical);
+    }
+    async stat(path) {
+        this.assertReady();
+        const canonical = this.assertAccessible(path, 'stat');
+        if (canonical === '/') {
+            return { type: 'directory', size: 0, mtime: new Date(0) };
+        }
+        if (this.isDirectory(canonical)) {
+            return { type: 'directory', size: 0, mtime: new Date(0) };
+        }
+        const cached = this.pageCache.get(canonical);
+        if (cached !== undefined) {
+            return { type: 'file', size: utf8.encode(cached).length, mtime: new Date(0) };
+        }
+        const chunks = this.chunksBySlug.get(canonical);
+        const size = chunks?.reduce((n, c) => n + utf8.encode(c.text).length, 0) ?? 0;
+        return { type: 'file', size, mtime: new Date(0) };
+    }
+    async lstat(path) {
+        return this.stat(path);
+    }
+    async readFile(path) {
+        this.assertReady();
+        const canonical = this.assertAccessible(path, 'open');
+        if (this.isDirectory(canonical))
+            throw eisdir('read', path);
+        if (!this.isFile(canonical))
+            throw enoent('open', path);
+        const cached = this.pageCache.get(canonical);
+        if (cached !== undefined)
+            return cached;
+        const chunks = this.chunksBySlug.get(canonical);
+        if (!chunks || chunks.length === 0) {
+            throw enoent('open', path);
+        }
+        const page = chunks.map((c) => c.text).join('');
+        this.pageCache.set(canonical, page);
+        return page;
+    }
+    async readFileBytes(path) {
+        return utf8.encode(await this.readFile(path));
+    }
+    async writeFile(_path, _content) {
+        throw eroFs('write', _path);
+    }
+    async writeFileBytes(path, _content) {
+        throw eroFs('write', path);
+    }
+    async appendFile(path, _content) {
+        throw eroFs('append', path);
+    }
+    async mkdir(path, _options) {
+        throw eroFs('mkdir', path);
+    }
+    async rm(path, _options) {
+        throw eroFs('rm', path);
+    }
+    async cp(src, dest, _options) {
+        throw eroFs('cp', `${src} -> ${dest}`);
+    }
+    async mv(src, dest) {
+        throw eroFs('rename', `${src} -> ${dest}`);
+    }
+    async symlink(_target, linkPath) {
+        throw eroFs('symlink', linkPath);
+    }
+    async readlink(path) {
+        throw enoent('readlink', path);
+    }
+    resolvePath(base, path) {
+        return resolveKnowledgePath(base, path);
+    }
+    async realpath(path) {
+        this.assertReady();
+        const canonical = this.assertAccessible(path, 'realpath');
+        return canonical;
+    }
+    async glob(pattern) {
+        this.assertReady();
+        const re = createGlobMatcher(pattern);
+        const hits = [];
+        for (const file of this.tree.files) {
+            if (re.test(file))
+                hits.push(file);
+        }
+        return hits.sort();
+    }
+}
+async function scanStoreEntries(store, indexName, filter) {
+    if (hasListEntries(store)) {
+        return store.listEntries(indexName, { filter });
+    }
+    const stats = await store.describeIndex(indexName);
+    const zeros = new Array(stats.dimension).fill(0);
+    const results = await store.query(indexName, {
+        queryVector: zeros,
+        topK: Math.max(stats.count, 1),
+        filter,
+        includeDocuments: true,
+    });
+    return results.map((r) => ({
+        id: r.id,
+        metadata: r.metadata,
+        document: r.document,
+    }));
+}

package/dist/fs/access.d.ts

@@ -0,0 +1,7 @@
+import type { VectorFilter } from '../types.js';
+export interface KnowledgeAccessFilter {
+    vectorFilter?: VectorFilter;
+    allowSlug?(slug: string): boolean;
+}
+export declare function combineFilters(a?: VectorFilter, b?: VectorFilter): VectorFilter | undefined;
+export declare function slugAllowed(slug: string, filter?: KnowledgeAccessFilter): boolean;

package/dist/fs/access.js

@@ -0,0 +1,12 @@
+export function combineFilters(a, b) {
+    if (!a)
+        return b;
+    if (!b)
+        return a;
+    return { $and: [a, b] };
+}
+export function slugAllowed(slug, filter) {
+    if (filter?.allowSlug && !filter.allowSlug(slug))
+        return false;
+    return true;
+}

package/dist/fs/index.d.ts

@@ -0,0 +1,6 @@
+export type { KnowledgeAccessFilter } from './access.js';
+export { combineFilters, slugAllowed } from './access.js';
+export { KnowledgeFs } from './KnowledgeFs.js';
+export type { KnowledgeFsOptions, KnowledgeSearchHit } from './KnowledgeFs.js';
+export { PAGE_META_KEY, CHUNK_INDEX_META_KEY, PATH_TREE_MANIFEST_ID, } from './path-tree.js';
+export type { ChunkRecord, PathTreeData, PathTreeManifest } from './path-tree.js';

package/dist/fs/index.js

@@ -0,0 +1,3 @@
+export { combineFilters, slugAllowed } from './access.js';
+export { KnowledgeFs } from './KnowledgeFs.js';
+export { PAGE_META_KEY, CHUNK_INDEX_META_KEY, PATH_TREE_MANIFEST_ID, } from './path-tree.js';

package/dist/fs/path-tree.d.ts

@@ -0,0 +1,32 @@
+export declare const PAGE_META_KEY = "page";
+export declare const CHUNK_INDEX_META_KEY = "chunk_index";
+export declare const PATH_TREE_MANIFEST_ID = "__path_tree__";
+export type PathTreeManifest = Record<string, {
+    isPublic?: boolean;
+    groups?: string[];
+}>;
+export interface ChunkRecord {
+    id: string;
+    slug: string;
+    chunkIndex: number;
+    text: string;
+}
+export interface PathTreeData {
+    files: Set<string>;
+    dirChildren: Map<string, string[]>;
+}
+export declare function normalizeKnowledgePath(path: string): string;
+export declare function joinKnowledgePath(parent: string, child: string): string;
+export declare function resolveKnowledgePath(base: string, path: string): string;
+export declare function pathUnderRoot(path: string, root: string): boolean;
+export declare function buildPathTree(slugs: Iterable<string>): PathTreeData;
+export declare function prunePathTree(tree: PathTreeData, allow: (slug: string) => boolean): PathTreeData;
+export declare function createGlobMatcher(pattern: string): RegExp;
+export declare function chunkRecordsFromEntries(entries: Array<{
+    id: string;
+    metadata?: Record<string, unknown>;
+    document?: string;
+}>): ChunkRecord[];
+export declare function groupChunksBySlug(records: ChunkRecord[]): Map<string, ChunkRecord[]>;
+export declare function parsePathTreeManifest(raw: string): string[];
+export declare function allChunkRecords(map: Map<string, ChunkRecord[]>): ChunkRecord[];

package/dist/fs/path-tree.js

@@ -0,0 +1,157 @@
+export const PAGE_META_KEY = 'page';
+export const CHUNK_INDEX_META_KEY = 'chunk_index';
+export const PATH_TREE_MANIFEST_ID = '__path_tree__';
+export function normalizeKnowledgePath(path) {
+    if (!path || path === '/')
+        return '/';
+    let normalized = path.endsWith('/') && path !== '/' ? path.slice(0, -1) : path;
+    if (!normalized.startsWith('/')) {
+        normalized = `/${normalized}`;
+    }
+    const parts = normalized.split('/').filter((p) => p && p !== '.');
+    const resolved = [];
+    for (const part of parts) {
+        if (part === '..') {
+            resolved.pop();
+        }
+        else {
+            resolved.push(part);
+        }
+    }
+    return resolved.length === 0 ? '/' : `/${resolved.join('/')}`;
+}
+export function joinKnowledgePath(parent, child) {
+    const base = normalizeKnowledgePath(parent);
+    if (child.startsWith('/'))
+        return normalizeKnowledgePath(child);
+    return normalizeKnowledgePath(base === '/' ? `/${child}` : `${base}/${child}`);
+}
+export function resolveKnowledgePath(base, path) {
+    if (path.startsWith('/'))
+        return normalizeKnowledgePath(path);
+    return joinKnowledgePath(base, path);
+}
+export function pathUnderRoot(path, root) {
+    const p = normalizeKnowledgePath(path);
+    const r = normalizeKnowledgePath(root);
+    if (r === '/')
+        return true;
+    return p === r || p.startsWith(`${r}/`);
+}
+export function buildPathTree(slugs) {
+    const files = new Set();
+    const dirChildren = new Map();
+    const addChild = (dir, name) => {
+        const normalizedDir = normalizeKnowledgePath(dir);
+        if (!dirChildren.has(normalizedDir)) {
+            dirChildren.set(normalizedDir, new Set());
+        }
+        dirChildren.get(normalizedDir).add(name);
+    };
+    for (const raw of slugs) {
+        const slug = normalizeKnowledgePath(raw);
+        if (slug === '/')
+            continue;
+        files.add(slug);
+        const parts = slug.slice(1).split('/');
+        let current = '/';
+        for (let i = 0; i < parts.length; i++) {
+            addChild(current, parts[i]);
+            if (i < parts.length - 1) {
+                current = joinKnowledgePath(current, parts[i]);
+            }
+        }
+    }
+    const sorted = new Map();
+    for (const [dir, children] of dirChildren) {
+        sorted.set(dir, [...children].sort());
+    }
+    return { files, dirChildren: sorted };
+}
+export function prunePathTree(tree, allow) {
+    const keptFiles = new Set();
+    for (const file of tree.files) {
+        if (allow(file))
+            keptFiles.add(file);
+    }
+    return buildPathTree(keptFiles);
+}
+export function createGlobMatcher(pattern) {
+    let i = 0;
+    let re = '^';
+    while (i < pattern.length) {
+        const ch = pattern[i];
+        if (ch === '*') {
+            if (pattern[i + 1] === '*') {
+                i += 2;
+                if (pattern[i] === '/') {
+                    re += '(?:.+/)?';
+                    i++;
+                }
+                else {
+                    re += '.*';
+                }
+            }
+            else {
+                re += '[^/]*';
+                i++;
+            }
+        }
+        else if (ch === '?') {
+            re += '[^/]';
+            i++;
+        }
+        else if ('.+^${}()|[]\\'.includes(ch)) {
+            re += `\\${ch}`;
+            i++;
+        }
+        else {
+            re += ch;
+            i++;
+        }
+    }
+    re += '$';
+    return new RegExp(re);
+}
+export function chunkRecordsFromEntries(entries) {
+    const records = [];
+    for (const entry of entries) {
+        const meta = entry.metadata ?? {};
+        const page = meta[PAGE_META_KEY];
+        const chunkIndex = meta[CHUNK_INDEX_META_KEY];
+        if (typeof page !== 'string' || page.length === 0)
+            continue;
+        if (typeof chunkIndex !== 'number' || !Number.isFinite(chunkIndex))
+            continue;
+        const slug = normalizeKnowledgePath(page);
+        records.push({
+            id: entry.id,
+            slug,
+            chunkIndex,
+            text: entry.document ?? '',
+        });
+    }
+    return records;
+}
+export function groupChunksBySlug(records) {
+    const map = new Map();
+    for (const record of records) {
+        const list = map.get(record.slug) ?? [];
+        list.push(record);
+        map.set(record.slug, list);
+    }
+    for (const [, list] of map) {
+        list.sort((a, b) => a.chunkIndex - b.chunkIndex);
+    }
+    return map;
+}
+export function parsePathTreeManifest(raw) {
+    const parsed = JSON.parse(raw);
+    return Object.keys(parsed).map((k) => normalizeKnowledgePath(k.startsWith('/') ? k : `/${k}`));
+}
+export function allChunkRecords(map) {
+    const out = [];
+    for (const list of map.values())
+        out.push(...list);
+    return out;
+}

package/dist/vectorStores/InMemoryVectorStore.d.ts

@@ -8,6 +8,13 @@ export declare class InMemoryVectorStore implements VectorStoreCore, VectorStore
     private indexes;
     createIndex(params: CreateIndexParams): Promise<void>;
     upsert(indexName: string, entries: VectorEntry[]): Promise<void>;
+    listEntries(indexName: string, params?: {
+        filter?: VectorFilter;
+    }): Promise<Array<{
+        id: string;
+        metadata?: Record<string, unknown>;
+        document?: string;
+    }>>;
     query(indexName: string, params: VectorQueryParams): Promise<VectorQueryResult[]>;
     listIndexes(): Promise<string[]>;
     deleteIndex(indexName: string): Promise<void>;

package/dist/vectorStores/InMemoryVectorStore.js

@@ -25,6 +25,17 @@ export class InMemoryVectorStore {
             index.entries.set(entry.id, { ...entry });
         }
     }
+    async listEntries(indexName, params) {
+        const index = this.getIndex(indexName);
+        const out = [];
+        for (const [id, entry] of index.entries) {
+            if (params?.filter && !matchFilter(entry.metadata ?? {}, params.filter)) {
+                continue;
+            }
+            out.push({ id, metadata: entry.metadata, document: entry.document });
+        }
+        return out;
+    }
     async query(indexName, params) {
         const index = this.getIndex(indexName);
         const topK = params.topK ?? 10;

package/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/kuralle/kuralle-agents.git",
     "directory": "packages/kuralle-rag"
   },
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "RAG primitives for Kuralle",
   "type": "module",
   "main": "dist/index.js",
@@ -16,6 +16,10 @@
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
     },
+    "./fs": {
+      "types": "./dist/fs/index.d.ts",
+      "default": "./dist/fs/index.js"
+    },
     "./filters": {
       "types": "./dist/filters/index.d.ts",
       "default": "./dist/filters/index.js"
@@ -25,6 +29,9 @@
       "default": "./dist/vectorStores/testing.js"
     }
   },
+  "dependencies": {
+    "@kuralle-agents/core": "0.6.0"
+  },
   "peerDependencies": {
     "ai": "^6.0.0",
     "zod": "^3.0.0"
@@ -34,7 +41,8 @@
     "ai": "^6.0.0",
     "bun-types": "^1.3.0",
     "typescript": "^5.3.0",
-    "zod": "^3.23.0"
+    "zod": "^3.23.0",
+    "@kuralle-agents/fs": "0.6.0"
   },
   "publishConfig": {
     "access": "public"
@@ -47,7 +55,7 @@
     "prebuild": "rm -rf dist",
     "build": "tsc -p tsconfig.json",
     "clean": "rm -rf dist",
-    "test": "bun test test/chunkers-tokens.test.ts test/vectorStores test/filters test/e2e-loaders.test.ts",
+    "test": "bun test test/chunkers-tokens.test.ts test/vectorStores test/filters test/e2e-loaders.test.ts test/knowledgefs.test.ts test/knowledgefs-grep.test.ts test/knowledgefs-rbac.test.ts test/knowledgefs-agent.test.ts",
     "test:live": "bun test test/e2e-tasktype.test.ts test/e2e-real.test.ts"
   }
 }