bluera-knowledge 0.9.21

package/src/db/lance.test.ts

@@ -0,0 +1,390 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { LanceStore } from './lance.js';
+import { createStoreId, createDocumentId } from '../types/brands.js';
+import { rm, mkdtemp } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+describe('LanceStore', () => {
+  let store: LanceStore;
+  let tempDir: string;
+  const storeId = createStoreId('test-store');
+
+  beforeAll(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'lance-test-'));
+    store = new LanceStore(tempDir);
+    await store.initialize(storeId);
+  });
+
+  afterAll(async () => {
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  it('adds and retrieves documents', async () => {
+    const doc = {
+      id: createDocumentId('doc-1'),
+      content: 'Test content',
+      vector: new Array(384).fill(0.1),
+      metadata: {
+        type: 'file' as const,
+        storeId,
+        indexedAt: new Date(),
+        path: '/test/file.txt',
+      },
+    };
+
+    await store.addDocuments(storeId, [doc]);
+
+    const results = await store.search(storeId, new Array(384).fill(0.1), 10);
+    expect(results.length).toBeGreaterThan(0);
+    expect(results[0]?.id).toBe('doc-1');
+  });
+
+  it('deletes documents', async () => {
+    const docId = createDocumentId('doc-to-delete');
+    const doc = {
+      id: docId,
+      content: 'Delete me',
+      vector: new Array(384).fill(0.2),
+      metadata: {
+        type: 'file' as const,
+        storeId,
+        indexedAt: new Date(),
+      },
+    };
+
+    await store.addDocuments(storeId, [doc]);
+    await store.deleteDocuments(storeId, [docId]);
+
+    const results = await store.search(storeId, new Array(384).fill(0.2), 10);
+    const found = results.find((r) => r.id === 'doc-to-delete');
+    expect(found).toBeUndefined();
+  });
+
+  it('performs full-text search', async () => {
+    const doc = {
+      id: createDocumentId('fts-doc'),
+      content: 'The quick brown fox jumps over the lazy dog',
+      vector: new Array(384).fill(0.1),
+      metadata: {
+        type: 'file' as const,
+        storeId,
+        indexedAt: new Date(),
+      },
+    };
+
+    await store.addDocuments(storeId, [doc]);
+    await store.createFtsIndex(storeId);
+
+    const results = await store.fullTextSearch(storeId, 'quick brown', 10);
+    expect(results.length).toBeGreaterThan(0);
+    expect(results[0]?.content).toContain('quick');
+  });
+
+  describe('search with threshold', () => {
+    it('filters results by threshold when provided', async () => {
+      const doc1 = {
+        id: createDocumentId('threshold-doc-1'),
+        content: 'exact match',
+        vector: new Array(384).fill(1.0),
+        metadata: {
+          type: 'file' as const,
+          storeId,
+          indexedAt: new Date(),
+        },
+      };
+
+      const doc2 = {
+        id: createDocumentId('threshold-doc-2'),
+        content: 'different content',
+        vector: new Array(384).fill(0.1),
+        metadata: {
+          type: 'file' as const,
+          storeId,
+          indexedAt: new Date(),
+        },
+      };
+
+      await store.addDocuments(storeId, [doc1, doc2]);
+
+      // Search with high similarity threshold
+      const results = await store.search(storeId, new Array(384).fill(1.0), 10, 0.9);
+
+      // Should filter out low-similarity results
+      expect(results.every(r => r.score >= 0.9)).toBe(true);
+    });
+
+    it('sets cosine distance type when threshold is provided', async () => {
+      const doc = {
+        id: createDocumentId('cosine-doc'),
+        content: 'test',
+        vector: new Array(384).fill(0.5),
+        metadata: {
+          type: 'file' as const,
+          storeId,
+          indexedAt: new Date(),
+        },
+      };
+
+      await store.addDocuments(storeId, [doc]);
+
+      // This should use cosine distance
+      const results = await store.search(storeId, new Array(384).fill(0.5), 10, 0.5);
+      expect(results).toBeDefined();
+    });
+
+    it('does not filter when threshold is undefined', async () => {
+      const doc = {
+        id: createDocumentId('no-threshold-doc'),
+        content: 'content',
+        vector: new Array(384).fill(0.3),
+        metadata: {
+          type: 'file' as const,
+          storeId,
+          indexedAt: new Date(),
+        },
+      };
+
+      await store.addDocuments(storeId, [doc]);
+
+      // Without threshold, all results returned
+      const results = await store.search(storeId, new Array(384).fill(0.1), 10);
+      expect(results).toBeDefined();
+    });
+  });
+
+  describe('fullTextSearch error handling', () => {
+    it('returns empty array when FTS index does not exist', async () => {
+      const newStoreId = createStoreId('no-fts-store');
+      await store.initialize(newStoreId);
+
+      const doc = {
+        id: createDocumentId('no-fts-doc'),
+        content: 'test content',
+        vector: new Array(384).fill(0.1),
+        metadata: {
+          type: 'file' as const,
+          storeId: newStoreId,
+          indexedAt: new Date(),
+        },
+      };
+
+      await store.addDocuments(newStoreId, [doc]);
+
+      // Don't create FTS index - should return empty array
+      const results = await store.fullTextSearch(newStoreId, 'test', 10);
+      expect(results).toEqual([]);
+    });
+  });
+
+  describe('table initialization', () => {
+    it('creates new table when table does not exist', async () => {
+      const newStoreId = createStoreId('new-table-store');
+
+      // Should create new table
+      await store.initialize(newStoreId);
+
+      const doc = {
+        id: createDocumentId('new-table-doc'),
+        content: 'test',
+        vector: new Array(384).fill(0.1),
+        metadata: {
+          type: 'file' as const,
+          storeId: newStoreId,
+          indexedAt: new Date(),
+        },
+      };
+
+      await store.addDocuments(newStoreId, [doc]);
+      const results = await store.search(newStoreId, new Array(384).fill(0.1), 10);
+      expect(results.length).toBeGreaterThan(0);
+    });
+
+    it('opens existing table when table already exists', async () => {
+      const existingStoreId = createStoreId('existing-table-store');
+
+      // Initialize once
+      await store.initialize(existingStoreId);
+
+      const doc1 = {
+        id: createDocumentId('existing-doc-1'),
+        content: 'first',
+        vector: new Array(384).fill(0.1),
+        metadata: {
+          type: 'file' as const,
+          storeId: existingStoreId,
+          indexedAt: new Date(),
+        },
+      };
+
+      await store.addDocuments(existingStoreId, [doc1]);
+
+      // Initialize again - should open existing table
+      await store.initialize(existingStoreId);
+
+      const doc2 = {
+        id: createDocumentId('existing-doc-2'),
+        content: 'second',
+        vector: new Array(384).fill(0.1),
+        metadata: {
+          type: 'file' as const,
+          storeId: existingStoreId,
+          indexedAt: new Date(),
+        },
+      };
+
+      await store.addDocuments(existingStoreId, [doc2]);
+
+      const results = await store.search(existingStoreId, new Array(384).fill(0.1), 10);
+      expect(results.length).toBeGreaterThanOrEqual(2);
+    });
+  });
+
+  describe('lazy table initialization', () => {
+    it('auto-initializes table when getTable is called for uninitialized store', async () => {
+      const lazyStoreId = createStoreId('lazy-init-store');
+
+      // Don't call initialize explicitly
+      const doc = {
+        id: createDocumentId('lazy-doc'),
+        content: 'lazy',
+        vector: new Array(384).fill(0.1),
+        metadata: {
+          type: 'file' as const,
+          storeId: lazyStoreId,
+          indexedAt: new Date(),
+        },
+      };
+
+      // This should trigger auto-initialization
+      await store.addDocuments(lazyStoreId, [doc]);
+
+      const results = await store.search(lazyStoreId, new Array(384).fill(0.1), 10);
+      expect(results.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe('deleteStore', () => {
+    it('deletes table and removes from cache', async () => {
+      const deleteStoreId = createStoreId('delete-store');
+      await store.initialize(deleteStoreId);
+
+      const doc = {
+        id: createDocumentId('delete-store-doc'),
+        content: 'to be deleted',
+        vector: new Array(384).fill(0.1),
+        metadata: {
+          type: 'file' as const,
+          storeId: deleteStoreId,
+          indexedAt: new Date(),
+        },
+      };
+
+      await store.addDocuments(deleteStoreId, [doc]);
+
+      // Delete the store
+      await store.deleteStore(deleteStoreId);
+
+      // Re-initializing should create new empty table
+      await store.initialize(deleteStoreId);
+      const results = await store.search(deleteStoreId, new Array(384).fill(0.1), 10);
+      expect(results.length).toBe(0);
+    });
+  });
+
+  describe('multiple documents operations', () => {
+    it('adds multiple documents at once', async () => {
+      const multiStoreId = createStoreId('multi-doc-store');
+      await store.initialize(multiStoreId);
+
+      const docs = [
+        {
+          id: createDocumentId('multi-1'),
+          content: 'first document',
+          vector: new Array(384).fill(0.1),
+          metadata: {
+            type: 'file' as const,
+            storeId: multiStoreId,
+            indexedAt: new Date(),
+          },
+        },
+        {
+          id: createDocumentId('multi-2'),
+          content: 'second document',
+          vector: new Array(384).fill(0.2),
+          metadata: {
+            type: 'file' as const,
+            storeId: multiStoreId,
+            indexedAt: new Date(),
+          },
+        },
+        {
+          id: createDocumentId('multi-3'),
+          content: 'third document',
+          vector: new Array(384).fill(0.3),
+          metadata: {
+            type: 'file' as const,
+            storeId: multiStoreId,
+            indexedAt: new Date(),
+          },
+        },
+      ];
+
+      await store.addDocuments(multiStoreId, docs);
+
+      const results = await store.search(multiStoreId, new Array(384).fill(0.2), 10);
+      expect(results.length).toBeGreaterThanOrEqual(3);
+    });
+
+    it('deletes multiple documents at once', async () => {
+      const multiDelStoreId = createStoreId('multi-del-store');
+      await store.initialize(multiDelStoreId);
+
+      const docs = [
+        {
+          id: createDocumentId('del-1'),
+          content: 'delete me 1',
+          vector: new Array(384).fill(0.1),
+          metadata: {
+            type: 'file' as const,
+            storeId: multiDelStoreId,
+            indexedAt: new Date(),
+          },
+        },
+        {
+          id: createDocumentId('del-2'),
+          content: 'delete me 2',
+          vector: new Array(384).fill(0.2),
+          metadata: {
+            type: 'file' as const,
+            storeId: multiDelStoreId,
+            indexedAt: new Date(),
+          },
+        },
+        {
+          id: createDocumentId('keep-3'),
+          content: 'keep me',
+          vector: new Array(384).fill(0.3),
+          metadata: {
+            type: 'file' as const,
+            storeId: multiDelStoreId,
+            indexedAt: new Date(),
+          },
+        },
+      ];
+
+      await store.addDocuments(multiDelStoreId, docs);
+      await store.deleteDocuments(multiDelStoreId, [
+        createDocumentId('del-1'),
+        createDocumentId('del-2')
+      ]);
+
+      const results = await store.search(multiDelStoreId, new Array(384).fill(0.2), 10);
+      const deletedIds = results.filter(r => r.id === 'del-1' || r.id === 'del-2');
+      expect(deletedIds.length).toBe(0);
+
+      const kept = results.find(r => r.id === 'keep-3');
+      expect(kept).toBeDefined();
+    });
+  });
+});

package/src/db/lance.ts

@@ -0,0 +1,164 @@
+import * as lancedb from '@lancedb/lancedb';
+import type { Table, Connection } from '@lancedb/lancedb';
+import type { Document, DocumentMetadata } from '../types/document.js';
+import type { StoreId, DocumentId } from '../types/brands.js';
+import { createDocumentId } from '../types/brands.js';
+
+interface LanceDocument {
+  id: string;
+  content: string;
+  vector: number[];
+  metadata: string; // JSON serialized
+  [key: string]: unknown;
+}
+
+interface SearchHit {
+  id: string;
+  content: string;
+  metadata: string;
+  _distance: number;
+}
+
+export class LanceStore {
+  private connection: Connection | null = null;
+  private readonly tables: Map<string, Table> = new Map();
+  private readonly dataDir: string;
+
+  constructor(dataDir: string) {
+    this.dataDir = dataDir;
+  }
+
+  async initialize(storeId: StoreId): Promise<void> {
+    if (this.connection === null) {
+      this.connection = await lancedb.connect(this.dataDir);
+    }
+
+    const tableName = this.getTableName(storeId);
+    const tableNames = await this.connection.tableNames();
+
+    if (!tableNames.includes(tableName)) {
+      // Create table with initial schema
+      const table = await this.connection.createTable(tableName, [
+        {
+          id: '__init__',
+          content: '',
+          vector: new Array(384).fill(0),
+          metadata: '{}',
+        },
+      ]);
+      // Delete the init row
+      await table.delete('id = "__init__"');
+      this.tables.set(tableName, table);
+    } else {
+      const table = await this.connection.openTable(tableName);
+      this.tables.set(tableName, table);
+    }
+  }
+
+  async addDocuments(storeId: StoreId, documents: Document[]): Promise<void> {
+    const table = await this.getTable(storeId);
+    const lanceDocuments: LanceDocument[] = documents.map((doc) => ({
+      id: doc.id,
+      content: doc.content,
+      vector: [...doc.vector],
+      metadata: JSON.stringify(doc.metadata),
+    }));
+    await table.add(lanceDocuments);
+  }
+
+  async deleteDocuments(storeId: StoreId, documentIds: DocumentId[]): Promise<void> {
+    const table = await this.getTable(storeId);
+    const idList = documentIds.map((id) => `"${id}"`).join(', ');
+    await table.delete(`id IN (${idList})`);
+  }
+
+  async search(
+    storeId: StoreId,
+    vector: number[],
+    limit: number,
+    threshold?: number
+  ): Promise<Array<{ id: DocumentId; content: string; score: number; metadata: DocumentMetadata }>> {
+    const table = await this.getTable(storeId);
+    let query = table.vectorSearch(vector).limit(limit);
+
+    if (threshold !== undefined) {
+      query = query.distanceType('cosine');
+    }
+
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    const results = (await query.toArray()) as SearchHit[];
+
+    return results
+      .filter((r) => {
+        if (threshold === undefined) return true;
+        const score = 1 - r._distance;
+        return score >= threshold;
+      })
+      .map((r) => ({
+        id: createDocumentId(r.id),
+        content: r.content,
+        score: 1 - r._distance,
+        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+        metadata: JSON.parse(r.metadata) as DocumentMetadata,
+      }));
+  }
+
+  async createFtsIndex(storeId: StoreId): Promise<void> {
+    const table = await this.getTable(storeId);
+    await table.createIndex('content', {
+      config: lancedb.Index.fts(),
+    });
+  }
+
+  async fullTextSearch(
+    storeId: StoreId,
+    query: string,
+    limit: number
+  ): Promise<Array<{ id: DocumentId; content: string; score: number; metadata: DocumentMetadata }>> {
+    const table = await this.getTable(storeId);
+
+    try {
+      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      const results = await table
+        .search(query, 'fts')
+        .limit(limit)
+        .toArray() as Array<{ id: string; content: string; metadata: string; score: number }>;
+
+      return results.map((r) => ({
+        id: createDocumentId(r.id),
+        content: r.content,
+        score: r.score,
+        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+        metadata: JSON.parse(r.metadata) as DocumentMetadata,
+      }));
+    } catch {
+      // FTS index may not exist, return empty
+      return [];
+    }
+  }
+
+  async deleteStore(storeId: StoreId): Promise<void> {
+    const tableName = this.getTableName(storeId);
+    if (this.connection !== null) {
+      await this.connection.dropTable(tableName);
+      this.tables.delete(tableName);
+    }
+  }
+
+  private getTableName(storeId: StoreId): string {
+    return `documents_${storeId}`;
+  }
+
+  private async getTable(storeId: StoreId): Promise<Table> {
+    const tableName = this.getTableName(storeId);
+    let table = this.tables.get(tableName);
+    if (table === undefined) {
+      await this.initialize(storeId);
+      table = this.tables.get(tableName);
+    }
+    if (table === undefined) {
+      throw new Error(`Table not found for store: ${storeId}`);
+    }
+    return table;
+  }
+}

package/src/defaults/repos.ts

@@ -0,0 +1,67 @@
+/**
+ * Default repositories for quick setup.
+ * These are Anthropic/Claude-related repositories that provide
+ * useful knowledge for Claude Code development.
+ */
+
+export interface DefaultRepo {
+  /** Git URL for cloning */
+  url: string;
+  /** Friendly name for the store */
+  name: string;
+  /** Description of what this repo contains */
+  description: string;
+  /** Tags for categorization */
+  tags: string[];
+}
+
+export const DEFAULT_REPOS: readonly DefaultRepo[] = [
+  {
+    url: 'git@github.com:ericbuess/claude-code-docs.git',
+    name: 'claude-code-docs',
+    description: 'Claude Code documentation',
+    tags: ['claude', 'docs', 'claude-code'],
+  },
+  {
+    url: 'git@github.com:anthropics/claude-code.git',
+    name: 'claude-code',
+    description: 'Claude Code CLI tool source',
+    tags: ['claude', 'cli', 'anthropic'],
+  },
+  {
+    url: 'git@github.com:anthropics/claude-agent-sdk-python.git',
+    name: 'claude-agent-sdk-python',
+    description: 'Claude Agent SDK for Python',
+    tags: ['claude', 'sdk', 'python', 'agents'],
+  },
+  {
+    url: 'git@github.com:anthropics/skills.git',
+    name: 'claude-skills',
+    description: 'Claude skills and capabilities',
+    tags: ['claude', 'skills'],
+  },
+  {
+    url: 'git@github.com:anthropics/claude-quickstarts.git',
+    name: 'claude-quickstarts',
+    description: 'Claude quickstart examples and tutorials',
+    tags: ['claude', 'examples', 'tutorials'],
+  },
+  {
+    url: 'git@github.com:anthropics/claude-plugins-official.git',
+    name: 'claude-plugins',
+    description: 'Official Claude plugins',
+    tags: ['claude', 'plugins'],
+  },
+  {
+    url: 'git@github.com:anthropics/claude-agent-sdk-typescript.git',
+    name: 'claude-agent-sdk-typescript',
+    description: 'Claude Agent SDK for TypeScript',
+    tags: ['claude', 'sdk', 'typescript', 'agents'],
+  },
+  {
+    url: 'git@github.com:anthropics/claude-agent-sdk-demos.git',
+    name: 'claude-agent-sdk-demos',
+    description: 'Claude Agent SDK demo applications',
+    tags: ['claude', 'sdk', 'demos', 'examples'],
+  },
+];