bluera-knowledge 0.10.0 → 0.11.0

package/src/cli/commands/sync.ts

@@ -0,0 +1,264 @@
+import { Command } from 'commander';
+import { createServices, destroyServices } from '../../services/index.js';
+import { StoreDefinitionService } from '../../services/store-definition.service.js';
+import {
+  isFileStoreDefinition,
+  isRepoStoreDefinition,
+  isWebStoreDefinition,
+} from '../../types/store-definition.js';
+import type { StoreService } from '../../services/store.service.js';
+import type { StoreDefinition } from '../../types/store-definition.js';
+import type { GlobalOptions } from '../program.js';
+
+interface SyncResult {
+  created: string[];
+  skipped: string[];
+  failed: Array<{ name: string; error: string }>;
+  orphans: string[];
+  pruned: string[];
+  dryRun: boolean;
+  wouldCreate: string[];
+  wouldPrune: string[];
+}
+
+/**
+ * Create a store from a definition
+ */
+async function createStoreFromDefinition(
+  def: StoreDefinition,
+  defService: StoreDefinitionService,
+  storeService: StoreService
+): Promise<{ success: true } | { success: false; error: string }> {
+  try {
+    if (isFileStoreDefinition(def)) {
+      const resolvedPath = defService.resolvePath(def.path);
+      const createResult = await storeService.create(
+        {
+          name: def.name,
+          type: 'file',
+          path: resolvedPath,
+          description: def.description,
+          tags: def.tags,
+        },
+        { skipDefinitionSync: true }
+      );
+      if (!createResult.success) {
+        return { success: false, error: createResult.error.message };
+      }
+      return { success: true };
+    }
+
+    if (isRepoStoreDefinition(def)) {
+      const createResult = await storeService.create(
+        {
+          name: def.name,
+          type: 'repo',
+          url: def.url,
+          branch: def.branch,
+          depth: def.depth,
+          description: def.description,
+          tags: def.tags,
+        },
+        { skipDefinitionSync: true }
+      );
+      if (!createResult.success) {
+        return { success: false, error: createResult.error.message };
+      }
+      return { success: true };
+    }
+
+    if (isWebStoreDefinition(def)) {
+      const createResult = await storeService.create(
+        {
+          name: def.name,
+          type: 'web',
+          url: def.url,
+          depth: def.depth,
+          description: def.description,
+          tags: def.tags,
+        },
+        { skipDefinitionSync: true }
+      );
+      if (!createResult.success) {
+        return { success: false, error: createResult.error.message };
+      }
+      return { success: true };
+    }
+
+    return { success: false, error: 'Unknown store definition type' };
+  } catch (error) {
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+export function createSyncCommand(getOptions: () => GlobalOptions): Command {
+  const sync = new Command('sync').description(
+    'Sync stores from definitions config (bootstrap on fresh clone)'
+  );
+
+  sync
+    .option('--dry-run', 'Show what would happen without making changes')
+    .option('--prune', 'Remove stores not in definitions')
+    .option('--reindex', 'Re-index existing stores after sync')
+    .action(async (options: { dryRun?: boolean; prune?: boolean; reindex?: boolean }) => {
+      const globalOpts = getOptions();
+      const projectRoot = globalOpts.projectRoot ?? process.cwd();
+
+      const defService = new StoreDefinitionService(projectRoot);
+      const services = await createServices(globalOpts.config, globalOpts.dataDir, projectRoot);
+
+      try {
+        const config = await defService.load();
+        const existingStores = await services.store.list();
+        const existingNames = new Set(existingStores.map((s) => s.name));
+        const definedNames = new Set(config.stores.map((d) => d.name));
+
+        const result: SyncResult = {
+          created: [],
+          skipped: [],
+          failed: [],
+          orphans: [],
+          pruned: [],
+          dryRun: options.dryRun === true,
+          wouldCreate: [],
+          wouldPrune: [],
+        };
+
+        // Process each definition
+        for (const def of config.stores) {
+          if (existingNames.has(def.name)) {
+            result.skipped.push(def.name);
+            continue;
+          }
+
+          if (options.dryRun === true) {
+            result.wouldCreate.push(def.name);
+            continue;
+          }
+
+          const createResult = await createStoreFromDefinition(def, defService, services.store);
+          if (createResult.success) {
+            result.created.push(def.name);
+          } else {
+            result.failed.push({ name: def.name, error: createResult.error });
+          }
+        }
+
+        // Find orphans
+        for (const store of existingStores) {
+          if (!definedNames.has(store.name)) {
+            result.orphans.push(store.name);
+          }
+        }
+
+        // Prune orphans if requested
+        if (options.prune === true && result.orphans.length > 0) {
+          if (options.dryRun === true) {
+            result.wouldPrune = [...result.orphans];
+          } else {
+            for (const orphanName of result.orphans) {
+              const store = await services.store.getByName(orphanName);
+              if (store !== undefined) {
+                const deleteResult = await services.store.delete(store.id, {
+                  skipDefinitionSync: true,
+                });
+                if (deleteResult.success) {
+                  result.pruned.push(orphanName);
+                }
+              }
+            }
+          }
+        }
+
+        // Output result
+        if (globalOpts.format === 'json') {
+          console.log(JSON.stringify(result, null, 2));
+        } else {
+          printHumanReadable(result, globalOpts.quiet === true);
+        }
+      } finally {
+        await destroyServices(services);
+      }
+    });
+
+  return sync;
+}
+
+function printHumanReadable(result: SyncResult, quiet: boolean): void {
+  if (quiet) {
+    // Just print created/pruned store names
+    for (const name of result.created) {
+      console.log(`created: ${name}`);
+    }
+    for (const name of result.pruned) {
+      console.log(`pruned: ${name}`);
+    }
+    for (const name of result.wouldCreate) {
+      console.log(`would create: ${name}`);
+    }
+    for (const name of result.wouldPrune) {
+      console.log(`would prune: ${name}`);
+    }
+    return;
+  }
+
+  if (result.dryRun) {
+    console.log('\n[DRY RUN] No changes made.\n');
+  } else {
+    console.log('\nSync completed.\n');
+  }
+
+  if (result.created.length > 0) {
+    console.log(`Created (${String(result.created.length)}):`);
+    for (const name of result.created) {
+      console.log(`  + ${name}`);
+    }
+  }
+
+  if (result.wouldCreate.length > 0) {
+    console.log(`Would create (${String(result.wouldCreate.length)}):`);
+    for (const name of result.wouldCreate) {
+      console.log(`  + ${name}`);
+    }
+  }
+
+  if (result.skipped.length > 0) {
+    console.log(`Skipped (already exist) (${String(result.skipped.length)}):`);
+    for (const name of result.skipped) {
+      console.log(`  - ${name}`);
+    }
+  }
+
+  if (result.failed.length > 0) {
+    console.log(`Failed (${String(result.failed.length)}):`);
+    for (const { name, error } of result.failed) {
+      console.log(`  ! ${name}: ${error}`);
+    }
+  }
+
+  if (result.orphans.length > 0) {
+    console.log(`Orphans (not in definitions) (${String(result.orphans.length)}):`);
+    for (const name of result.orphans) {
+      console.log(`  ? ${name}`);
+    }
+  }
+
+  if (result.pruned.length > 0) {
+    console.log(`Pruned (${String(result.pruned.length)}):`);
+    for (const name of result.pruned) {
+      console.log(`  x ${name}`);
+    }
+  }
+
+  if (result.wouldPrune.length > 0) {
+    console.log(`Would prune (${String(result.wouldPrune.length)}):`);
+    for (const name of result.wouldPrune) {
+      console.log(`  x ${name}`);
+    }
+  }
+
+  console.log('');
+}

package/src/cli/index.ts

@@ -5,3 +5,4 @@ export { createIndexCommand } from './commands/index-cmd.js';
 export { createServeCommand } from './commands/serve.js';
 export { createCrawlCommand } from './commands/crawl.js';
 export { createSetupCommand } from './commands/setup.js';
+export { createSyncCommand } from './commands/sync.js';

package/src/crawl/claude-client.test.ts

@@ -106,6 +106,62 @@ describe('ClaudeClient', () => {
       expect(result.reasoning).toBe('Found documentation pages');
     });
 
+    it('should extract structured_output from Claude CLI wrapper format', async () => {
+      const promise = client.determineCrawlUrls(
+        'https://example.com',
+        '<html>test</html>',
+        'Find all docs'
+      );
+
+      // Claude CLI with --json-schema returns this wrapper format
+      setTimeout(() => {
+        mockProcess.stdout.emit(
+          'data',
+          Buffer.from(
+            JSON.stringify({
+              type: 'result',
+              subtype: 'success',
+              result: '',
+              structured_output: {
+                urls: ['https://example.com/page1', 'https://example.com/page2'],
+                reasoning: 'Found documentation pages',
+              },
+            })
+          )
+        );
+        mockProcess.emit('close', 0);
+      }, 10);
+
+      const result = await promise;
+      expect(result.urls).toEqual(['https://example.com/page1', 'https://example.com/page2']);
+      expect(result.reasoning).toBe('Found documentation pages');
+    });
+
+    it('should fall back to raw response when structured_output is not an object', async () => {
+      const promise = client.determineCrawlUrls(
+        'https://example.com',
+        '<html>test</html>',
+        'Find all docs'
+      );
+
+      // When structured_output is not an object, use the raw response
+      // (which will fail validation if it doesn't have urls/reasoning)
+      setTimeout(() => {
+        mockProcess.stdout.emit(
+          'data',
+          Buffer.from(
+            JSON.stringify({
+              type: 'result',
+              structured_output: 'not an object',
+            })
+          )
+        );
+        mockProcess.emit('close', 0);
+      }, 10);
+
+      await expect(promise).rejects.toThrow('invalid crawl strategy');
+    });
+
     it('should call spawn with correct arguments for determineCrawlUrls', async () => {
       const promise = client.determineCrawlUrls(
         'https://example.com',

package/src/crawl/claude-client.ts

@@ -96,7 +96,11 @@ Return only URLs that are relevant to the instruction. If the instruction mentio
 
     try {
       const result = await this.callClaude(prompt, CRAWL_STRATEGY_SCHEMA);
-      const parsed: unknown = JSON.parse(result);
+      const rawParsed: unknown = JSON.parse(result);
+
+      // Claude CLI with --json-schema returns wrapper: {type, result, structured_output: {...}}
+      // Extract structured_output if present, otherwise use raw response
+      const parsed = this.extractStructuredOutput(rawParsed);
 
       // Validate and narrow type
       if (
@@ -232,4 +236,26 @@ ${this.truncateMarkdown(markdown, 100000)}`;
 
     return `${markdown.substring(0, maxLength)}\n\n[... content truncated ...]`;
   }
+
+  /**
+   * Type guard to check if value is a record (plain object)
+   */
+  private isRecord(value: unknown): value is Record<string, unknown> {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+  }
+
+  /**
+   * Extract structured_output from Claude CLI wrapper format if present.
+   * Claude CLI with --json-schema returns: {type, result, structured_output: {...}}
+   * This method extracts the inner structured_output, or returns the raw value if not wrapped.
+   */
+  private extractStructuredOutput(rawParsed: unknown): unknown {
+    if (this.isRecord(rawParsed) && 'structured_output' in rawParsed) {
+      const structuredOutput = rawParsed['structured_output'];
+      if (typeof structuredOutput === 'object') {
+        return structuredOutput;
+      }
+    }
+    return rawParsed;
+  }
 }

package/src/index.ts

@@ -3,6 +3,8 @@
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { Command } from 'commander';
+import { AdapterRegistry } from './analysis/adapter-registry.js';
+import { ZilAdapter } from './analysis/zil/index.js';
 import { createCrawlCommand } from './cli/commands/crawl.js';
 import { createIndexCommand } from './cli/commands/index-cmd.js';
 import { createMCPCommand } from './cli/commands/mcp.js';
@@ -16,8 +18,13 @@ import { createSearchCommand } from './cli/commands/search.js';
 import { createServeCommand } from './cli/commands/serve.js';
 import { createSetupCommand } from './cli/commands/setup.js';
 import { createStoreCommand } from './cli/commands/store.js';
+import { createSyncCommand } from './cli/commands/sync.js';
 import { createProgram, getGlobalOptions } from './cli/program.js';
 
+// Register built-in language adapters
+const registry = AdapterRegistry.getInstance();
+registry.register(new ZilAdapter());
+
 // Default paths
 const DEFAULT_DATA_DIR = join(homedir(), '.bluera', 'bluera-knowledge', 'data');
 const DEFAULT_CONFIG = join(homedir(), '.bluera', 'bluera-knowledge', 'config.json');
@@ -105,6 +112,7 @@ program.addCommand(createIndexCommand(() => getGlobalOptions(program)));
 program.addCommand(createServeCommand(() => getGlobalOptions(program)));
 program.addCommand(createCrawlCommand(() => getGlobalOptions(program)));
 program.addCommand(createSetupCommand(() => getGlobalOptions(program)));
+program.addCommand(createSyncCommand(() => getGlobalOptions(program)));
 program.addCommand(createMCPCommand(() => getGlobalOptions(program)));
 
 // Show comprehensive help when no arguments provided

package/src/mcp/commands/index.ts

@@ -9,11 +9,13 @@ import { jobCommands } from './job.commands.js';
 import { metaCommands } from './meta.commands.js';
 import { commandRegistry } from './registry.js';
 import { storeCommands } from './store.commands.js';
+import { syncCommands } from './sync.commands.js';
 
 // Register all commands
 commandRegistry.registerAll(storeCommands);
 commandRegistry.registerAll(jobCommands);
 commandRegistry.registerAll(metaCommands);
+commandRegistry.registerAll(syncCommands);
 
 // Re-export for convenience
 export { commandRegistry, executeCommand, generateHelp } from './registry.js';

package/src/mcp/commands/sync.commands.test.ts

@@ -0,0 +1,283 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { z } from 'zod';
+import { rm, mkdtemp, mkdir, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { syncCommands, handleStoresSync } from './sync.commands.js';
+import { StoreService } from '../../services/store.service.js';
+import { StoreDefinitionService } from '../../services/store-definition.service.js';
+import type { HandlerContext } from '../types.js';
+import type { ServiceContainer } from '../../services/index.js';
+import type { StoreDefinitionsConfig } from '../../types/store-definition.js';
+
+/**
+ * Create a minimal mock service container for testing
+ */
+function createMockServices(storeService: StoreService): ServiceContainer {
+  return {
+    store: storeService,
+    // Other services not needed for sync tests
+    config: {} as ServiceContainer['config'],
+    search: {} as ServiceContainer['search'],
+    index: {} as ServiceContainer['index'],
+    lance: {} as ServiceContainer['lance'],
+    embeddings: {} as ServiceContainer['embeddings'],
+    codeGraph: {} as ServiceContainer['codeGraph'],
+    pythonBridge: {} as ServiceContainer['pythonBridge'],
+  };
+}
+
+describe('sync.commands', () => {
+  describe('command definition', () => {
+    it('exports stores:sync command', () => {
+      const syncCmd = syncCommands.find((c) => c.name === 'stores:sync');
+      expect(syncCmd).toBeDefined();
+      expect(syncCmd?.description).toContain('Sync');
+    });
+
+    it('has correct args schema', () => {
+      const syncCmd = syncCommands.find((c) => c.name === 'stores:sync');
+      expect(syncCmd?.argsSchema).toBeDefined();
+
+      // Valid empty args
+      const result1 = syncCmd?.argsSchema?.safeParse({});
+      expect(result1?.success).toBe(true);
+
+      // Valid with options
+      const result2 = syncCmd?.argsSchema?.safeParse({
+        reindex: true,
+        prune: true,
+        dryRun: true,
+      });
+      expect(result2?.success).toBe(true);
+    });
+  });
+
+  describe('handleStoresSync', () => {
+    let projectRoot: string;
+    let dataDir: string;
+    let storeService: StoreService;
+    let defService: StoreDefinitionService;
+    let context: HandlerContext;
+
+    beforeEach(async () => {
+      projectRoot = await mkdtemp(join(tmpdir(), 'sync-test-'));
+      dataDir = join(projectRoot, '.bluera/bluera-knowledge/data');
+      defService = new StoreDefinitionService(projectRoot);
+      storeService = new StoreService(dataDir, { definitionService: defService });
+      await storeService.initialize();
+
+      context = {
+        services: createMockServices(storeService),
+        options: { projectRoot },
+      };
+    });
+
+    afterEach(async () => {
+      await rm(projectRoot, { recursive: true, force: true });
+    });
+
+    describe('creates missing stores', () => {
+      it('creates file store from definition', async () => {
+        // Create a directory to reference
+        const docsDir = join(projectRoot, 'docs');
+        await mkdir(docsDir, { recursive: true });
+
+        // Add definition manually (simulating config from git)
+        await defService.addDefinition({
+          type: 'file',
+          name: 'my-docs',
+          path: './docs',
+          description: 'Documentation',
+        });
+
+        const result = await handleStoresSync({}, context);
+        const response = JSON.parse(result.content[0].text);
+
+        expect(response.created).toContain('my-docs');
+        expect(response.skipped).toHaveLength(0);
+        expect(response.failed).toHaveLength(0);
+
+        // Verify store was created
+        const store = await storeService.getByName('my-docs');
+        expect(store).toBeDefined();
+        expect(store?.type).toBe('file');
+      });
+
+      it('creates web store from definition', async () => {
+        // Add web store definition
+        await defService.addDefinition({
+          type: 'web',
+          name: 'api-docs',
+          url: 'https://example.com/docs',
+          depth: 2,
+        });
+
+        const result = await handleStoresSync({}, context);
+        const response = JSON.parse(result.content[0].text);
+
+        expect(response.created).toContain('api-docs');
+
+        const store = await storeService.getByName('api-docs');
+        expect(store).toBeDefined();
+        expect(store?.type).toBe('web');
+      });
+    });
+
+    describe('skips existing stores', () => {
+      it('skips store that already exists', async () => {
+        const docsDir = join(projectRoot, 'docs');
+        await mkdir(docsDir, { recursive: true });
+
+        // Create store first (this auto-adds the definition via the integration)
+        await storeService.create({
+          name: 'existing-docs',
+          type: 'file',
+          path: docsDir,
+        });
+
+        // Definition was auto-added, so sync should skip this store
+        const result = await handleStoresSync({}, context);
+        const response = JSON.parse(result.content[0].text);
+
+        expect(response.skipped).toContain('existing-docs');
+        expect(response.created).toHaveLength(0);
+      });
+    });
+
+    describe('reports orphans', () => {
+      it('reports stores not in definitions', async () => {
+        const docsDir = join(projectRoot, 'docs');
+        await mkdir(docsDir, { recursive: true });
+
+        // Create store without definition (using skipDefinitionSync)
+        await storeService.create(
+          {
+            name: 'orphan-store',
+            type: 'file',
+            path: docsDir,
+          },
+          { skipDefinitionSync: true }
+        );
+
+        const result = await handleStoresSync({}, context);
+        const response = JSON.parse(result.content[0].text);
+
+        expect(response.orphans).toContain('orphan-store');
+      });
+    });
+
+    describe('dry run mode', () => {
+      it('does not create stores in dry run mode', async () => {
+        const docsDir = join(projectRoot, 'docs');
+        await mkdir(docsDir, { recursive: true });
+
+        await defService.addDefinition({
+          type: 'file',
+          name: 'dry-run-store',
+          path: './docs',
+        });
+
+        const result = await handleStoresSync({ dryRun: true }, context);
+        const response = JSON.parse(result.content[0].text);
+
+        expect(response.dryRun).toBe(true);
+        expect(response.wouldCreate).toContain('dry-run-store');
+
+        // Store should NOT exist
+        const store = await storeService.getByName('dry-run-store');
+        expect(store).toBeUndefined();
+      });
+    });
+
+    describe('prune mode', () => {
+      it('removes orphan stores when prune is true', async () => {
+        const docsDir = join(projectRoot, 'docs');
+        await mkdir(docsDir, { recursive: true });
+
+        // Create orphan store
+        await storeService.create(
+          {
+            name: 'to-prune',
+            type: 'file',
+            path: docsDir,
+          },
+          { skipDefinitionSync: true }
+        );
+
+        const result = await handleStoresSync({ prune: true }, context);
+        const response = JSON.parse(result.content[0].text);
+
+        expect(response.pruned).toContain('to-prune');
+
+        // Store should be deleted
+        const store = await storeService.getByName('to-prune');
+        expect(store).toBeUndefined();
+      });
+
+      it('does not prune in dry run mode', async () => {
+        const docsDir = join(projectRoot, 'docs');
+        await mkdir(docsDir, { recursive: true });
+
+        await storeService.create(
+          {
+            name: 'keep-me',
+            type: 'file',
+            path: docsDir,
+          },
+          { skipDefinitionSync: true }
+        );
+
+        const result = await handleStoresSync({ prune: true, dryRun: true }, context);
+        const response = JSON.parse(result.content[0].text);
+
+        expect(response.wouldPrune).toContain('keep-me');
+
+        // Store should still exist
+        const store = await storeService.getByName('keep-me');
+        expect(store).toBeDefined();
+      });
+    });
+
+    describe('error handling', () => {
+      it('continues on error and reports failures', async () => {
+        // Add definition for non-existent directory
+        await defService.addDefinition({
+          type: 'file',
+          name: 'bad-store',
+          path: './nonexistent',
+        });
+
+        // Also add a valid definition
+        const docsDir = join(projectRoot, 'docs');
+        await mkdir(docsDir, { recursive: true });
+        await defService.addDefinition({
+          type: 'file',
+          name: 'good-store',
+          path: './docs',
+        });
+
+        const result = await handleStoresSync({}, context);
+        const response = JSON.parse(result.content[0].text);
+
+        // Should have one failure and one success
+        expect(response.failed).toHaveLength(1);
+        expect(response.failed[0].name).toBe('bad-store');
+        expect(response.failed[0].error).toBeDefined();
+        expect(response.created).toContain('good-store');
+      });
+    });
+
+    describe('empty config', () => {
+      it('handles empty definitions gracefully', async () => {
+        const result = await handleStoresSync({}, context);
+        const response = JSON.parse(result.content[0].text);
+
+        expect(response.created).toHaveLength(0);
+        expect(response.skipped).toHaveLength(0);
+        expect(response.failed).toHaveLength(0);
+        expect(response.orphans).toHaveLength(0);
+      });
+    });
+  });
+});