bluera-knowledge 0.9.26 → 0.9.31

package/dist/index.js

@@ -1,12 +1,14 @@
 #!/usr/bin/env node
 import {
   runMCPServer
-} from "./chunk-J7J6LXOJ.js";
+} from "./chunk-OGEY66FZ.js";
 import {
   IntelligentCrawler
-} from "./chunk-BICFAWMN.js";
+} from "./chunk-2SJHNRXD.js";
 import {
   ASTParser,
+  ChunkingService,
+  classifyWebContentType,
   createDocumentId,
   createServices,
   createStoreId,
@@ -14,7 +16,7 @@ import {
   err,
   extractRepoName,
   ok
-} from "./chunk-5QMHZUC4.js";
+} from "./chunk-RWSXP3PQ.js";
 import "./chunk-L2YVNC63.js";
 
 // src/index.ts
@@ -85,12 +87,14 @@ function createStoreCommand(getOptions) {
   store.command("create <name>").description("Create a new store pointing to a local path or URL").requiredOption("-t, --type <type>", "Store type: file (local dir), repo (git), web (crawled site)").requiredOption("-s, --source <path>", "Local path for file/repo stores, URL for web stores").option("-d, --description <desc>", "Optional description for the store").option("--tags <tags>", "Comma-separated tags for filtering").action(async (name, options) => {
     const globalOpts = getOptions();
     const services = await createServices(globalOpts.config, globalOpts.dataDir);
+    let exitCode = 0;
     try {
+      const isUrl = options.source.startsWith("http://") || options.source.startsWith("https://");
       const result = await services.store.create({
         name,
         type: options.type,
-        path: options.type !== "web" ? options.source : void 0,
-        url: options.type === "web" ? options.source : void 0,
+        path: options.type === "file" || options.type === "repo" && !isUrl ? options.source : void 0,
+        url: options.type === "web" || options.type === "repo" && isUrl ? options.source : void 0,
         description: options.description,
         tags: options.tags?.split(",").map((t) => t.trim())
       });
@@ -104,11 +108,14 @@ Created store: ${result.data.name} (${result.data.id})
         }
       } else {
         console.error(`Error: ${result.error.message}`);
-        process.exit(1);
+        exitCode = 1;
       }
     } finally {
       await destroyServices(services);
     }
+    if (exitCode !== 0) {
+      process.exit(exitCode);
+    }
   });
   store.command("info <store>").description("Show store details: ID, type, path/URL, timestamps").action(async (storeIdOrName) => {
     const globalOpts = getOptions();
@@ -467,10 +474,33 @@ function createCrawlCommand(getOptions) {
   return new Command6("crawl").description("Crawl web pages with natural language control and index into store").argument("<url>", "URL to crawl").argument("<store>", "Target web store to add crawled content to").option("--crawl <instruction>", 'Natural language instruction for what to crawl (e.g., "all Getting Started pages")').option("--extract <instruction>", 'Natural language instruction for what to extract (e.g., "extract API references")').option("--simple", "Use simple BFS mode instead of intelligent crawling").option("--max-pages <number>", "Maximum number of pages to crawl", "50").option("--headless", "Use headless browser for JavaScript-rendered sites").action(async (url, storeIdOrName, cmdOptions) => {
     const globalOpts = getOptions();
     const services = await createServices(globalOpts.config, globalOpts.dataDir);
-    const store = await services.store.getByIdOrName(storeIdOrName);
-    if (!store || store.type !== "web") {
-      console.error(`Error: Web store not found: ${storeIdOrName}`);
-      process.exit(3);
+    let store;
+    let storeCreated = false;
+    const existingStore = await services.store.getByIdOrName(storeIdOrName);
+    if (!existingStore) {
+      const result = await services.store.create({
+        name: storeIdOrName,
+        type: "web",
+        url
+      });
+      if (!result.success) {
+        await destroyServices(services);
+        throw new Error(`Failed to create store: ${result.error.message}`);
+      }
+      const createdStore = result.data;
+      if (createdStore.type !== "web") {
+        throw new Error("Unexpected store type after creation");
+      }
+      store = createdStore;
+      storeCreated = true;
+      if (globalOpts.quiet !== true && globalOpts.format !== "json") {
+        console.log(`Created web store: ${store.name}`);
+      }
+    } else if (existingStore.type !== "web") {
+      await destroyServices(services);
+      throw new Error(`Store "${storeIdOrName}" exists but is not a web store (type: ${existingStore.type})`);
+    } else {
+      store = existingStore;
     }
     const maxPages = cmdOptions.maxPages !== void 0 ? parseInt(cmdOptions.maxPages) : 50;
     const isInteractive = process.stdout.isTTY && globalOpts.quiet !== true && globalOpts.format !== "json";
@@ -482,7 +512,9 @@ function createCrawlCommand(getOptions) {
       console.log(`Crawling ${url}`);
     }
     const crawler = new IntelligentCrawler();
+    const webChunker = ChunkingService.forContentType("web");
     let pagesIndexed = 0;
+    let chunksCreated = 0;
     crawler.on("progress", (progress) => {
       if (spinner) {
         if (progress.type === "strategy") {
@@ -508,22 +540,33 @@ function createCrawlCommand(getOptions) {
         ...cmdOptions.simple !== void 0 && { simple: cmdOptions.simple },
         useHeadless: cmdOptions.headless ?? false
       })) {
-        const contentToEmbed = result.extracted !== void 0 ? result.extracted : result.markdown;
-        const vector = await services.embeddings.embed(contentToEmbed);
-        docs.push({
-          id: createDocumentId(`${store.id}-${createHash("md5").update(result.url).digest("hex")}`),
-          content: contentToEmbed,
-          vector,
-          metadata: {
-            type: "web",
-            storeId: store.id,
-            url: result.url,
-            title: result.title,
-            extracted: result.extracted !== void 0,
-            depth: result.depth,
-            indexedAt: /* @__PURE__ */ new Date()
-          }
-        });
+        const contentToProcess = result.extracted !== void 0 ? result.extracted : result.markdown;
+        const chunks = webChunker.chunk(contentToProcess, `${result.url}.md`);
+        const fileType = classifyWebContentType(result.url, result.title);
+        const urlHash = createHash("md5").update(result.url).digest("hex");
+        for (const chunk of chunks) {
+          const chunkId = chunks.length > 1 ? `${store.id}-${urlHash}-${String(chunk.chunkIndex)}` : `${store.id}-${urlHash}`;
+          const vector = await services.embeddings.embed(chunk.content);
+          docs.push({
+            id: createDocumentId(chunkId),
+            content: chunk.content,
+            vector,
+            metadata: {
+              type: chunks.length > 1 ? "chunk" : "web",
+              storeId: store.id,
+              url: result.url,
+              title: result.title,
+              extracted: result.extracted !== void 0,
+              depth: result.depth,
+              indexedAt: /* @__PURE__ */ new Date(),
+              fileType,
+              chunkIndex: chunk.chunkIndex,
+              totalChunks: chunk.totalChunks,
+              sectionHeader: chunk.sectionHeader
+            }
+          });
+          chunksCreated++;
+        }
         pagesIndexed++;
       }
       if (docs.length > 0) {
@@ -535,8 +578,10 @@ function createCrawlCommand(getOptions) {
       const crawlResult = {
         success: true,
         store: store.name,
+        storeCreated,
         url,
         pagesCrawled: pagesIndexed,
+        chunksCreated,
         mode: cmdOptions.simple === true ? "simple" : "intelligent",
         hadCrawlInstruction: cmdOptions.crawl !== void 0,
         hadExtractInstruction: cmdOptions.extract !== void 0
@@ -544,9 +589,9 @@ function createCrawlCommand(getOptions) {
       if (globalOpts.format === "json") {
         console.log(JSON.stringify(crawlResult, null, 2));
       } else if (spinner !== void 0) {
-        spinner.succeed(`Crawled and indexed ${String(pagesIndexed)} pages`);
+        spinner.succeed(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
       } else if (globalOpts.quiet !== true) {
-        console.log(`Crawled and indexed ${String(pagesIndexed)} pages`);
+        console.log(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
       }
     } catch (error) {
       const message = `Crawl failed: ${error instanceof Error ? error.message : String(error)}`;