@absolutejs/absolute 0.19.0-beta.423 → 0.19.0-beta.424

package/dist/ai/index.js

@@ -2102,6 +2102,8 @@ var aiChat = (config) => {
 import { Elysia as Elysia2 } from "elysia";
 
 // src/ai/rag/ingestion.ts
+import { readdir, readFile } from "fs/promises";
+import { basename, extname, join, relative, resolve } from "path";
 var DEFAULT_MAX_CHUNK_LENGTH = 900;
 var DEFAULT_CHUNK_OVERLAP = 120;
 var DEFAULT_MIN_CHUNK_LENGTH = 80;
@@ -2147,6 +2149,35 @@ var stripMarkdown = (value) => {
 `);
   return normalizeWhitespace(stripped);
 };
+var markdownStructureUnits = (value) => {
+  const lines = value.replace(/\r\n?/g, `
+`).split(`
+`);
+  const sections = [];
+  let current = [];
+  for (const line of lines) {
+    if (/^\s*#{1,6}\s+/.test(line) && current.length > 0) {
+      sections.push(current.join(`
+`));
+      current = [];
+    }
+    current.push(line);
+  }
+  if (current.length > 0) {
+    sections.push(current.join(`
+`));
+  }
+  return sections.map((section) => stripMarkdown(section)).map((section) => normalizeWhitespace(section)).filter(Boolean);
+};
+var htmlStructureUnits = (value) => {
+  const marked = value.replace(/<(section|article|main|aside|nav|h[1-6])\b[^>]*>/gi, `
+
+__ABS_SECTION_BREAK__ `).replace(/<\/(section|article|main|aside|nav|h[1-6])>/gi, `
+
+`);
+  const normalized = stripHtml(marked);
+  return normalized.split(/__ABS_SECTION_BREAK__/).map((section) => normalizeWhitespace(section)).filter(Boolean);
+};
 var inferFormat = (document) => {
   if (document.format) {
     return document.format;
@@ -2192,6 +2223,21 @@ var fixedUnits = (text, maxChunkLength) => {
   }
   return units;
 };
+var sourceAwareUnits = (document, format, normalizedText) => {
+  switch (format) {
+    case "markdown": {
+      const sections = markdownStructureUnits(document.text);
+      return sections.length > 0 ? sections : paragraphUnits(normalizedText);
+    }
+    case "html": {
+      const sections = htmlStructureUnits(document.text);
+      return sections.length > 0 ? sections : paragraphUnits(normalizedText);
+    }
+    case "text":
+    default:
+      return paragraphUnits(normalizedText);
+  }
+};
 var overlapTail = (value, overlap) => {
   if (overlap <= 0 || value.length <= overlap) {
     return value;
@@ -2256,11 +2302,11 @@ var resolveChunkingOptions = (document, defaults) => {
     strategy
   };
 };
-var createChunkTexts = (text, options) => {
+var createChunkTexts = (document, format, text, options) => {
   if (text.length <= options.maxChunkLength) {
     return [text];
   }
-  const units = options.strategy === "fixed" ? fixedUnits(text, options.maxChunkLength) : options.strategy === "sentences" ? sentenceUnits(text) : paragraphUnits(text);
+  const units = options.strategy === "fixed" ? fixedUnits(text, options.maxChunkLength) : options.strategy === "source_aware" ? sourceAwareUnits(document, format, text) : options.strategy === "sentences" ? sentenceUnits(text) : paragraphUnits(text);
   return chunkFromUnits(units, options.maxChunkLength, options.chunkOverlap, options.minChunkLength);
 };
 var prepareRAGDocument = (document, defaultChunking) => {
@@ -2277,7 +2323,7 @@ var prepareRAGDocument = (document, defaultChunking) => {
     source,
     title
   };
-  const chunkTexts = createChunkTexts(normalizedText, chunking);
+  const chunkTexts = createChunkTexts(document, format, normalizedText, chunking);
   const chunks = chunkTexts.map((text, index) => ({
     chunkId: `${documentId}:${String(index + 1).padStart(3, "0")}`,
     metadata: {
@@ -2300,9 +2346,81 @@ var prepareRAGDocument = (document, defaultChunking) => {
   };
 };
 var prepareRAGDocuments = (input) => input.documents.map((document) => prepareRAGDocument(document, input.defaultChunking));
+var inferFormatFromPath = (path) => {
+  const extension = extname(path).toLowerCase();
+  if (extension === ".md" || extension === ".mdx") {
+    return "markdown";
+  }
+  if (extension === ".html" || extension === ".htm") {
+    return "html";
+  }
+  return "text";
+};
+var loadRAGDocumentFile = async (input) => {
+  const text = await readFile(input.path, "utf8");
+  return {
+    ...input,
+    format: input.format ?? inferFormatFromPath(input.path),
+    source: input.source ?? input.path,
+    text
+  };
+};
+var prepareRAGDocumentFile = async (input, defaultChunking) => prepareRAGDocument(await loadRAGDocumentFile(input), defaultChunking);
+var DEFAULT_DIRECTORY_EXTENSIONS = [
+  ".txt",
+  ".md",
+  ".mdx",
+  ".html",
+  ".htm"
+];
+var collectDirectoryFiles = async (directory, recursive, includeExtensions) => {
+  const entries = await readdir(directory, { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    const fullPath = join(directory, entry.name);
+    if (entry.isDirectory()) {
+      if (recursive) {
+        files.push(...await collectDirectoryFiles(fullPath, recursive, includeExtensions));
+      }
+      continue;
+    }
+    if (!entry.isFile()) {
+      continue;
+    }
+    const extension = extname(entry.name).toLowerCase();
+    if (includeExtensions.has(extension)) {
+      files.push(fullPath);
+    }
+  }
+  return files.sort();
+};
+var loadRAGDocumentsFromDirectory = async (input) => {
+  const root = resolve(input.directory);
+  const includeExtensions = new Set((input.includeExtensions ?? DEFAULT_DIRECTORY_EXTENSIONS).map((entry) => entry.startsWith(".") ? entry.toLowerCase() : `.${entry.toLowerCase()}`));
+  const files = await collectDirectoryFiles(root, input.recursive !== false, includeExtensions);
+  const documents = await Promise.all(files.map(async (path) => {
+    const source = relative(root, path).replace(/\\/g, "/");
+    const loaded = await loadRAGDocumentFile({
+      metadata: {
+        ...input.baseMetadata ?? {},
+        fileName: basename(path),
+        relativePath: source
+      },
+      path,
+      source
+    });
+    return loaded;
+  }));
+  return {
+    defaultChunking: input.defaultChunking,
+    documents
+  };
+};
+var prepareRAGDirectoryDocuments = async (input) => prepareRAGDocuments(await loadRAGDocumentsFromDirectory(input));
 var buildRAGUpsertInputFromDocuments = (input) => ({
   chunks: prepareRAGDocuments(input).flatMap((document) => document.chunks)
 });
+var buildRAGUpsertInputFromDirectory = async (input) => buildRAGUpsertInputFromDocuments(await loadRAGDocumentsFromDirectory(input));
 
 // src/ai/rag/collection.ts
 var DEFAULT_TOP_K = 6;
@@ -2895,7 +3013,7 @@ import { existsSync as existsSync2 } from "fs";
 import { existsSync, readFileSync } from "fs";
 import { createRequire } from "module";
 import { arch, platform } from "os";
-import { dirname, join } from "path";
+import { dirname, join as join2 } from "path";
 var require2 = createRequire(import.meta.url);
 var PLATFORM_PACKAGE_MAP = {
   "darwin-arm64": {
@@ -2943,7 +3061,7 @@ var resolveAbsoluteSQLiteVec = () => {
   try {
     const packageJsonPath = require2.resolve(`${packageInfo.packageName}/package.json`);
     const packageRoot = dirname(packageJsonPath);
-    const libraryPath = join(packageRoot, packageInfo.libraryFile);
+    const libraryPath = join2(packageRoot, packageInfo.libraryFile);
     const packageVersion = readPackageVersion(packageJsonPath);
     if (!existsSync(libraryPath)) {
       return {
@@ -3659,7 +3777,9 @@ export {
   ragChat,
   querySimilarity,
   prepareRAGDocuments,
+  prepareRAGDocumentFile,
   prepareRAGDocument,
+  prepareRAGDirectoryDocuments,
   parseAIMessage,
   openaiResponses,
   openaiCompatible,
@@ -3667,6 +3787,8 @@ export {
   moonshot,
   mistralai,
   meta,
+  loadRAGDocumentsFromDirectory,
+  loadRAGDocumentFile,
   ingestRAGDocuments,
   ingestDocuments,
   google,
@@ -3681,9 +3803,10 @@ export {
   createInMemoryRAGStore,
   createConversationManager,
   buildRAGUpsertInputFromDocuments,
+  buildRAGUpsertInputFromDirectory,
   alibaba,
   aiChat
 };
 
-//# debugId=D8D946A65C6DB67B64756E2164756E21
+//# debugId=1F6C91DA7227AEA564756E2164756E21
 //# sourceMappingURL=index.js.map