@vivantel/rag-core 2.0.0 → 2.1.1

package/README.md

@@ -15,34 +15,54 @@ npm install @vivantel/rag-core
 
 ## Quick Start
 
-### 1. Create a config file (`rag.config.ts`)
+### 1. Generate a config (recommended)
+
+```bash
+rag-update init
+```
+
+Scans your project for known file types and generates a `rag.config.ts` with the right strategy per pattern. You then fill in your embedding provider and vector store.
+
+### 2. Or write a config manually (`rag.config.ts`)
 
 ```typescript
-import { defineConfig, createChunker, tokenStrategy } from '@vivantel/rag-core';
-import { readFile } from 'fs/promises';
+import {
+  RAGPipelineConfig,
+  createChunker,
+  markdownHeadersStrategy,
+  tokenStrategy,
+} from '@vivantel/rag-core';
 
-export default defineConfig({
+const config: RAGPipelineConfig = {
   chunkers: [
-    createChunker({
-      name: 'my-chunker',
-      patterns: ['docs/**/*.md'],
-      async process(content, filePath, commitHash) {
-        return [{
-          content,
-          metadata: { type: 'doc', file: filePath },
-          sourceFile: filePath,
-          commitHash
-        }];
-      }
-    })
+    // Different directories can use different strategies
+    createChunker({ patterns: ['docs/**/*.md'], strategy: markdownHeadersStrategy() }),
+    createChunker({ patterns: ['rules/**/*.md'], strategy: wholeFileStrategy() }),
+    createChunker({ patterns: ['src/**/*.ts', 'src/**/*.tsx'], strategy: tokenStrategy() }),
   ],
-  
   embedder: new MyEmbedderProvider({ apiKey: process.env.API_KEY }),
-  vectorStore: new MyVectorStore({ url: process.env.DB_URL })
-});
+  vectorStore: new MyVectorStore({ url: process.env.DB_URL }),
+};
+
+export default config;
 ```
 
-### 2. Run the pipeline
+For custom chunking logic, use the `process` callback instead of `strategy`:
+
+```typescript
+createChunker({
+  name: 'custom',
+  patterns: ['**/*.txt'],
+  process: async (content, filePath, commitHash) => [{
+    content,
+    metadata: { file: filePath },
+    sourceFile: filePath,
+    commitHash,
+  }],
+})
+```
+
+### 3. Run the pipeline
 
 ```bash
 rag-update --config rag.config.ts
@@ -50,12 +70,12 @@ rag-update --config rag.config.ts
 
 ## Built-in Strategies
 
-### Chunk Strategies
-
-- `tokenStrategy({ maxTokens, overlap })` - Split by approximate token count
-- `markdownHeadersStrategy()` - Split by Markdown headers (##, ###)
-- `semanticStrategy()` - Split by sentence boundaries
-- `wholeFileStrategy()` - One chunk per file
+| Factory | Best for |
+|---------|----------|
+| `tokenStrategy({ maxTokens?, overlap? })` | Source code, structured text |
+| `markdownHeadersStrategy()` | Markdown documentation |
+| `semanticStrategy()` | Prose, articles |
+| `wholeFileStrategy()` | Small configs, rule files, YAML |
 
 ## License
 

package/dist/cli/init.d.ts

@@ -1,2 +1,9 @@
+interface ExtGroup {
+    exts: string[];
+    strategyFn: string;
+    name: string;
+}
+export declare function detectFileExtensions(cwd: string): Promise<ExtGroup[]>;
 export declare function runInit(): Promise<void>;
+export {};
 //# sourceMappingURL=init.d.ts.map

package/dist/cli/init.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAgOA,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAqE7C"}
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAMA,UAAU,QAAQ;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAoDD,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAI3E;AAsLD,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAsF7C"}

package/dist/cli/init.js

@@ -1,28 +1,60 @@
 import { checkbox, select, input } from "@inquirer/prompts";
 import { writeFile } from "fs/promises";
 import { existsSync } from "fs";
-const STRATEGY_INFO = {
-    token: {
-        fn: "tokenStrategy",
-        description: "Token-based chunking (source code, structured text)",
-        patterns: ["src/**/*.ts", "src/**/*.js"],
+import { readdir } from "fs/promises";
+import { join, extname } from "path";
+const EXT_GROUPS = [
+    {
+        exts: [".md", ".mdx"],
+        strategyFn: "markdownHeadersStrategy",
+        name: "markdown",
     },
-    markdown: {
-        fn: "markdownHeadersStrategy",
-        description: "Markdown header-aware chunking (documentation)",
-        patterns: ["docs/**/*.md", "*.md"],
-    },
-    semantic: {
-        fn: "semanticStrategy",
-        description: "Semantic paragraph chunking (prose, articles)",
-        patterns: ["**/*.txt"],
-    },
-    "whole-file": {
-        fn: "wholeFileStrategy",
-        description: "Treat each file as one chunk (small configs, YAML)",
-        patterns: ["**/*.yaml", "**/*.yml"],
-    },
-};
+    { exts: [".ts", ".tsx"], strategyFn: "tokenStrategy", name: "typescript" },
+    { exts: [".js", ".jsx"], strategyFn: "tokenStrategy", name: "javascript" },
+    { exts: [".py"], strategyFn: "tokenStrategy", name: "python" },
+    { exts: [".go"], strategyFn: "tokenStrategy", name: "go" },
+    { exts: [".cs"], strategyFn: "tokenStrategy", name: "csharp" },
+    { exts: [".java"], strategyFn: "tokenStrategy", name: "java" },
+    { exts: [".yaml", ".yml"], strategyFn: "wholeFileStrategy", name: "yaml" },
+    { exts: [".txt"], strategyFn: "semanticStrategy", name: "text" },
+];
+const IGNORED_DIRS = new Set([
+    "node_modules",
+    ".git",
+    "dist",
+    "build",
+    "coverage",
+    ".cache",
+    ".next",
+    "out",
+    ".turbo",
+]);
+async function collectExtensions(dir, found) {
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return;
+    }
+    for (const entry of entries) {
+        if (entry.isDirectory()) {
+            if (!IGNORED_DIRS.has(entry.name)) {
+                await collectExtensions(join(dir, entry.name), found);
+            }
+        }
+        else {
+            const ext = extname(entry.name).toLowerCase();
+            if (ext)
+                found.add(ext);
+        }
+    }
+}
+export async function detectFileExtensions(cwd) {
+    const found = new Set();
+    await collectExtensions(cwd, found);
+    return EXT_GROUPS.filter((g) => g.exts.some((e) => found.has(e)));
+}
 function buildEmbedderSection(provider) {
     switch (provider) {
         case "openai":
@@ -144,38 +176,27 @@ const vectorStore: VectorStore = {
 };`;
     }
 }
-function buildChunkersSection(strategies) {
-    if (strategies.length === 0) {
-        strategies = ["token"];
+function buildChunkersSection(groups) {
+    if (groups.length === 0) {
+        groups = [EXT_GROUPS.find((g) => g.name === "typescript")];
     }
-    const entries = strategies.map((s) => {
-        const info = STRATEGY_INFO[s];
-        const patterns = JSON.stringify(info.patterns);
-        return `    createChunker({
-      name: '${s}',
-      patterns: ${patterns},
-      process: async (content) => ${info.fn}()(content),
-    }),`;
-    });
-    return entries.join("\n");
+    return groups
+        .map((g) => {
+        const patterns = JSON.stringify(g.exts.map((e) => `**/*${e}`));
+        return `    createChunker({ patterns: ${patterns}, strategy: ${g.strategyFn}() }),`;
+    })
+        .join("\n");
 }
 function generateConfig(answers) {
-    const selectedStrategies = answers.strategies.length > 0 ? answers.strategies : ["token"];
-    const strategyFns = [
-        ...new Set(selectedStrategies.map((s) => STRATEGY_INFO[s].fn)),
-    ];
-    const imports = [
-        "RAGPipelineConfig",
-        "EmbeddingProvider",
-        "VectorStore",
-        "VectorSearchResult",
-        "createChunker",
-        ...strategyFns,
-    ].join(", ");
+    const strategyFns = [...new Set(answers.groups.map((g) => g.strategyFn))];
+    if (strategyFns.length === 0)
+        strategyFns.push("tokenStrategy");
+    const valueImports = ["createChunker", ...strategyFns].join(", ");
     const embedderSection = buildEmbedderSection(answers.embedder);
     const vectorStoreSection = buildVectorStoreSection(answers.vectorStore);
-    const chunkersSection = buildChunkersSection(selectedStrategies);
-    return `import { ${imports} } from '@vivantel/rag-core';
+    const chunkersSection = buildChunkersSection(answers.groups);
+    return `import type { RAGPipelineConfig, EmbeddingProvider, VectorStore, VectorSearchResult } from '@vivantel/rag-core';
+import { ${valueImports} } from '@vivantel/rag-core';
 
 ${embedderSection}
 
@@ -198,15 +219,31 @@ export default config;
 }
 export async function runInit() {
     console.log("\n🚀 RAG Config Generator\n");
-    const strategies = await checkbox({
-        message: "Which chunking strategies do you need?",
-        choices: [
-            { name: "Token (source code, structured text)", value: "token" },
-            { name: "Markdown headers (documentation)", value: "markdown" },
-            { name: "Semantic paragraphs (prose, articles)", value: "semantic" },
-            { name: "Whole file (small configs, YAML)", value: "whole-file" },
-        ],
-    });
+    console.log("Scanning project for file types...");
+    const detectedGroups = await detectFileExtensions(process.cwd());
+    let selectedGroups;
+    if (detectedGroups.length > 0) {
+        const confirmed = await checkbox({
+            message: "Detected file types — select which to index:",
+            choices: detectedGroups.map((g) => ({
+                name: `${g.name} (${g.exts.join(", ")}) → ${g.strategyFn}`,
+                value: g,
+                checked: true,
+            })),
+        });
+        selectedGroups = confirmed;
+    }
+    else {
+        console.log("No known file types detected. Choose strategies manually:");
+        const strategyNames = await checkbox({
+            message: "Which chunking strategies do you need?",
+            choices: EXT_GROUPS.map((g) => ({
+                name: `${g.name} (${g.strategyFn})`,
+                value: g,
+            })),
+        });
+        selectedGroups = strategyNames;
+    }
     const embedder = await select({
         message: "Which embedding provider?",
         choices: [
@@ -247,7 +284,7 @@ export async function runInit() {
         }
     }
     const config = generateConfig({
-        strategies,
+        groups: selectedGroups,
         embedder,
         vectorStore,
         outputPath,

package/dist/cli/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAShC,MAAM,aAAa,GAGf;IACF,KAAK,EAAE;QACL,EAAE,EAAE,eAAe;QACnB,WAAW,EAAE,qDAAqD;QAClE,QAAQ,EAAE,CAAC,aAAa,EAAE,aAAa,CAAC;KACzC;IACD,QAAQ,EAAE;QACR,EAAE,EAAE,yBAAyB;QAC7B,WAAW,EAAE,gDAAgD;QAC7D,QAAQ,EAAE,CAAC,cAAc,EAAE,MAAM,CAAC;KACnC;IACD,QAAQ,EAAE;QACR,EAAE,EAAE,kBAAkB;QACtB,WAAW,EAAE,+CAA+C;QAC5D,QAAQ,EAAE,CAAC,UAAU,CAAC;KACvB;IACD,YAAY,EAAE;QACZ,EAAE,EAAE,mBAAmB;QACvB,WAAW,EAAE,oDAAoD;QACjE,QAAQ,EAAE,CAAC,WAAW,EAAE,UAAU,CAAC;KACpC;CACF,CAAC;AAEF,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ;YACX,OAAO;;;;;;;;;;;;;;;;;GAiBV,CAAC;QAEA,KAAK,eAAe;YAClB,OAAO;;;;;;;;;;;;;;;;;;GAkBV,CAAC;QAEA;YACE,OAAO;;;;;;;;;;;;;GAaV,CAAC;IACF,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAa;IAC5C,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,UAAU;YACb,OAAO;;;;;;;;;;;;;;;;;GAiBV,CAAC;QAEA,KAAK,UAAU;YACb,OAAO;;;;;;;;;;;;;;GAcV,CAAC;QAEA;YACE,OAAO;;;;;;;;;;;;;;;;;;;;;;GAsBV,CAAC;IACF,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,UAAoB;IAChD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,UAAU,GAAG,CAAC,OAAO,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACnC,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/C,OAAO;eACI,CAAC;kBACE,QAAQ;oCACU,IAAI,CAAC,EAAE;QACnC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,cAAc,CAAC,OAAoB;IAC1C,MAAM,kBAAkB,GACtB,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG;QAClB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;KAC/D,CAAC;IAEF,MAAM,OAAO,GAAG;QACd,mBAAmB;QACnB,mBAAmB;QACnB,aAAa;QACb,oBAAoB;QACpB,eAAe;QACf,GAAG,WAAW;KACf,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,MAAM,eAAe,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/D,MAAM,kBAAkB,GAAG,uBAAuB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACxE,MAAM,eAAe,GAAG,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;IAEjE,OAAO,YAAY,OAAO;;EAE1B,eAAe;;EAEf,kBAAkB;;;;EAIlB,eAAe;;;;;;;;;;;CAWhB,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;IAE3C,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC;QAChC,OAAO,EAAE,wCAAwC;QACjD,OAAO,EAAE;YACP,EAAE,IAAI,EAAE,sCAAsC,EAAE,KAAK,EAAE,OAAO,EAAE;YAChE,EAAE,IAAI,EAAE,kCAAkC,EAAE,KAAK,EAAE,UAAU,EAAE;YAC/D,EAAE,IAAI,EAAE,uCAAuC,EAAE,KAAK,EAAE,UAAU,EAAE;YACpE,EAAE,IAAI,EAAE,kCAAkC,EAAE,KAAK,EAAE,YAAY,EAAE;SAClE;KACF,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC;QAC5B,OAAO,EAAE,2BAA2B;QACpC,OAAO,EAAE;YACP,EAAE,IAAI,EAAE,iCAAiC,EAAE,KAAK,EAAE,QAAQ,EAAE;YAC5D;gBACE,IAAI,EAAE,2CAA2C;gBACjD,KAAK,EAAE,eAAe;aACvB;YACD,EAAE,IAAI,EAAE,2CAA2C,EAAE,KAAK,EAAE,QAAQ,EAAE;SACvE;KACF,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC;QAC/B,OAAO,EAAE,qBAAqB;QAC9B,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,kDAAkD;gBACxD,KAAK,EAAE,UAAU;aAClB;YACD,EAAE,IAAI,EAAE,wCAAwC,EAAE,KAAK,EAAE,UAAU,EAAE;YACrE,EAAE,IAAI,EAAE,2CAA2C,EAAE,KAAK,EAAE,QAAQ,EAAE;SACvE;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC;QAC7B,OAAO,EAAE,kCAAkC;QAC3C,OAAO,EAAE,iBAAiB;KAC3B,CAAC,CAAC;IAEH,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC;YAC7B,OAAO,EAAE,GAAG,UAAU,6BAA6B;YACnD,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,KAAK,EAAE;gBACxC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE;aACpC;SACF,CAAC,CAAC;QACH,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YAC9B,OAAO;QACT,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC;QAC5B,UAAU;QACV,QAAQ;QACR,WAAW;QACX,UAAU;KACX,CAAC,CAAC;IACH,MAAM,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAE7C,OAAO,CAAC,GAAG,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;AAC3D,CAAC"}
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAQrC,MAAM,UAAU,GAAe;IAC7B;QACE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,yBAAyB;QACrC,IAAI,EAAE,UAAU;KACjB;IACD,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,EAAE;IAC1D,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE;CACjE,CAAC;AAEF,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,cAAc;IACd,MAAM;IACN,MAAM;IACN,OAAO;IACP,UAAU;IACV,QAAQ;IACR,OAAO;IACP,KAAK;IACL,QAAQ;CACT,CAAC,CAAC;AAEH,KAAK,UAAU,iBAAiB,CAC9B,GAAW,EACX,KAAkB;IAElB,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClC,MAAM,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YAC9C,IAAI,GAAG;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAW;IACpD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACpC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpE,CAAC;AASD,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ;YACX,OAAO;;;;;;;;;;;;;;;;;GAiBV,CAAC;QAEA,KAAK,eAAe;YAClB,OAAO;;;;;;;;;;;;;;;;;;GAkBV,CAAC;QAEA;YACE,OAAO;;;;;;;;;;;;;GAaV,CAAC;IACF,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAa;IAC5C,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,UAAU;YACb,OAAO;;;;;;;;;;;;;;;;;GAiBV,CAAC;QAEA,KAAK,UAAU;YACb,OAAO;;;;;;;;;;;;;;GAcV,CAAC;QAEA;YACE,OAAO;;;;;;;;;;;;;;;;;;;;;;GAsBV,CAAC;IACF,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAkB;IAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,MAAM;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/D,OAAO,iCAAiC,QAAQ,eAAe,CAAC,CAAC,UAAU,QAAQ,CAAC;IACtF,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,OAAoB;IAC1C,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1E,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAEhE,MAAM,YAAY,GAAG,CAAC,eAAe,EAAE,GAAG,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAElE,MAAM,eAAe,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/D,MAAM,kBAAkB,GAAG,uBAAuB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACxE,MAAM,eAAe,GAAG,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAE7D,OAAO;WACE,YAAY;;EAErB,eAAe;;EAEf,kBAAkB;;;;EAIlB,eAAe;;;;;;;;;;;CAWhB,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;IAE3C,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAEjE,IAAI,cAA0B,CAAC;IAE/B,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC;YAC/B,OAAO,EAAE,8CAA8C;YACvD,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE;gBAC1D,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,cAAc,GAAG,SAAuB,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;QACzE,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC;YACnC,OAAO,EAAE,wCAAwC;YACjD,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9B,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,UAAU,GAAG;gBACnC,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,cAAc,GAAG,aAA2B,CAAC;IAC/C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC;QAC5B,OAAO,EAAE,2BAA2B;QACpC,OAAO,EAAE;YACP,EAAE,IAAI,EAAE,iCAAiC,EAAE,KAAK,EAAE,QAAQ,EAAE;YAC5D;gBACE,IAAI,EAAE,2CAA2C;gBACjD,KAAK,EAAE,eAAe;aACvB;YACD,EAAE,IAAI,EAAE,2CAA2C,EAAE,KAAK,EAAE,QAAQ,EAAE;SACvE;KACF,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC;QAC/B,OAAO,EAAE,qBAAqB;QAC9B,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,kDAAkD;gBACxD,KAAK,EAAE,UAAU;aAClB;YACD,EAAE,IAAI,EAAE,wCAAwC,EAAE,KAAK,EAAE,UAAU,EAAE;YACrE,EAAE,IAAI,EAAE,2CAA2C,EAAE,KAAK,EAAE,QAAQ,EAAE;SACvE;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC;QAC7B,OAAO,EAAE,kCAAkC;QAC3C,OAAO,EAAE,iBAAiB;KAC3B,CAAC,CAAC;IAEH,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC;YAC7B,OAAO,EAAE,GAAG,UAAU,6BAA6B;YACnD,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,KAAK,EAAE;gBACxC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE;aACpC;SACF,CAAC,CAAC;QACH,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YAC9B,OAAO;QACT,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC;QAC5B,MAAM,EAAE,cAAc;QACtB,QAAQ;QACR,WAAW;QACX,UAAU;KACX,CAAC,CAAC;IACH,MAAM,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAE7C,OAAO,CAAC,GAAG,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;AAC3D,CAAC"}

package/dist/helpers/create-chunker.d.ts

@@ -1,9 +1,18 @@
-import { FileChunker, Chunk } from "../interfaces/index.js";
-export interface CreateChunkerOptions {
+import { FileChunker, ChunkStrategy, Chunk } from "../interfaces/index.js";
+type WithStrategy = {
+    strategy: ChunkStrategy;
+    process?: never;
+    name?: string;
+};
+type WithProcess = {
+    process: (content: string, filePath: string, commitHash: string) => Promise<Chunk[]>;
+    strategy?: never;
     name: string;
+};
+export type CreateChunkerOptions = {
     patterns: string[];
-    process: (content: string, filePath: string, commitHash: string) => Promise<Chunk[]>;
     canProcess?: (filePath: string, content?: string) => Promise<boolean>;
-}
+} & (WithStrategy | WithProcess);
 export declare function createChunker(options: CreateChunkerOptions): FileChunker;
+export {};
 //# sourceMappingURL=create-chunker.d.ts.map

package/dist/helpers/create-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-chunker.d.ts","sourceRoot":"","sources":["../../src/helpers/create-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAE5D,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,EAAE,CACP,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,KACf,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IACtB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACvE;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,WAAW,CA0BxE"}
+{"version":3,"file":"create-chunker.d.ts","sourceRoot":"","sources":["../../src/helpers/create-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAE3E,KAAK,YAAY,GAAG;IAClB,QAAQ,EAAE,aAAa,CAAC;IACxB,OAAO,CAAC,EAAE,KAAK,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,KAAK,WAAW,GAAG;IACjB,OAAO,EAAE,CACP,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,KACf,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IACtB,QAAQ,CAAC,EAAE,KAAK,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACvE,GAAG,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC;AAEjC,wBAAgB,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,WAAW,CA0CxE"}

package/dist/helpers/create-chunker.js

@@ -1,6 +1,17 @@
 export function createChunker(options) {
+    let name;
+    let processContent;
+    if (options.strategy != null) {
+        name = options.name ?? `${options.strategy.name}:${options.patterns[0]}`;
+        const strategy = options.strategy;
+        processContent = (content, filePath) => strategy.chunk(content, filePath);
+    }
+    else {
+        name = options.name;
+        processContent = options.process;
+    }
     return {
-        name: options.name,
+        name,
         patterns: options.patterns,
         async chunk(filePath, commitHash) {
             const { readFile } = await import("fs/promises");
@@ -11,7 +22,7 @@ export function createChunker(options) {
                     return [];
                 }
             }
-            return options.process(content, filePath, commitHash);
+            return processContent(content, filePath, commitHash);
         },
         async canProcess(filePath, content) {
             if (options.canProcess) {

package/dist/helpers/create-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"create-chunker.js","sourceRoot":"","sources":["../../src/helpers/create-chunker.ts"],"names":[],"mappings":"AAaA,MAAM,UAAU,aAAa,CAAC,OAA6B;IACzD,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAE1B,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,UAAkB;YAC9C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAElD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC/D,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;YAED,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QACxD,CAAC;QAED,KAAK,CAAC,UAAU,CAAC,QAAgB,EAAE,OAAgB;YACjD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC/C,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"create-chunker.js","sourceRoot":"","sources":["../../src/helpers/create-chunker.ts"],"names":[],"mappings":"AAuBA,MAAM,UAAU,aAAa,CAAC,OAA6B;IACzD,IAAI,IAAY,CAAC;IACjB,IAAI,cAIiB,CAAC;IAEtB,IAAI,OAAO,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;QAC7B,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,cAAc,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC5E,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACpB,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;IACnC,CAAC;IAED,OAAO;QACL,IAAI;QACJ,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAE1B,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,UAAkB;YAC9C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAElD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC/D,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;YAED,OAAO,cAAc,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QACvD,CAAC;QAED,KAAK,CAAC,UAAU,CAAC,QAAgB,EAAE,OAAgB;YACjD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC/C,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vivantel/rag-core",
-  "version": "2.0.0",
+  "version": "2.1.1",
   "type": "module",
   "workspaces": [
     "packages/*"