@vivantel/rag-core 1.1.3 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -26
- package/dist/bin/rag-update.d.ts +3 -0
- package/dist/bin/rag-update.d.ts.map +1 -0
- package/dist/bin/rag-update.js +116 -0
- package/dist/bin/rag-update.js.map +1 -0
- package/dist/cli/init.d.ts +9 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +305 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/validate.d.ts +2 -0
- package/dist/cli/validate.d.ts.map +1 -0
- package/dist/cli/validate.js +54 -0
- package/dist/cli/validate.js.map +1 -0
- package/dist/config-loader.d.ts.map +1 -1
- package/dist/config-loader.js +73 -5
- package/dist/config-loader.js.map +1 -1
- package/dist/core/chunk-processor.d.ts +1 -1
- package/dist/core/chunk-processor.d.ts.map +1 -1
- package/dist/core/chunk-processor.js +23 -1
- package/dist/core/chunk-processor.js.map +1 -1
- package/dist/core/embedder.d.ts +5 -1
- package/dist/core/embedder.d.ts.map +1 -1
- package/dist/core/embedder.js +33 -29
- package/dist/core/embedder.js.map +1 -1
- package/dist/core/errors.d.ts +16 -0
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/errors.js +17 -0
- package/dist/core/errors.js.map +1 -0
- package/dist/core/orchestrator.d.ts +8 -0
- package/dist/core/orchestrator.d.ts.map +1 -1
- package/dist/core/orchestrator.js +134 -39
- package/dist/core/orchestrator.js.map +1 -1
- package/dist/core/plugin-discovery.d.ts +19 -0
- package/dist/core/plugin-discovery.d.ts.map +1 -0
- package/dist/core/plugin-discovery.js +47 -0
- package/dist/core/plugin-discovery.js.map +1 -0
- package/dist/core/telemetry.d.ts +61 -0
- package/dist/core/telemetry.d.ts.map +1 -0
- package/dist/core/telemetry.js +50 -0
- package/dist/core/telemetry.js.map +1 -0
- package/dist/core/uploader.d.ts +5 -1
- package/dist/core/uploader.d.ts.map +1 -1
- package/dist/core/uploader.js +11 -4
- package/dist/core/uploader.js.map +1 -1
- package/dist/core/utils.d.ts +7 -0
- package/dist/core/utils.d.ts.map +1 -1
- package/dist/core/utils.js +35 -0
- package/dist/core/utils.js.map +1 -1
- package/dist/helpers/create-chunker.d.ts +13 -4
- package/dist/helpers/create-chunker.d.ts.map +1 -1
- package/dist/helpers/create-chunker.js +13 -2
- package/dist/helpers/create-chunker.js.map +1 -1
- package/dist/index.d.ts +14 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -2
- package/dist/index.js.map +1 -1
- package/dist/interfaces/embedder.d.ts +2 -0
- package/dist/interfaces/embedder.d.ts.map +1 -1
- package/dist/interfaces/vector-store.d.ts +2 -0
- package/dist/interfaces/vector-store.d.ts.map +1 -1
- package/package.json +8 -2
package/README.md
CHANGED
|
@@ -15,34 +15,54 @@ npm install @vivantel/rag-core
|
|
|
15
15
|
|
|
16
16
|
## Quick Start
|
|
17
17
|
|
|
18
|
-
### 1.
|
|
18
|
+
### 1. Generate a config (recommended)
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
rag-update init
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Scans your project for known file types and generates a `rag.config.ts` with the right strategy per pattern. You then fill in your embedding provider and vector store.
|
|
25
|
+
|
|
26
|
+
### 2. Or write a config manually (`rag.config.ts`)
|
|
19
27
|
|
|
20
28
|
```typescript
|
|
21
|
-
import {
|
|
22
|
-
|
|
29
|
+
import {
|
|
30
|
+
RAGPipelineConfig,
|
|
31
|
+
createChunker,
|
|
32
|
+
markdownHeadersStrategy,
|
|
33
|
+
tokenStrategy,
|
|
34
|
+
} from '@vivantel/rag-core';
|
|
23
35
|
|
|
24
|
-
|
|
36
|
+
const config: RAGPipelineConfig = {
|
|
25
37
|
chunkers: [
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return [{
|
|
31
|
-
content,
|
|
32
|
-
metadata: { type: 'doc', file: filePath },
|
|
33
|
-
sourceFile: filePath,
|
|
34
|
-
commitHash
|
|
35
|
-
}];
|
|
36
|
-
}
|
|
37
|
-
})
|
|
38
|
+
// Different directories can use different strategies
|
|
39
|
+
createChunker({ patterns: ['docs/**/*.md'], strategy: markdownHeadersStrategy() }),
|
|
40
|
+
createChunker({ patterns: ['rules/**/*.md'], strategy: wholeFileStrategy() }),
|
|
41
|
+
createChunker({ patterns: ['src/**/*.ts', 'src/**/*.tsx'], strategy: tokenStrategy() }),
|
|
38
42
|
],
|
|
39
|
-
|
|
40
43
|
embedder: new MyEmbedderProvider({ apiKey: process.env.API_KEY }),
|
|
41
|
-
vectorStore: new MyVectorStore({ url: process.env.DB_URL })
|
|
42
|
-
}
|
|
44
|
+
vectorStore: new MyVectorStore({ url: process.env.DB_URL }),
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export default config;
|
|
43
48
|
```
|
|
44
49
|
|
|
45
|
-
|
|
50
|
+
For custom chunking logic, use the `process` callback instead of `strategy`:
|
|
51
|
+
|
|
52
|
+
```typescript
|
|
53
|
+
createChunker({
|
|
54
|
+
name: 'custom',
|
|
55
|
+
patterns: ['**/*.txt'],
|
|
56
|
+
process: async (content, filePath, commitHash) => [{
|
|
57
|
+
content,
|
|
58
|
+
metadata: { file: filePath },
|
|
59
|
+
sourceFile: filePath,
|
|
60
|
+
commitHash,
|
|
61
|
+
}],
|
|
62
|
+
})
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### 3. Run the pipeline
|
|
46
66
|
|
|
47
67
|
```bash
|
|
48
68
|
rag-update --config rag.config.ts
|
|
@@ -50,12 +70,12 @@ rag-update --config rag.config.ts
|
|
|
50
70
|
|
|
51
71
|
## Built-in Strategies
|
|
52
72
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
73
|
+
| Factory | Best for |
|
|
74
|
+
|---------|----------|
|
|
75
|
+
| `tokenStrategy({ maxTokens?, overlap? })` | Source code, structured text |
|
|
76
|
+
| `markdownHeadersStrategy()` | Markdown documentation |
|
|
77
|
+
| `semanticStrategy()` | Prose, articles |
|
|
78
|
+
| `wholeFileStrategy()` | Small configs, rule files, YAML |
|
|
59
79
|
|
|
60
80
|
## License
|
|
61
81
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rag-update.d.ts","sourceRoot":"","sources":["../../src/bin/rag-update.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { Command } from "commander";
|
|
3
|
+
import { config } from "dotenv";
|
|
4
|
+
import { loadConfig } from "../config-loader.js";
|
|
5
|
+
import { Orchestrator } from "../core/orchestrator.js";
|
|
6
|
+
import { RagError } from "../core/errors.js";
|
|
7
|
+
import { runInit } from "../cli/init.js";
|
|
8
|
+
import { runValidate } from "../cli/validate.js";
|
|
9
|
+
config();
|
|
10
|
+
const program = new Command();
|
|
11
|
+
function handleError(error) {
|
|
12
|
+
console.error("❌ Error:", error instanceof Error ? error.message : error);
|
|
13
|
+
if (error instanceof RagError && error.suggestion) {
|
|
14
|
+
console.error(" 💡", error.suggestion);
|
|
15
|
+
}
|
|
16
|
+
process.exit(1);
|
|
17
|
+
}
|
|
18
|
+
async function runOnce(options) {
|
|
19
|
+
const cfg = await loadConfig(options.config);
|
|
20
|
+
const orchestrator = new Orchestrator({
|
|
21
|
+
...cfg,
|
|
22
|
+
options: {
|
|
23
|
+
...cfg.options,
|
|
24
|
+
force: options.force || cfg.options?.force,
|
|
25
|
+
skipUpload: options.noUpload || cfg.options?.skipUpload,
|
|
26
|
+
dryRun: options.dryRun || cfg.options?.dryRun,
|
|
27
|
+
chunksFile: options.chunksOut || cfg.options?.chunksFile,
|
|
28
|
+
embeddingsFile: options.embeddingsOut || cfg.options?.embeddingsFile,
|
|
29
|
+
},
|
|
30
|
+
});
|
|
31
|
+
await orchestrator.run();
|
|
32
|
+
}
|
|
33
|
+
program
|
|
34
|
+
.name("rag-update")
|
|
35
|
+
.description("Update RAG index with latest changes")
|
|
36
|
+
.version("2.0.0")
|
|
37
|
+
.option("-c, --config <path>", "Path to config file", "./rag.config.ts")
|
|
38
|
+
.option("-f, --force", "Force full rebuild", false)
|
|
39
|
+
.option("--no-upload", "Skip upload to vector store", false)
|
|
40
|
+
.option("--dry-run", "Show what would change without uploading", false)
|
|
41
|
+
.option("--chunks-out <path>", "Output path for chunks.json")
|
|
42
|
+
.option("--embeddings-out <path>", "Output path for embeddings.json")
|
|
43
|
+
.option("--watch", "Re-run pipeline on file changes", false)
|
|
44
|
+
.action(async () => {
|
|
45
|
+
const opts = program.opts();
|
|
46
|
+
const runOptions = {
|
|
47
|
+
config: opts.config,
|
|
48
|
+
force: opts.force,
|
|
49
|
+
noUpload: !opts.upload,
|
|
50
|
+
dryRun: opts.dryRun,
|
|
51
|
+
chunksOut: opts.chunksOut,
|
|
52
|
+
embeddingsOut: opts.embeddingsOut,
|
|
53
|
+
};
|
|
54
|
+
console.log("🚀 RAG Update Tool\n");
|
|
55
|
+
try {
|
|
56
|
+
await runOnce(runOptions);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
handleError(error);
|
|
60
|
+
}
|
|
61
|
+
if (!opts.watch)
|
|
62
|
+
return;
|
|
63
|
+
// Watch mode
|
|
64
|
+
const { default: chokidar } = await import("chokidar");
|
|
65
|
+
const cfg = await loadConfig(opts.config).catch(() => null);
|
|
66
|
+
const patterns = cfg
|
|
67
|
+
? cfg.chunkers.flatMap((c) => c.patterns)
|
|
68
|
+
: [];
|
|
69
|
+
const watched = [opts.config, ...patterns];
|
|
70
|
+
console.log("\n👁️ Watching for changes...");
|
|
71
|
+
let debounce = null;
|
|
72
|
+
chokidar
|
|
73
|
+
.watch(watched, { ignoreInitial: true })
|
|
74
|
+
.on("all", (event, path) => {
|
|
75
|
+
if (debounce)
|
|
76
|
+
clearTimeout(debounce);
|
|
77
|
+
debounce = setTimeout(async () => {
|
|
78
|
+
console.log(`\n🔄 Change detected (${event}: ${path}), re-running...\n`);
|
|
79
|
+
try {
|
|
80
|
+
await runOnce(runOptions);
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
console.error("❌ Error:", error instanceof Error ? error.message : error);
|
|
84
|
+
}
|
|
85
|
+
}, 500);
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
program
|
|
89
|
+
.command("init")
|
|
90
|
+
.description("Generate a rag.config.ts template interactively")
|
|
91
|
+
.action(async () => {
|
|
92
|
+
try {
|
|
93
|
+
await runInit();
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
if (error.name === "ExitPromptError") {
|
|
97
|
+
console.log("\nCancelled.");
|
|
98
|
+
process.exit(0);
|
|
99
|
+
}
|
|
100
|
+
handleError(error);
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
program
|
|
104
|
+
.command("validate")
|
|
105
|
+
.description("Validate config without running the pipeline")
|
|
106
|
+
.option("-c, --config <path>", "Path to config file", "./rag.config.ts")
|
|
107
|
+
.action(async (opts) => {
|
|
108
|
+
try {
|
|
109
|
+
await runValidate(opts.config);
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
handleError(error);
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
program.parse();
|
|
116
|
+
//# sourceMappingURL=rag-update.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rag-update.js","sourceRoot":"","sources":["../../src/bin/rag-update.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD,MAAM,EAAE,CAAC;AAET,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1E,IAAI,KAAK,YAAY,QAAQ,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QAClD,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,OAOtB;IACC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC;QACpC,GAAG,GAAG;QACN,OAAO,EAAE;YACP,GAAG,GAAG,CAAC,OAAO;YACd,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,OAAO,EAAE,KAAK;YAC1C,UAAU,EAAE,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,OAAO,EAAE,UAAU;YACvD,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,MAAM;YAC7C,UAAU,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,OAAO,EAAE,UAAU;YACxD,cAAc,EAAE,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC,OAAO,EAAE,cAAc;SACrE;KACF,CAAC,CAAC;IACH,MAAM,YAAY,CAAC,GAAG,EAAE,CAAC;AAC3B,CAAC;AAED,OAAO;KACJ,IAAI,CAAC,YAAY,CAAC;KAClB,WAAW,CAAC,sCAAsC,CAAC;KACnD,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,CAAC,qBAAqB,EAAE,qBAAqB,EAAE,iBAAiB,CAAC;KACvE,MAAM,CAAC,aAAa,EAAE,oBAAoB,EAAE,KAAK,CAAC;KAClD,MAAM,CAAC,aAAa,EAAE,6BAA6B,EAAE,KAAK,CAAC;KAC3D,MAAM,CAAC,WAAW,EAAE,0CAA0C,EAAE,KAAK,CAAC;KACtE,MAAM,CAAC,qBAAqB,EAAE,6BAA6B,CAAC;KAC5D,MAAM,CAAC,yBAAyB,EAAE,iCAAiC,CAAC;KACpE,MAAM,CAAC,SAAS,EAAE,iCAAiC,EAAE,KAAK,CAAC;KAC3D,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAQrB,CAAC;IAEL,MAAM,UAAU,GAAG;QACjB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,QAAQ,EAAE,CAAC,IAAI,CAAC,MAAM;QACtB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;KAClC,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAEpC,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,WAAW,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,KAAK;QAAE,OAAO;IAExB,aAAa;IACb,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAa,GAAG;QAC5B,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;QACzC,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAE9C,IAAI,QAAQ,GAAyC,IAAI,CAAC;IAC1D,QAAQ;SACL,KAAK,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;SACvC,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QACzB,IAAI,QAAQ;YAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;QACrC,QAAQ,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;YAC/B,OAAO,CAAC,GAAG,CACT,yBAAyB,KAAK,KAAK,IAAI,oBAAoB,CAC5D,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CACX,UAAU,EACV,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAC/C,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,GAAG,CAAC,CAAC;IACV,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,iDAAiD,CAAC;KAC9D,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,IAAI,CAAC;QACH,MAAM,OAAO,EAAE,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAChE,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,WAAW,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,UAAU,CAAC;KACnB,WAAW,CAAC,8CAA8C,CAAC;KAC3D,MAAM,CAAC,qBAAqB,EAAE,qBAAqB,EAAE,iBAAiB,CAAC;KACvE,MAAM,CAAC,KAAK,EAAE,IAAwB,EAAE,EAAE;IACzC,IAAI,CAAC;QACH,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,WAAW,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
interface ExtGroup {
|
|
2
|
+
exts: string[];
|
|
3
|
+
strategyFn: string;
|
|
4
|
+
name: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function detectFileExtensions(cwd: string): Promise<ExtGroup[]>;
|
|
7
|
+
export declare function runInit(): Promise<void>;
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=init.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAMA,UAAU,QAAQ;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAoDD,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAI3E;AA4LD,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAsF7C"}
|
package/dist/cli/init.js
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
import { checkbox, select, input } from "@inquirer/prompts";
|
|
2
|
+
import { writeFile } from "fs/promises";
|
|
3
|
+
import { existsSync } from "fs";
|
|
4
|
+
import { readdir } from "fs/promises";
|
|
5
|
+
import { join, extname } from "path";
|
|
6
|
+
const EXT_GROUPS = [
|
|
7
|
+
{
|
|
8
|
+
exts: [".md", ".mdx"],
|
|
9
|
+
strategyFn: "markdownHeadersStrategy",
|
|
10
|
+
name: "markdown",
|
|
11
|
+
},
|
|
12
|
+
{ exts: [".ts", ".tsx"], strategyFn: "tokenStrategy", name: "typescript" },
|
|
13
|
+
{ exts: [".js", ".jsx"], strategyFn: "tokenStrategy", name: "javascript" },
|
|
14
|
+
{ exts: [".py"], strategyFn: "tokenStrategy", name: "python" },
|
|
15
|
+
{ exts: [".go"], strategyFn: "tokenStrategy", name: "go" },
|
|
16
|
+
{ exts: [".cs"], strategyFn: "tokenStrategy", name: "csharp" },
|
|
17
|
+
{ exts: [".java"], strategyFn: "tokenStrategy", name: "java" },
|
|
18
|
+
{ exts: [".yaml", ".yml"], strategyFn: "wholeFileStrategy", name: "yaml" },
|
|
19
|
+
{ exts: [".txt"], strategyFn: "semanticStrategy", name: "text" },
|
|
20
|
+
];
|
|
21
|
+
const IGNORED_DIRS = new Set([
|
|
22
|
+
"node_modules",
|
|
23
|
+
".git",
|
|
24
|
+
"dist",
|
|
25
|
+
"build",
|
|
26
|
+
"coverage",
|
|
27
|
+
".cache",
|
|
28
|
+
".next",
|
|
29
|
+
"out",
|
|
30
|
+
".turbo",
|
|
31
|
+
]);
|
|
32
|
+
async function collectExtensions(dir, found) {
|
|
33
|
+
let entries;
|
|
34
|
+
try {
|
|
35
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
for (const entry of entries) {
|
|
41
|
+
if (entry.isDirectory()) {
|
|
42
|
+
if (!IGNORED_DIRS.has(entry.name)) {
|
|
43
|
+
await collectExtensions(join(dir, entry.name), found);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
const ext = extname(entry.name).toLowerCase();
|
|
48
|
+
if (ext)
|
|
49
|
+
found.add(ext);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
export async function detectFileExtensions(cwd) {
|
|
54
|
+
const found = new Set();
|
|
55
|
+
await collectExtensions(cwd, found);
|
|
56
|
+
return EXT_GROUPS.filter((g) => g.exts.some((e) => found.has(e)));
|
|
57
|
+
}
|
|
58
|
+
function buildEmbedderSection(provider) {
|
|
59
|
+
switch (provider) {
|
|
60
|
+
case "openai":
|
|
61
|
+
return `// Install: npm install openai
|
|
62
|
+
// import OpenAI from 'openai';
|
|
63
|
+
// const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
|
64
|
+
|
|
65
|
+
const embedder: EmbeddingProvider = {
|
|
66
|
+
name: 'openai',
|
|
67
|
+
dimensions: 1536, // text-embedding-3-small
|
|
68
|
+
async embed(text: string): Promise<number[]> {
|
|
69
|
+
// const res = await openai.embeddings.create({ model: 'text-embedding-3-small', input: text });
|
|
70
|
+
// return res.data[0].embedding;
|
|
71
|
+
throw new Error('Configure your OpenAI embedder — uncomment and adapt the lines above');
|
|
72
|
+
},
|
|
73
|
+
async embedBatch(texts: string[]): Promise<number[][]> {
|
|
74
|
+
// const res = await openai.embeddings.create({ model: 'text-embedding-3-small', input: texts });
|
|
75
|
+
// return res.data.map(d => d.embedding);
|
|
76
|
+
throw new Error('Configure your OpenAI embedder — uncomment and adapt the lines above');
|
|
77
|
+
},
|
|
78
|
+
};`;
|
|
79
|
+
case "github-models":
|
|
80
|
+
return `// Requires GITHUB_TOKEN in your environment (.env or CI secret)
|
|
81
|
+
|
|
82
|
+
const embedder: EmbeddingProvider = {
|
|
83
|
+
name: 'github-models',
|
|
84
|
+
dimensions: 1536, // text-embedding-3-small
|
|
85
|
+
async embed(text: string): Promise<number[]> {
|
|
86
|
+
const res = await fetch('https://models.inference.ai.azure.com/embeddings', {
|
|
87
|
+
method: 'POST',
|
|
88
|
+
headers: {
|
|
89
|
+
'Content-Type': 'application/json',
|
|
90
|
+
Authorization: \`Bearer \${process.env.GITHUB_TOKEN}\`,
|
|
91
|
+
},
|
|
92
|
+
body: JSON.stringify({ model: 'text-embedding-3-small', input: text }),
|
|
93
|
+
});
|
|
94
|
+
if (!res.ok) throw new Error(\`GitHub Models error: \${res.statusText}\`);
|
|
95
|
+
const json = await res.json() as { data: Array<{ embedding: number[] }> };
|
|
96
|
+
return json.data[0].embedding;
|
|
97
|
+
},
|
|
98
|
+
};`;
|
|
99
|
+
default:
|
|
100
|
+
return `const embedder: EmbeddingProvider = {
|
|
101
|
+
name: 'custom',
|
|
102
|
+
dimensions: 1536, // update to match your model's output size
|
|
103
|
+
async embed(text: string): Promise<number[]> {
|
|
104
|
+
// TODO: call your embedding API and return a number[]
|
|
105
|
+
void text;
|
|
106
|
+
throw new Error('Not implemented');
|
|
107
|
+
},
|
|
108
|
+
// Optional batch method for better performance:
|
|
109
|
+
// async embedBatch(texts: string[]): Promise<number[][]> {
|
|
110
|
+
// // TODO: batch embed and return number[][]
|
|
111
|
+
// throw new Error('Not implemented');
|
|
112
|
+
// },
|
|
113
|
+
};`;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
function buildVectorStoreSection(store) {
|
|
117
|
+
switch (store) {
|
|
118
|
+
case "supabase":
|
|
119
|
+
return `// Install: npm install @vivantel/rag-store-supabase @supabase/supabase-js
|
|
120
|
+
// import { SupabaseVectorStore } from '@vivantel/rag-store-supabase';
|
|
121
|
+
// const vectorStore = new SupabaseVectorStore({
|
|
122
|
+
// url: process.env.SUPABASE_URL!,
|
|
123
|
+
// key: process.env.SUPABASE_KEY!,
|
|
124
|
+
// table: 'documents',
|
|
125
|
+
// });
|
|
126
|
+
|
|
127
|
+
const vectorStore: VectorStore = {
|
|
128
|
+
name: 'supabase',
|
|
129
|
+
async initialize() {
|
|
130
|
+
throw new Error('Configure your Supabase vector store — uncomment the lines above');
|
|
131
|
+
},
|
|
132
|
+
async upsert(_docs) { throw new Error('Not implemented'); },
|
|
133
|
+
async deleteBySourceFile(_files) { throw new Error('Not implemented'); },
|
|
134
|
+
async getCurrentState() { return new Map(); },
|
|
135
|
+
async search(_embedding, _topK) { return []; },
|
|
136
|
+
};`;
|
|
137
|
+
case "pinecone":
|
|
138
|
+
return `// Install: npm install @pinecone-database/pinecone
|
|
139
|
+
// import { Pinecone } from '@pinecone-database/pinecone';
|
|
140
|
+
// const pc = new Pinecone({ apiKey: process.env.PINECONE_API_KEY! });
|
|
141
|
+
// const index = pc.index(process.env.PINECONE_INDEX!);
|
|
142
|
+
|
|
143
|
+
const vectorStore: VectorStore = {
|
|
144
|
+
name: 'pinecone',
|
|
145
|
+
async initialize() {
|
|
146
|
+
throw new Error('Configure your Pinecone vector store — uncomment the lines above');
|
|
147
|
+
},
|
|
148
|
+
async upsert(_docs) { throw new Error('Not implemented'); },
|
|
149
|
+
async deleteBySourceFile(_files) { throw new Error('Not implemented'); },
|
|
150
|
+
async getCurrentState() { return new Map(); },
|
|
151
|
+
async search(_embedding, _topK) { return []; },
|
|
152
|
+
};`;
|
|
153
|
+
default:
|
|
154
|
+
return `const vectorStore: VectorStore = {
|
|
155
|
+
name: 'custom',
|
|
156
|
+
async initialize(): Promise<void> {
|
|
157
|
+
// TODO: initialize your vector store connection
|
|
158
|
+
},
|
|
159
|
+
async upsert(docs): Promise<void> {
|
|
160
|
+
// TODO: store \`docs\` in your vector database
|
|
161
|
+
void docs;
|
|
162
|
+
},
|
|
163
|
+
async deleteBySourceFile(files): Promise<void> {
|
|
164
|
+
// TODO: remove all documents whose sourceFile is in \`files\`
|
|
165
|
+
void files;
|
|
166
|
+
},
|
|
167
|
+
async getCurrentState(): Promise<Map<string, string>> {
|
|
168
|
+
// TODO: return Map<sourceFile, commitHash> of what is currently stored
|
|
169
|
+
return new Map();
|
|
170
|
+
},
|
|
171
|
+
async search(embedding, topK): Promise<VectorSearchResult[]> {
|
|
172
|
+
// TODO: return the topK most similar documents
|
|
173
|
+
void embedding; void topK;
|
|
174
|
+
return [];
|
|
175
|
+
},
|
|
176
|
+
};`;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
function buildChunkersSection(groups) {
|
|
180
|
+
if (groups.length === 0) {
|
|
181
|
+
groups = [EXT_GROUPS.find((g) => g.name === "typescript")];
|
|
182
|
+
}
|
|
183
|
+
return groups
|
|
184
|
+
.map((g) => {
|
|
185
|
+
const patterns = JSON.stringify(g.exts.map((e) => `**/*${e}`));
|
|
186
|
+
return ` createChunker({ patterns: ${patterns}, strategy: ${g.strategyFn}() }),`;
|
|
187
|
+
})
|
|
188
|
+
.join("\n");
|
|
189
|
+
}
|
|
190
|
+
function generateConfig(answers) {
|
|
191
|
+
const strategyFns = [...new Set(answers.groups.map((g) => g.strategyFn))];
|
|
192
|
+
if (strategyFns.length === 0)
|
|
193
|
+
strategyFns.push("tokenStrategy");
|
|
194
|
+
const imports = [
|
|
195
|
+
"RAGPipelineConfig",
|
|
196
|
+
"EmbeddingProvider",
|
|
197
|
+
"VectorStore",
|
|
198
|
+
"VectorSearchResult",
|
|
199
|
+
"createChunker",
|
|
200
|
+
...strategyFns,
|
|
201
|
+
].join(", ");
|
|
202
|
+
const embedderSection = buildEmbedderSection(answers.embedder);
|
|
203
|
+
const vectorStoreSection = buildVectorStoreSection(answers.vectorStore);
|
|
204
|
+
const chunkersSection = buildChunkersSection(answers.groups);
|
|
205
|
+
return `import { ${imports} } from '@vivantel/rag-core';
|
|
206
|
+
|
|
207
|
+
${embedderSection}
|
|
208
|
+
|
|
209
|
+
${vectorStoreSection}
|
|
210
|
+
|
|
211
|
+
const config: RAGPipelineConfig = {
|
|
212
|
+
chunkers: [
|
|
213
|
+
${chunkersSection}
|
|
214
|
+
],
|
|
215
|
+
embedder,
|
|
216
|
+
vectorStore,
|
|
217
|
+
options: {
|
|
218
|
+
chunksFile: './docs/rag/chunks.json',
|
|
219
|
+
embeddingsFile: './docs/rag/embeddings.json',
|
|
220
|
+
},
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
export default config;
|
|
224
|
+
`;
|
|
225
|
+
}
|
|
226
|
+
export async function runInit() {
|
|
227
|
+
console.log("\n🚀 RAG Config Generator\n");
|
|
228
|
+
console.log("Scanning project for file types...");
|
|
229
|
+
const detectedGroups = await detectFileExtensions(process.cwd());
|
|
230
|
+
let selectedGroups;
|
|
231
|
+
if (detectedGroups.length > 0) {
|
|
232
|
+
const confirmed = await checkbox({
|
|
233
|
+
message: "Detected file types — select which to index:",
|
|
234
|
+
choices: detectedGroups.map((g) => ({
|
|
235
|
+
name: `${g.name} (${g.exts.join(", ")}) → ${g.strategyFn}`,
|
|
236
|
+
value: g,
|
|
237
|
+
checked: true,
|
|
238
|
+
})),
|
|
239
|
+
});
|
|
240
|
+
selectedGroups = confirmed;
|
|
241
|
+
}
|
|
242
|
+
else {
|
|
243
|
+
console.log("No known file types detected. Choose strategies manually:");
|
|
244
|
+
const strategyNames = await checkbox({
|
|
245
|
+
message: "Which chunking strategies do you need?",
|
|
246
|
+
choices: EXT_GROUPS.map((g) => ({
|
|
247
|
+
name: `${g.name} (${g.strategyFn})`,
|
|
248
|
+
value: g,
|
|
249
|
+
})),
|
|
250
|
+
});
|
|
251
|
+
selectedGroups = strategyNames;
|
|
252
|
+
}
|
|
253
|
+
const embedder = await select({
|
|
254
|
+
message: "Which embedding provider?",
|
|
255
|
+
choices: [
|
|
256
|
+
{ name: "OpenAI (text-embedding-3-small)", value: "openai" },
|
|
257
|
+
{
|
|
258
|
+
name: "GitHub Models (Azure-compatible endpoint)",
|
|
259
|
+
value: "github-models",
|
|
260
|
+
},
|
|
261
|
+
{ name: "Custom (implement the interface yourself)", value: "custom" },
|
|
262
|
+
],
|
|
263
|
+
});
|
|
264
|
+
const vectorStore = await select({
|
|
265
|
+
message: "Which vector store?",
|
|
266
|
+
choices: [
|
|
267
|
+
{
|
|
268
|
+
name: "Supabase pgvector (@vivantel/rag-store-supabase)",
|
|
269
|
+
value: "supabase",
|
|
270
|
+
},
|
|
271
|
+
{ name: "Pinecone (@pinecone-database/pinecone)", value: "pinecone" },
|
|
272
|
+
{ name: "Custom (implement the interface yourself)", value: "custom" },
|
|
273
|
+
],
|
|
274
|
+
});
|
|
275
|
+
const outputPath = await input({
|
|
276
|
+
message: "Output path for the config file?",
|
|
277
|
+
default: "./rag.config.ts",
|
|
278
|
+
});
|
|
279
|
+
if (existsSync(outputPath)) {
|
|
280
|
+
const overwrite = await select({
|
|
281
|
+
message: `${outputPath} already exists. Overwrite?`,
|
|
282
|
+
choices: [
|
|
283
|
+
{ name: "Yes, overwrite", value: "yes" },
|
|
284
|
+
{ name: "No, cancel", value: "no" },
|
|
285
|
+
],
|
|
286
|
+
});
|
|
287
|
+
if (overwrite === "no") {
|
|
288
|
+
console.log("\n❌ Cancelled.");
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
const config = generateConfig({
|
|
293
|
+
groups: selectedGroups,
|
|
294
|
+
embedder,
|
|
295
|
+
vectorStore,
|
|
296
|
+
outputPath,
|
|
297
|
+
});
|
|
298
|
+
await writeFile(outputPath, config, "utf-8");
|
|
299
|
+
console.log(`\n✅ Created ${outputPath}`);
|
|
300
|
+
console.log("\nNext steps:");
|
|
301
|
+
console.log(" 1. Fill in the TODO sections in the config file");
|
|
302
|
+
console.log(" 2. Run `rag-update validate` to check the config");
|
|
303
|
+
console.log(" 3. Run `rag-update` to start indexing\n");
|
|
304
|
+
}
|
|
305
|
+
//# sourceMappingURL=init.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAQrC,MAAM,UAAU,GAAe;IAC7B;QACE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,yBAAyB;QACrC,IAAI,EAAE,UAAU;KACjB;IACD,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,EAAE;IAC1D,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE;CACjE,CAAC;AAEF,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,cAAc;IACd,MAAM;IACN,MAAM;IACN,OAAO;IACP,UAAU;IACV,QAAQ;IACR,OAAO;IACP,KAAK;IACL,QAAQ;CACT,CAAC,CAAC;AAEH,KAAK,UAAU,iBAAiB,CAC9B,GAAW,EACX,KAAkB;IAElB,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClC,MAAM,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YAC9C,IAAI,GAAG;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAW;IACpD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACpC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpE,CAAC;AASD,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ;YACX,OAAO;;;;;;;;;;;;;;;;;GAiBV,CAAC;QAEA,KAAK,eAAe;YAClB,OAAO;;;;;;;;;;;;;;;;;;GAkBV,CAAC;QAEA;YACE,OAAO;;;;;;;;;;;;;GAaV,CAAC;IACF,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAa;IAC5C,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,UAAU;YACb,OAAO;;;;;;;;;;;;;;;;;GAiBV,CAAC;QAEA,KAAK,UAAU;YACb,OAAO;;;;;;;;;;;;;;GAcV,CAAC;QAEA;YACE,OAAO;;;;;;;;;;;;;;;;;;;;;;GAsBV,CAAC;IACF,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAkB;IAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,MAAM;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/D,OAAO,iCAAiC,QAAQ,eAAe,CAAC,CAAC,UAAU,QAAQ,CAAC;IACtF,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,OAAoB;IAC1C,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1E,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAEhE,MAAM,OAAO,GAAG;QACd,mBAAmB;QACnB,mBAAmB;QACnB,aAAa;QACb,oBAAoB;QACpB,eAAe;QACf,GAAG,WAAW;KACf,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,MAAM,eAAe,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/D,MAAM,kBAAkB,GAAG,uBAAuB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACxE,MAAM,eAAe,GAAG,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAE7D,OAAO,YAAY,OAAO;;EAE1B,eAAe;;EAEf,kBAAkB;;;;EAIlB,eAAe;;;;;;;;;;;CAWhB,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;IAE3C,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAEjE,IAAI,cAA0B,CAAC;IAE/B,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC;YAC/B,OAAO,EAAE,8CAA8C;YACvD,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE;gBAC1D,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,cAAc,GAAG,SAAuB,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;QACzE,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC;YACnC,OAAO,EAAE,wCAAwC;YACjD,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9B,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,UAAU,GAAG;gBACnC,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,cAAc,GAAG,aAA2B,CAAC;IAC/C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC;QAC5B,OAAO,EAAE,2BAA2B;QACpC,OAAO,EAAE;YACP,EAAE,IAAI,EAAE,iCAAiC,EAAE,KAAK,EAAE,QAAQ,EAAE;YAC5D;gBACE,IAAI,EAAE,2CAA2C;gBACjD,KAAK,EAAE,eAAe;aACvB;YACD,EAAE,IAAI,EAAE,2CAA2C,EAAE,KAAK,EAAE,QAAQ,EAAE;SACvE;KACF,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC;QAC/B,OAAO,EAAE,qBAAqB;QAC9B,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,kDAAkD;gBACxD,KAAK,EAAE,UAAU;aAClB;YACD,EAAE,IAAI,EAAE,wCAAwC,EAAE,KAAK,EAAE,UAAU,EAAE;YACrE,EAAE,IAAI,EAAE,2CAA2C,EAAE,KAAK,EAAE,QAAQ,EAAE;SACvE;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC;QAC7B,OAAO,EAAE,kCAAkC;QAC3C,OAAO,EAAE,iBAAiB;KAC3B,CAAC,CAAC;IAEH,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC;YAC7B,OAAO,EAAE,GAAG,UAAU,6BAA6B;YACnD,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,KAAK,EAAE;gBACxC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE;aACpC;SACF,CAAC,CAAC;QACH,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YAC9B,OAAO;QACT,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC;QAC5B,MAAM,EAAE,cAAc;QACtB,QAAQ;QACR,WAAW;QACX,UAAU;KACX,CAAC,CAAC;IACH,MAAM,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAE7C,OAAO,CAAC,GAAG,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;AAC3D,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/cli/validate.ts"],"names":[],"mappings":"AAKA,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAwDnE"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { glob } from "glob";
|
|
2
|
+
import { loadConfig } from "../config-loader.js";
|
|
3
|
+
import { GitTracker } from "../core/git-tracker.js";
|
|
4
|
+
import { ConfigError } from "../core/errors.js";
|
|
5
|
+
export async function runValidate(configPath) {
|
|
6
|
+
console.log(`\n🔍 Validating RAG config: ${configPath}\n`);
|
|
7
|
+
let config;
|
|
8
|
+
try {
|
|
9
|
+
process.stdout.write("📦 Loading config... ");
|
|
10
|
+
config = await loadConfig(configPath);
|
|
11
|
+
console.log("✅");
|
|
12
|
+
}
|
|
13
|
+
catch (err) {
|
|
14
|
+
console.log("❌");
|
|
15
|
+
throw new ConfigError(`Failed to load config: ${err instanceof Error ? err.message : String(err)}`, {
|
|
16
|
+
suggestion: "Run `rag-update init` to generate a new config file.",
|
|
17
|
+
cause: err,
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
console.log("\n📂 Scanning files...\n");
|
|
21
|
+
let hasWarnings = false;
|
|
22
|
+
for (const chunker of config.chunkers) {
|
|
23
|
+
const files = await glob(chunker.patterns, { nodir: true });
|
|
24
|
+
const unique = [...new Set(files)];
|
|
25
|
+
const patternsDisplay = chunker.patterns.join(", ");
|
|
26
|
+
console.log(` Chunker: ${chunker.name} (patterns: ${patternsDisplay})`);
|
|
27
|
+
if (unique.length === 0) {
|
|
28
|
+
console.log(` ⚠️ No files matched — check your patterns\n`);
|
|
29
|
+
hasWarnings = true;
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
console.log(` ✅ Found ${unique.length} matching file(s)\n`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
const gitTracker = new GitTracker(config.chunkers);
|
|
36
|
+
const allFiles = await gitTracker.getAllTrackedFiles();
|
|
37
|
+
console.log(` Total: ${allFiles.length} file(s) tracked across ${config.chunkers.length} chunker(s)`);
|
|
38
|
+
console.log("\n🔌 Checking vector store...");
|
|
39
|
+
try {
|
|
40
|
+
await config.vectorStore.initialize();
|
|
41
|
+
console.log(" ✅ Vector store connected");
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
console.log(" ⚠️ Could not connect — check credentials");
|
|
45
|
+
hasWarnings = true;
|
|
46
|
+
}
|
|
47
|
+
if (hasWarnings) {
|
|
48
|
+
console.log("\n⚠️ Config loaded with warnings.");
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
console.log("\n✅ Config is valid!");
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=validate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../src/cli/validate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAClD,OAAO,CAAC,GAAG,CAAC,+BAA+B,UAAU,IAAI,CAAC,CAAC;IAE3D,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC9C,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjB,MAAM,IAAI,WAAW,CACnB,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC5E;YACE,UAAU,EAAE,sDAAsD;YAClE,KAAK,EAAE,GAAG;SACX,CACF,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QACnC,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,IAAI,eAAe,eAAe,GAAG,CAAC,CAAC;QAEzE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC9D,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,kBAAkB,EAAE,CAAC;IACvD,OAAO,CAAC,GAAG,CACT,YAAY,QAAQ,CAAC,MAAM,2BAA2B,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,CAC1F,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;QAC3D,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IACpD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACtC,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config-loader.d.ts","sourceRoot":"","sources":["../src/config-loader.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"config-loader.d.ts","sourceRoot":"","sources":["../src/config-loader.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAqF3D,wBAAsB,UAAU,CAC9B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,iBAAiB,CAAC,CAgB5B"}
|