@aiconnect/easy-rag 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +181 -0
- package/dist/chunker/csv.d.ts +4 -0
- package/dist/chunker/csv.d.ts.map +1 -0
- package/dist/chunker/csv.js +10 -0
- package/dist/chunker/csv.js.map +1 -0
- package/dist/chunker/index.d.ts +5 -0
- package/dist/chunker/index.d.ts.map +1 -0
- package/dist/chunker/index.js +20 -0
- package/dist/chunker/index.js.map +1 -0
- package/dist/chunker/markdown.d.ts +4 -0
- package/dist/chunker/markdown.d.ts.map +1 -0
- package/dist/chunker/markdown.js +35 -0
- package/dist/chunker/markdown.js.map +1 -0
- package/dist/chunker/pdf.d.ts +4 -0
- package/dist/chunker/pdf.d.ts.map +1 -0
- package/dist/chunker/pdf.js +26 -0
- package/dist/chunker/pdf.js.map +1 -0
- package/dist/chunker/types.d.ts +12 -0
- package/dist/chunker/types.d.ts.map +1 -0
- package/dist/chunker/types.js +2 -0
- package/dist/chunker/types.js.map +1 -0
- package/dist/commands/init.d.ts +2 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +120 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/serve.d.ts +2 -0
- package/dist/commands/serve.d.ts.map +1 -0
- package/dist/commands/serve.js +17 -0
- package/dist/commands/serve.js.map +1 -0
- package/dist/config/index.d.ts +9 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +66 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/types.d.ts +5 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/embeddings/index.d.ts +3 -0
- package/dist/embeddings/index.d.ts.map +1 -0
- package/dist/embeddings/index.js +2 -0
- package/dist/embeddings/index.js.map +1 -0
- package/dist/embeddings/openai.d.ts +2 -0
- package/dist/embeddings/openai.d.ts.map +1 -0
- package/dist/embeddings/openai.js +57 -0
- package/dist/embeddings/openai.js.map +1 -0
- package/dist/embeddings/types.d.ts +15 -0
- package/dist/embeddings/types.d.ts.map +1 -0
- package/dist/embeddings/types.js +2 -0
- package/dist/embeddings/types.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +248 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer/index.d.ts +4 -0
- package/dist/indexer/index.d.ts.map +1 -0
- package/dist/indexer/index.js +3 -0
- package/dist/indexer/index.js.map +1 -0
- package/dist/indexer/orchestrator.d.ts +3 -0
- package/dist/indexer/orchestrator.d.ts.map +1 -0
- package/dist/indexer/orchestrator.js +106 -0
- package/dist/indexer/orchestrator.js.map +1 -0
- package/dist/indexer/scanner.d.ts +2 -0
- package/dist/indexer/scanner.d.ts.map +1 -0
- package/dist/indexer/scanner.js +34 -0
- package/dist/indexer/scanner.js.map +1 -0
- package/dist/indexer/types.d.ts +12 -0
- package/dist/indexer/types.d.ts.map +1 -0
- package/dist/indexer/types.js +2 -0
- package/dist/indexer/types.js.map +1 -0
- package/dist/parsers/csv.d.ts +3 -0
- package/dist/parsers/csv.d.ts.map +1 -0
- package/dist/parsers/csv.js +63 -0
- package/dist/parsers/csv.js.map +1 -0
- package/dist/parsers/fileTypeDetector.d.ts +3 -0
- package/dist/parsers/fileTypeDetector.d.ts.map +1 -0
- package/dist/parsers/fileTypeDetector.js +16 -0
- package/dist/parsers/fileTypeDetector.js.map +1 -0
- package/dist/parsers/index.d.ts +3 -0
- package/dist/parsers/index.d.ts.map +1 -0
- package/dist/parsers/index.js +18 -0
- package/dist/parsers/index.js.map +1 -0
- package/dist/parsers/markdown.d.ts +3 -0
- package/dist/parsers/markdown.d.ts.map +1 -0
- package/dist/parsers/markdown.js +30 -0
- package/dist/parsers/markdown.js.map +1 -0
- package/dist/parsers/pdf.d.ts +3 -0
- package/dist/parsers/pdf.d.ts.map +1 -0
- package/dist/parsers/pdf.js +22 -0
- package/dist/parsers/pdf.js.map +1 -0
- package/dist/parsers/types.d.ts +17 -0
- package/dist/parsers/types.d.ts.map +1 -0
- package/dist/parsers/types.js +2 -0
- package/dist/parsers/types.js.map +1 -0
- package/dist/query/embedding.d.ts +2 -0
- package/dist/query/embedding.d.ts.map +1 -0
- package/dist/query/embedding.js +6 -0
- package/dist/query/embedding.js.map +1 -0
- package/dist/query/index.d.ts +3 -0
- package/dist/query/index.d.ts.map +1 -0
- package/dist/query/index.js +86 -0
- package/dist/query/index.js.map +1 -0
- package/dist/query/search.d.ts +6 -0
- package/dist/query/search.d.ts.map +1 -0
- package/dist/query/search.js +45 -0
- package/dist/query/search.js.map +1 -0
- package/dist/query/types.d.ts +19 -0
- package/dist/query/types.d.ts.map +1 -0
- package/dist/query/types.js +2 -0
- package/dist/query/types.js.map +1 -0
- package/dist/vector-store/chroma-server.d.ts +10 -0
- package/dist/vector-store/chroma-server.d.ts.map +1 -0
- package/dist/vector-store/chroma-server.js +102 -0
- package/dist/vector-store/chroma-server.js.map +1 -0
- package/dist/vector-store/chromadb.d.ts +8 -0
- package/dist/vector-store/chromadb.d.ts.map +1 -0
- package/dist/vector-store/chromadb.js +98 -0
- package/dist/vector-store/chromadb.js.map +1 -0
- package/dist/vector-store/index.d.ts +4 -0
- package/dist/vector-store/index.d.ts.map +1 -0
- package/dist/vector-store/index.js +3 -0
- package/dist/vector-store/index.js.map +1 -0
- package/dist/vector-store/types.d.ts +12 -0
- package/dist/vector-store/types.d.ts.map +1 -0
- package/dist/vector-store/types.js +2 -0
- package/dist/vector-store/types.js.map +1 -0
- package/dist/vector-store/utils.d.ts +2 -0
- package/dist/vector-store/utils.d.ts.map +1 -0
- package/dist/vector-store/utils.js +17 -0
- package/dist/vector-store/utils.js.map +1 -0
- package/package.json +57 -0
- package/skills/easy-rag/SKILL.md +198 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 AI Connect
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# @aiconnect/easy-rag
|
|
2
|
+
|
|
3
|
+
A TypeScript CLI for local RAG (Retrieval-Augmented Generation) — index documents and query them with natural language.
|
|
4
|
+
|
|
5
|
+
Uses **OpenAI embeddings** and **ChromaDB** for vector storage. No external infrastructure needed.
|
|
6
|
+
|
|
7
|
+
## Quick Start
|
|
8
|
+
|
|
9
|
+
**Prerequisites:** Node.js >= 18 and an [OpenAI API key](https://platform.openai.com/api-keys).
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
# Install
|
|
13
|
+
npm install -g @aiconnect/easy-rag
|
|
14
|
+
|
|
15
|
+
# Set up your API key (interactive)
|
|
16
|
+
easy-rag init
|
|
17
|
+
|
|
18
|
+
# Index a folder and query it
|
|
19
|
+
easy-rag index ./my-docs
|
|
20
|
+
easy-rag query "What is the refund policy?"
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Example output:
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
$ easy-rag index ./my-docs
|
|
27
|
+
Scanning ./my-docs...
|
|
28
|
+
Found 3 files (2 .md, 1 .pdf)
|
|
29
|
+
Chunking documents... 12 chunks created
|
|
30
|
+
Generating embeddings... done
|
|
31
|
+
Indexed 12 chunks into collection "my-docs"
|
|
32
|
+
|
|
33
|
+
$ easy-rag query "What is the refund policy?"
|
|
34
|
+
Result 1:
|
|
35
|
+
Refunds are available within 30 days of purchase.
|
|
36
|
+
Contact support@example.com to initiate a refund request.
|
|
37
|
+
|
|
38
|
+
Result 2:
|
|
39
|
+
All subscription plans include a 30-day money-back guarantee...
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Commands
|
|
43
|
+
|
|
44
|
+
### `easy-rag init`
|
|
45
|
+
|
|
46
|
+
Interactive setup — prompts for your OpenAI API key and embedding model. Saves to `~/.easy-rag/config.json`.
|
|
47
|
+
|
|
48
|
+
You can also skip `init` and set an environment variable instead:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
export OPENAI_API_KEY="sk-..."
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### `easy-rag index <folder>`
|
|
55
|
+
|
|
56
|
+
Scans a folder recursively and indexes all supported files into ChromaDB.
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
easy-rag index ./knowledge-base
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
- **Supported formats:** `.pdf`, `.md`, `.csv`
|
|
63
|
+
- **Chunking:** Automatic — Markdown by heading, PDF by paragraph, CSV by row
|
|
64
|
+
- **Collection name:** Derived from the folder name
|
|
65
|
+
- Re-indexing the same folder creates a new collection with a unique suffix
|
|
66
|
+
|
|
67
|
+
### `easy-rag query <question>`
|
|
68
|
+
|
|
69
|
+
Search indexed documents using natural language.
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
easy-rag query "How do I configure the API?"
|
|
73
|
+
easy-rag query --top 3 --metadata "quarterly revenue"
|
|
74
|
+
easy-rag query --collection my-docs "setup instructions"
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
| Option | Description | Default |
|
|
78
|
+
|--------|-------------|---------|
|
|
79
|
+
| `-t, --top <n>` | Number of results | 5 |
|
|
80
|
+
| `-m, --metadata` | Show source file, score, chunk index | off |
|
|
81
|
+
| `-c, --collection <name>` | Search specific collection | all |
|
|
82
|
+
|
|
83
|
+
### `easy-rag collections`
|
|
84
|
+
|
|
85
|
+
List all indexed collections with document counts.
|
|
86
|
+
|
|
87
|
+
### `easy-rag delete <collection>`
|
|
88
|
+
|
|
89
|
+
Delete a collection. Use `--force` to skip confirmation.
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
easy-rag delete my-docs --force
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### `easy-rag serve`
|
|
96
|
+
|
|
97
|
+
Start ChromaDB in the foreground. Useful if you want a persistent server across multiple `index`/`query` calls instead of auto-starting one per command.
|
|
98
|
+
|
|
99
|
+
### `easy-rag parse <file>`
|
|
100
|
+
|
|
101
|
+
Parse a single file and print extracted text. Useful for debugging.
|
|
102
|
+
|
|
103
|
+
## How It Works
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
Documents (.pdf, .md, .csv)
|
|
107
|
+
│
|
|
108
|
+
▼
|
|
109
|
+
┌─────────┐ ┌─────────┐ ┌─────────────┐ ┌──────────┐
|
|
110
|
+
│ Parse │ → │ Chunk │ → │ Embed │ → │ Store │
|
|
111
|
+
│ │ │ │ │ (OpenAI) │ │ (ChromaDB)│
|
|
112
|
+
└─────────┘ └─────────┘ └─────────────┘ └──────────┘
|
|
113
|
+
│
|
|
114
|
+
Query ─── Embed ───┘
|
|
115
|
+
│
|
|
116
|
+
▼
|
|
117
|
+
Relevant chunks
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
1. **Parse** — Extract text from PDF, Markdown, or CSV
|
|
121
|
+
2. **Chunk** — Split intelligently by heading, paragraph, or row
|
|
122
|
+
3. **Embed** — Generate vectors via OpenAI (`text-embedding-3-large`)
|
|
123
|
+
4. **Store** — Save in ChromaDB (runs locally, auto-started)
|
|
124
|
+
5. **Query** — Embed the question and find the closest chunks
|
|
125
|
+
|
|
126
|
+
## Configuration
|
|
127
|
+
|
|
128
|
+
### API Key
|
|
129
|
+
|
|
130
|
+
| Method | How | Priority |
|
|
131
|
+
|--------|-----|----------|
|
|
132
|
+
| `easy-rag init` | Interactive prompt, saved to `~/.easy-rag/config.json` | 2nd |
|
|
133
|
+
| Environment variable | `export OPENAI_API_KEY="sk-..."` | 1st (overrides config) |
|
|
134
|
+
|
|
135
|
+
### Embedding Model
|
|
136
|
+
|
|
137
|
+
Set during `easy-rag init` or via `EMBEDDING_MODEL` env var.
|
|
138
|
+
|
|
139
|
+
| Model | Notes |
|
|
140
|
+
|-------|-------|
|
|
141
|
+
| `text-embedding-3-large` | Default, best performance |
|
|
142
|
+
| `text-embedding-3-small` | Faster, cheaper |
|
|
143
|
+
| `text-embedding-ada-002` | Legacy |
|
|
144
|
+
|
|
145
|
+
### ChromaDB
|
|
146
|
+
|
|
147
|
+
ChromaDB is the local vector store. **It starts automatically** — no setup needed.
|
|
148
|
+
|
|
149
|
+
- Data is stored in `~/.easy-rag/chromadb/` (auto-created on first use)
|
|
150
|
+
- To connect to an external instance: `export CHROMA_URL="http://my-chroma-host:8000"`
|
|
151
|
+
|
|
152
|
+
## Development
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
git clone https://github.com/johnjohn-aic/easy-rag.git
|
|
156
|
+
cd easy-rag
|
|
157
|
+
npm install
|
|
158
|
+
|
|
159
|
+
npm run build # Compile TypeScript
|
|
160
|
+
npm test # Run tests
|
|
161
|
+
npx tsx src/index.ts <command> # Dev mode (no build needed)
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### Project Structure
|
|
165
|
+
|
|
166
|
+
```
|
|
167
|
+
src/
|
|
168
|
+
index.ts # CLI entrypoint (commander)
|
|
169
|
+
parsers/ # PDF, Markdown, CSV text extraction
|
|
170
|
+
chunker/ # Content-aware text splitting
|
|
171
|
+
embeddings/ # OpenAI embedding generation
|
|
172
|
+
vector-store/ # ChromaDB client + server management
|
|
173
|
+
query/ # Vector similarity search
|
|
174
|
+
indexer/ # Orchestrates parse → chunk → embed → store
|
|
175
|
+
config/ # Global config (~/.easy-rag/config.json)
|
|
176
|
+
commands/ # CLI command implementations (init, serve)
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
## License
|
|
180
|
+
|
|
181
|
+
MIT
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/chunker/csv.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,wBAAgB,UAAU,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,eAAe,EAAE,CAQtE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/chunker/csv.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,UAAU,CAAC,SAAwB;IACjD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACpC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,QAAQ,EAAE;YACR,GAAG,GAAG,CAAC,QAAQ;YACf,UAAU,EAAE,KAAK;SAClB;KACF,CAAC,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { ParsedDocument, CSVDocument } from '../parsers/types.js';
|
|
2
|
+
import type { ChunkedDocument } from './types.js';
|
|
3
|
+
export declare function chunkDocument(document: ParsedDocument | CSVDocument[]): ChunkedDocument[];
|
|
4
|
+
export type { ChunkedDocument, ChunkMetadata } from './types.js';
|
|
5
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunker/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAKlD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,cAAc,GAAG,WAAW,EAAE,GAAG,eAAe,EAAE,CAiBzF;AAED,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { markdownChunker } from './markdown.js';
|
|
2
|
+
import { pdfChunker } from './pdf.js';
|
|
3
|
+
import { csvChunker } from './csv.js';
|
|
4
|
+
export function chunkDocument(document) {
|
|
5
|
+
if (Array.isArray(document)) {
|
|
6
|
+
return csvChunker(document);
|
|
7
|
+
}
|
|
8
|
+
const { metadata } = document;
|
|
9
|
+
switch (metadata.contentType) {
|
|
10
|
+
case 'markdown':
|
|
11
|
+
return markdownChunker(document);
|
|
12
|
+
case 'pdf':
|
|
13
|
+
return pdfChunker(document);
|
|
14
|
+
case 'csv':
|
|
15
|
+
throw new Error('CSV documents should be passed as array to chunkDocument');
|
|
16
|
+
default:
|
|
17
|
+
throw new Error(`Unsupported content type: ${metadata.contentType}`);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunker/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,MAAM,UAAU,aAAa,CAAC,QAAwC;IACpE,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAAC;IAE9B,QAAQ,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC7B,KAAK,UAAU;YACb,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAC;QACnC,KAAK,KAAK;YACR,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC9B,KAAK,KAAK;YACR,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E;YACE,MAAM,IAAI,KAAK,CAAC,6BAA6B,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACzE,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/chunker/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,wBAAgB,eAAe,CAAC,QAAQ,EAAE,cAAc,GAAG,eAAe,EAAE,CAuC3E"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
export function markdownChunker(document) {
|
|
2
|
+
const chunks = [];
|
|
3
|
+
const { content, metadata } = document;
|
|
4
|
+
const headingRegex = /^(#{1,3})\s+(.+)$/gm;
|
|
5
|
+
const headingMatches = [...content.matchAll(headingRegex)];
|
|
6
|
+
if (headingMatches.length === 0) {
|
|
7
|
+
chunks.push({
|
|
8
|
+
content,
|
|
9
|
+
metadata: {
|
|
10
|
+
...metadata,
|
|
11
|
+
chunkIndex: 0
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
return chunks;
|
|
15
|
+
}
|
|
16
|
+
for (let i = 0; i < headingMatches.length; i++) {
|
|
17
|
+
const match = headingMatches[i];
|
|
18
|
+
const sectionTitle = match[2];
|
|
19
|
+
const startOffset = match.index;
|
|
20
|
+
const endOffset = i < headingMatches.length - 1
|
|
21
|
+
? headingMatches[i + 1].index
|
|
22
|
+
: content.length;
|
|
23
|
+
const sectionContent = content.slice(startOffset, endOffset);
|
|
24
|
+
chunks.push({
|
|
25
|
+
content: sectionContent,
|
|
26
|
+
metadata: {
|
|
27
|
+
...metadata,
|
|
28
|
+
chunkIndex: i,
|
|
29
|
+
sectionTitle
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return chunks;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=markdown.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/chunker/markdown.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,eAAe,CAAC,QAAwB;IACtD,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAAC;IAEvC,MAAM,YAAY,GAAG,qBAAqB,CAAC;IAC3C,MAAM,cAAc,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;IAE3D,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC;YACV,OAAO;YACP,QAAQ,EAAE;gBACR,GAAG,QAAQ;gBACX,UAAU,EAAE,CAAC;aACd;SACF,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;QAChC,MAAM,SAAS,GAAG,CAAC,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC;YAC7C,CAAC,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK;YAC7B,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAEnB,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAE7D,MAAM,CAAC,IAAI,CAAC;YACV,OAAO,EAAE,cAAc;YACvB,QAAQ,EAAE;gBACR,GAAG,QAAQ;gBACX,UAAU,EAAE,CAAC;gBACb,YAAY;aACb;SACF,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pdf.d.ts","sourceRoot":"","sources":["../../src/chunker/pdf.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,wBAAgB,UAAU,CAAC,QAAQ,EAAE,cAAc,GAAG,eAAe,EAAE,CAgCtE"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export function pdfChunker(document) {
|
|
2
|
+
const chunks = [];
|
|
3
|
+
const { content, metadata } = document;
|
|
4
|
+
const paragraphs = content.split(/\n\n+/);
|
|
5
|
+
let currentPosition = 0;
|
|
6
|
+
for (let i = 0; i < paragraphs.length; i++) {
|
|
7
|
+
const paragraph = paragraphs[i].trim();
|
|
8
|
+
if (paragraph.length === 0) {
|
|
9
|
+
continue;
|
|
10
|
+
}
|
|
11
|
+
const startPosition = content.indexOf(paragraph, currentPosition);
|
|
12
|
+
const endPosition = startPosition + paragraph.length;
|
|
13
|
+
chunks.push({
|
|
14
|
+
content: paragraph,
|
|
15
|
+
metadata: {
|
|
16
|
+
...metadata,
|
|
17
|
+
chunkIndex: i,
|
|
18
|
+
startPosition,
|
|
19
|
+
endPosition
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
currentPosition = endPosition;
|
|
23
|
+
}
|
|
24
|
+
return chunks;
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=pdf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../src/chunker/pdf.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,UAAU,CAAC,QAAwB;IACjD,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAAC;IAEvC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAE1C,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;QAClE,MAAM,WAAW,GAAG,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC;QAErD,MAAM,CAAC,IAAI,CAAC;YACV,OAAO,EAAE,SAAS;YAClB,QAAQ,EAAE;gBACR,GAAG,QAAQ;gBACX,UAAU,EAAE,CAAC;gBACb,aAAa;gBACb,WAAW;aACZ;SACF,CAAC,CAAC;QAEH,eAAe,GAAG,WAAW,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { DocumentMetadata } from '../parsers/types.js';
|
|
2
|
+
export interface ChunkMetadata extends DocumentMetadata {
|
|
3
|
+
chunkIndex: number;
|
|
4
|
+
sectionTitle?: string;
|
|
5
|
+
startPosition?: number;
|
|
6
|
+
endPosition?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface ChunkedDocument {
|
|
9
|
+
content: string;
|
|
10
|
+
metadata: ChunkMetadata;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/chunker/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,MAAM,WAAW,aAAc,SAAQ,gBAAgB;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,CAAC;CACzB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/chunker/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAYA,wBAAsB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAsH1C"}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import readline from 'node:readline/promises';
|
|
2
|
+
import { saveConfig, loadConfig, CONFIG_PATH } from '../config/index.js';
|
|
3
|
+
const MODELS = [
|
|
4
|
+
{ value: 'text-embedding-3-large', label: 'text-embedding-3-large (default, best performance)' },
|
|
5
|
+
{ value: 'text-embedding-3-small', label: 'text-embedding-3-small (faster, cheaper)' },
|
|
6
|
+
{ value: 'text-embedding-ada-002', label: 'text-embedding-ada-002 (legacy model)' },
|
|
7
|
+
];
|
|
8
|
+
const MAX_ATTEMPTS = 3;
|
|
9
|
+
export async function init() {
|
|
10
|
+
const rl = readline.createInterface({
|
|
11
|
+
input: process.stdin,
|
|
12
|
+
output: process.stdout,
|
|
13
|
+
});
|
|
14
|
+
try {
|
|
15
|
+
console.log('EasyRAG Configuration Setup');
|
|
16
|
+
console.log('────────────────────────────\n');
|
|
17
|
+
const existingConfig = await loadConfig();
|
|
18
|
+
const hasExistingConfig = Object.keys(existingConfig).length > 0;
|
|
19
|
+
if (hasExistingConfig) {
|
|
20
|
+
console.log(`A configuration file already exists at ${CONFIG_PATH}`);
|
|
21
|
+
console.log('Running init will overwrite your current settings.\n');
|
|
22
|
+
const continueAnswer = await rl.question('Continue? [y/N]: ');
|
|
23
|
+
if (continueAnswer.toLowerCase() !== 'y') {
|
|
24
|
+
console.log('Configuration cancelled.');
|
|
25
|
+
process.exit(0);
|
|
26
|
+
}
|
|
27
|
+
console.log('');
|
|
28
|
+
}
|
|
29
|
+
let apiKey = '';
|
|
30
|
+
if (existingConfig.openai_api_key) {
|
|
31
|
+
const maskedKey = maskApiKey(existingConfig.openai_api_key);
|
|
32
|
+
console.log(`Current OpenAI API key: ${maskedKey}`);
|
|
33
|
+
}
|
|
34
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
35
|
+
const remainingAttempts = MAX_ATTEMPTS - attempt;
|
|
36
|
+
apiKey = await rl.question('Enter your OpenAI API key (sk-...): ');
|
|
37
|
+
if (apiKey === '') {
|
|
38
|
+
if (attempt === MAX_ATTEMPTS) {
|
|
39
|
+
console.log('Too many failed attempts. Aborting initialization.');
|
|
40
|
+
process.exit(2);
|
|
41
|
+
}
|
|
42
|
+
const continueVal = attempt < MAX_ATTEMPTS;
|
|
43
|
+
if (continueVal) {
|
|
44
|
+
console.log(`Please try again (${remainingAttempts} attempts remaining):\n`);
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (!apiKey.startsWith('sk-')) {
|
|
49
|
+
console.log('Invalid API key format. Key must start with "sk-"');
|
|
50
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
51
|
+
console.log(`Please try again (${remainingAttempts} attempts remaining):\n`);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
console.log('Too many failed attempts. Aborting initialization.');
|
|
56
|
+
process.exit(2);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
console.log('');
|
|
62
|
+
console.log('Select embedding model:');
|
|
63
|
+
MODELS.forEach((model, index) => {
|
|
64
|
+
console.log(` ${index + 1}. ${model.label}`);
|
|
65
|
+
});
|
|
66
|
+
console.log('');
|
|
67
|
+
let modelSelection = '';
|
|
68
|
+
let model = MODELS[0].value;
|
|
69
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
70
|
+
const remainingAttempts = MAX_ATTEMPTS - attempt;
|
|
71
|
+
modelSelection = await rl.question('Enter choice [1-3] (default: 1): ');
|
|
72
|
+
if (modelSelection === '' || modelSelection === '1') {
|
|
73
|
+
model = MODELS[0].value;
|
|
74
|
+
break;
|
|
75
|
+
}
|
|
76
|
+
const choice = parseInt(modelSelection, 10);
|
|
77
|
+
if (choice >= 1 && choice <= MODELS.length) {
|
|
78
|
+
model = MODELS[choice - 1].value;
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
82
|
+
console.log('Invalid choice. Please enter a number between 1 and 3.');
|
|
83
|
+
console.log(`Please try again (${remainingAttempts} attempts remaining):\n`);
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
console.log('Too many failed attempts. Aborting initialization.');
|
|
87
|
+
process.exit(2);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const newConfig = {
|
|
91
|
+
openai_api_key: apiKey,
|
|
92
|
+
embedding_model: model,
|
|
93
|
+
};
|
|
94
|
+
try {
|
|
95
|
+
await saveConfig(newConfig);
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
console.error(`\nError: Cannot write to ${CONFIG_PATH}`);
|
|
99
|
+
console.error(`Permission denied or directory does not exist.`);
|
|
100
|
+
process.exit(3);
|
|
101
|
+
}
|
|
102
|
+
console.log('');
|
|
103
|
+
console.log(`✓ Configuration saved to ${CONFIG_PATH}`);
|
|
104
|
+
console.log(`✓ OpenAI API key: ${maskApiKey(apiKey)} (masked)`);
|
|
105
|
+
console.log(`✓ Embedding model: ${model}`);
|
|
106
|
+
console.log('');
|
|
107
|
+
console.log('You can now use EasyRAG without setting environment variables.');
|
|
108
|
+
console.log('To change settings later, run \'easy-rag init\' again.');
|
|
109
|
+
}
|
|
110
|
+
finally {
|
|
111
|
+
rl.close();
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
function maskApiKey(key) {
|
|
115
|
+
if (key.length <= 8) {
|
|
116
|
+
return 'sk-***';
|
|
117
|
+
}
|
|
118
|
+
return `${key.substring(0, 7)}***`;
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=init.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,wBAAwB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGzE,MAAM,MAAM,GAAG;IACb,EAAE,KAAK,EAAE,wBAAwB,EAAE,KAAK,EAAE,oDAAoD,EAAE;IAChG,EAAE,KAAK,EAAE,wBAAwB,EAAE,KAAK,EAAE,0CAA0C,EAAE;IACtF,EAAE,KAAK,EAAE,wBAAwB,EAAE,KAAK,EAAE,uCAAuC,EAAE;CACpF,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,CAAC;AAEvB,MAAM,CAAC,KAAK,UAAU,IAAI;IACxB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAE9C,MAAM,cAAc,GAAG,MAAM,UAAU,EAAE,CAAC;QAC1C,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAEjE,IAAI,iBAAiB,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,0CAA0C,WAAW,EAAE,CAAC,CAAC;YACrE,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;YAEpE,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;YAC9D,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,GAAG,EAAE,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;gBACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,IAAI,cAAc,CAAC,cAAc,EAAE,CAAC;YAClC,MAAM,SAAS,GAAG,UAAU,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;YAC5D,OAAO,CAAC,GAAG,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;QACtD,CAAC;QAED,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;YACzD,MAAM,iBAAiB,GAAG,YAAY,GAAG,OAAO,CAAC;YACjD,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC,CAAC;YAEnE,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;gBAClB,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;oBAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAClB,CAAC;gBACD,MAAM,WAAW,GAAG,OAAO,GAAG,YAAY,CAAC;gBAC3C,IAAI,WAAW,EAAE,CAAC;oBAChB,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,yBAAyB,CAAC,CAAC;oBAC7E,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;gBACjE,IAAI,OAAO,GAAG,YAAY,EAAE,CAAC;oBAC3B,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,yBAAyB,CAAC,CAAC;oBAC7E,SAAS;gBACX,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;oBAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;YACD,MAAM;QACR,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;QACvC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEhB,IAAI,cAAc,GAAG,EAAE,CAAC;QACxB,IAAI,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAE5B,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;YACzD,MAAM,iBAAiB,GAAG,YAAY,GAAG,OAAO,CAAC;YACjD,cAAc,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC,CAAC;YAExE,IAAI,cAAc,KAAK,EAAE,IAAI,cAAc,KAAK,GAAG,EAAE,CAAC;gBACpD,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;gBACxB,MAAM;YACR,CAAC;YAED,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;YAC5C,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC3C,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;gBACjC,MAAM;YACR,CAAC;YAED,IAAI,OAAO,GAAG,YAAY,EAAE,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;gBACtE,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,yBAAyB,CAAC,CAAC;YAC/E,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAkB;YAC/B,cAAc,EAAE,MAAM;YACtB,eAAe,EAAE,KAAK;SACvB,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,4BAA4B,WAAW,EAAE,CAAC,CAAC;YACzD,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAChE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,4BAA4B,WAAW,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,qBAAqB,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;QAC9E,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IACxE,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC;AACrC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serve.d.ts","sourceRoot":"","sources":["../../src/commands/serve.ts"],"names":[],"mappings":"AAEA,wBAAsB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAiB3C"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { ensureChromaRunning } from '../vector-store/chroma-server.js';
|
|
2
|
+
export async function serve() {
|
|
3
|
+
// ensureChromaRunning handles checking if it's already running,
|
|
4
|
+
// or starting it if it's not. It also sets up signal handlers
|
|
5
|
+
// to clean up the child process on exit.
|
|
6
|
+
const success = await ensureChromaRunning();
|
|
7
|
+
if (!success) {
|
|
8
|
+
console.error('Failed to ensure ChromaDB is running.');
|
|
9
|
+
process.exit(1);
|
|
10
|
+
}
|
|
11
|
+
console.log('Server is active. Use Ctrl+C to stop.');
|
|
12
|
+
// Keep the Node.js event loop alive indefinitely.
|
|
13
|
+
// The process will only exit when it receives a signal (SIGINT/SIGTERM),
|
|
14
|
+
// which is handled by ensureChromaRunning's cleanup logic.
|
|
15
|
+
await new Promise(() => { });
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=serve.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serve.js","sourceRoot":"","sources":["../../src/commands/serve.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAEvE,MAAM,CAAC,KAAK,UAAU,KAAK;IACzB,gEAAgE;IAChE,8DAA8D;IAC9D,yCAAyC;IACzC,MAAM,OAAO,GAAG,MAAM,mBAAmB,EAAE,CAAC;IAE5C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;QACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IAErD,kDAAkD;IAClD,yEAAyE;IACzE,2DAA2D;IAC3D,MAAM,IAAI,OAAO,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { EasyRagConfig } from './types.js';
|
|
2
|
+
export declare const CONFIG_PATH: string;
|
|
3
|
+
export declare function loadConfig(): Promise<EasyRagConfig>;
|
|
4
|
+
export declare function saveConfig(config: EasyRagConfig): Promise<void>;
|
|
5
|
+
export declare function getEffectiveConfig(): Promise<EasyRagConfig>;
|
|
6
|
+
export declare function getOpenAIKey(): Promise<string | undefined>;
|
|
7
|
+
export declare function getEmbeddingModel(): Promise<string>;
|
|
8
|
+
export declare function clearCache(): void;
|
|
9
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,eAAO,MAAM,WAAW,QAAsD,CAAC;AAM/E,wBAAsB,UAAU,IAAI,OAAO,CAAC,aAAa,CAAC,CAqBzD;AAED,wBAAsB,UAAU,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAWrE;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,aAAa,CAAC,CAgBjE;AAED,wBAAsB,YAAY,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAGhE;AAED,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,CAGzD;AAED,wBAAgB,UAAU,IAAI,IAAI,CAEjC"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import os from 'os';
|
|
4
|
+
export const CONFIG_PATH = path.join(os.homedir(), '.easy-rag', 'config.json');
|
|
5
|
+
const DEFAULT_EMBEDDING_MODEL = 'text-embedding-3-large';
|
|
6
|
+
let cachedConfig = null;
|
|
7
|
+
export async function loadConfig() {
|
|
8
|
+
if (cachedConfig) {
|
|
9
|
+
return cachedConfig;
|
|
10
|
+
}
|
|
11
|
+
try {
|
|
12
|
+
const configContent = await fs.readFile(CONFIG_PATH, 'utf-8');
|
|
13
|
+
const parsedConfig = JSON.parse(configContent);
|
|
14
|
+
cachedConfig = parsedConfig;
|
|
15
|
+
return parsedConfig;
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
if (error.code === 'ENOENT') {
|
|
19
|
+
return {};
|
|
20
|
+
}
|
|
21
|
+
if (error.name === 'SyntaxError') {
|
|
22
|
+
console.warn(`Invalid JSON in ${CONFIG_PATH}: ${error.message}`);
|
|
23
|
+
return {};
|
|
24
|
+
}
|
|
25
|
+
console.warn(`Cannot read ${CONFIG_PATH}: ${error.message}`);
|
|
26
|
+
return {};
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export async function saveConfig(config) {
|
|
30
|
+
const configDir = path.dirname(CONFIG_PATH);
|
|
31
|
+
try {
|
|
32
|
+
await fs.mkdir(configDir, { recursive: true });
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
throw new Error(`Cannot create config directory: ${error.message}`);
|
|
36
|
+
}
|
|
37
|
+
const configContent = JSON.stringify(config, null, 2);
|
|
38
|
+
await fs.writeFile(CONFIG_PATH, configContent, { mode: 0o600 });
|
|
39
|
+
cachedConfig = config;
|
|
40
|
+
}
|
|
41
|
+
export async function getEffectiveConfig() {
|
|
42
|
+
const fileConfig = await loadConfig();
|
|
43
|
+
const envConfig = {};
|
|
44
|
+
if (process.env.OPENAI_API_KEY && process.env.OPENAI_API_KEY.trim() !== '') {
|
|
45
|
+
envConfig.openai_api_key = process.env.OPENAI_API_KEY;
|
|
46
|
+
}
|
|
47
|
+
if (process.env.EMBEDDING_MODEL && process.env.EMBEDDING_MODEL.trim() !== '') {
|
|
48
|
+
envConfig.embedding_model = process.env.EMBEDDING_MODEL;
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
openai_api_key: envConfig.openai_api_key || fileConfig.openai_api_key,
|
|
52
|
+
embedding_model: envConfig.embedding_model || fileConfig.embedding_model,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
export async function getOpenAIKey() {
|
|
56
|
+
const config = await getEffectiveConfig();
|
|
57
|
+
return config.openai_api_key;
|
|
58
|
+
}
|
|
59
|
+
export async function getEmbeddingModel() {
|
|
60
|
+
const config = await getEffectiveConfig();
|
|
61
|
+
return config.embedding_model || DEFAULT_EMBEDDING_MODEL;
|
|
62
|
+
}
|
|
63
|
+
export function clearCache() {
|
|
64
|
+
cachedConfig = null;
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;AAE/E,MAAM,uBAAuB,GAAG,wBAAwB,CAAC;AAEzD,IAAI,YAAY,GAAyB,IAAI,CAAC;AAE9C,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAC9D,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAkB,CAAC;QAChE,YAAY,GAAG,YAAY,CAAC;QAC5B,OAAO,YAAY,CAAC;IACtB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,IAAK,KAAe,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YAC5C,OAAO,CAAC,IAAI,CAAC,mBAAmB,WAAW,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5E,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,eAAe,WAAW,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,MAAqB;IACpD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC5C,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,mCAAoC,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtD,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChE,YAAY,GAAG,MAAM,CAAC;AACxB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,MAAM,UAAU,GAAG,MAAM,UAAU,EAAE,CAAC;IACtC,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3E,SAAS,CAAC,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IACxD,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC7E,SAAS,CAAC,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IAC1D,CAAC;IAED,OAAO;QACL,cAAc,EAAE,SAAS,CAAC,cAAc,IAAI,UAAU,CAAC,cAAc;QACrE,eAAe,EAAE,SAAS,CAAC,eAAe,IAAI,UAAU,CAAC,eAAe;KACzE,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,MAAM,MAAM,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC1C,OAAO,MAAM,CAAC,cAAc,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,MAAM,MAAM,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC1C,OAAO,MAAM,CAAC,eAAe,IAAI,uBAAuB,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/config/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B"}
|