@aiconnect/easy-rag 0.3.1 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -28
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +105 -3
- package/dist/commands/init.js.map +1 -1
- package/dist/config/types.d.ts +3 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/eval/index.d.ts +10 -0
- package/dist/eval/index.d.ts.map +1 -0
- package/dist/eval/index.js +37 -0
- package/dist/eval/index.js.map +1 -0
- package/dist/eval/metrics.d.ts +12 -0
- package/dist/eval/metrics.d.ts.map +1 -0
- package/dist/eval/metrics.js +43 -0
- package/dist/eval/metrics.js.map +1 -0
- package/dist/eval/reporter.d.ts +7 -0
- package/dist/eval/reporter.d.ts.map +1 -0
- package/dist/eval/reporter.js +105 -0
- package/dist/eval/reporter.js.map +1 -0
- package/dist/eval/runner.d.ts +8 -0
- package/dist/eval/runner.d.ts.map +1 -0
- package/dist/eval/runner.js +43 -0
- package/dist/eval/runner.js.map +1 -0
- package/dist/eval/test-cases.d.ts +3 -0
- package/dist/eval/test-cases.d.ts.map +1 -0
- package/dist/eval/test-cases.js +98 -0
- package/dist/eval/test-cases.js.map +1 -0
- package/dist/eval/types.d.ts +49 -0
- package/dist/eval/types.d.ts.map +1 -0
- package/dist/eval/types.js +2 -0
- package/dist/eval/types.js.map +1 -0
- package/dist/index.js +10 -45
- package/dist/index.js.map +1 -1
- package/dist/indexer/orchestrator.d.ts.map +1 -1
- package/dist/indexer/orchestrator.js +24 -6
- package/dist/indexer/orchestrator.js.map +1 -1
- package/dist/query/hybrid.d.ts +17 -0
- package/dist/query/hybrid.d.ts.map +1 -0
- package/dist/query/hybrid.js +116 -0
- package/dist/query/hybrid.js.map +1 -0
- package/dist/query/index.d.ts.map +1 -1
- package/dist/query/index.js +81 -24
- package/dist/query/index.js.map +1 -1
- package/dist/query/search.d.ts +7 -4
- package/dist/query/search.d.ts.map +1 -1
- package/dist/query/search.js +45 -40
- package/dist/query/search.js.map +1 -1
- package/dist/query/types.d.ts +8 -0
- package/dist/query/types.d.ts.map +1 -1
- package/dist/vector-store/index.d.ts +3 -2
- package/dist/vector-store/index.d.ts.map +1 -1
- package/dist/vector-store/index.js +2 -2
- package/dist/vector-store/index.js.map +1 -1
- package/dist/vector-store/tokenizer.d.ts +3 -0
- package/dist/vector-store/tokenizer.d.ts.map +1 -0
- package/dist/vector-store/tokenizer.js +91 -0
- package/dist/vector-store/tokenizer.js.map +1 -0
- package/dist/vector-store/utils.d.ts +4 -1
- package/dist/vector-store/utils.d.ts.map +1 -1
- package/dist/vector-store/utils.js +17 -6
- package/dist/vector-store/utils.js.map +1 -1
- package/dist/vector-store/zvec.d.ts +15 -0
- package/dist/vector-store/zvec.d.ts.map +1 -0
- package/dist/vector-store/zvec.js +136 -0
- package/dist/vector-store/zvec.js.map +1 -0
- package/package.json +5 -4
- package/skills/easy-rag/SKILL.md +40 -9
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
A TypeScript CLI for local RAG (Retrieval-Augmented Generation) — index documents and query them with natural language.
|
|
4
4
|
|
|
5
|
-
Uses **OpenAI embeddings** and **
|
|
5
|
+
Uses **OpenAI embeddings** and **Zvec** (embedded vector database) for storage. No external infrastructure or server needed.
|
|
6
6
|
|
|
7
7
|
## Quick Start
|
|
8
8
|
|
|
@@ -43,7 +43,16 @@ Result 2:
|
|
|
43
43
|
|
|
44
44
|
### `easy-rag init`
|
|
45
45
|
|
|
46
|
-
Interactive setup — prompts for your OpenAI API key and
|
|
46
|
+
Interactive setup — prompts for your OpenAI API key, embedding model, and advanced search settings. Saves to `~/.easy-rag/config.json`.
|
|
47
|
+
|
|
48
|
+
**Basic Setup:**
|
|
49
|
+
- OpenAI API key
|
|
50
|
+
- Embedding model (text-embedding-3-large, -small, or -ada-002)
|
|
51
|
+
|
|
52
|
+
**Advanced Settings (optional):**
|
|
53
|
+
- Search mode: auto, hybrid, vector, or bm25 (default: auto)
|
|
54
|
+
- Vector weight: 0.0-1.0 (default: 0.6)
|
|
55
|
+
- BM25 weight: 0.0-1.0 (default: 0.4)
|
|
47
56
|
|
|
48
57
|
You can also skip `init` and set an environment variable instead:
|
|
49
58
|
|
|
@@ -53,7 +62,7 @@ export OPENAI_API_KEY="sk-..."
|
|
|
53
62
|
|
|
54
63
|
### `easy-rag index <folder>`
|
|
55
64
|
|
|
56
|
-
Scans a folder recursively and indexes all supported files into
|
|
65
|
+
Scans a folder recursively and indexes all supported files into Zvec.
|
|
57
66
|
|
|
58
67
|
```bash
|
|
59
68
|
easy-rag index ./knowledge-base
|
|
@@ -66,12 +75,14 @@ easy-rag index ./knowledge-base
|
|
|
66
75
|
|
|
67
76
|
### `easy-rag query <question>`
|
|
68
77
|
|
|
69
|
-
Search indexed documents using
|
|
78
|
+
Search indexed documents using auto mode (RRF fusion) by default.
|
|
70
79
|
|
|
71
80
|
```bash
|
|
72
|
-
easy-rag query "How do I configure
|
|
81
|
+
easy-rag query "How do I configure API?"
|
|
73
82
|
easy-rag query --top 3 --metadata "quarterly revenue"
|
|
74
83
|
easy-rag query --collection my-docs "setup instructions"
|
|
84
|
+
easy-rag query --mode vector "semantic search only"
|
|
85
|
+
easy-rag query --mode bm25 "exact keyword match"
|
|
75
86
|
```
|
|
76
87
|
|
|
77
88
|
| Option | Description | Default |
|
|
@@ -79,6 +90,7 @@ easy-rag query --collection my-docs "setup instructions"
|
|
|
79
90
|
| `-t, --top <n>` | Number of results | 5 |
|
|
80
91
|
| `-m, --metadata` | Show source file, score, chunk index | off |
|
|
81
92
|
| `-c, --collection <name>` | Search specific collection | all |
|
|
93
|
+
| `--mode <mode>` | Search mode: auto, hybrid, vector, or bm25 | auto |
|
|
82
94
|
|
|
83
95
|
### `easy-rag collections`
|
|
84
96
|
|
|
@@ -92,36 +104,84 @@ Delete a collection. Use `--force` to skip confirmation.
|
|
|
92
104
|
easy-rag delete my-docs --force
|
|
93
105
|
```
|
|
94
106
|
|
|
95
|
-
### `easy-rag serve`
|
|
96
|
-
|
|
97
|
-
Start ChromaDB in the foreground. Useful if you want a persistent server across multiple `index`/`query` calls instead of auto-starting one per command.
|
|
98
|
-
|
|
99
107
|
### `easy-rag parse <file>`
|
|
100
108
|
|
|
101
109
|
Parse a single file and print extracted text. Useful for debugging.
|
|
102
110
|
|
|
103
|
-
## How It Works
|
|
111
|
+
## How It Works
|
|
104
112
|
|
|
105
113
|
```
|
|
106
114
|
Documents (.pdf, .md, .csv)
|
|
107
115
|
│
|
|
108
116
|
▼
|
|
109
|
-
┌─────────┐ ┌─────────┐
|
|
110
|
-
│ Parse │ → │ Chunk │ → │ Embed
|
|
111
|
-
│ │ │ │ │ (OpenAI)
|
|
112
|
-
└─────────┘ └─────────┘
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
117
|
+
┌─────────┐ ┌─────────┐ ┌──────────────┐ ┌────────┐
|
|
118
|
+
│ Parse │ → │ Chunk │ → │ Embed │ → │ Store │
|
|
119
|
+
│ │ │ │ │ (OpenAI) │ │ (Zvec) │
|
|
120
|
+
└─────────┘ └─────────┘ └──────────────┘ └────────┘
|
|
121
|
+
│ │
|
|
122
|
+
│ Dense + Sparse vectors
|
|
123
|
+
│
|
|
124
|
+
Query ─── Embed ───┘
|
|
125
|
+
│
|
|
126
|
+
┌───────────────┼───────────────┐
|
|
127
|
+
▼ ▼ ▼
|
|
128
|
+
Dense Search Sparse Search RRF Merge
|
|
129
|
+
│ │ │
|
|
130
|
+
└───────────────┼───────────────┘
|
|
131
|
+
▼
|
|
132
|
+
Relevant chunks
|
|
118
133
|
```
|
|
119
134
|
|
|
120
135
|
1. **Parse** — Extract text from PDF, Markdown, or CSV
|
|
121
136
|
2. **Chunk** — Split intelligently by heading, paragraph, or row
|
|
122
|
-
3. **Embed** — Generate vectors via OpenAI (`text-embedding-3-large`)
|
|
123
|
-
4. **Store** — Save in
|
|
124
|
-
5. **Query** — Embed the question and find the closest chunks
|
|
137
|
+
3. **Embed** — Generate dense vectors via OpenAI (`text-embedding-3-large`) + sparse vectors (TF-hash)
|
|
138
|
+
4. **Store** — Save in Zvec (embedded, no server needed)
|
|
139
|
+
5. **Query** — Embed the question and find the closest chunks via hybrid search
|
|
140
|
+
|
|
141
|
+
## Hybrid Search
|
|
142
|
+
|
|
143
|
+
EasyRAG uses **hybrid search** combining dense vector similarity and sparse keyword matching via Reciprocal Rank Fusion (RRF).
|
|
144
|
+
|
|
145
|
+
### Why Hybrid Search?
|
|
146
|
+
|
|
147
|
+
- **Dense search** captures semantic meaning but can miss exact keywords
|
|
148
|
+
- **Sparse search** excels at keyword matches but lacks semantic understanding
|
|
149
|
+
- **Hybrid search** gives the best of both worlds — like production RAG systems
|
|
150
|
+
|
|
151
|
+
### How It Works
|
|
152
|
+
|
|
153
|
+
```
|
|
154
|
+
score(d) = vector_weight / (60 + vector_rank) + bm25_weight / (60 + bm25_rank)
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### Search Modes
|
|
158
|
+
|
|
159
|
+
| Mode | Description | Use Case |
|
|
160
|
+
|------|-------------|----------|
|
|
161
|
+
| `auto` (default) | Dense + Sparse via RRF, no weights needed | General queries |
|
|
162
|
+
| `hybrid` | Dense + Sparse with configurable weights | Fine-tuned relevance |
|
|
163
|
+
| `vector` | Dense vector similarity only | Semantic/abstract queries |
|
|
164
|
+
| `bm25` | Sparse keyword only | Exact terms, acronyms, names |
|
|
165
|
+
|
|
166
|
+
In `hybrid` mode, weights are configurable: vector=0.6, bm25=0.4 (via config.json).
|
|
167
|
+
In `auto` mode, weights are fixed at 0.5/0.5 (RRF with equal weights, no configuration needed).
|
|
168
|
+
|
|
169
|
+
> **Note:** `auto` currently uses manual RRF fusion because the Zvec Node SDK (`@zvec/zvec`) does not yet expose native multi-vector queries or rerankers. When the SDK adds this support, `auto` will delegate fusion to the engine.
|
|
170
|
+
|
|
171
|
+
Example:
|
|
172
|
+
```bash
|
|
173
|
+
# Auto search (default — RRF fusion, no config needed)
|
|
174
|
+
easy-rag query "refund policy"
|
|
175
|
+
|
|
176
|
+
# Hybrid with custom weights from config
|
|
177
|
+
easy-rag query --mode hybrid "refund policy"
|
|
178
|
+
|
|
179
|
+
# Vector-only for semantic queries
|
|
180
|
+
easy-rag query --mode vector "how does it work?"
|
|
181
|
+
|
|
182
|
+
# BM25-only for exact keywords
|
|
183
|
+
easy-rag query --mode bm25 "API endpoint /v1/users"
|
|
184
|
+
```
|
|
125
185
|
|
|
126
186
|
## Configuration
|
|
127
187
|
|
|
@@ -142,12 +202,24 @@ Set during `easy-rag init` or via `EMBEDDING_MODEL` env var.
|
|
|
142
202
|
| `text-embedding-3-small` | Faster, cheaper |
|
|
143
203
|
| `text-embedding-ada-002` | Legacy |
|
|
144
204
|
|
|
145
|
-
###
|
|
205
|
+
### Advanced Search Settings
|
|
206
|
+
|
|
207
|
+
Configure during `easy-rag init` or edit `~/.easy-rag/config.json`.
|
|
208
|
+
|
|
209
|
+
| Setting | Description | Default |
|
|
210
|
+
|---------|-------------|---------|
|
|
211
|
+
| `search_mode` | Search mode: auto, hybrid, vector, or bm25 | auto |
|
|
212
|
+
| `vector_weight` | Weight for vector results in hybrid mode (0.0-1.0) | 0.6 |
|
|
213
|
+
| `bm25_weight` | Weight for sparse results in hybrid mode (0.0-1.0) | 0.4 |
|
|
214
|
+
|
|
215
|
+
Weights don't need to sum to 1.0 — they are used independently in the RRF formula.
|
|
216
|
+
|
|
217
|
+
### Zvec
|
|
146
218
|
|
|
147
|
-
|
|
219
|
+
Zvec is the embedded vector store — no server or external process needed.
|
|
148
220
|
|
|
149
|
-
- Data is stored in `~/.easy-rag/
|
|
150
|
-
-
|
|
221
|
+
- Data is stored in `~/.easy-rag/collections/` (auto-created on first use)
|
|
222
|
+
- Each collection is a directory on disk, fully portable
|
|
151
223
|
|
|
152
224
|
## Development
|
|
153
225
|
|
|
@@ -169,8 +241,8 @@ src/
|
|
|
169
241
|
parsers/ # PDF, Markdown, CSV text extraction
|
|
170
242
|
chunker/ # Content-aware text splitting
|
|
171
243
|
embeddings/ # OpenAI embedding generation
|
|
172
|
-
vector-store/ #
|
|
173
|
-
query/ #
|
|
244
|
+
vector-store/ # Zvec embedded vector store + sparse tokenizer
|
|
245
|
+
query/ # Dense, sparse, and hybrid search
|
|
174
246
|
indexer/ # Orchestrates parse → chunk → embed → store
|
|
175
247
|
config/ # Global config (~/.easy-rag/config.json)
|
|
176
248
|
commands/ # CLI command implementations (init, serve)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAqDA,wBAAsB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAgG1C"}
|
package/dist/commands/init.js
CHANGED
|
@@ -5,6 +5,14 @@ const MODELS = [
|
|
|
5
5
|
{ value: 'text-embedding-3-small', label: 'text-embedding-3-small (faster, cheaper)' },
|
|
6
6
|
{ value: 'text-embedding-ada-002', label: 'text-embedding-ada-002 (legacy model)' },
|
|
7
7
|
];
|
|
8
|
+
const SEARCH_MODES = [
|
|
9
|
+
{ value: 'auto', label: 'auto (default - RRF fusion, no weights needed)' },
|
|
10
|
+
{ value: 'hybrid', label: 'hybrid (weighted fusion - configurable weights)' },
|
|
11
|
+
{ value: 'vector', label: 'vector (vector search only)' },
|
|
12
|
+
{ value: 'bm25', label: 'bm25 (keyword search only)' },
|
|
13
|
+
];
|
|
14
|
+
const DEFAULT_VECTOR_WEIGHT = 0.6;
|
|
15
|
+
const DEFAULT_BM25_WEIGHT = 0.4;
|
|
8
16
|
const MAX_ATTEMPTS = 3;
|
|
9
17
|
function detectExistingKey(existingConfig) {
|
|
10
18
|
const envKey = process.env.OPENAI_API_KEY?.trim();
|
|
@@ -74,10 +82,25 @@ export async function init() {
|
|
|
74
82
|
else {
|
|
75
83
|
model = await promptModel(rl);
|
|
76
84
|
}
|
|
85
|
+
// --- Advanced Settings ---
|
|
86
|
+
let searchMode = existingConfig.search_mode ?? 'auto';
|
|
87
|
+
let vectorWeight = existingConfig.vector_weight ?? DEFAULT_VECTOR_WEIGHT;
|
|
88
|
+
let bm25Weight = existingConfig.bm25_weight ?? DEFAULT_BM25_WEIGHT;
|
|
89
|
+
const advancedAnswer = await rl.question('Configure advanced settings? [y/N]: ');
|
|
90
|
+
if (advancedAnswer.toLowerCase() === 'y') {
|
|
91
|
+
console.log('');
|
|
92
|
+
console.log('=== Advanced Settings ===\n');
|
|
93
|
+
searchMode = await promptSearchMode(rl, existingConfig.search_mode);
|
|
94
|
+
vectorWeight = await promptVectorWeight(rl, existingConfig.vector_weight);
|
|
95
|
+
bm25Weight = await promptBm25Weight(rl, existingConfig.bm25_weight);
|
|
96
|
+
}
|
|
77
97
|
// --- Save ---
|
|
78
98
|
const newConfig = {
|
|
79
99
|
openai_api_key: apiKey,
|
|
80
100
|
embedding_model: model,
|
|
101
|
+
search_mode: searchMode,
|
|
102
|
+
vector_weight: vectorWeight,
|
|
103
|
+
bm25_weight: bm25Weight,
|
|
81
104
|
};
|
|
82
105
|
try {
|
|
83
106
|
await saveConfig(newConfig);
|
|
@@ -88,9 +111,13 @@ export async function init() {
|
|
|
88
111
|
process.exit(3);
|
|
89
112
|
}
|
|
90
113
|
console.log('');
|
|
91
|
-
console.log(`✓ Configuration saved to ${CONFIG_PATH}`);
|
|
92
|
-
console.log(
|
|
93
|
-
console.log(
|
|
114
|
+
console.log(`✓ Configuration saved to ${CONFIG_PATH}\n`);
|
|
115
|
+
console.log('Settings:');
|
|
116
|
+
console.log(` OpenAI API key: ${maskApiKey(apiKey)} (masked)`);
|
|
117
|
+
console.log(` Embedding model: ${model}`);
|
|
118
|
+
console.log(` Search mode: ${searchMode}`);
|
|
119
|
+
console.log(` Vector weight: ${vectorWeight}`);
|
|
120
|
+
console.log(` BM25 weight: ${bm25Weight}`);
|
|
94
121
|
console.log('');
|
|
95
122
|
console.log('You can now use EasyRAG without setting environment variables.');
|
|
96
123
|
console.log('To change settings later, run \'easy-rag init\' again.');
|
|
@@ -156,6 +183,81 @@ async function promptModel(rl) {
|
|
|
156
183
|
// Unreachable, but satisfies TypeScript
|
|
157
184
|
process.exit(2);
|
|
158
185
|
}
|
|
186
|
+
async function promptSearchMode(rl, existingValue) {
|
|
187
|
+
const defaultValue = existingValue ?? 'auto';
|
|
188
|
+
const defaultIndex = SEARCH_MODES.findIndex(m => m.value === defaultValue);
|
|
189
|
+
console.log('Search Mode:');
|
|
190
|
+
SEARCH_MODES.forEach((m, index) => {
|
|
191
|
+
console.log(` ${index + 1}. ${m.label}`);
|
|
192
|
+
});
|
|
193
|
+
console.log('');
|
|
194
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
195
|
+
const remainingAttempts = MAX_ATTEMPTS - attempt;
|
|
196
|
+
const selection = await rl.question(`Enter choice [1-${SEARCH_MODES.length}] (default: ${defaultIndex + 1}): `);
|
|
197
|
+
if (selection === '') {
|
|
198
|
+
return defaultValue;
|
|
199
|
+
}
|
|
200
|
+
const choice = parseInt(selection, 10);
|
|
201
|
+
if (choice >= 1 && choice <= SEARCH_MODES.length) {
|
|
202
|
+
return SEARCH_MODES[choice - 1].value;
|
|
203
|
+
}
|
|
204
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
205
|
+
console.log(`Invalid choice. Please enter a number between 1 and ${SEARCH_MODES.length}.`);
|
|
206
|
+
console.log(`Please try again (${remainingAttempts} attempts remaining):\n`);
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
console.log('Too many failed attempts. Aborting initialization.');
|
|
210
|
+
process.exit(2);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
process.exit(2);
|
|
214
|
+
}
|
|
215
|
+
async function promptVectorWeight(rl, existingValue) {
|
|
216
|
+
const defaultValue = existingValue ?? DEFAULT_VECTOR_WEIGHT;
|
|
217
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
218
|
+
const remainingAttempts = MAX_ATTEMPTS - attempt;
|
|
219
|
+
const input = await rl.question(`Vector Weight (0.0-1.0) [${defaultValue}]: `);
|
|
220
|
+
if (input === '') {
|
|
221
|
+
return defaultValue;
|
|
222
|
+
}
|
|
223
|
+
const value = parseFloat(input);
|
|
224
|
+
if (!isNaN(value) && value >= 0.0 && value <= 1.0) {
|
|
225
|
+
return value;
|
|
226
|
+
}
|
|
227
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
228
|
+
console.log('Invalid value. Please enter a number between 0.0 and 1.0.');
|
|
229
|
+
console.log(`Please try again (${remainingAttempts} attempts remaining):\n`);
|
|
230
|
+
}
|
|
231
|
+
else {
|
|
232
|
+
console.log('Too many failed attempts. Aborting initialization.');
|
|
233
|
+
process.exit(2);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
process.exit(2);
|
|
237
|
+
}
|
|
238
|
+
async function promptBm25Weight(rl, existingValue) {
|
|
239
|
+
const defaultValue = existingValue ?? DEFAULT_BM25_WEIGHT;
|
|
240
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
241
|
+
const remainingAttempts = MAX_ATTEMPTS - attempt;
|
|
242
|
+
const input = await rl.question(`BM25 Weight (0.0-1.0) [${defaultValue}]: `);
|
|
243
|
+
if (input === '') {
|
|
244
|
+
return defaultValue;
|
|
245
|
+
}
|
|
246
|
+
const value = parseFloat(input);
|
|
247
|
+
if (!isNaN(value) && value >= 0.0 && value <= 1.0) {
|
|
248
|
+
return value;
|
|
249
|
+
}
|
|
250
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
251
|
+
console.log('Invalid value. Please enter a number between 0.0 and 1.0.');
|
|
252
|
+
console.log(`Please try again (${remainingAttempts} attempts remaining):\n`);
|
|
253
|
+
}
|
|
254
|
+
else {
|
|
255
|
+
console.log('Too many failed attempts. Aborting initialization.');
|
|
256
|
+
process.exit(2);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
process.exit(2);
|
|
260
|
+
}
|
|
159
261
|
function maskApiKey(key) {
|
|
160
262
|
if (key.length <= 8) {
|
|
161
263
|
return 'sk-***';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,wBAAwB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGzE,MAAM,MAAM,GAAG;IACb,EAAE,KAAK,EAAE,wBAAwB,EAAE,KAAK,EAAE,oDAAoD,EAAE;IAChG,EAAE,KAAK,EAAE,wBAAwB,EAAE,KAAK,EAAE,0CAA0C,EAAE;IACtF,EAAE,KAAK,EAAE,wBAAwB,EAAE,KAAK,EAAE,uCAAuC,EAAE;CACpF,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,CAAC;AAOvB,SAAS,iBAAiB,CAAC,cAA6B;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC;IAClD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC1C,CAAC;IACD,IAAI,cAAc,CAAC,cAAc,EAAE,CAAC;QAClC,OAAO,EAAE,KAAK,EAAE,cAAc,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,cAA6B;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC;IACrD,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC5C,CAAC;IACD,IAAI,cAAc,CAAC,eAAe,EAAE,CAAC;QACnC,OAAO,EAAE,KAAK,EAAE,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACrE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,MAAwB;IAC3C,OAAO,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC;AACnE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI;IACxB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAE9C,MAAM,cAAc,GAAG,MAAM,UAAU,EAAE,CAAC;QAE1C,kBAAkB;QAClB,MAAM,WAAW,GAAG,iBAAiB,CAAC,cAAc,CAAC,CAAC;QACtD,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,MAAM,EAAE,CAAC,CAAC;YACpF,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;YAE3D,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,GAAG,EAAE,CAAC;gBACjC,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,CAAC,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,0BAA0B;QAC1B,MAAM,aAAa,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAC;QAC1D,IAAI,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAE5B,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,aAAa,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,aAAa,CAAC,KAAK,CAAC;YACnG,OAAO,CAAC,GAAG,CAAC,0BAA0B,WAAW,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,UAAU,EAAE,CAAC,CAAC;YAC3F,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC;YAE7D,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,GAAG,EAAE,CAAC;gBACjC,KAAK,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC;QAED,eAAe;QACf,MAAM,SAAS,GAAkB;YAC/B,cAAc,EAAE,MAAM;YACtB,eAAe,EAAE,KAAK;
|
|
1
|
+
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,wBAAwB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGzE,MAAM,MAAM,GAAG;IACb,EAAE,KAAK,EAAE,wBAAwB,EAAE,KAAK,EAAE,oDAAoD,EAAE;IAChG,EAAE,KAAK,EAAE,wBAAwB,EAAE,KAAK,EAAE,0CAA0C,EAAE;IACtF,EAAE,KAAK,EAAE,wBAAwB,EAAE,KAAK,EAAE,uCAAuC,EAAE;CACpF,CAAC;AAEF,MAAM,YAAY,GAAG;IACnB,EAAE,KAAK,EAAE,MAAe,EAAE,KAAK,EAAE,gDAAgD,EAAE;IACnF,EAAE,KAAK,EAAE,QAAiB,EAAE,KAAK,EAAE,iDAAiD,EAAE;IACtF,EAAE,KAAK,EAAE,QAAiB,EAAE,KAAK,EAAE,6BAA6B,EAAE;IAClE,EAAE,KAAK,EAAE,MAAe,EAAE,KAAK,EAAE,4BAA4B,EAAE;CAChE,CAAC;AAEF,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC,MAAM,YAAY,GAAG,CAAC,CAAC;AAOvB,SAAS,iBAAiB,CAAC,cAA6B;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC;IAClD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC1C,CAAC;IACD,IAAI,cAAc,CAAC,cAAc,EAAE,CAAC;QAClC,OAAO,EAAE,KAAK,EAAE,cAAc,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,cAA6B;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC;IACrD,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC5C,CAAC;IACD,IAAI,cAAc,CAAC,eAAe,EAAE,CAAC;QACnC,OAAO,EAAE,KAAK,EAAE,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACrE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,MAAwB;IAC3C,OAAO,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC;AACnE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI;IACxB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAE9C,MAAM,cAAc,GAAG,MAAM,UAAU,EAAE,CAAC;QAE1C,kBAAkB;QAClB,MAAM,WAAW,GAAG,iBAAiB,CAAC,cAAc,CAAC,CAAC;QACtD,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,MAAM,EAAE,CAAC,CAAC;YACpF,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;YAE3D,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,GAAG,EAAE,CAAC;gBACjC,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,CAAC,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,0BAA0B;QAC1B,MAAM,aAAa,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAC;QAC1D,IAAI,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAE5B,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,aAAa,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,aAAa,CAAC,KAAK,CAAC;YACnG,OAAO,CAAC,GAAG,CAAC,0BAA0B,WAAW,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,UAAU,EAAE,CAAC,CAAC;YAC3F,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC;YAE7D,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,GAAG,EAAE,CAAC;gBACjC,KAAK,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,MAAM,WAAW,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC;QAED,4BAA4B;QAC5B,IAAI,UAAU,GAA0C,cAAc,CAAC,WAAW,IAAI,MAAM,CAAC;QAC7F,IAAI,YAAY,GAAG,cAAc,CAAC,aAAa,IAAI,qBAAqB,CAAC;QACzE,IAAI,UAAU,GAAG,cAAc,CAAC,WAAW,IAAI,mBAAmB,CAAC;QAEnE,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC,CAAC;QACjF,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,GAAG,EAAE,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAE3C,UAAU,GAAG,MAAM,gBAAgB,CAAC,EAAE,EAAE,cAAc,CAAC,WAAW,CAAC,CAAC;YACpE,YAAY,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,cAAc,CAAC,aAAa,CAAC,CAAC;YAC1E,UAAU,GAAG,MAAM,gBAAgB,CAAC,EAAE,EAAE,cAAc,CAAC,WAAW,CAAC,CAAC;QACtE,CAAC;QAED,eAAe;QACf,MAAM,SAAS,GAAkB;YAC/B,cAAc,EAAE,MAAM;YACtB,eAAe,EAAE,KAAK;YACtB,WAAW,EAAE,UAAU;YACvB,aAAa,EAAE,YAAY;YAC3B,WAAW,EAAE,UAAU;SACxB,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,4BAA4B,WAAW,EAAE,CAAC,CAAC;YACzD,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAChE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,4BAA4B,WAAW,IAAI,CAAC,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,qBAAqB,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,oBAAoB,YAAY,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;QAC9E,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IACxE,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,EAAsB;IAChD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,YAAY,GAAG,OAAO,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC,CAAC;QAEzE,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAClB,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,yBAAyB,CAAC,CAAC;YAC7E,SAAS;QACX,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;YACjE,IAAI,OAAO,GAAG,YAAY,EAAE,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,yBAAyB,CAAC,CAAC;gBAC7E,SAAS;YACX,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,wCAAwC;IACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,EAAsB;IAC/C,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;QAC1B,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,YAAY,GAAG,OAAO,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC,CAAC;QAEzE,IAAI,SAAS,KAAK,EAAE,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YAC1C,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACzB,CAAC;QAED,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;QAClC,CAAC;QAED,IAAI,OAAO,GAAG,YAAY,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;YACtE,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,yBAAyB,CAAC,CAAC;QAC/E,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,EAAsB,EAAE,aAAqD;IAC3G,MAAM,YAAY,GAAG,aAAa,IAAI,MAAM,CAAC;IAC7C,MAAM,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC;IAE3E,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC5B,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;QAChC,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,YAAY,GAAG,OAAO,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,mBAAmB,YAAY,CAAC,MAAM,eAAe,YAAY,GAAG,CAAC,KAAK,CAAC,CAAC;QAEhH,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YACrB,OAAO,YAAY,CAAC;QACtB,CAAC;QAED,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACjD,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;QACxC,CAAC;QAED,IAAI,OAAO,GAAG,YAAY,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,uDAAuD,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;YAC3F,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,yBAAyB,CAAC,CAAC;QAC/E,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,EAAsB,EAAE,aAAsB;IAC9E,MAAM,YAAY,GAAG,aAAa,IAAI,qBAAqB,CAAC;IAE5D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,YAAY,GAAG,OAAO,CAAC;QACjD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,4BAA4B,YAAY,KAAK,CAAC,CAAC;QAE/E,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;YACjB,OAAO,YAAY,CAAC;QACtB,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,GAAG,YAAY,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;YACzE,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,yBAAyB,CAAC,CAAC;QAC/E,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,EAAsB,EAAE,aAAsB;IAC5E,MAAM,YAAY,GAAG,aAAa,IAAI,mBAAmB,CAAC;IAE1D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,YAAY,GAAG,OAAO,CAAC;QACjD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,0BAA0B,YAAY,KAAK,CAAC,CAAC;QAE7E,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;YACjB,OAAO,YAAY,CAAC;QACtB,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,GAAG,YAAY,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;YACzE,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,yBAAyB,CAAC,CAAC;QAC/E,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC;AACrC,CAAC"}
|
package/dist/config/types.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/config/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/config/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;CACrD"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { EvalSummary } from './types.js';
|
|
2
|
+
export interface EvaluationOptions {
|
|
3
|
+
collection: string;
|
|
4
|
+
top?: number;
|
|
5
|
+
negativeThreshold?: number;
|
|
6
|
+
outputDir?: string;
|
|
7
|
+
comparePath?: string;
|
|
8
|
+
}
|
|
9
|
+
export declare function runEvaluation(options: EvaluationOptions): Promise<EvalSummary>;
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CA2CpF"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { TEST_CASES } from './test-cases.js';
|
|
2
|
+
import { runTestCase } from './runner.js';
|
|
3
|
+
import { buildSummary, printSummary, printComparison, saveReport, loadPreviousReport } from './reporter.js';
|
|
4
|
+
export async function runEvaluation(options) {
|
|
5
|
+
const { collection, top = 5, negativeThreshold = 0.35, outputDir = 'docs', comparePath, } = options;
|
|
6
|
+
console.log(`\nRunning evaluation against collection: ${collection}`);
|
|
7
|
+
console.log(`Cases: ${TEST_CASES.length} | Top-K: ${top} | Neg. threshold: ${negativeThreshold}\n`);
|
|
8
|
+
const results = [];
|
|
9
|
+
for (const testCase of TEST_CASES) {
|
|
10
|
+
process.stdout.write(` [${testCase.id}] ${testCase.query.slice(0, 60)}... `);
|
|
11
|
+
try {
|
|
12
|
+
const result = await runTestCase(testCase, { collection, top, negativeThreshold });
|
|
13
|
+
results.push(result);
|
|
14
|
+
const allPass = result.overallPass;
|
|
15
|
+
console.log(allPass ? 'PASS' : 'FAIL');
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
console.log('ERROR');
|
|
19
|
+
console.error(` ${error instanceof Error ? error.message : String(error)}`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
const summary = buildSummary(collection, negativeThreshold, results);
|
|
23
|
+
printSummary(summary);
|
|
24
|
+
if (comparePath) {
|
|
25
|
+
try {
|
|
26
|
+
const previous = await loadPreviousReport(comparePath);
|
|
27
|
+
printComparison(summary, previous);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
console.warn(`Warning: Could not load comparison file: ${comparePath}`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const savedPath = await saveReport(summary, outputDir);
|
|
34
|
+
console.log(`Report saved to: ${savedPath}\n`);
|
|
35
|
+
return summary;
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAW5G,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAA0B;IAC5D,MAAM,EACJ,UAAU,EACV,GAAG,GAAG,CAAC,EACP,iBAAiB,GAAG,IAAI,EACxB,SAAS,GAAG,MAAM,EAClB,WAAW,GACZ,GAAG,OAAO,CAAC;IAEZ,OAAO,CAAC,GAAG,CAAC,4CAA4C,UAAU,EAAE,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CAAC,UAAU,UAAU,CAAC,MAAM,aAAa,GAAG,sBAAsB,iBAAiB,IAAI,CAAC,CAAC;IAEpG,MAAM,OAAO,GAAG,EAAE,CAAC;IAEnB,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,EAAE,KAAK,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC9E,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACnF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;IACrE,YAAY,CAAC,OAAO,CAAC,CAAC;IAEtB,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,WAAW,CAAC,CAAC;YACvD,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,4CAA4C,WAAW,EAAE,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,oBAAoB,SAAS,IAAI,CAAC,CAAC;IAE/C,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { HybridQueryResult } from '../query/index.js';
|
|
2
|
+
export declare function computeMRR(results: HybridQueryResult[], expectedSources: string[]): number;
|
|
3
|
+
export declare function computePrecisionAtK(results: HybridQueryResult[], expectedSources: string[], k: number): number;
|
|
4
|
+
export declare function hasRelevantInTopK(results: HybridQueryResult[], expectedSources: string[], k: number): boolean;
|
|
5
|
+
export declare function getTopScore(results: HybridQueryResult[]): number;
|
|
6
|
+
export declare function buildResultDetails(results: HybridQueryResult[], expectedSources: string[]): Array<{
|
|
7
|
+
rank: number;
|
|
8
|
+
sourcePath: string;
|
|
9
|
+
score: number;
|
|
10
|
+
relevant: boolean;
|
|
11
|
+
}>;
|
|
12
|
+
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/eval/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAQ3D,wBAAgB,UAAU,CAAC,OAAO,EAAE,iBAAiB,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE,GAAG,MAAM,CAQ1F;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,iBAAiB,EAAE,EAC5B,eAAe,EAAE,MAAM,EAAE,EACzB,CAAC,EAAE,MAAM,GACR,MAAM,CAKR;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,iBAAiB,EAAE,EAC5B,eAAe,EAAE,MAAM,EAAE,EACzB,CAAC,EAAE,MAAM,GACR,OAAO,CAGT;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAIhE;AAED,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,iBAAiB,EAAE,EAC5B,eAAe,EAAE,MAAM,EAAE,GACxB,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,CAAC,CAO/E"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
function isRelevant(result, expectedSources) {
|
|
2
|
+
if (expectedSources.length === 0)
|
|
3
|
+
return false;
|
|
4
|
+
const src = result.metadata.sourcePath.toLowerCase();
|
|
5
|
+
return expectedSources.some((s) => src.includes(s.toLowerCase()));
|
|
6
|
+
}
|
|
7
|
+
export function computeMRR(results, expectedSources) {
|
|
8
|
+
if (expectedSources.length === 0)
|
|
9
|
+
return 0;
|
|
10
|
+
for (let i = 0; i < results.length; i++) {
|
|
11
|
+
if (isRelevant(results[i], expectedSources)) {
|
|
12
|
+
return 1 / (i + 1);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
return 0;
|
|
16
|
+
}
|
|
17
|
+
export function computePrecisionAtK(results, expectedSources, k) {
|
|
18
|
+
if (expectedSources.length === 0)
|
|
19
|
+
return 0;
|
|
20
|
+
const topK = results.slice(0, k);
|
|
21
|
+
const relevant = topK.filter((r) => isRelevant(r, expectedSources)).length;
|
|
22
|
+
return relevant / k;
|
|
23
|
+
}
|
|
24
|
+
export function hasRelevantInTopK(results, expectedSources, k) {
|
|
25
|
+
if (expectedSources.length === 0)
|
|
26
|
+
return false;
|
|
27
|
+
return results.slice(0, k).some((r) => isRelevant(r, expectedSources));
|
|
28
|
+
}
|
|
29
|
+
export function getTopScore(results) {
|
|
30
|
+
if (results.length === 0)
|
|
31
|
+
return 0;
|
|
32
|
+
const r = results[0];
|
|
33
|
+
return r.hybridScore ?? r.similarity ?? 0;
|
|
34
|
+
}
|
|
35
|
+
export function buildResultDetails(results, expectedSources) {
|
|
36
|
+
return results.map((r, idx) => ({
|
|
37
|
+
rank: idx + 1,
|
|
38
|
+
sourcePath: r.metadata.sourcePath,
|
|
39
|
+
score: r.hybridScore ?? r.similarity ?? 0,
|
|
40
|
+
relevant: isRelevant(r, expectedSources),
|
|
41
|
+
}));
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/eval/metrics.ts"],"names":[],"mappings":"AAEA,SAAS,UAAU,CAAC,MAAyB,EAAE,eAAyB;IACtE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC;IACrD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,OAA4B,EAAE,eAAyB;IAChF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAA4B,EAC5B,eAAyB,EACzB,CAAS;IAET,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC;IAC3E,OAAO,QAAQ,GAAG,CAAC,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,OAA4B,EAC5B,eAAyB,EACzB,CAAS;IAET,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/C,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAA4B;IACtD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACrB,OAAO,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,OAA4B,EAC5B,eAAyB;IAEzB,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC9B,IAAI,EAAE,GAAG,GAAG,CAAC;QACb,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU;QACjC,KAAK,EAAE,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC;QACzC,QAAQ,EAAE,UAAU,CAAC,CAAC,EAAE,eAAe,CAAC;KACzC,CAAC,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { EvalSummary, EvalCaseResult } from './types.js';
|
|
2
|
+
export declare function buildSummary(collection: string, negativeThreshold: number, cases: EvalCaseResult[]): EvalSummary;
|
|
3
|
+
export declare function printSummary(summary: EvalSummary): void;
|
|
4
|
+
export declare function printComparison(current: EvalSummary, previous: EvalSummary): void;
|
|
5
|
+
export declare function saveReport(summary: EvalSummary, outputDir: string): Promise<string>;
|
|
6
|
+
export declare function loadPreviousReport(comparePath: string): Promise<EvalSummary>;
|
|
7
|
+
//# sourceMappingURL=reporter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/eval/reporter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAc9D,wBAAgB,YAAY,CAC1B,UAAU,EAAE,MAAM,EAClB,iBAAiB,EAAE,MAAM,EACzB,KAAK,EAAE,cAAc,EAAE,GACtB,WAAW,CAoCb;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,IAAI,CA0CvD;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,GAAG,IAAI,CAYjF;AAED,wBAAsB,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAOzF;AAED,wBAAsB,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAGlF"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
function pad(s, n) {
|
|
4
|
+
return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
|
|
5
|
+
}
|
|
6
|
+
function fmt(n, decimals = 3) {
|
|
7
|
+
return n.toFixed(decimals);
|
|
8
|
+
}
|
|
9
|
+
function passLabel(pass) {
|
|
10
|
+
return pass ? 'PASS' : 'FAIL';
|
|
11
|
+
}
|
|
12
|
+
export function buildSummary(collection, negativeThreshold, cases) {
|
|
13
|
+
const passed = cases.filter((c) => c.overallPass).length;
|
|
14
|
+
const allModes = cases.flatMap((c) => c.modes);
|
|
15
|
+
const positiveModes = allModes.filter((_, i) => {
|
|
16
|
+
const caseIdx = cases.findIndex((c) => c.modes.includes(allModes[i]));
|
|
17
|
+
return caseIdx !== -1 && !cases[caseIdx].isNegative;
|
|
18
|
+
});
|
|
19
|
+
const avgMrr = positiveModes.length > 0
|
|
20
|
+
? positiveModes.reduce((sum, m) => sum + m.mrr, 0) / positiveModes.length
|
|
21
|
+
: 0;
|
|
22
|
+
const avgP1 = positiveModes.length > 0
|
|
23
|
+
? positiveModes.reduce((sum, m) => sum + m.precisionAt1, 0) / positiveModes.length
|
|
24
|
+
: 0;
|
|
25
|
+
const avgP3 = positiveModes.length > 0
|
|
26
|
+
? positiveModes.reduce((sum, m) => sum + m.precisionAt3, 0) / positiveModes.length
|
|
27
|
+
: 0;
|
|
28
|
+
return {
|
|
29
|
+
collection,
|
|
30
|
+
timestamp: new Date().toISOString(),
|
|
31
|
+
negativeThreshold,
|
|
32
|
+
totalCases: cases.length,
|
|
33
|
+
totalPassed: passed,
|
|
34
|
+
totalFailed: cases.length - passed,
|
|
35
|
+
passRate: cases.length > 0 ? passed / cases.length : 0,
|
|
36
|
+
avgMrr,
|
|
37
|
+
avgPrecisionAt1: avgP1,
|
|
38
|
+
avgPrecisionAt3: avgP3,
|
|
39
|
+
cases,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
export function printSummary(summary) {
|
|
43
|
+
const sep = '─'.repeat(100);
|
|
44
|
+
console.log('\n' + sep);
|
|
45
|
+
console.log(` easy-rag Evaluation Report`);
|
|
46
|
+
console.log(` Collection : ${summary.collection}`);
|
|
47
|
+
console.log(` Timestamp : ${summary.timestamp}`);
|
|
48
|
+
console.log(` Neg. threshold: ${summary.negativeThreshold}`);
|
|
49
|
+
console.log(sep);
|
|
50
|
+
const header = [
|
|
51
|
+
pad('ID', 32),
|
|
52
|
+
pad('Mode', 8),
|
|
53
|
+
pad('MRR', 6),
|
|
54
|
+
pad('P@1', 5),
|
|
55
|
+
pad('P@3', 5),
|
|
56
|
+
pad('TopScore', 10),
|
|
57
|
+
pad('Result', 6),
|
|
58
|
+
].join(' ');
|
|
59
|
+
console.log(header);
|
|
60
|
+
console.log(sep);
|
|
61
|
+
for (const c of summary.cases) {
|
|
62
|
+
const prefix = c.isNegative ? '[NEG] ' : '';
|
|
63
|
+
for (const m of c.modes) {
|
|
64
|
+
const row = [
|
|
65
|
+
pad(prefix + c.id, 32),
|
|
66
|
+
pad(m.mode, 8),
|
|
67
|
+
pad(fmt(m.mrr), 6),
|
|
68
|
+
pad(fmt(m.precisionAt1, 2), 5),
|
|
69
|
+
pad(fmt(m.precisionAt3, 2), 5),
|
|
70
|
+
pad(fmt(m.topScore, 4), 10),
|
|
71
|
+
passLabel(m.pass),
|
|
72
|
+
].join(' ');
|
|
73
|
+
console.log(row);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
console.log(sep);
|
|
77
|
+
console.log(` Total: ${summary.totalCases} cases | Passed: ${summary.totalPassed} | Failed: ${summary.totalFailed} | Pass rate: ${(summary.passRate * 100).toFixed(1)}%`);
|
|
78
|
+
console.log(` Avg MRR: ${fmt(summary.avgMrr)} | Avg P@1: ${fmt(summary.avgPrecisionAt1, 2)} | Avg P@3: ${fmt(summary.avgPrecisionAt3, 2)}`);
|
|
79
|
+
console.log(sep + '\n');
|
|
80
|
+
}
|
|
81
|
+
export function printComparison(current, previous) {
|
|
82
|
+
const delta = (a, b) => {
|
|
83
|
+
const d = a - b;
|
|
84
|
+
const sign = d >= 0 ? '+' : '';
|
|
85
|
+
return `${sign}${d.toFixed(3)}`;
|
|
86
|
+
};
|
|
87
|
+
console.log('\n Comparison vs previous run:');
|
|
88
|
+
console.log(` Pass rate : ${(previous.passRate * 100).toFixed(1)}% → ${(current.passRate * 100).toFixed(1)}% (${delta(current.passRate, previous.passRate)})`);
|
|
89
|
+
console.log(` Avg MRR : ${previous.avgMrr.toFixed(3)} → ${current.avgMrr.toFixed(3)} (${delta(current.avgMrr, previous.avgMrr)})`);
|
|
90
|
+
console.log(` Avg P@1 : ${previous.avgPrecisionAt1.toFixed(2)} → ${current.avgPrecisionAt1.toFixed(2)} (${delta(current.avgPrecisionAt1, previous.avgPrecisionAt1)})`);
|
|
91
|
+
console.log(` Avg P@3 : ${previous.avgPrecisionAt3.toFixed(2)} → ${current.avgPrecisionAt3.toFixed(2)} (${delta(current.avgPrecisionAt3, previous.avgPrecisionAt3)})\n`);
|
|
92
|
+
}
|
|
93
|
+
export async function saveReport(summary, outputDir) {
|
|
94
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
95
|
+
const ts = summary.timestamp.replace(/[:.]/g, '-');
|
|
96
|
+
const filename = `eval-results-${ts}.json`;
|
|
97
|
+
const filePath = path.join(outputDir, filename);
|
|
98
|
+
await fs.writeFile(filePath, JSON.stringify(summary, null, 2), 'utf-8');
|
|
99
|
+
return filePath;
|
|
100
|
+
}
|
|
101
|
+
export async function loadPreviousReport(comparePath) {
|
|
102
|
+
const raw = await fs.readFile(comparePath, 'utf-8');
|
|
103
|
+
return JSON.parse(raw);
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=reporter.js.map
|