@aiconnect/easy-rag 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +181 -0
- package/dist/chunker/csv.d.ts +4 -0
- package/dist/chunker/csv.d.ts.map +1 -0
- package/dist/chunker/csv.js +10 -0
- package/dist/chunker/csv.js.map +1 -0
- package/dist/chunker/index.d.ts +5 -0
- package/dist/chunker/index.d.ts.map +1 -0
- package/dist/chunker/index.js +20 -0
- package/dist/chunker/index.js.map +1 -0
- package/dist/chunker/markdown.d.ts +4 -0
- package/dist/chunker/markdown.d.ts.map +1 -0
- package/dist/chunker/markdown.js +35 -0
- package/dist/chunker/markdown.js.map +1 -0
- package/dist/chunker/pdf.d.ts +4 -0
- package/dist/chunker/pdf.d.ts.map +1 -0
- package/dist/chunker/pdf.js +26 -0
- package/dist/chunker/pdf.js.map +1 -0
- package/dist/chunker/types.d.ts +12 -0
- package/dist/chunker/types.d.ts.map +1 -0
- package/dist/chunker/types.js +2 -0
- package/dist/chunker/types.js.map +1 -0
- package/dist/commands/init.d.ts +2 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +120 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/serve.d.ts +2 -0
- package/dist/commands/serve.d.ts.map +1 -0
- package/dist/commands/serve.js +17 -0
- package/dist/commands/serve.js.map +1 -0
- package/dist/config/index.d.ts +9 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +66 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/types.d.ts +5 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/embeddings/index.d.ts +3 -0
- package/dist/embeddings/index.d.ts.map +1 -0
- package/dist/embeddings/index.js +2 -0
- package/dist/embeddings/index.js.map +1 -0
- package/dist/embeddings/openai.d.ts +2 -0
- package/dist/embeddings/openai.d.ts.map +1 -0
- package/dist/embeddings/openai.js +57 -0
- package/dist/embeddings/openai.js.map +1 -0
- package/dist/embeddings/types.d.ts +15 -0
- package/dist/embeddings/types.d.ts.map +1 -0
- package/dist/embeddings/types.js +2 -0
- package/dist/embeddings/types.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +248 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer/index.d.ts +4 -0
- package/dist/indexer/index.d.ts.map +1 -0
- package/dist/indexer/index.js +3 -0
- package/dist/indexer/index.js.map +1 -0
- package/dist/indexer/orchestrator.d.ts +3 -0
- package/dist/indexer/orchestrator.d.ts.map +1 -0
- package/dist/indexer/orchestrator.js +106 -0
- package/dist/indexer/orchestrator.js.map +1 -0
- package/dist/indexer/scanner.d.ts +2 -0
- package/dist/indexer/scanner.d.ts.map +1 -0
- package/dist/indexer/scanner.js +34 -0
- package/dist/indexer/scanner.js.map +1 -0
- package/dist/indexer/types.d.ts +12 -0
- package/dist/indexer/types.d.ts.map +1 -0
- package/dist/indexer/types.js +2 -0
- package/dist/indexer/types.js.map +1 -0
- package/dist/parsers/csv.d.ts +3 -0
- package/dist/parsers/csv.d.ts.map +1 -0
- package/dist/parsers/csv.js +63 -0
- package/dist/parsers/csv.js.map +1 -0
- package/dist/parsers/fileTypeDetector.d.ts +3 -0
- package/dist/parsers/fileTypeDetector.d.ts.map +1 -0
- package/dist/parsers/fileTypeDetector.js +16 -0
- package/dist/parsers/fileTypeDetector.js.map +1 -0
- package/dist/parsers/index.d.ts +3 -0
- package/dist/parsers/index.d.ts.map +1 -0
- package/dist/parsers/index.js +18 -0
- package/dist/parsers/index.js.map +1 -0
- package/dist/parsers/markdown.d.ts +3 -0
- package/dist/parsers/markdown.d.ts.map +1 -0
- package/dist/parsers/markdown.js +30 -0
- package/dist/parsers/markdown.js.map +1 -0
- package/dist/parsers/pdf.d.ts +3 -0
- package/dist/parsers/pdf.d.ts.map +1 -0
- package/dist/parsers/pdf.js +22 -0
- package/dist/parsers/pdf.js.map +1 -0
- package/dist/parsers/types.d.ts +17 -0
- package/dist/parsers/types.d.ts.map +1 -0
- package/dist/parsers/types.js +2 -0
- package/dist/parsers/types.js.map +1 -0
- package/dist/query/embedding.d.ts +2 -0
- package/dist/query/embedding.d.ts.map +1 -0
- package/dist/query/embedding.js +6 -0
- package/dist/query/embedding.js.map +1 -0
- package/dist/query/index.d.ts +3 -0
- package/dist/query/index.d.ts.map +1 -0
- package/dist/query/index.js +86 -0
- package/dist/query/index.js.map +1 -0
- package/dist/query/search.d.ts +6 -0
- package/dist/query/search.d.ts.map +1 -0
- package/dist/query/search.js +45 -0
- package/dist/query/search.js.map +1 -0
- package/dist/query/types.d.ts +19 -0
- package/dist/query/types.d.ts.map +1 -0
- package/dist/query/types.js +2 -0
- package/dist/query/types.js.map +1 -0
- package/dist/vector-store/chroma-server.d.ts +10 -0
- package/dist/vector-store/chroma-server.d.ts.map +1 -0
- package/dist/vector-store/chroma-server.js +102 -0
- package/dist/vector-store/chroma-server.js.map +1 -0
- package/dist/vector-store/chromadb.d.ts +8 -0
- package/dist/vector-store/chromadb.d.ts.map +1 -0
- package/dist/vector-store/chromadb.js +98 -0
- package/dist/vector-store/chromadb.js.map +1 -0
- package/dist/vector-store/index.d.ts +4 -0
- package/dist/vector-store/index.d.ts.map +1 -0
- package/dist/vector-store/index.js +3 -0
- package/dist/vector-store/index.js.map +1 -0
- package/dist/vector-store/types.d.ts +12 -0
- package/dist/vector-store/types.d.ts.map +1 -0
- package/dist/vector-store/types.js +2 -0
- package/dist/vector-store/types.js.map +1 -0
- package/dist/vector-store/utils.d.ts +2 -0
- package/dist/vector-store/utils.d.ts.map +1 -0
- package/dist/vector-store/utils.js +17 -0
- package/dist/vector-store/utils.js.map +1 -0
- package/package.json +57 -0
- package/skills/easy-rag/SKILL.md +198 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
export async function parseCSV(filePath) {
|
|
3
|
+
try {
|
|
4
|
+
const content = await fs.readFile(filePath, 'utf-8');
|
|
5
|
+
const lines = content.trim().split('\n');
|
|
6
|
+
if (lines.length === 0) {
|
|
7
|
+
return [];
|
|
8
|
+
}
|
|
9
|
+
const headers = parseCSVLine(lines[0]);
|
|
10
|
+
const documents = [];
|
|
11
|
+
for (let i = 1; i < lines.length; i++) {
|
|
12
|
+
const values = parseCSVLine(lines[i]);
|
|
13
|
+
let documentContent;
|
|
14
|
+
if (headers.length > 0 && values.length === headers.length) {
|
|
15
|
+
documentContent = headers.map((header, index) => `${header}: ${values[index]}`).join('\n');
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
documentContent = values.join(', ');
|
|
19
|
+
}
|
|
20
|
+
documents.push({
|
|
21
|
+
content: documentContent,
|
|
22
|
+
metadata: {
|
|
23
|
+
sourcePath: filePath,
|
|
24
|
+
contentType: 'csv',
|
|
25
|
+
rowNumber: i
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
return documents;
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
if (error instanceof Error) {
|
|
33
|
+
throw new Error(`Failed to parse CSV file: ${error.message}`);
|
|
34
|
+
}
|
|
35
|
+
throw new Error('Failed to parse CSV file: Unknown error');
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function parseCSVLine(line) {
|
|
39
|
+
const result = [];
|
|
40
|
+
let current = '';
|
|
41
|
+
let inQuotes = false;
|
|
42
|
+
for (let i = 0; i < line.length; i++) {
|
|
43
|
+
const char = line[i];
|
|
44
|
+
if (char === '"') {
|
|
45
|
+
inQuotes = !inQuotes;
|
|
46
|
+
}
|
|
47
|
+
else if (char === ',' && !inQuotes) {
|
|
48
|
+
result.push(current.trim());
|
|
49
|
+
current = '';
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
current += char;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
result.push(current.trim());
|
|
56
|
+
return result.map(value => {
|
|
57
|
+
if (value.startsWith('"') && value.endsWith('"')) {
|
|
58
|
+
return value.slice(1, -1);
|
|
59
|
+
}
|
|
60
|
+
return value;
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=csv.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/parsers/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,aAAa,CAAC;AAG7B,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAAgB;IAC7C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEzC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,SAAS,GAAkB,EAAE,CAAC;QAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,eAAuB,CAAC;YAE5B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC3D,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,MAAM,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7F,CAAC;iBAAM,CAAC;gBACN,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,OAAO,EAAE,eAAe;gBACxB,QAAQ,EAAE;oBACR,UAAU,EAAE,QAAQ;oBACpB,WAAW,EAAE,KAAK;oBAClB,SAAS,EAAE,CAAC;iBACb;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAErB,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,QAAQ,GAAG,CAAC,QAAQ,CAAC;QACvB,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5B,OAAO,GAAG,EAAE,CAAC;QACf,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAE5B,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACxB,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fileTypeDetector.d.ts","sourceRoot":"","sources":["../../src/parsers/fileTypeDetector.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAczD"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
export function detectFileType(filePath) {
|
|
3
|
+
const ext = path.extname(filePath).toLowerCase();
|
|
4
|
+
switch (ext) {
|
|
5
|
+
case '.pdf':
|
|
6
|
+
return 'pdf';
|
|
7
|
+
case '.md':
|
|
8
|
+
case '.markdown':
|
|
9
|
+
return 'markdown';
|
|
10
|
+
case '.csv':
|
|
11
|
+
return 'csv';
|
|
12
|
+
default:
|
|
13
|
+
throw new Error(`Unsupported file extension: ${ext}. Supported formats: .pdf, .md, .markdown, .csv`);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=fileTypeDetector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fileTypeDetector.js","sourceRoot":"","sources":["../../src/parsers/fileTypeDetector.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAGxB,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAEjD,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,MAAM;YACT,OAAO,KAAK,CAAC;QACf,KAAK,KAAK,CAAC;QACX,KAAK,WAAW;YACd,OAAO,UAAU,CAAC;QACpB,KAAK,MAAM;YACT,OAAO,KAAK,CAAC;QACf;YACE,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,iDAAiD,CAAC,CAAC;IACzG,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/parsers/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAM9D,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,WAAW,EAAE,CAAC,CAa7F"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { detectFileType } from './fileTypeDetector.js';
|
|
2
|
+
import { parsePDF } from './pdf.js';
|
|
3
|
+
import { parseMarkdown } from './markdown.js';
|
|
4
|
+
import { parseCSV } from './csv.js';
|
|
5
|
+
export async function parseDocument(filePath) {
|
|
6
|
+
const fileType = detectFileType(filePath);
|
|
7
|
+
switch (fileType) {
|
|
8
|
+
case 'pdf':
|
|
9
|
+
return await parsePDF(filePath);
|
|
10
|
+
case 'markdown':
|
|
11
|
+
return await parseMarkdown(filePath);
|
|
12
|
+
case 'csv':
|
|
13
|
+
return await parseCSV(filePath);
|
|
14
|
+
default:
|
|
15
|
+
throw new Error(`Unsupported file type: ${fileType}`);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/parsers/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB;IAClD,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IAE1C,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,KAAK;YACR,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAClC,KAAK,UAAU;YACb,OAAO,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;QACvC,KAAK,KAAK;YACR,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAClC;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,QAAQ,EAAE,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/parsers/markdown.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CA4B7E"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
export async function parseMarkdown(filePath) {
|
|
3
|
+
try {
|
|
4
|
+
const content = await fs.readFile(filePath, 'utf-8');
|
|
5
|
+
const lines = content.split('\n');
|
|
6
|
+
const sections = [];
|
|
7
|
+
const headingRegex = /^(#{1,3})\s+(.+)$/;
|
|
8
|
+
for (const line of lines) {
|
|
9
|
+
const match = line.match(headingRegex);
|
|
10
|
+
if (match) {
|
|
11
|
+
sections.push(match[2]);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
return {
|
|
15
|
+
content,
|
|
16
|
+
metadata: {
|
|
17
|
+
sourcePath: filePath,
|
|
18
|
+
contentType: 'markdown',
|
|
19
|
+
sections: sections.length > 0 ? sections : undefined
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
if (error instanceof Error) {
|
|
25
|
+
throw new Error(`Failed to parse Markdown file: ${error.message}`);
|
|
26
|
+
}
|
|
27
|
+
throw new Error('Failed to parse Markdown file: Unknown error');
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=markdown.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/parsers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,aAAa,CAAC;AAG7B,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB;IAClD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,mBAAmB,CAAC;QAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACvC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO;YACP,QAAQ,EAAE;gBACR,UAAU,EAAE,QAAQ;gBACpB,WAAW,EAAE,UAAU;gBACvB,QAAQ,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;aACrD;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,kCAAkC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pdf.d.ts","sourceRoot":"","sources":["../../src/parsers/pdf.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAkBxE"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import pdfParse from 'pdf-parse';
|
|
3
|
+
export async function parsePDF(filePath) {
|
|
4
|
+
try {
|
|
5
|
+
const dataBuffer = await fs.readFile(filePath);
|
|
6
|
+
const data = await pdfParse(dataBuffer);
|
|
7
|
+
return {
|
|
8
|
+
content: data.text,
|
|
9
|
+
metadata: {
|
|
10
|
+
sourcePath: filePath,
|
|
11
|
+
contentType: 'pdf'
|
|
12
|
+
}
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
if (error instanceof Error) {
|
|
17
|
+
throw new Error(`Failed to parse PDF file: ${error.message}`);
|
|
18
|
+
}
|
|
19
|
+
throw new Error('Failed to parse PDF file: Unknown error');
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=pdf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../src/parsers/pdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,QAAQ,MAAM,WAAW,CAAC;AAGjC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAAgB;IAC7C,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;QAExC,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,IAAI;YAClB,QAAQ,EAAE;gBACR,UAAU,EAAE,QAAQ;gBACpB,WAAW,EAAE,KAAK;aACnB;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface ParsedDocument {
|
|
2
|
+
content: string;
|
|
3
|
+
metadata: DocumentMetadata;
|
|
4
|
+
}
|
|
5
|
+
export interface DocumentMetadata {
|
|
6
|
+
sourcePath: string;
|
|
7
|
+
contentType: 'pdf' | 'markdown' | 'csv';
|
|
8
|
+
sections?: string[];
|
|
9
|
+
rowNumber?: number;
|
|
10
|
+
}
|
|
11
|
+
export type FileType = 'pdf' | 'markdown' | 'csv';
|
|
12
|
+
export interface CSVDocument extends ParsedDocument {
|
|
13
|
+
metadata: DocumentMetadata & {
|
|
14
|
+
rowNumber: number;
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/parsers/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,gBAAgB,CAAC;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,KAAK,GAAG,UAAU,GAAG,KAAK,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,QAAQ,GAAG,KAAK,GAAG,UAAU,GAAG,KAAK,CAAC;AAElD,MAAM,WAAW,WAAY,SAAQ,cAAc;IACjD,QAAQ,EAAE,gBAAgB,GAAG;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;CACpD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/parsers/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/query/embedding.ts"],"names":[],"mappings":"AAEA,wBAAsB,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAG7E"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../src/query/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,KAAa;IACxD,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/query/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAe,YAAY,EAAE,MAAM,YAAY,CAAC;AAI5D,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiEzF"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { generateQueryEmbedding } from './embedding.js';
|
|
2
|
+
import { listAllCollections, getCollection, searchChromaDB } from './search.js';
|
|
3
|
+
export async function executeQuery(question, options) {
|
|
4
|
+
if (!process.env.OPENAI_API_KEY) {
|
|
5
|
+
console.error('Error: OPENAI_API_KEY environment variable is required');
|
|
6
|
+
console.error('\nTo set your API key:');
|
|
7
|
+
console.error(' export OPENAI_API_KEY=your-api-key-here');
|
|
8
|
+
console.error('\nOr pass it when running the command:');
|
|
9
|
+
console.error(' OPENAI_API_KEY=your-api-key-here easy-rag query "<question>"');
|
|
10
|
+
process.exit(1);
|
|
11
|
+
}
|
|
12
|
+
const queryEmbedding = await generateQueryEmbedding(question);
|
|
13
|
+
const allResults = [];
|
|
14
|
+
if (options.collection) {
|
|
15
|
+
try {
|
|
16
|
+
const collection = await getCollection(options.collection);
|
|
17
|
+
const results = await searchChromaDB(collection, queryEmbedding, options.top);
|
|
18
|
+
allResults.push(...results);
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
if (error instanceof Error && error.message.includes('Collection')) {
|
|
22
|
+
console.error(`Error: Collection "${options.collection}" not found`);
|
|
23
|
+
console.error('\nAvailable collections:');
|
|
24
|
+
const collections = await listAllCollections();
|
|
25
|
+
if (collections.length === 0) {
|
|
26
|
+
console.error(' (no collections found)');
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
collections.forEach((c) => console.error(` - ${c}`));
|
|
30
|
+
}
|
|
31
|
+
process.exit(1);
|
|
32
|
+
}
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
const collectionNames = await listAllCollections();
|
|
38
|
+
if (collectionNames.length === 0) {
|
|
39
|
+
console.error('Error: No collections found in ChromaDB');
|
|
40
|
+
console.error('\nPlease run "easy-rag index <folder>" to create a collection first.');
|
|
41
|
+
process.exit(1);
|
|
42
|
+
}
|
|
43
|
+
for (const name of collectionNames) {
|
|
44
|
+
try {
|
|
45
|
+
const collection = await getCollection(name);
|
|
46
|
+
const results = await searchChromaDB(collection, queryEmbedding, options.top);
|
|
47
|
+
allResults.push(...results);
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
console.error(`Warning: Failed to search collection "${name}": ${error instanceof Error ? error.message : String(error)}`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
if (allResults.length === 0) {
|
|
55
|
+
console.log('No results found.');
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
allResults.sort((a, b) => b.similarity - a.similarity);
|
|
59
|
+
const topResults = allResults.slice(0, options.top);
|
|
60
|
+
if (options.includeMetadata) {
|
|
61
|
+
displayResultsWithMetadata(topResults);
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
displayResults(topResults);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
function displayResults(results) {
|
|
68
|
+
console.log();
|
|
69
|
+
results.forEach((result, index) => {
|
|
70
|
+
console.log(`--- Result ${index + 1} ---`);
|
|
71
|
+
console.log(result.content);
|
|
72
|
+
console.log();
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
function displayResultsWithMetadata(results) {
|
|
76
|
+
console.log();
|
|
77
|
+
results.forEach((result, index) => {
|
|
78
|
+
console.log(`--- Result ${index + 1} ---`);
|
|
79
|
+
console.log(`Source: ${result.metadata.sourcePath}`);
|
|
80
|
+
console.log(`Similarity: ${(result.similarity * 100).toFixed(1)}%`);
|
|
81
|
+
console.log(`Chunk: ${result.metadata.chunkIndex}`);
|
|
82
|
+
console.log(`\nContent:\n${result.content}`);
|
|
83
|
+
console.log();
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/query/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,OAAqB;IACxE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;QACxE,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACxC,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACxD,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAChF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IAE9D,MAAM,UAAU,GAAkB,EAAE,CAAC;IAErC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAC3D,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YAC9E,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACnE,OAAO,CAAC,KAAK,CAAC,sBAAsB,OAAO,CAAC,UAAU,aAAa,CAAC,CAAC;gBACrE,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAC1C,MAAM,WAAW,GAAG,MAAM,kBAAkB,EAAE,CAAC;gBAC/C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC7B,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAC5C,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;gBACxD,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QACnD,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACzD,OAAO,CAAC,KAAK,CAAC,sEAAsE,CAAC,CAAC;YACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC9E,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,yCAAyC,IAAI,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7H,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IAED,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAEpD,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;QAC5B,0BAA0B,CAAC,UAAU,CAAC,CAAC;IACzC,CAAC;SAAM,CAAC;QACN,cAAc,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,OAAsB;IAC5C,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAChC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,0BAA0B,CAAC,OAAsB;IACxD,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAChC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { type Collection } from 'chromadb';
|
|
2
|
+
import type { QueryResult } from './types.js';
|
|
3
|
+
export declare function listAllCollections(): Promise<string[]>;
|
|
4
|
+
export declare function getCollection(collectionName: string): Promise<Collection>;
|
|
5
|
+
export declare function searchChromaDB(collection: Collection, queryEmbedding: number[], top: number): Promise<QueryResult[]>;
|
|
6
|
+
//# sourceMappingURL=search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/query/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAM9C,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAG5D;AAED,wBAAsB,aAAa,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAK/E;AAED,wBAAsB,cAAc,CAClC,UAAU,EAAE,UAAU,EACtB,cAAc,EAAE,MAAM,EAAE,EACxB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,WAAW,EAAE,CAAC,CAiCxB"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { ChromaClient } from 'chromadb';
|
|
2
|
+
const client = new ChromaClient({
|
|
3
|
+
path: process.env.CHROMA_URL || 'http://localhost:8000',
|
|
4
|
+
});
|
|
5
|
+
export async function listAllCollections() {
|
|
6
|
+
const collections = await client.listCollections();
|
|
7
|
+
return collections.map((c) => String(c));
|
|
8
|
+
}
|
|
9
|
+
export async function getCollection(collectionName) {
|
|
10
|
+
const sanitized = collectionName
|
|
11
|
+
.replace(/[^a-zA-Z0-9_-]/g, '_')
|
|
12
|
+
.toLowerCase();
|
|
13
|
+
return await client.getCollection({ name: sanitized });
|
|
14
|
+
}
|
|
15
|
+
export async function searchChromaDB(collection, queryEmbedding, top) {
|
|
16
|
+
const result = await collection.query({
|
|
17
|
+
queryEmbeddings: [queryEmbedding],
|
|
18
|
+
nResults: top,
|
|
19
|
+
});
|
|
20
|
+
const queryResults = [];
|
|
21
|
+
if (result.documents && result.documents[0]) {
|
|
22
|
+
for (let i = 0; i < result.documents[0].length; i++) {
|
|
23
|
+
const content = result.documents[0][i];
|
|
24
|
+
const metadata = result.metadatas?.[0]?.[i];
|
|
25
|
+
const distance = result.distances?.[0]?.[i];
|
|
26
|
+
if (content && metadata) {
|
|
27
|
+
queryResults.push({
|
|
28
|
+
content,
|
|
29
|
+
metadata: {
|
|
30
|
+
sourcePath: String(metadata.sourcePath),
|
|
31
|
+
contentType: String(metadata.contentType),
|
|
32
|
+
chunkIndex: Number(metadata.chunkIndex),
|
|
33
|
+
sectionTitle: metadata.sectionTitle !== undefined ? String(metadata.sectionTitle) : undefined,
|
|
34
|
+
startPosition: metadata.startPosition !== undefined ? Number(metadata.startPosition) : undefined,
|
|
35
|
+
endPosition: metadata.endPosition !== undefined ? Number(metadata.endPosition) : undefined,
|
|
36
|
+
rowNumber: metadata.rowNumber !== undefined ? Number(metadata.rowNumber) : undefined,
|
|
37
|
+
},
|
|
38
|
+
similarity: distance !== undefined ? Math.max(0, 1 - distance) : 0,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return queryResults;
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/query/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAmB,MAAM,UAAU,CAAC;AAGzD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC;IAC9B,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,uBAAuB;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,eAAe,EAAE,CAAC;IACnD,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,cAAsB;IACxD,MAAM,SAAS,GAAG,cAAc;SAC7B,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,WAAW,EAAE,CAAC;IACjB,OAAO,MAAM,MAAM,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,UAAsB,EACtB,cAAwB,EACxB,GAAW;IAEX,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC;QACpC,eAAe,EAAE,CAAC,cAAc,CAAC;QACjC,QAAQ,EAAE,GAAG;KACd,CAAC,CAAC;IAEH,MAAM,YAAY,GAAkB,EAAE,CAAC;IAEvC,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAA4B,CAAC;YACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAE5C,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACxB,YAAY,CAAC,IAAI,CAAC;oBAChB,OAAO;oBACP,QAAQ,EAAE;wBACR,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;wBACvC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC;wBACzC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;wBACvC,YAAY,EAAE,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS;wBAC7F,aAAa,EAAE,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;wBAChG,WAAW,EAAE,QAAQ,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS;wBAC1F,SAAS,EAAE,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;qBACrF;oBACD,UAAU,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;iBACnE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export interface QueryResult {
|
|
2
|
+
content: string;
|
|
3
|
+
metadata: {
|
|
4
|
+
sourcePath: string;
|
|
5
|
+
contentType: string;
|
|
6
|
+
chunkIndex: number;
|
|
7
|
+
sectionTitle?: string;
|
|
8
|
+
startPosition?: number;
|
|
9
|
+
endPosition?: number;
|
|
10
|
+
rowNumber?: number;
|
|
11
|
+
};
|
|
12
|
+
similarity: number;
|
|
13
|
+
}
|
|
14
|
+
export interface QueryOptions {
|
|
15
|
+
top: number;
|
|
16
|
+
includeMetadata: boolean;
|
|
17
|
+
collection?: string;
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/query/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE;QACR,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/query/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ensures ChromaDB is running. If not, starts it automatically.
|
|
3
|
+
* Returns `true` if ChromaDB is ready, `false` otherwise.
|
|
4
|
+
*/
|
|
5
|
+
export declare function ensureChromaRunning(): Promise<boolean>;
|
|
6
|
+
/**
|
|
7
|
+
* Stops the ChromaDB server if it was started by this process.
|
|
8
|
+
*/
|
|
9
|
+
export declare function stopChroma(): void;
|
|
10
|
+
//# sourceMappingURL=chroma-server.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chroma-server.d.ts","sourceRoot":"","sources":["../../src/vector-store/chroma-server.ts"],"names":[],"mappings":"AAwDA;;;GAGG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,OAAO,CAAC,CA4C5D;AAED;;GAEG;AACH,wBAAgB,UAAU,IAAI,IAAI,CAKjC"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { spawn } from 'child_process';
|
|
2
|
+
import { fileURLToPath } from 'url';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import fs from 'fs';
|
|
5
|
+
import { ensureChromaDataDirectory } from './utils.js';
|
|
6
|
+
const CHROMA_URL = process.env.CHROMA_URL || 'http://localhost:8000';
|
|
7
|
+
let chromaProcess = null;
|
|
8
|
+
/**
|
|
9
|
+
* Checks if the ChromaDB server is reachable.
|
|
10
|
+
*/
|
|
11
|
+
async function isChromaRunning() {
|
|
12
|
+
try {
|
|
13
|
+
const response = await fetch(`${CHROMA_URL}/api/v2/heartbeat`);
|
|
14
|
+
return response.ok;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Waits until ChromaDB responds to heartbeat, with timeout.
|
|
22
|
+
*/
|
|
23
|
+
async function waitForChroma(timeoutMs = 15_000) {
|
|
24
|
+
const start = Date.now();
|
|
25
|
+
while (Date.now() - start < timeoutMs) {
|
|
26
|
+
if (await isChromaRunning())
|
|
27
|
+
return true;
|
|
28
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
29
|
+
}
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Resolves the chroma binary path.
|
|
34
|
+
* Looks in the package's own node_modules first, then cwd's node_modules, then falls back to npx.
|
|
35
|
+
*/
|
|
36
|
+
function resolveChromaBin() {
|
|
37
|
+
// 1. Try the package's own node_modules (works when installed globally)
|
|
38
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
39
|
+
const packageRoot = path.resolve(__dirname, '..', '..');
|
|
40
|
+
const packageBin = path.join(packageRoot, 'node_modules', '.bin', 'chroma');
|
|
41
|
+
if (fs.existsSync(packageBin)) {
|
|
42
|
+
return { command: packageBin, args: [] };
|
|
43
|
+
}
|
|
44
|
+
// 2. Try cwd's node_modules (works during development)
|
|
45
|
+
const cwdBin = path.join(process.cwd(), 'node_modules', '.bin', 'chroma');
|
|
46
|
+
if (fs.existsSync(cwdBin)) {
|
|
47
|
+
return { command: cwdBin, args: [] };
|
|
48
|
+
}
|
|
49
|
+
// 3. Fall back to npx
|
|
50
|
+
return { command: 'npx', args: ['chroma'] };
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Ensures ChromaDB is running. If not, starts it automatically.
|
|
54
|
+
* Returns `true` if ChromaDB is ready, `false` otherwise.
|
|
55
|
+
*/
|
|
56
|
+
export async function ensureChromaRunning() {
|
|
57
|
+
if (await isChromaRunning()) {
|
|
58
|
+
return true;
|
|
59
|
+
}
|
|
60
|
+
console.log('ChromaDB is not running. Starting server...');
|
|
61
|
+
const chromadbPath = await ensureChromaDataDirectory();
|
|
62
|
+
const port = new URL(CHROMA_URL).port || '8000';
|
|
63
|
+
const { command, args } = resolveChromaBin();
|
|
64
|
+
chromaProcess = spawn(command, [...args, 'run', '--path', chromadbPath, '--port', port], {
|
|
65
|
+
stdio: 'ignore',
|
|
66
|
+
detached: false,
|
|
67
|
+
});
|
|
68
|
+
chromaProcess.on('error', (err) => {
|
|
69
|
+
console.error(`Failed to start ChromaDB: ${err.message}`);
|
|
70
|
+
});
|
|
71
|
+
chromaProcess.on('exit', (code) => {
|
|
72
|
+
if (code !== null && code !== 0) {
|
|
73
|
+
console.error(`ChromaDB exited with code ${code}`);
|
|
74
|
+
}
|
|
75
|
+
chromaProcess = null;
|
|
76
|
+
});
|
|
77
|
+
// Register cleanup handlers
|
|
78
|
+
const cleanup = () => stopChroma();
|
|
79
|
+
process.on('exit', cleanup);
|
|
80
|
+
process.on('SIGINT', () => { cleanup(); process.exit(0); });
|
|
81
|
+
process.on('SIGTERM', () => { cleanup(); process.exit(0); });
|
|
82
|
+
const ready = await waitForChroma();
|
|
83
|
+
if (ready) {
|
|
84
|
+
console.log('ChromaDB server started successfully.\n');
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
console.error('ChromaDB failed to start within 15 seconds.');
|
|
88
|
+
stopChroma();
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Stops the ChromaDB server if it was started by this process.
|
|
95
|
+
*/
|
|
96
|
+
export function stopChroma() {
|
|
97
|
+
if (chromaProcess && !chromaProcess.killed) {
|
|
98
|
+
chromaProcess.kill('SIGTERM');
|
|
99
|
+
chromaProcess = null;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=chroma-server.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chroma-server.js","sourceRoot":"","sources":["../../src/vector-store/chroma-server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAqB,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAEvD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,uBAAuB,CAAC;AACrE,IAAI,aAAa,GAAwB,IAAI,CAAC;AAE9C;;GAEG;AACH,KAAK,UAAU,eAAe;IAC5B,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,UAAU,mBAAmB,CAAC,CAAC;QAC/D,OAAO,QAAQ,CAAC,EAAE,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAAC,SAAS,GAAG,MAAM;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;QACtC,IAAI,MAAM,eAAe,EAAE;YAAE,OAAO,IAAI,CAAC;QACzC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB;IACvB,wEAAwE;IACxE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IACxD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC5E,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAC3C,CAAC;IAED,uDAAuD;IACvD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC1E,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACvC,CAAC;IAED,sBAAsB;IACtB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;IACvC,IAAI,MAAM,eAAe,EAAE,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;IAE3D,MAAM,YAAY,GAAG,MAAM,yBAAyB,EAAE,CAAC;IACvD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC;IAChD,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAE7C,aAAa,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,CAAC,EAAE;QACvF,KAAK,EAAE,QAAQ;QACf,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;IAEH,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QAChC,OAAO,CAAC,KAAK,CAAC,6BAA6B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAChC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,CAAC,KAAK,CAAC,6BAA6B,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,4BAA4B;IAC5B,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC;IACnC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5B,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7D,MAAM,KAAK,GAAG,MAAM,aAAa,EAAE,CAAC;IAEpC,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;IACzD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAC7D,UAAU,EAAE,CAAC;QACb,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU;IACxB,IAAI,aAAa,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;QAC3C,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type Collection } from 'chromadb';
|
|
2
|
+
import type { VectorChunk, CollectionInfo } from './types.js';
|
|
3
|
+
export declare function getOrCreateCollection(baseName: string): Promise<Collection>;
|
|
4
|
+
export declare function storeEmbeddings(collection: Collection, chunks: VectorChunk[]): Promise<void>;
|
|
5
|
+
export declare function initializeChromaDB(): Promise<void>;
|
|
6
|
+
export declare function listCollections(): Promise<CollectionInfo[]>;
|
|
7
|
+
export declare function deleteCollection(collectionName: string): Promise<void>;
|
|
8
|
+
//# sourceMappingURL=chromadb.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chromadb.d.ts","sourceRoot":"","sources":["../../src/vector-store/chromadb.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAO9D,wBAAsB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CA6BjF;AAED,wBAAsB,eAAe,CACnC,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,WAAW,EAAE,GACpB,OAAO,CAAC,IAAI,CAAC,CAqCf;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAExD;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,CAqBjE;AAED,wBAAsB,gBAAgB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAK5E"}
|