mcp-store-db 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +133 -0
  2. package/dist/index.d.ts +51 -0
  3. package/dist/index.d.ts.map +1 -0
  4. package/dist/index.js +179 -0
  5. package/dist/index.js.map +1 -0
  6. package/dist/storage/chunkStore.d.ts +44 -0
  7. package/dist/storage/chunkStore.d.ts.map +1 -0
  8. package/dist/storage/chunkStore.js +138 -0
  9. package/dist/storage/chunkStore.js.map +1 -0
  10. package/dist/storage/documentStore.d.ts +33 -0
  11. package/dist/storage/documentStore.d.ts.map +1 -0
  12. package/dist/storage/documentStore.js +156 -0
  13. package/dist/storage/documentStore.js.map +1 -0
  14. package/dist/storage/vectorStore.d.ts +37 -0
  15. package/dist/storage/vectorStore.d.ts.map +1 -0
  16. package/dist/storage/vectorStore.js +168 -0
  17. package/dist/storage/vectorStore.js.map +1 -0
  18. package/dist/test.d.ts +2 -0
  19. package/dist/test.d.ts.map +1 -0
  20. package/dist/test.js +87 -0
  21. package/dist/test.js.map +1 -0
  22. package/dist/utils/analyser/log_analyser.d.ts +8 -0
  23. package/dist/utils/analyser/log_analyser.d.ts.map +1 -0
  24. package/dist/utils/analyser/log_analyser.js +21 -0
  25. package/dist/utils/analyser/log_analyser.js.map +1 -0
  26. package/dist/utils/backup/backup.d.ts +3 -0
  27. package/dist/utils/backup/backup.d.ts.map +1 -0
  28. package/dist/utils/backup/backup.js +38 -0
  29. package/dist/utils/backup/backup.js.map +1 -0
  30. package/dist/utils/calculate_file_size.d.ts +2 -0
  31. package/dist/utils/calculate_file_size.d.ts.map +1 -0
  32. package/dist/utils/calculate_file_size.js +17 -0
  33. package/dist/utils/calculate_file_size.js.map +1 -0
  34. package/dist/utils/chunker/recursive_chunker.d.ts +3 -0
  35. package/dist/utils/chunker/recursive_chunker.d.ts.map +1 -0
  36. package/dist/utils/chunker/recursive_chunker.js +43 -0
  37. package/dist/utils/chunker/recursive_chunker.js.map +1 -0
  38. package/dist/utils/copy_utils/copy_file.d.ts +29 -0
  39. package/dist/utils/copy_utils/copy_file.d.ts.map +1 -0
  40. package/dist/utils/copy_utils/copy_file.js +150 -0
  41. package/dist/utils/copy_utils/copy_file.js.map +1 -0
  42. package/dist/utils/copy_utils/copy_folder.d.ts +17 -0
  43. package/dist/utils/copy_utils/copy_folder.d.ts.map +1 -0
  44. package/dist/utils/copy_utils/copy_folder.js +88 -0
  45. package/dist/utils/copy_utils/copy_folder.js.map +1 -0
  46. package/dist/utils/csv/parser.d.ts +3 -0
  47. package/dist/utils/csv/parser.d.ts.map +1 -0
  48. package/dist/utils/csv/parser.js +48 -0
  49. package/dist/utils/csv/parser.js.map +1 -0
  50. package/dist/utils/extract_error_message.d.ts +4 -0
  51. package/dist/utils/extract_error_message.d.ts.map +1 -0
  52. package/dist/utils/extract_error_message.js +11 -0
  53. package/dist/utils/extract_error_message.js.map +1 -0
  54. package/dist/utils/mime_check.d.ts +3 -0
  55. package/dist/utils/mime_check.d.ts.map +1 -0
  56. package/dist/utils/mime_check.js +15 -0
  57. package/dist/utils/mime_check.js.map +1 -0
  58. package/dist/utils/pdf/parser.d.ts +16 -0
  59. package/dist/utils/pdf/parser.d.ts.map +1 -0
  60. package/dist/utils/pdf/parser.js +26 -0
  61. package/dist/utils/pdf/parser.js.map +1 -0
  62. package/dist/utils/text/parser.d.ts +13 -0
  63. package/dist/utils/text/parser.d.ts.map +1 -0
  64. package/dist/utils/text/parser.js +38 -0
  65. package/dist/utils/text/parser.js.map +1 -0
  66. package/dist/utils/text_normalizer.d.ts +2 -0
  67. package/dist/utils/text_normalizer.d.ts.map +1 -0
  68. package/dist/utils/text_normalizer.js +8 -0
  69. package/dist/utils/text_normalizer.js.map +1 -0
  70. package/package.json +57 -0
package/README.md ADDED
@@ -0,0 +1,133 @@
1
+ # mcp-store-db
2
+
3
+ A lightweight vector database and RAG storage layer for Node.js. Supports document ingestion, chunking, embedding, and semantic search — all file-based with zero external database dependencies.
4
+
5
+ ## Installation
6
+
7
+ ```bash
8
+ npm install mcp-store-db
9
+ ```
10
+
11
+ ## Quick Start
12
+
13
+ ```typescript
14
+ import MiniVectorDatabase from 'mcp-store-db';
15
+
16
+ const db = new MiniVectorDatabase({
17
+ location: './data',
18
+ name: 'my-store',
19
+ chunkConfiguration: {
20
+ overlap: 0,
21
+ size: 500,
22
+ separators: ['\n\n', '\n', '. ', ' '],
23
+ },
24
+ vectorConfigurations: {
25
+ cacheDir: './models',
26
+ searchLimit: 5,
27
+ model: 'Xenova/all-MiniLM-L6-v2',
28
+ },
29
+ });
30
+
31
+ // Initialize (downloads model on first run)
32
+ await db.init();
33
+
34
+ // Ingest a document (supports .txt, .pdf, .csv)
35
+ const docId = await db.ingest('./documents/example.txt');
36
+
37
+ // Semantic search
38
+ const results = await db.query('what is machine learning');
39
+ console.log(results);
40
+ // [{ content: '...', score: 0.82, chunkId: '...', documentId: '...' }]
41
+
42
+ // List all documents
43
+ const docs = await db.listDoc();
44
+
45
+ // Get a specific document
46
+ const doc = await db.getDoc('document-id');
47
+
48
+ // Delete a document (cascades to chunks and vectors)
49
+ await db.deleteDoc('document-id');
50
+ ```
51
+
52
+ ## How It Works
53
+
54
+ ```
55
+ Ingest: File → Document Store → Chunk Store → Vector Store (embeddings)
56
+ Query: Text → Embed → Cosine Similarity Search → Return ranked chunks
57
+ ```
58
+
59
+ The library runs a three-layer pipeline:
60
+
61
+ 1. **Document Store** — stores raw documents with metadata as JSON files
62
+ 2. **Chunk Store** — splits documents into smaller segments using recursive chunking
63
+ 3. **Vector Store** — generates embeddings using [Xenova/transformers](https://github.com/xenova/transformers.js) and performs cosine similarity search
64
+
65
+ All data is stored as JSON files on disk. No database server required.
66
+
67
+ ## API
68
+
69
+ ### `new MiniVectorDatabase(props)`
70
+
71
+ | Property | Type | Description |
72
+ |----------|------|-------------|
73
+ | `location` | `string` | Directory path for storing data |
74
+ | `name` | `string` | Name of the database instance |
75
+ | `chunkConfiguration.size` | `number` | Maximum chunk size in characters |
76
+ | `chunkConfiguration.overlap` | `number` | Character overlap between chunks |
77
+ | `chunkConfiguration.separators` | `string[]` | Split boundaries in priority order |
78
+ | `vectorConfigurations.cacheDir` | `string` | Directory to cache the embedding model |
79
+ | `vectorConfigurations.searchLimit` | `number` | Max results returned by `query()` |
80
+ | `vectorConfigurations.model` | `SupportedModels` | Embedding model name |
81
+
82
+ ### Methods
83
+
84
+ | Method | Returns | Description |
85
+ |--------|---------|-------------|
86
+ | `init()` | `Promise<void>` | Initialize stores and load embedding model |
87
+ | `ingest(filePath)` | `Promise<string \| string[] \| null>` | Ingest a file (.txt, .pdf, .csv) and return document ID(s) |
88
+ | `query(text)` | `Promise<QueryResult[]>` | Semantic search — returns ranked chunks with scores |
89
+ | `listDoc()` | `Promise<DocumentType[]>` | List all documents |
90
+ | `getDoc(id)` | `Promise<DocumentType \| undefined>` | Get a document by ID |
91
+ | `deleteDoc(id)` | `Promise<void>` | Delete a document and its chunks/vectors |
92
+
93
+ ### Events
94
+
95
+ The class extends `EventEmitter` and emits:
96
+
97
+ | Event | Values | Description |
98
+ |-------|--------|-------------|
99
+ | `status` | `'in-progress'`, `'ready'` | Initialization status |
100
+ | `ingest` | `'in-progress'`, `'done'` | Ingestion progress |
101
+ | `query` | `'in-progress'`, `'done'` | Query progress |
102
+ | `delete` | `'in-progress'`, `'done'` | Deletion progress |
103
+ | `error` | `string` | Error message |
104
+
105
+ ### Exported Types
106
+
107
+ ```typescript
108
+ import MiniVectorDatabase, {
109
+ type MiniVectorDatabaseProps,
110
+ type QueryResult,
111
+ type DocumentType,
112
+ type SupportedModels,
113
+ } from 'mcp-store-db';
114
+ ```
115
+
116
+ ## Supported File Types
117
+
118
+ - `.txt` — Plain text files
119
+ - `.pdf` — PDF documents
120
+ - `.csv` — CSV files (requires `content` and `metadata` columns)
121
+
122
+ ## Supported Models
123
+
124
+ - `Xenova/all-MiniLM-L6-v2` — 384-dimensional embeddings, good balance of speed and quality
125
+
126
+ ## Requirements
127
+
128
+ - Node.js >= 18
129
+ - ESM project (`"type": "module"` in package.json)
130
+
131
+ ## License
132
+
133
+ MIT
@@ -0,0 +1,51 @@
1
+ import { type DocumentType } from './storage/documentStore.js';
2
+ import { EventEmitter } from 'node:events';
3
+ export type SupportedModels = 'Xenova/all-MiniLM-L6-v2';
4
+ export type QueryResult = {
5
+ content: string;
6
+ score: number;
7
+ chunkId: string;
8
+ documentId: string;
9
+ };
10
+ export type MiniVectorDatabaseProps = {
11
+ location: string;
12
+ name: string;
13
+ chunkConfiguration: {
14
+ overlap: number;
15
+ size: number;
16
+ separators: string[];
17
+ };
18
+ vectorConfigurations: {
19
+ cacheDir: string;
20
+ searchLimit: number;
21
+ model: SupportedModels;
22
+ };
23
+ };
24
+ declare class MiniVectorDatabase extends EventEmitter {
25
+ location: string;
26
+ name: string;
27
+ chunkConfiguration: {
28
+ overlap: number;
29
+ size: number;
30
+ separators: string[];
31
+ };
32
+ vectorConfigurations: {
33
+ cacheDir: string;
34
+ searchLimit: number;
35
+ model: SupportedModels;
36
+ };
37
+ private doc;
38
+ private chunk;
39
+ private vector;
40
+ isReady: boolean;
41
+ constructor({ location, name, chunkConfiguration, vectorConfigurations, }: MiniVectorDatabaseProps);
42
+ init(): Promise<void>;
43
+ ingest(filePath: string): Promise<string | string[] | null>;
44
+ query(text: string): Promise<QueryResult[]>;
45
+ deleteDoc(id: string): Promise<void>;
46
+ listDoc(): Promise<DocumentType[]>;
47
+ getDoc(id: string): Promise<DocumentType | undefined>;
48
+ }
49
+ export { type DocumentType };
50
+ export default MiniVectorDatabase;
51
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAsB,EAAE,KAAK,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAG9E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAI3C,MAAM,MAAM,eAAe,GAAG,yBAAyB,CAAC;AAExD,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF,oBAAoB,EAAE;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,eAAe,CAAC;KACxB,CAAC;CACH,CAAC;AAEF,cAAM,kBAAmB,SAAQ,YAAY;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF,oBAAoB,EAAE;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,eAAe,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,GAAG,CAAiB;IAC5B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,EAAE,OAAO,CAAC;gBACL,EACV,QAAQ,EACR,IAAI,EACJ,kBAAkB,EAClB,oBAAoB,GACrB,EAAE,uBAAuB;IAQpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA8CrB,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,CAAC;IAsC3D,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAgC3C,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBpC,OAAO,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IAYlC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC;CAW5D;AAED,OAAO,EAAE,KAAK,YAAY,EAAE,CAAC;AAC7B,eAAe,kBAAkB,CAAC"}
package/dist/index.js ADDED
@@ -0,0 +1,179 @@
1
+ import DocumentStore, {} from './storage/documentStore.js';
2
+ import Chunkstore from './storage/chunkStore.js';
3
+ import VectorStore from './storage/vectorStore.js';
4
+ import { EventEmitter } from 'node:events';
5
+ import { extractErrorMessage } from './utils/extract_error_message.js';
6
+ import checkFileType from './utils/mime_check.js';
7
+ class MiniVectorDatabase extends EventEmitter {
8
+ location;
9
+ name;
10
+ chunkConfiguration;
11
+ vectorConfigurations;
12
+ doc;
13
+ chunk;
14
+ vector;
15
+ isReady;
16
+ constructor({ location, name, chunkConfiguration, vectorConfigurations, }) {
17
+ super();
18
+ this.location = location;
19
+ this.name = name;
20
+ this.chunkConfiguration = chunkConfiguration;
21
+ this.vectorConfigurations = vectorConfigurations;
22
+ this.isReady = false;
23
+ }
24
+ async init() {
25
+ try {
26
+ this.emit('status', 'in-progress');
27
+ this.doc = new DocumentStore({
28
+ location: this.location,
29
+ name: this.name,
30
+ });
31
+ await this.doc.init();
32
+ this.chunk = new Chunkstore({
33
+ location: this.location,
34
+ document: this.doc,
35
+ name: this.name,
36
+ chunk_overlap: this.chunkConfiguration.overlap || 0,
37
+ chunk_size: this.chunkConfiguration.size || 500,
38
+ separators: this.chunkConfiguration.separators || [
39
+ '\n\n',
40
+ '\n',
41
+ '. ',
42
+ ' ',
43
+ ],
44
+ });
45
+ await this.chunk.init();
46
+ this.vector = new VectorStore({
47
+ location: this.location,
48
+ cacheDir: this.vectorConfigurations.cacheDir || './models',
49
+ name: this.name,
50
+ chunk: this.chunk,
51
+ searchlimit: this.vectorConfigurations.searchLimit || 5,
52
+ });
53
+ await this.vector.init(this.vectorConfigurations.model || 'Xenova/all-MiniLM-L6-v2');
54
+ this.emit('status', 'ready');
55
+ this.isReady = true;
56
+ }
57
+ catch (error) {
58
+ this.emit('error', extractErrorMessage(error));
59
+ console.error(error);
60
+ throw error;
61
+ }
62
+ }
63
+ async ingest(filePath) {
64
+ try {
65
+ if (!this.isReady)
66
+ throw new Error('Vector Db not ready');
67
+ this.emit('ingest', 'in-progress');
68
+ const file_type = checkFileType(filePath);
69
+ let dataid = null;
70
+ if (file_type === 'Text File') {
71
+ dataid = await this.doc.textInjector(filePath);
72
+ }
73
+ else if (file_type === 'PDF File') {
74
+ dataid = await this.doc.pdfInjector(filePath);
75
+ }
76
+ else if (file_type === 'CSV File') {
77
+ dataid = await this.doc.csvInjector(filePath);
78
+ }
79
+ else {
80
+ throw new Error('File type not supported');
81
+ }
82
+ if (!dataid)
83
+ throw new Error('Parser failed');
84
+ let chunks = [];
85
+ if (Array.isArray(dataid)) {
86
+ chunks = await this.chunk.addMultiple({ documentId: dataid });
87
+ }
88
+ else if (dataid) {
89
+ chunks = await this.chunk.add({ documentId: dataid });
90
+ }
91
+ if (chunks?.length) {
92
+ await this.vector.embeddIds({ chunkId: chunks });
93
+ this.emit('ingest', 'done');
94
+ return dataid;
95
+ }
96
+ return null;
97
+ }
98
+ catch (error) {
99
+ this.emit('error', extractErrorMessage(error));
100
+ console.error(error);
101
+ throw error;
102
+ }
103
+ }
104
+ async query(text) {
105
+ try {
106
+ if (!this.isReady)
107
+ throw new Error('Vector Db not ready');
108
+ this.emit('query', 'in-progress');
109
+ const data = await this.vector.searchVectors(text);
110
+ if (!data)
111
+ return [];
112
+ const result = await Promise.all(data.map(async (item) => {
113
+ const [chunkId, score] = item;
114
+ const chunk = await this.chunk.get(chunkId);
115
+ if (!chunk)
116
+ throw new Error(`Data integrity error: chunk not found for id ${chunkId}`);
117
+ return {
118
+ content: chunk.content,
119
+ score,
120
+ chunkId,
121
+ documentId: chunk.documentId,
122
+ };
123
+ }));
124
+ this.emit('query', 'done');
125
+ return result;
126
+ }
127
+ catch (error) {
128
+ this.emit('error', extractErrorMessage(error));
129
+ console.error(error);
130
+ throw error;
131
+ }
132
+ }
133
+ async deleteDoc(id) {
134
+ try {
135
+ if (!this.isReady)
136
+ throw new Error('Vector Db not ready');
137
+ this.emit('delete', 'in-progress');
138
+ await this.doc.deleteItem(id);
139
+ const chunks = (await this.chunk.list()).filter((item) => item.documentId === id);
140
+ await this.chunk.deleteMultipleItem(chunks);
141
+ await this.vector.deleteMultipleItem(chunks);
142
+ this.emit('delete', 'done');
143
+ }
144
+ catch (error) {
145
+ this.emit('error', extractErrorMessage(error));
146
+ console.error(error);
147
+ throw error;
148
+ }
149
+ }
150
+ async listDoc() {
151
+ try {
152
+ if (!this.isReady)
153
+ throw new Error('Vector Db not ready');
154
+ const docs = this.doc.list();
155
+ return docs;
156
+ }
157
+ catch (error) {
158
+ this.emit('error', extractErrorMessage(error));
159
+ console.error(error);
160
+ throw error;
161
+ }
162
+ }
163
+ async getDoc(id) {
164
+ try {
165
+ if (!this.isReady)
166
+ throw new Error('Vector Db not ready');
167
+ const doc = this.doc.get(id);
168
+ return doc;
169
+ }
170
+ catch (error) {
171
+ this.emit('error', extractErrorMessage(error));
172
+ console.error(error);
173
+ throw error;
174
+ }
175
+ }
176
+ }
177
+ export {};
178
+ export default MiniVectorDatabase;
179
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,EAAE,EAAqB,MAAM,4BAA4B,CAAC;AAC9E,OAAO,UAAU,MAAM,yBAAyB,CAAC;AACjD,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,aAAa,MAAM,uBAAuB,CAAC;AA0BlD,MAAM,kBAAmB,SAAQ,YAAY;IAC3C,QAAQ,CAAS;IACjB,IAAI,CAAS;IACb,kBAAkB,CAIhB;IACF,oBAAoB,CAIlB;IACM,GAAG,CAAiB;IACpB,KAAK,CAAc;IACnB,MAAM,CAAe;IAC7B,OAAO,CAAU;IACjB,YAAY,EACV,QAAQ,EACR,IAAI,EACJ,kBAAkB,EAClB,oBAAoB,GACI;QACxB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;IACvB,CAAC;IACD,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YACnC,IAAI,CAAC,GAAG,GAAG,IAAI,aAAa,CAAC;gBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YAEtB,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC;gBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,IAAI,CAAC,GAAG;gBAClB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,aAAa,EAAE,IAAI,CAAC,kBAAkB,CAAC,OAAO,IAAI,CAAC;gBACnD,UAAU,EAAE,IAAI,CAAC,kBAAkB,CAAC,IAAI,IAAI,GAAG;gBAC/C,UAAU,EAAE,IAAI,CAAC,kBAAkB,CAAC,UAAU,IAAI;oBAChD,MAAM;oBACN,IAAI;oBACJ,IAAI;oBACJ,GAAG;iBACJ;aACF,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAExB,IAAI,CAAC,MAAM,GAAG,IAAI,WAAW,CAAC;gBAC5B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,IAAI,CAAC,oBAAoB,CAAC,QAAQ,IAAI,UAAU;gBAC1D,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,WAAW,EAAE,IAAI,CAAC,oBAAoB,CAAC,WAAW,IAAI,CAAC;aACxD,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACpB,IAAI,CAAC,oBAAoB,CAAC,KAAK,IAAI,yBAAyB,CAC7D,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB;QAC3B,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAC1D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YACnC,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC1C,IAAI,MAAM,GAAG,IAAI,CAAC;YAClB,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;gBAC9B,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;iBAAM,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;gBACpC,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;gBACpC,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC;YAED,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YAE9C,IAAI,MAAM,GAAyB,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;YAChE,CAAC;iBAAM,IAAI,MAAM,EAAE,CAAC;gBAClB,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;YACxD,CAAC;YAED,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;gBACnB,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;gBACjD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAC5B,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAC1D,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,CAAC,IAAI;gBAAE,OAAO,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBACtB,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC;gBAC9B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,CAAC,KAAK;oBACR,MAAM,IAAI,KAAK,CACb,gDAAgD,OAAO,EAAE,CAC1D,CAAC;gBAEJ,OAAO;oBACL,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;oBACL,OAAO;oBACP,UAAU,EAAE,KAAK,CAAC,UAAU;iBAC7B,CAAC;YACJ,CAAC,CAAC,CACH,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,EAAU;QACxB,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAC1D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YACnC,MAAM,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC9B,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC7C,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,EAAE,CACjC,CAAC;YACF,MAAM,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7B,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF;AAED,OAAO,EAAqB,CAAC;AAC7B,eAAe,kBAAkB,CAAC"}
@@ -0,0 +1,44 @@
1
+ import type DocumentStore from './documentStore.js';
2
+ type Props = {
3
+ location: string;
4
+ name: string;
5
+ chunk_size: number;
6
+ chunk_overlap: number;
7
+ separators: string[];
8
+ document: DocumentStore;
9
+ };
10
+ export type ChunkType = {
11
+ id: string;
12
+ content: string;
13
+ documentId: string;
14
+ index: number;
15
+ };
16
+ type ChunkTypeRequest = {
17
+ documentId: string;
18
+ };
19
+ type ChunkTypeRequestMultiple = {
20
+ documentId: string[];
21
+ };
22
+ declare class Chunkstore {
23
+ location: string;
24
+ name: string;
25
+ path: string;
26
+ chunk_size: number;
27
+ chunk_overlap: number;
28
+ separators: string[];
29
+ document: DocumentStore;
30
+ constructor({ location, name, chunk_size, chunk_overlap, separators, document, }: Props);
31
+ private openFile;
32
+ init(): Promise<void>;
33
+ private readFile;
34
+ private createFile;
35
+ add(data: ChunkTypeRequest): Promise<string[]>;
36
+ addMultiple(data: ChunkTypeRequestMultiple): Promise<string[]>;
37
+ get(data: string): Promise<ChunkType | undefined>;
38
+ list(): Promise<ChunkType[]>;
39
+ deleteItem(data: string): Promise<void>;
40
+ deleteMultipleItem(data: ChunkType[]): Promise<void>;
41
+ delete(): Promise<void>;
42
+ }
43
+ export default Chunkstore;
44
+ //# sourceMappingURL=chunkStore.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"chunkStore.d.ts","sourceRoot":"","sources":["../../src/storage/chunkStore.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,aAAa,MAAM,oBAAoB,CAAC;AAEpD,KAAK,KAAK,GAAG;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AACF,MAAM,MAAM,SAAS,GAAG;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AACF,KAAK,gBAAgB,GAAG;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AACF,KAAK,wBAAwB,GAAG;IAC9B,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB,CAAC;AAEF,cAAM,UAAU;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,EAAE,aAAa,CAAC;gBACZ,EACV,QAAQ,EACR,IAAI,EACJ,UAAU,EACV,aAAa,EACb,UAAU,EACV,QAAQ,GACT,EAAE,KAAK;YAUM,QAAQ;IAShB,IAAI;YAYI,QAAQ;YAUR,UAAU;IAYlB,GAAG,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAiC9C,WAAW,CAAC,IAAI,EAAE,wBAAwB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAc9D,GAAG,CAAC,IAAI,EAAE,MAAM;IAMhB,IAAI;IAKJ,UAAU,CAAC,IAAI,EAAE,MAAM;IAMvB,kBAAkB,CAAC,IAAI,EAAE,SAAS,EAAE;IAOpC,MAAM;CASb;AAED,eAAe,UAAU,CAAC"}
@@ -0,0 +1,138 @@
1
+ import fs from 'node:fs/promises';
2
+ import { extractErrorMessage } from '../utils/extract_error_message.js';
3
+ import crypto from 'node:crypto';
4
+ import recursiveChunker from '../utils/chunker/recursive_chunker.js';
5
+ class Chunkstore {
6
+ location;
7
+ name;
8
+ path;
9
+ chunk_size;
10
+ chunk_overlap;
11
+ separators;
12
+ document;
13
+ constructor({ location, name, chunk_size, chunk_overlap, separators, document, }) {
14
+ this.location = location;
15
+ this.name = name;
16
+ this.path = `${this.location}/chunks/${this.name}.json`;
17
+ this.chunk_size = chunk_size;
18
+ this.chunk_overlap = chunk_overlap;
19
+ this.separators = separators;
20
+ this.document = document;
21
+ }
22
+ async openFile() {
23
+ try {
24
+ await fs.access(this.path);
25
+ return true;
26
+ }
27
+ catch {
28
+ return false;
29
+ }
30
+ }
31
+ async init() {
32
+ const exists = await this.openFile();
33
+ if (exists) {
34
+ return;
35
+ }
36
+ if (this.location && this.name) {
37
+ await this.createFile(null);
38
+ }
39
+ else {
40
+ throw new Error('Data locations not available');
41
+ }
42
+ }
43
+ async readFile() {
44
+ try {
45
+ const contents = await fs.readFile(this.path, { encoding: 'utf8' });
46
+ return JSON.parse(contents);
47
+ }
48
+ catch (error) {
49
+ console.error(extractErrorMessage(error));
50
+ throw error;
51
+ }
52
+ }
53
+ async createFile(data) {
54
+ try {
55
+ await fs.mkdir(`${this.location}/chunks`, { recursive: true });
56
+ await fs.writeFile(this.path, JSON.stringify(data ?? []), {
57
+ flag: data ? 'w' : 'wx',
58
+ });
59
+ }
60
+ catch (error) {
61
+ console.error(extractErrorMessage(error));
62
+ throw error;
63
+ }
64
+ }
65
+ async add(data) {
66
+ try {
67
+ const contents = await this.readFile();
68
+ if (data.documentId) {
69
+ const doc = await this.document.get(data.documentId);
70
+ if (doc?.content) {
71
+ const chunks = recursiveChunker(doc.content, this.chunk_size, this.chunk_overlap, this.separators);
72
+ const chunkData = chunks.map((item, inx) => {
73
+ const id = crypto.randomUUID();
74
+ return {
75
+ id,
76
+ content: item,
77
+ documentId: data.documentId,
78
+ index: inx,
79
+ };
80
+ });
81
+ await this.createFile([...contents, ...chunkData]);
82
+ return chunkData.map((c) => c.id);
83
+ }
84
+ }
85
+ return [];
86
+ }
87
+ catch (err) {
88
+ console.error(err);
89
+ throw err;
90
+ }
91
+ }
92
+ async addMultiple(data) {
93
+ try {
94
+ const allIds = [];
95
+ for (const docId of data.documentId) {
96
+ const ids = await this.add({ documentId: docId });
97
+ allIds.push(...ids);
98
+ }
99
+ return allIds;
100
+ }
101
+ catch (error) {
102
+ console.error(error);
103
+ throw error;
104
+ }
105
+ }
106
+ async get(data) {
107
+ const contents = await this.readFile();
108
+ const req = contents.find((item) => item.id === data);
109
+ return req;
110
+ }
111
+ async list() {
112
+ const contents = await this.readFile();
113
+ return contents;
114
+ }
115
+ async deleteItem(data) {
116
+ const contents = await this.readFile();
117
+ const req = contents.filter((item) => item.id !== data);
118
+ await this.createFile(req);
119
+ }
120
+ async deleteMultipleItem(data) {
121
+ const contents = await this.readFile();
122
+ const ids = new Set(data.map((d) => d.id));
123
+ const req = contents.filter((item) => !ids.has(item.id));
124
+ await this.createFile(req);
125
+ }
126
+ async delete() {
127
+ try {
128
+ await fs.unlink(this.path);
129
+ console.log(`Chunk deleted: ${this.name}`);
130
+ }
131
+ catch (err) {
132
+ console.error(err);
133
+ throw err;
134
+ }
135
+ }
136
+ }
137
+ export default Chunkstore;
138
+ //# sourceMappingURL=chunkStore.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"chunkStore.js","sourceRoot":"","sources":["../../src/storage/chunkStore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,gBAAgB,MAAM,uCAAuC,CAAC;AAwBrE,MAAM,UAAU;IACd,QAAQ,CAAS;IACjB,IAAI,CAAS;IACb,IAAI,CAAS;IACb,UAAU,CAAS;IACnB,aAAa,CAAS;IACtB,UAAU,CAAW;IACrB,QAAQ,CAAgB;IACxB,YAAY,EACV,QAAQ,EACR,IAAI,EACJ,UAAU,EACV,aAAa,EACb,UAAU,EACV,QAAQ,GACF;QACN,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,WAAW,IAAI,CAAC,IAAI,OAAO,CAAC;QACxD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,QAAQ;QACpB,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACrC,IAAI,MAAM,EAAE,CAAC;YACX,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ;QACpB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;YACpE,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1C,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,IAAwB;QAC/C,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE;gBACxD,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI;aACxB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1C,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAsB;QAC9B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACvC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACrD,IAAI,GAAG,EAAE,OAAO,EAAE,CAAC;oBACjB,MAAM,MAAM,GAAG,gBAAgB,CAC7B,GAAG,CAAC,OAAO,EACX,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,UAAU,CAChB,CAAC;oBAEF,MAAM,SAAS,GAAgB,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;wBACtD,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;wBAC/B,OAAO;4BACL,EAAE;4BACF,OAAO,EAAE,IAAI;4BACb,UAAU,EAAE,IAAI,CAAC,UAAU;4BAC3B,KAAK,EAAE,GAAG;yBACX,CAAC;oBACJ,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;oBACnD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACpC,CAAC;YACH,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAA8B;QAC9C,IAAI,CAAC;YACH,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;YACtB,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAY;QACpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;QACtD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;QACxD,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,IAAiB;QACxC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAED,eAAe,UAAU,CAAC"}
@@ -0,0 +1,33 @@
1
+ type Props = {
2
+ location: string;
3
+ name: string;
4
+ };
5
+ export type DocumentType = {
6
+ id: string;
7
+ content: string;
8
+ metadata: Record<string, string>;
9
+ };
10
+ type DocumentTypeRequest = {
11
+ content: string;
12
+ metadata: Record<string, string>;
13
+ };
14
+ declare class DocumentStore {
15
+ location: string;
16
+ name: string;
17
+ path: string;
18
+ constructor({ location, name }: Props);
19
+ private openFile;
20
+ init(): Promise<void>;
21
+ private readFile;
22
+ private createFile;
23
+ add(data: DocumentTypeRequest): Promise<void>;
24
+ get(data: string): Promise<DocumentType | undefined>;
25
+ list(): Promise<DocumentType[]>;
26
+ deleteItem(data: string): Promise<void>;
27
+ delete(): Promise<void>;
28
+ csvInjector(source: string): Promise<string[]>;
29
+ textInjector(source: string): Promise<string | null>;
30
+ pdfInjector(source: string): Promise<string | null>;
31
+ }
32
+ export default DocumentStore;
33
+ //# sourceMappingURL=documentStore.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"documentStore.d.ts","sourceRoot":"","sources":["../../src/storage/documentStore.ts"],"names":[],"mappings":"AAOA,KAAK,KAAK,GAAG;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AACF,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,CAAC;AACF,KAAK,mBAAmB,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,CAAC;AAEF,cAAM,aAAa;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;gBACD,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,KAAK;YAMvB,QAAQ;IAShB,IAAI;YAYI,QAAQ;YAUR,UAAU;IAYlB,GAAG,CAAC,IAAI,EAAE,mBAAmB;IAO7B,GAAG,CAAC,IAAI,EAAE,MAAM;IAMhB,IAAI;IAKJ,UAAU,CAAC,IAAI,EAAE,MAAM;IAMvB,MAAM;IAUN,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAyB9C,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAoBpD,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;CAmB1D;AAED,eAAe,aAAa,CAAC"}