doc2vec 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -11
- package/dist/content-processor.js +725 -0
- package/dist/database.js +507 -0
- package/dist/doc2vec.js +48 -1012
- package/dist/types.js +2 -0
- package/dist/utils.js +118 -0
- package/package.json +8 -2
- package/Dockerfile +0 -29
- package/config.yaml +0 -167
- package/doc2vec.ts +0 -1734
- package/logger.ts +0 -287
- package/mcp/Dockerfile +0 -14
- package/mcp/README.md +0 -166
- package/mcp/package-lock.json +0 -1754
- package/mcp/package.json +0 -41
- package/mcp/src/index.ts +0 -214
- package/mcp/tsconfig.json +0 -16
- package/tsconfig.json +0 -18
package/logger.ts
DELETED
|
@@ -1,287 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Enhanced Logger for structured and consistent logging
|
|
3
|
-
* Compatible with CommonJS and ESM environments
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Logger levels with their corresponding numeric values
|
|
8
|
-
*/
|
|
9
|
-
enum LogLevel {
|
|
10
|
-
DEBUG = 0,
|
|
11
|
-
INFO = 1,
|
|
12
|
-
WARN = 2,
|
|
13
|
-
ERROR = 3,
|
|
14
|
-
NONE = 100
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
/**
|
|
18
|
-
* Configuration options for the Logger
|
|
19
|
-
*/
|
|
20
|
-
interface LoggerConfig {
|
|
21
|
-
level: LogLevel;
|
|
22
|
-
useTimestamp: boolean;
|
|
23
|
-
useColor: boolean;
|
|
24
|
-
logToFile?: string;
|
|
25
|
-
prettyPrint?: boolean;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Basic color functions that don't rely on external packages
|
|
30
|
-
*/
|
|
31
|
-
const colors = {
|
|
32
|
-
gray: (text: string) => `\x1b[90m${text}\x1b[0m`,
|
|
33
|
-
blue: (text: string) => `\x1b[34m${text}\x1b[0m`,
|
|
34
|
-
yellow: (text: string) => `\x1b[33m${text}\x1b[0m`,
|
|
35
|
-
red: (text: string) => `\x1b[31m${text}\x1b[0m`,
|
|
36
|
-
green: (text: string) => `\x1b[32m${text}\x1b[0m`,
|
|
37
|
-
reset: (text: string) => `\x1b[0m${text}\x1b[0m`
|
|
38
|
-
};
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* Enhanced Logger class with color-coding, timestamp, and formatting
|
|
42
|
-
*/
|
|
43
|
-
class Logger {
|
|
44
|
-
private config: LoggerConfig;
|
|
45
|
-
private moduleName: string;
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Create a new Logger instance
|
|
49
|
-
*
|
|
50
|
-
* @param moduleName Name of the module using this logger
|
|
51
|
-
* @param config Logger configuration options
|
|
52
|
-
*/
|
|
53
|
-
constructor(moduleName: string, config?: Partial<LoggerConfig>) {
|
|
54
|
-
this.moduleName = moduleName;
|
|
55
|
-
this.config = {
|
|
56
|
-
level: LogLevel.INFO,
|
|
57
|
-
useTimestamp: true,
|
|
58
|
-
useColor: true,
|
|
59
|
-
prettyPrint: true,
|
|
60
|
-
...config
|
|
61
|
-
};
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
/**
|
|
65
|
-
* Format a log message with timestamp, level, and module information
|
|
66
|
-
*
|
|
67
|
-
* @param level Log level for this message
|
|
68
|
-
* @param message The message to log
|
|
69
|
-
* @param args Additional arguments to include
|
|
70
|
-
* @returns Formatted log message
|
|
71
|
-
*/
|
|
72
|
-
private formatMessage(level: string, message: string, args: any[] = []): string {
|
|
73
|
-
const timestamp = this.config.useTimestamp ?
|
|
74
|
-
`[${new Date().toISOString()}] ` : '';
|
|
75
|
-
|
|
76
|
-
const modulePrefix = this.moduleName ?
|
|
77
|
-
`[${this.moduleName}] ` : '';
|
|
78
|
-
|
|
79
|
-
const levelFormatted = `[${level.padEnd(5)}]`;
|
|
80
|
-
|
|
81
|
-
let formattedMessage = `${timestamp}${levelFormatted} ${modulePrefix}${message}`;
|
|
82
|
-
|
|
83
|
-
if (args.length > 0) {
|
|
84
|
-
const argsString = args.map(arg => {
|
|
85
|
-
if (arg instanceof Error) {
|
|
86
|
-
return `\n--- Error Details ---\nMessage: ${arg.message}\nStack:\n${arg.stack}\n--- End Error ---`;
|
|
87
|
-
}
|
|
88
|
-
else if (this.config.prettyPrint && typeof arg === 'object' && arg !== null) {
|
|
89
|
-
try {
|
|
90
|
-
return JSON.stringify(arg, null, 2);
|
|
91
|
-
} catch (e) {
|
|
92
|
-
return "[Unserializable Object]";
|
|
93
|
-
}
|
|
94
|
-
} else {
|
|
95
|
-
return String(arg);
|
|
96
|
-
}
|
|
97
|
-
}).join('\n');
|
|
98
|
-
|
|
99
|
-
if (this.config.prettyPrint) {
|
|
100
|
-
formattedMessage += `\n${argsString}`;
|
|
101
|
-
} else {
|
|
102
|
-
formattedMessage += ` ${args.map(String).join(' ')}`;
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
return formattedMessage;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/**
|
|
110
|
-
* Apply color to a message based on log level
|
|
111
|
-
*
|
|
112
|
-
* @param level Log level
|
|
113
|
-
* @param message Message to color
|
|
114
|
-
* @returns Colored message
|
|
115
|
-
*/
|
|
116
|
-
private colorize(level: LogLevel, message: string): string {
|
|
117
|
-
if (!this.config.useColor) return message;
|
|
118
|
-
|
|
119
|
-
switch (level) {
|
|
120
|
-
case LogLevel.DEBUG:
|
|
121
|
-
return colors.gray(message);
|
|
122
|
-
case LogLevel.INFO:
|
|
123
|
-
return colors.blue(message);
|
|
124
|
-
case LogLevel.WARN:
|
|
125
|
-
return colors.yellow(message);
|
|
126
|
-
case LogLevel.ERROR:
|
|
127
|
-
return colors.red(message);
|
|
128
|
-
default:
|
|
129
|
-
return message;
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
/**
|
|
134
|
-
* Log a debug message
|
|
135
|
-
*
|
|
136
|
-
* @param message Message to log
|
|
137
|
-
* @param args Additional arguments
|
|
138
|
-
*/
|
|
139
|
-
debug(message: string, ...args: any[]): void {
|
|
140
|
-
if (this.config.level <= LogLevel.DEBUG) {
|
|
141
|
-
const formattedMessage = this.formatMessage('DEBUG', message, args);
|
|
142
|
-
console.log(this.colorize(LogLevel.DEBUG, formattedMessage));
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
/**
|
|
147
|
-
* Log an info message
|
|
148
|
-
*
|
|
149
|
-
* @param message Message to log
|
|
150
|
-
* @param args Additional arguments
|
|
151
|
-
*/
|
|
152
|
-
info(message: string, ...args: any[]): void {
|
|
153
|
-
if (this.config.level <= LogLevel.INFO) {
|
|
154
|
-
const formattedMessage = this.formatMessage('INFO', message, args);
|
|
155
|
-
console.log(this.colorize(LogLevel.INFO, formattedMessage));
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
/**
|
|
160
|
-
* Log a warning message
|
|
161
|
-
*
|
|
162
|
-
* @param message Message to log
|
|
163
|
-
* @param args Additional arguments
|
|
164
|
-
*/
|
|
165
|
-
warn(message: string, ...args: any[]): void {
|
|
166
|
-
if (this.config.level <= LogLevel.WARN) {
|
|
167
|
-
const formattedMessage = this.formatMessage('WARN', message, args);
|
|
168
|
-
console.warn(this.colorize(LogLevel.WARN, formattedMessage));
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
/**
|
|
173
|
-
* Log an error message
|
|
174
|
-
*
|
|
175
|
-
* @param message Message to log
|
|
176
|
-
* @param args Additional arguments
|
|
177
|
-
*/
|
|
178
|
-
error(message: string, ...args: any[]): void {
|
|
179
|
-
if (this.config.level <= LogLevel.ERROR) {
|
|
180
|
-
const formattedMessage = this.formatMessage('ERROR', message, args);
|
|
181
|
-
console.error(this.colorize(LogLevel.ERROR, formattedMessage));
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
/**
|
|
186
|
-
* Create a child logger with a more specific module name
|
|
187
|
-
*
|
|
188
|
-
* @param subModule Name of the sub-module
|
|
189
|
-
* @returns New logger instance
|
|
190
|
-
*/
|
|
191
|
-
child(subModule: string): Logger {
|
|
192
|
-
return new Logger(`${this.moduleName}:${subModule}`, this.config);
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
/**
|
|
196
|
-
* Format a section header to clearly separate logical parts of execution
|
|
197
|
-
*
|
|
198
|
-
* @param title Section title
|
|
199
|
-
* @returns Logger instance for chaining
|
|
200
|
-
*/
|
|
201
|
-
section(title: string): Logger {
|
|
202
|
-
if (this.config.level <= LogLevel.INFO) {
|
|
203
|
-
const separator = '='.repeat(Math.max(80 - title.length - 4, 10));
|
|
204
|
-
const message = `${separator} ${title} ${separator}`;
|
|
205
|
-
console.log(this.colorize(LogLevel.INFO, message));
|
|
206
|
-
}
|
|
207
|
-
return this;
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
/**
|
|
211
|
-
* Create a progress indicator
|
|
212
|
-
*
|
|
213
|
-
* @param title Title of the operation
|
|
214
|
-
* @param total Total number of items to process
|
|
215
|
-
* @returns Object with update and complete methods
|
|
216
|
-
*/
|
|
217
|
-
progress(title: string, total: number) {
|
|
218
|
-
let current = 0;
|
|
219
|
-
const startTime = Date.now();
|
|
220
|
-
|
|
221
|
-
const update = (increment = 1, message?: string) => {
|
|
222
|
-
if (this.config.level > LogLevel.INFO) return;
|
|
223
|
-
|
|
224
|
-
current += increment;
|
|
225
|
-
const percentage = Math.min(Math.floor((current / total) * 100), 100);
|
|
226
|
-
const elapsed = (Date.now() - startTime) / 1000;
|
|
227
|
-
let rate = current / elapsed;
|
|
228
|
-
|
|
229
|
-
let timeRemaining = '';
|
|
230
|
-
if (rate > 0 && current < total) {
|
|
231
|
-
const remainingSecs = (total - current) / rate;
|
|
232
|
-
timeRemaining = `, ETA: ${Math.floor(remainingSecs / 60)}m ${Math.floor(remainingSecs % 60)}s`;
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
const progressBar = this.createProgressBar(percentage);
|
|
236
|
-
|
|
237
|
-
const statusMsg = message ? ` - ${message}` : '';
|
|
238
|
-
console.log(this.colorize(
|
|
239
|
-
LogLevel.INFO,
|
|
240
|
-
this.formatMessage('INFO', `${title}: ${progressBar} ${percentage}% (${current}/${total}${timeRemaining})${statusMsg}`)
|
|
241
|
-
));
|
|
242
|
-
};
|
|
243
|
-
|
|
244
|
-
const complete = (message = 'Completed') => {
|
|
245
|
-
if (this.config.level > LogLevel.INFO) return;
|
|
246
|
-
|
|
247
|
-
const elapsed = (Date.now() - startTime) / 1000;
|
|
248
|
-
const rate = total / elapsed;
|
|
249
|
-
|
|
250
|
-
console.log(this.colorize(
|
|
251
|
-
LogLevel.INFO,
|
|
252
|
-
this.formatMessage('INFO', `${title}: ${this.createProgressBar(100)} 100% (${total}/${total}) - ${message} in ${elapsed.toFixed(2)}s (${rate.toFixed(2)} items/sec)`)
|
|
253
|
-
));
|
|
254
|
-
};
|
|
255
|
-
|
|
256
|
-
return { update, complete };
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
/**
|
|
260
|
-
* Create a visual progress bar
|
|
261
|
-
*
|
|
262
|
-
* @param percentage Completion percentage
|
|
263
|
-
* @returns Visual progress bar
|
|
264
|
-
*/
|
|
265
|
-
private createProgressBar(percentage: number): string {
|
|
266
|
-
const width = 20;
|
|
267
|
-
const completeChars = Math.floor((percentage / 100) * width);
|
|
268
|
-
const incompleteChars = width - completeChars;
|
|
269
|
-
|
|
270
|
-
let bar = '[';
|
|
271
|
-
if (this.config.useColor) {
|
|
272
|
-
bar += colors.green('='.repeat(completeChars));
|
|
273
|
-
bar += ' '.repeat(incompleteChars);
|
|
274
|
-
} else {
|
|
275
|
-
bar += '='.repeat(completeChars);
|
|
276
|
-
bar += ' '.repeat(incompleteChars);
|
|
277
|
-
}
|
|
278
|
-
bar += ']';
|
|
279
|
-
|
|
280
|
-
return bar;
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
// Create a default logger instance
|
|
285
|
-
const defaultLogger = new Logger('app');
|
|
286
|
-
|
|
287
|
-
export { Logger, LogLevel, defaultLogger };
|
package/mcp/Dockerfile
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
FROM node:20-slim AS base
|
|
2
|
-
|
|
3
|
-
# Copy project files
|
|
4
|
-
COPY package.json /usr/src/app/
|
|
5
|
-
COPY package-lock.json /usr/src/app/
|
|
6
|
-
COPY tsconfig.json /usr/src/app/
|
|
7
|
-
COPY src/index.ts /usr/src/app/src/
|
|
8
|
-
COPY *.db /data/
|
|
9
|
-
WORKDIR /usr/src/app
|
|
10
|
-
|
|
11
|
-
RUN npm install
|
|
12
|
-
RUN npm run build
|
|
13
|
-
|
|
14
|
-
ENTRYPOINT ["node", "build/index.js"]
|
package/mcp/README.md
DELETED
|
@@ -1,166 +0,0 @@
|
|
|
1
|
-
# SQLite Vector Documentation Query MCP Server
|
|
2
|
-
|
|
3
|
-
This is a Model Context Protocol (MCP) server that enables querying documentation stored in SQLite databases with vector embeddings. The server uses OpenAI's embedding API to convert natural language queries into vector embeddings and performs semantic search against documentation stored in SQLite databases.
|
|
4
|
-
|
|
5
|
-
## Features
|
|
6
|
-
|
|
7
|
-
- Vector-based semantic search for documentation
|
|
8
|
-
- Filters by product name and version
|
|
9
|
-
- Uses OpenAI's embedding API for query embedding generation
|
|
10
|
-
- Fully compatible with the Model Context Protocol
|
|
11
|
-
- Simple Express-based API with Server-Sent Events (SSE) for real-time communication
|
|
12
|
-
|
|
13
|
-
## Prerequisites
|
|
14
|
-
|
|
15
|
-
- Node.js 20 or higher
|
|
16
|
-
- OpenAI API key
|
|
17
|
-
- Documentation stored in SQLite vector databases (using `sqlite-vec`)
|
|
18
|
-
|
|
19
|
-
## Environment Variables
|
|
20
|
-
|
|
21
|
-
| Variable | Description | Default |
|
|
22
|
-
|----------|-------------|---------|
|
|
23
|
-
| `OPENAI_API_KEY` | Your OpenAI API key (required) | - |
|
|
24
|
-
| `SQLITE_DB_DIR` | Directory containing SQLite databases | Current directory |
|
|
25
|
-
| `PORT` | Port to run the server on | 3001 |
|
|
26
|
-
|
|
27
|
-
## Local Setup and Running
|
|
28
|
-
|
|
29
|
-
1. Install dependencies:
|
|
30
|
-
```bash
|
|
31
|
-
npm install
|
|
32
|
-
```
|
|
33
|
-
|
|
34
|
-
2. Create a `.env` file with required environment variables:
|
|
35
|
-
```
|
|
36
|
-
OPENAI_API_KEY=your_openai_api_key
|
|
37
|
-
SQLITE_DB_DIR=/path/to/databases
|
|
38
|
-
PORT=3001
|
|
39
|
-
```
|
|
40
|
-
|
|
41
|
-
3. Build the TypeScript code:
|
|
42
|
-
```bash
|
|
43
|
-
npm run build
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
4. Start the server:
|
|
47
|
-
```bash
|
|
48
|
-
npm start
|
|
49
|
-
```
|
|
50
|
-
|
|
51
|
-
## Docker Setup
|
|
52
|
-
|
|
53
|
-
### Building the Docker Image
|
|
54
|
-
|
|
55
|
-
```bash
|
|
56
|
-
docker build -t sqlite-vec-mcp-server:latest .
|
|
57
|
-
```
|
|
58
|
-
|
|
59
|
-
This is going to include any `*.db` files in the `/data` directory of the image.
|
|
60
|
-
|
|
61
|
-
### Running with Docker
|
|
62
|
-
|
|
63
|
-
```bash
|
|
64
|
-
docker run -p 3001:3001 \
|
|
65
|
-
-e OPENAI_API_KEY=your_openai_api_key \
|
|
66
|
-
sqlite-vec-mcp-server:latest
|
|
67
|
-
```
|
|
68
|
-
|
|
69
|
-
### Create a Secret for the OpenAI API Key
|
|
70
|
-
|
|
71
|
-
```bash
|
|
72
|
-
kubectl create secret generic mcp-secrets \
|
|
73
|
-
--from-literal=OPENAI_API_KEY=your_openai_api_key
|
|
74
|
-
```
|
|
75
|
-
|
|
76
|
-
### Create a ConfigMap for Database Configuration
|
|
77
|
-
|
|
78
|
-
```bash
|
|
79
|
-
kubectl create configmap mcp-config \
|
|
80
|
-
--from-literal=SQLITE_DB_DIR=/data \
|
|
81
|
-
--from-literal=PORT=3001
|
|
82
|
-
```
|
|
83
|
-
|
|
84
|
-
### Create a Deployment
|
|
85
|
-
|
|
86
|
-
Create a file named `deployment.yaml`:
|
|
87
|
-
|
|
88
|
-
```yaml
|
|
89
|
-
apiVersion: apps/v1
|
|
90
|
-
kind: Deployment
|
|
91
|
-
metadata:
|
|
92
|
-
name: mcp-sqlite-vec
|
|
93
|
-
labels:
|
|
94
|
-
app: mcp-sqlite-vec
|
|
95
|
-
spec:
|
|
96
|
-
replicas: 1
|
|
97
|
-
selector:
|
|
98
|
-
matchLabels:
|
|
99
|
-
app: mcp-sqlite-vec
|
|
100
|
-
template:
|
|
101
|
-
metadata:
|
|
102
|
-
labels:
|
|
103
|
-
app: mcp-sqlite-vec
|
|
104
|
-
spec:
|
|
105
|
-
containers:
|
|
106
|
-
- name: mcp-sqlite-vec
|
|
107
|
-
image: sqlite-vec-mcp-server:latest
|
|
108
|
-
imagePullPolicy: Always
|
|
109
|
-
ports:
|
|
110
|
-
- containerPort: 3001
|
|
111
|
-
env:
|
|
112
|
-
- name: OPENAI_API_KEY
|
|
113
|
-
valueFrom:
|
|
114
|
-
secretKeyRef:
|
|
115
|
-
name: mcp-secrets
|
|
116
|
-
key: OPENAI_API_KEY
|
|
117
|
-
- name: SQLITE_DB_DIR
|
|
118
|
-
valueFrom:
|
|
119
|
-
configMapKeyRef:
|
|
120
|
-
name: mcp-config
|
|
121
|
-
key: SQLITE_DB_DIR
|
|
122
|
-
- name: PORT
|
|
123
|
-
valueFrom:
|
|
124
|
-
configMapKeyRef:
|
|
125
|
-
name: mcp-config
|
|
126
|
-
key: PORT
|
|
127
|
-
```
|
|
128
|
-
|
|
129
|
-
Apply it:
|
|
130
|
-
```bash
|
|
131
|
-
kubectl apply -f deployment.yaml
|
|
132
|
-
```
|
|
133
|
-
|
|
134
|
-
### Create a Service
|
|
135
|
-
|
|
136
|
-
Create a file named `service.yaml`:
|
|
137
|
-
|
|
138
|
-
```yaml
|
|
139
|
-
apiVersion: v1
|
|
140
|
-
kind: Service
|
|
141
|
-
metadata:
|
|
142
|
-
name: mcp-sqlite-vec
|
|
143
|
-
spec:
|
|
144
|
-
selector:
|
|
145
|
-
app: mcp-sqlite-vec
|
|
146
|
-
ports:
|
|
147
|
-
- port: 3001
|
|
148
|
-
targetPort: 3001
|
|
149
|
-
type: ClusterIP
|
|
150
|
-
```
|
|
151
|
-
|
|
152
|
-
Apply it:
|
|
153
|
-
```bash
|
|
154
|
-
kubectl apply -f service.yaml
|
|
155
|
-
```
|
|
156
|
-
|
|
157
|
-
## Using the MCP Server
|
|
158
|
-
|
|
159
|
-
The server implements a tool called `query-documentation` that can be used to query documentation.
|
|
160
|
-
|
|
161
|
-
### Tool Parameters
|
|
162
|
-
|
|
163
|
-
- `queryText` (string, required): The natural language query to search for
|
|
164
|
-
- `productName` (string, required): The name of the product documentation database to search within
|
|
165
|
-
- `version` (string, optional): The specific version of the product documentation
|
|
166
|
-
- `limit` (number, optional, default: 4): Maximum number of results to return
|