doc2vec 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -12
- package/dist/content-processor.js +725 -0
- package/dist/database.js +507 -0
- package/dist/doc2vec.js +48 -964
- package/dist/types.js +2 -0
- package/dist/utils.js +118 -0
- package/package.json +10 -4
- package/Dockerfile +0 -29
- package/config.yaml +0 -164
- package/dist/doc2vec copy.js +0 -1437
- package/dist/logger copy.js +0 -246
- package/doc2vec.ts +0 -1677
- package/tsconfig.json +0 -18
package/dist/logger copy.js
DELETED
|
@@ -1,246 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* Enhanced Logger for structured and consistent logging
|
|
4
|
-
* Compatible with CommonJS and ESM environments
|
|
5
|
-
*/
|
|
6
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.defaultLogger = exports.LogLevel = exports.Logger = void 0;
|
|
8
|
-
/**
|
|
9
|
-
* Logger levels with their corresponding numeric values
|
|
10
|
-
*/
|
|
11
|
-
var LogLevel;
|
|
12
|
-
(function (LogLevel) {
|
|
13
|
-
LogLevel[LogLevel["DEBUG"] = 0] = "DEBUG";
|
|
14
|
-
LogLevel[LogLevel["INFO"] = 1] = "INFO";
|
|
15
|
-
LogLevel[LogLevel["WARN"] = 2] = "WARN";
|
|
16
|
-
LogLevel[LogLevel["ERROR"] = 3] = "ERROR";
|
|
17
|
-
LogLevel[LogLevel["NONE"] = 100] = "NONE";
|
|
18
|
-
})(LogLevel || (exports.LogLevel = LogLevel = {}));
|
|
19
|
-
/**
|
|
20
|
-
* Basic color functions that don't rely on external packages
|
|
21
|
-
*/
|
|
22
|
-
const colors = {
|
|
23
|
-
gray: (text) => `\x1b[90m${text}\x1b[0m`,
|
|
24
|
-
blue: (text) => `\x1b[34m${text}\x1b[0m`,
|
|
25
|
-
yellow: (text) => `\x1b[33m${text}\x1b[0m`,
|
|
26
|
-
red: (text) => `\x1b[31m${text}\x1b[0m`,
|
|
27
|
-
green: (text) => `\x1b[32m${text}\x1b[0m`,
|
|
28
|
-
reset: (text) => `\x1b[0m${text}\x1b[0m`
|
|
29
|
-
};
|
|
30
|
-
/**
|
|
31
|
-
* Enhanced Logger class with color-coding, timestamp, and formatting
|
|
32
|
-
*/
|
|
33
|
-
class Logger {
|
|
34
|
-
/**
|
|
35
|
-
* Create a new Logger instance
|
|
36
|
-
*
|
|
37
|
-
* @param moduleName Name of the module using this logger
|
|
38
|
-
* @param config Logger configuration options
|
|
39
|
-
*/
|
|
40
|
-
constructor(moduleName, config) {
|
|
41
|
-
this.moduleName = moduleName;
|
|
42
|
-
this.config = {
|
|
43
|
-
level: LogLevel.INFO,
|
|
44
|
-
useTimestamp: true,
|
|
45
|
-
useColor: true,
|
|
46
|
-
prettyPrint: true,
|
|
47
|
-
...config
|
|
48
|
-
};
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
* Format a log message with timestamp, level, and module information
|
|
52
|
-
*
|
|
53
|
-
* @param level Log level for this message
|
|
54
|
-
* @param message The message to log
|
|
55
|
-
* @param args Additional arguments to include
|
|
56
|
-
* @returns Formatted log message
|
|
57
|
-
*/
|
|
58
|
-
formatMessage(level, message, args = []) {
|
|
59
|
-
const timestamp = this.config.useTimestamp ?
|
|
60
|
-
`[${new Date().toISOString()}] ` : '';
|
|
61
|
-
const modulePrefix = this.moduleName ?
|
|
62
|
-
`[${this.moduleName}] ` : '';
|
|
63
|
-
const levelFormatted = `[${level.padEnd(5)}]`;
|
|
64
|
-
let formattedMessage = `${timestamp}${levelFormatted} ${modulePrefix}${message}`;
|
|
65
|
-
if (args.length > 0) {
|
|
66
|
-
const argsString = args.map(arg => {
|
|
67
|
-
if (arg instanceof Error) {
|
|
68
|
-
return `\n--- Error Details ---\nMessage: ${arg.message}\nStack:\n${arg.stack}\n--- End Error ---`;
|
|
69
|
-
}
|
|
70
|
-
else if (this.config.prettyPrint && typeof arg === 'object' && arg !== null) {
|
|
71
|
-
try {
|
|
72
|
-
return JSON.stringify(arg, null, 2);
|
|
73
|
-
}
|
|
74
|
-
catch (e) {
|
|
75
|
-
return "[Unserializable Object]";
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
else {
|
|
79
|
-
return String(arg);
|
|
80
|
-
}
|
|
81
|
-
}).join('\n');
|
|
82
|
-
if (this.config.prettyPrint) {
|
|
83
|
-
formattedMessage += `\n${argsString}`;
|
|
84
|
-
}
|
|
85
|
-
else {
|
|
86
|
-
formattedMessage += ` ${args.map(String).join(' ')}`;
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
return formattedMessage;
|
|
90
|
-
}
|
|
91
|
-
/**
|
|
92
|
-
* Apply color to a message based on log level
|
|
93
|
-
*
|
|
94
|
-
* @param level Log level
|
|
95
|
-
* @param message Message to color
|
|
96
|
-
* @returns Colored message
|
|
97
|
-
*/
|
|
98
|
-
colorize(level, message) {
|
|
99
|
-
if (!this.config.useColor)
|
|
100
|
-
return message;
|
|
101
|
-
switch (level) {
|
|
102
|
-
case LogLevel.DEBUG:
|
|
103
|
-
return colors.gray(message);
|
|
104
|
-
case LogLevel.INFO:
|
|
105
|
-
return colors.blue(message);
|
|
106
|
-
case LogLevel.WARN:
|
|
107
|
-
return colors.yellow(message);
|
|
108
|
-
case LogLevel.ERROR:
|
|
109
|
-
return colors.red(message);
|
|
110
|
-
default:
|
|
111
|
-
return message;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
/**
|
|
115
|
-
* Log a debug message
|
|
116
|
-
*
|
|
117
|
-
* @param message Message to log
|
|
118
|
-
* @param args Additional arguments
|
|
119
|
-
*/
|
|
120
|
-
debug(message, ...args) {
|
|
121
|
-
if (this.config.level <= LogLevel.DEBUG) {
|
|
122
|
-
const formattedMessage = this.formatMessage('DEBUG', message, args);
|
|
123
|
-
console.log(this.colorize(LogLevel.DEBUG, formattedMessage));
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
/**
|
|
127
|
-
* Log an info message
|
|
128
|
-
*
|
|
129
|
-
* @param message Message to log
|
|
130
|
-
* @param args Additional arguments
|
|
131
|
-
*/
|
|
132
|
-
info(message, ...args) {
|
|
133
|
-
if (this.config.level <= LogLevel.INFO) {
|
|
134
|
-
const formattedMessage = this.formatMessage('INFO', message, args);
|
|
135
|
-
console.log(this.colorize(LogLevel.INFO, formattedMessage));
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
/**
|
|
139
|
-
* Log a warning message
|
|
140
|
-
*
|
|
141
|
-
* @param message Message to log
|
|
142
|
-
* @param args Additional arguments
|
|
143
|
-
*/
|
|
144
|
-
warn(message, ...args) {
|
|
145
|
-
if (this.config.level <= LogLevel.WARN) {
|
|
146
|
-
const formattedMessage = this.formatMessage('WARN', message, args);
|
|
147
|
-
console.warn(this.colorize(LogLevel.WARN, formattedMessage));
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
/**
|
|
151
|
-
* Log an error message
|
|
152
|
-
*
|
|
153
|
-
* @param message Message to log
|
|
154
|
-
* @param args Additional arguments
|
|
155
|
-
*/
|
|
156
|
-
error(message, ...args) {
|
|
157
|
-
if (this.config.level <= LogLevel.ERROR) {
|
|
158
|
-
const formattedMessage = this.formatMessage('ERROR', message, args);
|
|
159
|
-
console.error(this.colorize(LogLevel.ERROR, formattedMessage));
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
/**
|
|
163
|
-
* Create a child logger with a more specific module name
|
|
164
|
-
*
|
|
165
|
-
* @param subModule Name of the sub-module
|
|
166
|
-
* @returns New logger instance
|
|
167
|
-
*/
|
|
168
|
-
child(subModule) {
|
|
169
|
-
return new Logger(`${this.moduleName}:${subModule}`, this.config);
|
|
170
|
-
}
|
|
171
|
-
/**
|
|
172
|
-
* Format a section header to clearly separate logical parts of execution
|
|
173
|
-
*
|
|
174
|
-
* @param title Section title
|
|
175
|
-
* @returns Logger instance for chaining
|
|
176
|
-
*/
|
|
177
|
-
section(title) {
|
|
178
|
-
if (this.config.level <= LogLevel.INFO) {
|
|
179
|
-
const separator = '='.repeat(Math.max(80 - title.length - 4, 10));
|
|
180
|
-
const message = `${separator} ${title} ${separator}`;
|
|
181
|
-
console.log(this.colorize(LogLevel.INFO, message));
|
|
182
|
-
}
|
|
183
|
-
return this;
|
|
184
|
-
}
|
|
185
|
-
/**
|
|
186
|
-
* Create a progress indicator
|
|
187
|
-
*
|
|
188
|
-
* @param title Title of the operation
|
|
189
|
-
* @param total Total number of items to process
|
|
190
|
-
* @returns Object with update and complete methods
|
|
191
|
-
*/
|
|
192
|
-
progress(title, total) {
|
|
193
|
-
let current = 0;
|
|
194
|
-
const startTime = Date.now();
|
|
195
|
-
const update = (increment = 1, message) => {
|
|
196
|
-
if (this.config.level > LogLevel.INFO)
|
|
197
|
-
return;
|
|
198
|
-
current += increment;
|
|
199
|
-
const percentage = Math.min(Math.floor((current / total) * 100), 100);
|
|
200
|
-
const elapsed = (Date.now() - startTime) / 1000;
|
|
201
|
-
let rate = current / elapsed;
|
|
202
|
-
let timeRemaining = '';
|
|
203
|
-
if (rate > 0 && current < total) {
|
|
204
|
-
const remainingSecs = (total - current) / rate;
|
|
205
|
-
timeRemaining = `, ETA: ${Math.floor(remainingSecs / 60)}m ${Math.floor(remainingSecs % 60)}s`;
|
|
206
|
-
}
|
|
207
|
-
const progressBar = this.createProgressBar(percentage);
|
|
208
|
-
const statusMsg = message ? ` - ${message}` : '';
|
|
209
|
-
console.log(this.colorize(LogLevel.INFO, this.formatMessage('INFO', `${title}: ${progressBar} ${percentage}% (${current}/${total}${timeRemaining})${statusMsg}`)));
|
|
210
|
-
};
|
|
211
|
-
const complete = (message = 'Completed') => {
|
|
212
|
-
if (this.config.level > LogLevel.INFO)
|
|
213
|
-
return;
|
|
214
|
-
const elapsed = (Date.now() - startTime) / 1000;
|
|
215
|
-
const rate = total / elapsed;
|
|
216
|
-
console.log(this.colorize(LogLevel.INFO, this.formatMessage('INFO', `${title}: ${this.createProgressBar(100)} 100% (${total}/${total}) - ${message} in ${elapsed.toFixed(2)}s (${rate.toFixed(2)} items/sec)`)));
|
|
217
|
-
};
|
|
218
|
-
return { update, complete };
|
|
219
|
-
}
|
|
220
|
-
/**
|
|
221
|
-
* Create a visual progress bar
|
|
222
|
-
*
|
|
223
|
-
* @param percentage Completion percentage
|
|
224
|
-
* @returns Visual progress bar
|
|
225
|
-
*/
|
|
226
|
-
createProgressBar(percentage) {
|
|
227
|
-
const width = 20;
|
|
228
|
-
const completeChars = Math.floor((percentage / 100) * width);
|
|
229
|
-
const incompleteChars = width - completeChars;
|
|
230
|
-
let bar = '[';
|
|
231
|
-
if (this.config.useColor) {
|
|
232
|
-
bar += colors.green('='.repeat(completeChars));
|
|
233
|
-
bar += ' '.repeat(incompleteChars);
|
|
234
|
-
}
|
|
235
|
-
else {
|
|
236
|
-
bar += '='.repeat(completeChars);
|
|
237
|
-
bar += ' '.repeat(incompleteChars);
|
|
238
|
-
}
|
|
239
|
-
bar += ']';
|
|
240
|
-
return bar;
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
exports.Logger = Logger;
|
|
244
|
-
// Create a default logger instance
|
|
245
|
-
const defaultLogger = new Logger('app');
|
|
246
|
-
exports.defaultLogger = defaultLogger;
|