@okf/ootils 1.3.6 → 1.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/node.d.mts +297 -3
- package/dist/node.d.ts +297 -3
- package/dist/node.js +627 -270
- package/dist/node.mjs +648 -270
- package/package.json +3 -1
package/dist/node.mjs
CHANGED
|
@@ -1,295 +1,341 @@
|
|
|
1
|
+
var __create = Object.create;
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
1
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
6
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
2
7
|
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
3
8
|
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
4
9
|
}) : x)(function(x) {
|
|
5
10
|
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
6
11
|
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
7
12
|
});
|
|
13
|
+
var __esm = (fn, res) => function __init() {
|
|
14
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
15
|
+
};
|
|
8
16
|
var __commonJS = (cb, mod) => function __require2() {
|
|
9
17
|
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
10
18
|
};
|
|
19
|
+
var __export = (target, all) => {
|
|
20
|
+
for (var name in all)
|
|
21
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
22
|
+
};
|
|
23
|
+
var __copyProps = (to, from, except, desc) => {
|
|
24
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
25
|
+
for (let key of __getOwnPropNames(from))
|
|
26
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
27
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
28
|
+
}
|
|
29
|
+
return to;
|
|
30
|
+
};
|
|
31
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
32
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
33
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
34
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
35
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
36
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
37
|
+
mod
|
|
38
|
+
));
|
|
39
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
11
40
|
|
|
12
|
-
// src/
|
|
13
|
-
var
|
|
14
|
-
"src/
|
|
41
|
+
// src/bullmq/WorkerManager.js
|
|
42
|
+
var require_WorkerManager = __commonJS({
|
|
43
|
+
"src/bullmq/WorkerManager.js"(exports, module) {
|
|
15
44
|
"use strict";
|
|
16
|
-
var
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
kp_contributed_by: {
|
|
39
|
-
type: Schema.Types.ObjectId,
|
|
40
|
-
ref: "user"
|
|
41
|
-
},
|
|
42
|
-
valuePath: String,
|
|
43
|
-
documentId: {
|
|
44
|
-
type: Schema.Types.ObjectId
|
|
45
|
-
}
|
|
46
|
-
},
|
|
47
|
-
// Main content data - using Schema.Types.Mixed for dynamic structure
|
|
48
|
-
main: {
|
|
49
|
-
type: Schema.Types.Mixed,
|
|
50
|
-
default: {}
|
|
51
|
-
},
|
|
52
|
-
// Annotation specific details
|
|
53
|
-
annotations: {
|
|
54
|
-
tags: {
|
|
55
|
-
type: Map,
|
|
56
|
-
_id: false,
|
|
57
|
-
of: new Schema({
|
|
58
|
-
collectionId: String,
|
|
59
|
-
data: [
|
|
60
|
-
{
|
|
61
|
-
_id: { type: Schema.Types.ObjectId },
|
|
62
|
-
display: String,
|
|
63
|
-
tagId: String
|
|
64
|
-
}
|
|
65
|
-
]
|
|
66
|
-
})
|
|
67
|
-
},
|
|
68
|
-
fragment: {
|
|
69
|
-
isLexical: Boolean,
|
|
70
|
-
editorState: Object,
|
|
71
|
-
allText: String
|
|
72
|
-
},
|
|
73
|
-
annoKey: String,
|
|
74
|
-
author: {
|
|
75
|
-
id: { type: Schema.Types.ObjectId },
|
|
76
|
-
name: String
|
|
45
|
+
var WorkerManager2 = class {
|
|
46
|
+
constructor(workers = []) {
|
|
47
|
+
this.workers = workers;
|
|
48
|
+
this.activeWorkers = [];
|
|
49
|
+
}
|
|
50
|
+
startAllWorkers() {
|
|
51
|
+
if (this.workers.length === 0) {
|
|
52
|
+
console.log("No workers provided to start");
|
|
53
|
+
return [];
|
|
54
|
+
}
|
|
55
|
+
console.log("\u{1F680} Starting all workers...");
|
|
56
|
+
this.workers.forEach((WorkerClass) => {
|
|
57
|
+
try {
|
|
58
|
+
const workerInstance = new WorkerClass();
|
|
59
|
+
workerInstance.start();
|
|
60
|
+
this.activeWorkers.push({
|
|
61
|
+
name: WorkerClass.name,
|
|
62
|
+
instance: workerInstance
|
|
63
|
+
});
|
|
64
|
+
console.log(`\u2705 Started ${WorkerClass.name}`);
|
|
65
|
+
} catch (error) {
|
|
66
|
+
console.error(`\u274C Failed to start ${WorkerClass.name}:`, error);
|
|
77
67
|
}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
timestamps: true,
|
|
100
|
-
toJSON: { virtuals: true },
|
|
101
|
-
toObject: { virtuals: true },
|
|
102
|
-
strict: false
|
|
103
|
-
// This allows for flexible document structure beyond the defined schema
|
|
68
|
+
});
|
|
69
|
+
console.log(`\u{1F389} Successfully started ${this.activeWorkers.length} workers`);
|
|
70
|
+
return this.activeWorkers;
|
|
71
|
+
}
|
|
72
|
+
async shutdown() {
|
|
73
|
+
console.log("\u{1F6D1} Stopping all workers...");
|
|
74
|
+
const stopPromises = this.activeWorkers.map(
|
|
75
|
+
({ name, instance }) => instance.stop().catch(
|
|
76
|
+
(err) => console.error(`\u274C Error stopping ${name}:`, err)
|
|
77
|
+
)
|
|
78
|
+
);
|
|
79
|
+
await Promise.all(stopPromises);
|
|
80
|
+
this.activeWorkers = [];
|
|
81
|
+
console.log("\u2705 All workers stopped");
|
|
82
|
+
}
|
|
83
|
+
getStatus() {
|
|
84
|
+
return this.activeWorkers.map(({ name, instance }) => ({
|
|
85
|
+
name,
|
|
86
|
+
queueId: instance.config?.id || "unknown",
|
|
87
|
+
isActive: !!instance.worker
|
|
88
|
+
}));
|
|
104
89
|
}
|
|
105
|
-
);
|
|
106
|
-
AnnotationSchema.index({ "meta.contentType": 1 });
|
|
107
|
-
AnnotationSchema.index({ "meta.documentId": 1 });
|
|
108
|
-
AnnotationSchema.index({ createdAt: -1 });
|
|
109
|
-
AnnotationSchema.index({ "annotations.annoKey": 1 });
|
|
110
|
-
AnnotationSchema.pre("save", function(next) {
|
|
111
|
-
this.updatedAt = /* @__PURE__ */ new Date();
|
|
112
|
-
next();
|
|
113
|
-
});
|
|
114
|
-
AnnotationSchema.methods.getMainField = function(fieldPath) {
|
|
115
|
-
if (!fieldPath) return null;
|
|
116
|
-
const parts = fieldPath.split(".");
|
|
117
|
-
let value = this.main;
|
|
118
|
-
for (const part of parts) {
|
|
119
|
-
if (!value || typeof value !== "object") return null;
|
|
120
|
-
value = value[part];
|
|
121
|
-
}
|
|
122
|
-
return value;
|
|
123
90
|
};
|
|
124
|
-
|
|
125
|
-
return this.main.title || "Untitled Annotation";
|
|
126
|
-
});
|
|
127
|
-
module.exports = AnnotationSchema;
|
|
91
|
+
module.exports = { WorkerManager: WorkerManager2 };
|
|
128
92
|
}
|
|
129
93
|
});
|
|
130
94
|
|
|
131
|
-
// src/
|
|
132
|
-
var
|
|
133
|
-
"src/
|
|
95
|
+
// src/bullmq/BaseProducer.js
|
|
96
|
+
var require_BaseProducer = __commonJS({
|
|
97
|
+
"src/bullmq/BaseProducer.js"(exports, module) {
|
|
134
98
|
"use strict";
|
|
135
|
-
var
|
|
136
|
-
var
|
|
137
|
-
{
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
99
|
+
var { Queue } = __require("bullmq");
|
|
100
|
+
var BaseProducer2 = class {
|
|
101
|
+
constructor(config) {
|
|
102
|
+
this.config = config;
|
|
103
|
+
this.queue = new Queue(config.id, config.queueConfig);
|
|
104
|
+
}
|
|
105
|
+
async execute(params) {
|
|
106
|
+
throw new Error("execute() method must be implemented by subclass");
|
|
107
|
+
}
|
|
108
|
+
async addBulkJobs(jobs) {
|
|
109
|
+
try {
|
|
110
|
+
const results = await this.queue.addBulk(jobs);
|
|
111
|
+
console.log(`\u2705 ${this.constructor.name}: Successfully queued ${results.length} jobs`);
|
|
112
|
+
return results;
|
|
113
|
+
} catch (error) {
|
|
114
|
+
console.error(`\u274C ${this.constructor.name}: Failed to queue jobs:`, error);
|
|
115
|
+
throw error;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
async addJob(name, data, opts = {}) {
|
|
119
|
+
try {
|
|
120
|
+
const job = await this.queue.add(name, data, opts);
|
|
121
|
+
console.log(`\u2705 ${this.constructor.name}: Successfully queued job ${job.id}`);
|
|
122
|
+
return job;
|
|
123
|
+
} catch (error) {
|
|
124
|
+
console.error(`\u274C ${this.constructor.name}: Failed to queue job:`, error);
|
|
125
|
+
throw error;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
async getStatus() {
|
|
129
|
+
const waiting = await this.queue.getWaiting();
|
|
130
|
+
const active = await this.queue.getActive();
|
|
131
|
+
const completed = await this.queue.getCompleted();
|
|
132
|
+
const failed = await this.queue.getFailed();
|
|
133
|
+
return {
|
|
134
|
+
waiting: waiting.length,
|
|
135
|
+
active: active.length,
|
|
136
|
+
completed: completed.length,
|
|
137
|
+
failed: failed.length,
|
|
138
|
+
total: waiting.length + active.length + completed.length + failed.length
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
async getJobDetails(limit = 10) {
|
|
142
|
+
const waiting = await this.queue.getWaiting(0, limit - 1);
|
|
143
|
+
const active = await this.queue.getActive(0, limit - 1);
|
|
144
|
+
const completed = await this.queue.getCompleted(0, limit - 1);
|
|
145
|
+
const failed = await this.queue.getFailed(0, limit - 1);
|
|
146
|
+
return {
|
|
147
|
+
waiting: waiting.map((job) => ({
|
|
148
|
+
id: job.id,
|
|
149
|
+
batchIndex: job.data?.batchIndex,
|
|
150
|
+
totalBatches: job.data?.totalBatches,
|
|
151
|
+
rawRecordsCount: job.data?.rawBatchData?.length
|
|
152
|
+
})),
|
|
153
|
+
active: active.map((job) => ({
|
|
154
|
+
id: job.id,
|
|
155
|
+
batchIndex: job.data?.batchIndex,
|
|
156
|
+
totalBatches: job.data?.totalBatches,
|
|
157
|
+
progress: job.progress,
|
|
158
|
+
rawRecordsCount: job.data?.rawBatchData?.length
|
|
159
|
+
})),
|
|
160
|
+
completed: completed.map((job) => ({
|
|
161
|
+
id: job.id,
|
|
162
|
+
batchIndex: job.returnvalue?.batchIndex,
|
|
163
|
+
totalProcessed: job.returnvalue?.totalItemsProcessed,
|
|
164
|
+
summary: job.returnvalue?.summary
|
|
165
|
+
})),
|
|
166
|
+
failed: failed.map((job) => ({
|
|
167
|
+
id: job.id,
|
|
168
|
+
batchIndex: job.data?.batchIndex,
|
|
169
|
+
error: job.failedReason,
|
|
170
|
+
attempts: job.attemptsMade
|
|
171
|
+
}))
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
async stop() {
|
|
175
|
+
if (this.queue) {
|
|
176
|
+
await this.queue.close();
|
|
177
|
+
console.log(`\u{1F6D1} ${this.constructor.name} queue closed`);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
module.exports = { BaseProducer: BaseProducer2 };
|
|
162
182
|
}
|
|
163
183
|
});
|
|
164
184
|
|
|
165
|
-
// src/
|
|
166
|
-
var
|
|
167
|
-
"src/
|
|
185
|
+
// src/bullmq/BaseWorker.js
|
|
186
|
+
var require_BaseWorker = __commonJS({
|
|
187
|
+
"src/bullmq/BaseWorker.js"(exports, module) {
|
|
168
188
|
"use strict";
|
|
169
|
-
var
|
|
170
|
-
var
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
{
|
|
207
|
-
|
|
189
|
+
var { Worker } = __require("bullmq");
|
|
190
|
+
var BaseWorker2 = class {
|
|
191
|
+
constructor(config) {
|
|
192
|
+
this.config = config;
|
|
193
|
+
this.worker = null;
|
|
194
|
+
}
|
|
195
|
+
async execute(job) {
|
|
196
|
+
throw new Error("execute() method must be implemented by subclass");
|
|
197
|
+
}
|
|
198
|
+
start() {
|
|
199
|
+
this.worker = new Worker(
|
|
200
|
+
this.config.id,
|
|
201
|
+
this.execute.bind(this),
|
|
202
|
+
this.config.workerConfig
|
|
203
|
+
);
|
|
204
|
+
this.setupEventHandlers();
|
|
205
|
+
console.log(`\u{1F680} ${this.constructor.name} started for queue: ${this.config.id}`);
|
|
206
|
+
return this.worker;
|
|
207
|
+
}
|
|
208
|
+
setupEventHandlers() {
|
|
209
|
+
this.worker.on("completed", (job, returnValue) => {
|
|
210
|
+
this.onCompleted(job, returnValue);
|
|
211
|
+
});
|
|
212
|
+
this.worker.on("failed", (job, err) => {
|
|
213
|
+
this.onFailed(job, err);
|
|
214
|
+
});
|
|
215
|
+
this.worker.on("progress", (job, progress) => {
|
|
216
|
+
this.onProgress(job, progress);
|
|
217
|
+
});
|
|
218
|
+
this.worker.on("error", (error) => {
|
|
219
|
+
this.onError(error);
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
onCompleted(job, returnValue) {
|
|
223
|
+
console.log(`\u2705 Job ${job.id} completed in ${this.constructor.name}`);
|
|
224
|
+
}
|
|
225
|
+
onFailed(job, err) {
|
|
226
|
+
console.error(`\u{1F4A5} Job ${job.id} failed in ${this.constructor.name}:`, err.message);
|
|
227
|
+
}
|
|
228
|
+
onProgress(job, progress) {
|
|
229
|
+
console.log(`\u{1F4CA} Job ${job.id} progress in ${this.constructor.name}: ${progress}%`);
|
|
230
|
+
}
|
|
231
|
+
onError(error) {
|
|
232
|
+
console.error(`\u274C Worker error in ${this.constructor.name}:`, error);
|
|
233
|
+
}
|
|
234
|
+
async stop() {
|
|
235
|
+
if (this.worker) {
|
|
236
|
+
await this.worker.close();
|
|
237
|
+
console.log(`\u{1F6D1} ${this.constructor.name} stopped`);
|
|
208
238
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
enable: Boolean
|
|
237
|
-
},
|
|
238
|
-
participantModule: {
|
|
239
|
-
enable: Boolean
|
|
240
|
-
},
|
|
241
|
-
formFieldNumbering: {
|
|
242
|
-
enable: Boolean
|
|
243
|
-
},
|
|
244
|
-
postPblRedirPath: Object,
|
|
245
|
-
templateIndex: Object,
|
|
246
|
-
sharing: {
|
|
247
|
-
enable: Boolean,
|
|
248
|
-
trackShareCount: {
|
|
249
|
-
type: Boolean,
|
|
250
|
-
default: false
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
module.exports = { BaseWorker: BaseWorker2 };
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
// src/bullmq/GLOBAL_BULLMQ_CONFIG.js
|
|
246
|
+
var GLOBAL_BULLMQ_CONFIG_exports = {};
|
|
247
|
+
__export(GLOBAL_BULLMQ_CONFIG_exports, {
|
|
248
|
+
BASE_BULLMQ_CONFIG: () => BASE_BULLMQ_CONFIG
|
|
249
|
+
});
|
|
250
|
+
var BASE_BULLMQ_CONFIG;
|
|
251
|
+
var init_GLOBAL_BULLMQ_CONFIG = __esm({
|
|
252
|
+
"src/bullmq/GLOBAL_BULLMQ_CONFIG.js"() {
|
|
253
|
+
"use strict";
|
|
254
|
+
BASE_BULLMQ_CONFIG = {
|
|
255
|
+
PLUGIN__MAD_USERS_SYNC_QUEUE: {
|
|
256
|
+
id: "plugin--mad-users-sync-queue",
|
|
257
|
+
queueConfig: {
|
|
258
|
+
defaultJobOptions: {
|
|
259
|
+
backoff: {
|
|
260
|
+
type: "exponential",
|
|
261
|
+
delay: 2e3
|
|
262
|
+
},
|
|
263
|
+
attempts: 3,
|
|
264
|
+
removeOnComplete: 100,
|
|
265
|
+
removeOnFail: 50
|
|
251
266
|
}
|
|
252
267
|
},
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
268
|
+
workerConfig: {
|
|
269
|
+
concurrency: 1,
|
|
270
|
+
// Process jobs one at a time to avoid race conditions
|
|
271
|
+
limiter: {
|
|
272
|
+
max: 10,
|
|
273
|
+
// Max 10 jobs per...
|
|
274
|
+
duration: 6e4
|
|
275
|
+
// ...60 seconds (rate limiting)
|
|
257
276
|
}
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
277
|
+
}
|
|
278
|
+
},
|
|
279
|
+
// Chunk Processing Queue
|
|
280
|
+
CHUNK_PROCESSING_QUEUE: {
|
|
281
|
+
id: "chunk-processing-queue",
|
|
282
|
+
queueConfig: {
|
|
283
|
+
defaultJobOptions: {
|
|
284
|
+
backoff: {
|
|
285
|
+
type: "exponential",
|
|
286
|
+
delay: 2e3
|
|
287
|
+
},
|
|
288
|
+
attempts: 3,
|
|
289
|
+
removeOnComplete: 100,
|
|
290
|
+
removeOnFail: 50
|
|
267
291
|
}
|
|
268
292
|
},
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
293
|
+
workerConfig: {
|
|
294
|
+
concurrency: 10,
|
|
295
|
+
// Process 10 jobs at once for chunk processing
|
|
296
|
+
limiter: {
|
|
297
|
+
max: 5,
|
|
298
|
+
// Max 50 jobs per...
|
|
299
|
+
duration: 6e4
|
|
300
|
+
// ...60 seconds (higher throughput for chunking)
|
|
277
301
|
}
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
});
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
// src/bullmq/GET_GLOBAL_BULLMQ_CONFIG.js
|
|
309
|
+
var require_GET_GLOBAL_BULLMQ_CONFIG = __commonJS({
|
|
310
|
+
"src/bullmq/GET_GLOBAL_BULLMQ_CONFIG.js"(exports, module) {
|
|
311
|
+
"use strict";
|
|
312
|
+
var { default: Redis } = __require("ioredis");
|
|
313
|
+
var { BASE_BULLMQ_CONFIG: BASE_BULLMQ_CONFIG2 } = (init_GLOBAL_BULLMQ_CONFIG(), __toCommonJS(GLOBAL_BULLMQ_CONFIG_exports));
|
|
314
|
+
function GET_GLOBAL_BULLMQ_CONFIG2({ env, redisCredentials }) {
|
|
315
|
+
const redisConnectionForBullMQ = new Redis({
|
|
316
|
+
host: redisCredentials.REDIS_HOST,
|
|
317
|
+
port: redisCredentials.REDIS_PORT,
|
|
318
|
+
password: redisCredentials.REDIS_PASSWORD,
|
|
319
|
+
maxRetriesPerRequest: null
|
|
320
|
+
});
|
|
321
|
+
return Object.keys(BASE_BULLMQ_CONFIG2).reduce((acc, key) => ({
|
|
322
|
+
...acc,
|
|
323
|
+
[key]: {
|
|
324
|
+
...BASE_BULLMQ_CONFIG2[key],
|
|
325
|
+
id: BASE_BULLMQ_CONFIG2[key].id + `_${env}`,
|
|
326
|
+
// suffix env to queue to keep it unique for each environment
|
|
327
|
+
queueConfig: {
|
|
328
|
+
connection: redisConnectionForBullMQ,
|
|
329
|
+
...BASE_BULLMQ_CONFIG2[key].queueConfig
|
|
330
|
+
},
|
|
331
|
+
workerConfig: {
|
|
332
|
+
connection: redisConnectionForBullMQ,
|
|
333
|
+
...BASE_BULLMQ_CONFIG2[key].workerConfig
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}), {});
|
|
337
|
+
}
|
|
338
|
+
module.exports = { GET_GLOBAL_BULLMQ_CONFIG: GET_GLOBAL_BULLMQ_CONFIG2 };
|
|
293
339
|
}
|
|
294
340
|
});
|
|
295
341
|
|
|
@@ -663,10 +709,317 @@ var getDbByTenant = ({
|
|
|
663
709
|
throw new Error("getDbByTenant : mongodb object doesnt exist");
|
|
664
710
|
};
|
|
665
711
|
|
|
712
|
+
// src/models/AIChat.ts
|
|
713
|
+
import mongoose3 from "mongoose";
|
|
714
|
+
|
|
715
|
+
// src/models/Annotations.ts
|
|
716
|
+
import { Schema } from "mongoose";
|
|
717
|
+
var AnnotationSchema = new Schema(
|
|
718
|
+
{
|
|
719
|
+
// Tags section - dynamic structure for various tag categories
|
|
720
|
+
tags: {
|
|
721
|
+
type: Map,
|
|
722
|
+
of: new Schema({
|
|
723
|
+
data: [
|
|
724
|
+
{
|
|
725
|
+
_id: { type: Schema.Types.ObjectId },
|
|
726
|
+
display: String,
|
|
727
|
+
tagId: String
|
|
728
|
+
}
|
|
729
|
+
],
|
|
730
|
+
collectionId: String
|
|
731
|
+
}, {
|
|
732
|
+
_id: false
|
|
733
|
+
})
|
|
734
|
+
},
|
|
735
|
+
// Meta information
|
|
736
|
+
meta: {
|
|
737
|
+
contentType: { type: String, required: true },
|
|
738
|
+
kp_contributed_by: {
|
|
739
|
+
type: Schema.Types.ObjectId,
|
|
740
|
+
ref: "user"
|
|
741
|
+
},
|
|
742
|
+
valuePath: String,
|
|
743
|
+
documentId: {
|
|
744
|
+
type: Schema.Types.ObjectId
|
|
745
|
+
}
|
|
746
|
+
},
|
|
747
|
+
// Main content data - using Schema.Types.Mixed for dynamic structure
|
|
748
|
+
main: {
|
|
749
|
+
type: Schema.Types.Mixed,
|
|
750
|
+
default: {}
|
|
751
|
+
},
|
|
752
|
+
// Annotation specific details
|
|
753
|
+
annotations: {
|
|
754
|
+
tags: {
|
|
755
|
+
type: Map,
|
|
756
|
+
of: new Schema({
|
|
757
|
+
collectionId: String,
|
|
758
|
+
data: [
|
|
759
|
+
{
|
|
760
|
+
_id: { type: Schema.Types.ObjectId },
|
|
761
|
+
display: String,
|
|
762
|
+
tagId: String
|
|
763
|
+
}
|
|
764
|
+
]
|
|
765
|
+
}, {
|
|
766
|
+
_id: false
|
|
767
|
+
})
|
|
768
|
+
},
|
|
769
|
+
fragment: {
|
|
770
|
+
isLexical: Boolean,
|
|
771
|
+
editorState: Object,
|
|
772
|
+
allText: String
|
|
773
|
+
},
|
|
774
|
+
annoKey: String,
|
|
775
|
+
author: {
|
|
776
|
+
id: { type: Schema.Types.ObjectId },
|
|
777
|
+
name: String
|
|
778
|
+
}
|
|
779
|
+
},
|
|
780
|
+
embeddings: Array,
|
|
781
|
+
contentEnhancedText: String,
|
|
782
|
+
// // Optional chunk related fields
|
|
783
|
+
// chunk: {
|
|
784
|
+
// embeddings: Schema.Types.Mixed,
|
|
785
|
+
// contentEnhancedText: String
|
|
786
|
+
// },
|
|
787
|
+
clusterId: String,
|
|
788
|
+
kp_date_published: Date,
|
|
789
|
+
createdAt: {
|
|
790
|
+
type: Date,
|
|
791
|
+
default: Date.now
|
|
792
|
+
},
|
|
793
|
+
updatedAt: {
|
|
794
|
+
type: Date,
|
|
795
|
+
default: Date.now
|
|
796
|
+
},
|
|
797
|
+
topicId: [{ type: Schema.Types.ObjectId, ref: "generatedTopics" }],
|
|
798
|
+
translations: Object
|
|
799
|
+
},
|
|
800
|
+
{
|
|
801
|
+
timestamps: true,
|
|
802
|
+
toJSON: { virtuals: true },
|
|
803
|
+
toObject: { virtuals: true },
|
|
804
|
+
strict: false
|
|
805
|
+
// This allows for flexible document structure beyond the defined schema
|
|
806
|
+
}
|
|
807
|
+
);
|
|
808
|
+
AnnotationSchema.index({ "meta.contentType": 1 });
|
|
809
|
+
AnnotationSchema.index({ "meta.documentId": 1 });
|
|
810
|
+
AnnotationSchema.index({ createdAt: -1 });
|
|
811
|
+
AnnotationSchema.index({ "annotations.annoKey": 1 });
|
|
812
|
+
AnnotationSchema.index({ "kp_date_published": -1 });
|
|
813
|
+
AnnotationSchema.pre("save", function(next) {
|
|
814
|
+
this.updatedAt = /* @__PURE__ */ new Date();
|
|
815
|
+
next();
|
|
816
|
+
});
|
|
817
|
+
AnnotationSchema.methods.getMainField = function(fieldPath) {
|
|
818
|
+
if (!fieldPath) return null;
|
|
819
|
+
const parts = fieldPath.split(".");
|
|
820
|
+
let value = this.main;
|
|
821
|
+
for (const part of parts) {
|
|
822
|
+
if (!value || typeof value !== "object") return null;
|
|
823
|
+
value = value[part];
|
|
824
|
+
}
|
|
825
|
+
return value;
|
|
826
|
+
};
|
|
827
|
+
AnnotationSchema.virtual("displayTitle").get(function() {
|
|
828
|
+
return this.main?.title || "Untitled Annotation";
|
|
829
|
+
});
|
|
830
|
+
var Annotations_default = AnnotationSchema;
|
|
831
|
+
|
|
832
|
+
// src/models/AIChat.ts
|
|
833
|
+
var AIChatSchema = new mongoose3.Schema(
|
|
834
|
+
{
|
|
835
|
+
userId: { type: mongoose3.Schema.Types.ObjectId, index: { unique: true } },
|
|
836
|
+
createdAt: Date,
|
|
837
|
+
lastAcitivity: Date,
|
|
838
|
+
messages: [
|
|
839
|
+
{
|
|
840
|
+
id: String,
|
|
841
|
+
author: {
|
|
842
|
+
type: String,
|
|
843
|
+
enum: ["system", "user", "assistant"]
|
|
844
|
+
},
|
|
845
|
+
content: {
|
|
846
|
+
type: { type: String },
|
|
847
|
+
value: String
|
|
848
|
+
},
|
|
849
|
+
vectorSearchInfo: {
|
|
850
|
+
chunks: [Annotations_default]
|
|
851
|
+
},
|
|
852
|
+
args: {
|
|
853
|
+
query: String,
|
|
854
|
+
reframedQuery: String,
|
|
855
|
+
summary: Object
|
|
856
|
+
},
|
|
857
|
+
userFeedback: {
|
|
858
|
+
reaction: { type: String, enum: ["positive", "negative"] },
|
|
859
|
+
comments: String
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
]
|
|
863
|
+
},
|
|
864
|
+
{ collection: "aiChat" }
|
|
865
|
+
);
|
|
866
|
+
var AIChat_default = AIChatSchema;
|
|
867
|
+
|
|
868
|
+
// src/models/PlatformConfigs.ts
|
|
869
|
+
import mongoose4 from "mongoose";
|
|
870
|
+
var platformConfigTypes = [
|
|
871
|
+
"roles",
|
|
872
|
+
"nav",
|
|
873
|
+
"deployment",
|
|
874
|
+
"userAgreement",
|
|
875
|
+
"localeData",
|
|
876
|
+
"theme",
|
|
877
|
+
"AI"
|
|
878
|
+
];
|
|
879
|
+
var PlatformConfigsSchema = new mongoose4.Schema(
|
|
880
|
+
{
|
|
881
|
+
type: {
|
|
882
|
+
type: String,
|
|
883
|
+
enum: platformConfigTypes,
|
|
884
|
+
unique: true
|
|
885
|
+
},
|
|
886
|
+
roles: Array,
|
|
887
|
+
data: Object
|
|
888
|
+
},
|
|
889
|
+
{ collection: "platformConfigs" }
|
|
890
|
+
);
|
|
891
|
+
var PlatformConfigs_default = PlatformConfigsSchema;
|
|
892
|
+
|
|
893
|
+
// src/models/Tpl.ts
|
|
894
|
+
import mongoose5 from "mongoose";
|
|
895
|
+
var TplSchema = new mongoose5.Schema({
|
|
896
|
+
dateFirstPublished: Date,
|
|
897
|
+
dateCreated: Date,
|
|
898
|
+
dateLastPublished: Date,
|
|
899
|
+
dateLastEdited: Date,
|
|
900
|
+
status: {
|
|
901
|
+
type: String,
|
|
902
|
+
default: "published",
|
|
903
|
+
// only cuz we dont want to go and add this property in all databases
|
|
904
|
+
enum: ["unpublished", "editPublished", "published"]
|
|
905
|
+
},
|
|
906
|
+
version: {
|
|
907
|
+
type: Number,
|
|
908
|
+
default: 0
|
|
909
|
+
},
|
|
910
|
+
versionPublishedBy: {
|
|
911
|
+
type: mongoose5.Schema.Types.ObjectId,
|
|
912
|
+
ref: "user"
|
|
913
|
+
// reference to the 'user' model
|
|
914
|
+
},
|
|
915
|
+
firstPublishedBy: {
|
|
916
|
+
type: mongoose5.Schema.Types.ObjectId,
|
|
917
|
+
ref: "user"
|
|
918
|
+
// reference to the 'user' model
|
|
919
|
+
},
|
|
920
|
+
kp_content_type: {
|
|
921
|
+
type: String,
|
|
922
|
+
required: true,
|
|
923
|
+
unique: true
|
|
924
|
+
},
|
|
925
|
+
category: {
|
|
926
|
+
//to deprecate and turn into 'layout'
|
|
927
|
+
type: String,
|
|
928
|
+
default: "knowledgeResources2"
|
|
929
|
+
},
|
|
930
|
+
kp_settings: [
|
|
931
|
+
{
|
|
932
|
+
type: Object
|
|
933
|
+
}
|
|
934
|
+
],
|
|
935
|
+
kp_templates: {
|
|
936
|
+
type: Object
|
|
937
|
+
},
|
|
938
|
+
tplMeta: Object,
|
|
939
|
+
tplLocales: Object,
|
|
940
|
+
indexed: Object,
|
|
941
|
+
drafts: {
|
|
942
|
+
active: Object
|
|
943
|
+
},
|
|
944
|
+
rollbacks: Object,
|
|
945
|
+
//for 'remembering' hidden configurations
|
|
946
|
+
// OTHER CONFIGS
|
|
947
|
+
listing: Object,
|
|
948
|
+
//listing page configurations. this is new, currently only used in nct
|
|
949
|
+
general: {
|
|
950
|
+
content: {
|
|
951
|
+
title: String,
|
|
952
|
+
singular: String,
|
|
953
|
+
ctaText: String,
|
|
954
|
+
listingDesc: String
|
|
955
|
+
},
|
|
956
|
+
allowQuickTagCreation: { enable: Boolean },
|
|
957
|
+
segment: String,
|
|
958
|
+
settingsUIStyle: String,
|
|
959
|
+
hasUpdateType: Boolean,
|
|
960
|
+
annotation: {
|
|
961
|
+
enable: Boolean
|
|
962
|
+
},
|
|
963
|
+
participantModule: {
|
|
964
|
+
enable: Boolean
|
|
965
|
+
},
|
|
966
|
+
formFieldNumbering: {
|
|
967
|
+
enable: Boolean
|
|
968
|
+
},
|
|
969
|
+
postPblRedirPath: Object,
|
|
970
|
+
templateIndex: Object,
|
|
971
|
+
sharing: {
|
|
972
|
+
enable: Boolean,
|
|
973
|
+
trackShareCount: {
|
|
974
|
+
type: Boolean,
|
|
975
|
+
default: false
|
|
976
|
+
}
|
|
977
|
+
},
|
|
978
|
+
viewsCount: {
|
|
979
|
+
enable: {
|
|
980
|
+
type: Boolean,
|
|
981
|
+
default: false
|
|
982
|
+
}
|
|
983
|
+
},
|
|
984
|
+
comments: {
|
|
985
|
+
enable: Boolean
|
|
986
|
+
},
|
|
987
|
+
reactions: {
|
|
988
|
+
type: Map,
|
|
989
|
+
of: {
|
|
990
|
+
enable: Boolean,
|
|
991
|
+
icon: String
|
|
992
|
+
}
|
|
993
|
+
},
|
|
994
|
+
csvExport: {
|
|
995
|
+
enable: Boolean,
|
|
996
|
+
excludeFields: Array,
|
|
997
|
+
enableUpdateExport: Boolean,
|
|
998
|
+
fieldsToSortAtEnd: Array,
|
|
999
|
+
fetchBatches: {
|
|
1000
|
+
enable: Boolean,
|
|
1001
|
+
batchSize: Number
|
|
1002
|
+
}
|
|
1003
|
+
},
|
|
1004
|
+
onboardingFlow: Object,
|
|
1005
|
+
//only on profile tpls
|
|
1006
|
+
selfServeSurveyConfig: Object,
|
|
1007
|
+
//tci helpers - these exist only to show / not show certain UIs in the tci
|
|
1008
|
+
disableKPSettings: Boolean
|
|
1009
|
+
}
|
|
1010
|
+
//general contenttype configs. mostly the stuff inside platformConfigs > contentTypes
|
|
1011
|
+
}, {
|
|
1012
|
+
toJSON: { virtuals: true },
|
|
1013
|
+
// So `res.json()` and other `JSON.stringify()` functions include virtuals
|
|
1014
|
+
toObject: { virtuals: true }
|
|
1015
|
+
// So `toObject()` output includes virtuals
|
|
1016
|
+
});
|
|
1017
|
+
TplSchema.virtual("layout").get(function() {
|
|
1018
|
+
return this.category;
|
|
1019
|
+
});
|
|
1020
|
+
var Tpl_default = TplSchema;
|
|
1021
|
+
|
|
666
1022
|
// src/db/getModelByTenant.ts
|
|
667
|
-
var AnnotationsSchema = require_Annotations();
|
|
668
|
-
var PlatformConfigsSchema = require_PlatformConfigs();
|
|
669
|
-
var TplSchema = require_Tpl();
|
|
670
1023
|
var getModelByTenant = ({
|
|
671
1024
|
tenant,
|
|
672
1025
|
modelName,
|
|
@@ -685,22 +1038,28 @@ var getModelByTenant = ({
|
|
|
685
1038
|
}
|
|
686
1039
|
return db.model(modelName);
|
|
687
1040
|
};
|
|
688
|
-
var getAnnotationsModelByTenant = ({ tenant, env, mongodb, dbConfigs }) => getModelByTenant({
|
|
1041
|
+
var getAnnotationsModelByTenant = ({ tenant, env, mongodb, dbConfigs, modelName }) => getModelByTenant({
|
|
689
1042
|
tenant,
|
|
690
|
-
modelName: "annotations",
|
|
691
|
-
schema:
|
|
1043
|
+
modelName: modelName || "annotations",
|
|
1044
|
+
schema: Annotations_default,
|
|
692
1045
|
env
|
|
693
1046
|
});
|
|
694
1047
|
var getPlatformConfigsModelByTenant = ({ tenant, env, mongodb, dbConfigs }) => getModelByTenant({
|
|
695
1048
|
tenant,
|
|
696
1049
|
modelName: "platformConfigs",
|
|
697
|
-
schema:
|
|
1050
|
+
schema: PlatformConfigs_default,
|
|
698
1051
|
env
|
|
699
1052
|
});
|
|
700
1053
|
var getTplModelByTenant = ({ tenant, env, mongodb, dbConfigs }) => getModelByTenant({
|
|
701
1054
|
tenant,
|
|
702
1055
|
modelName: "tpl",
|
|
703
|
-
schema:
|
|
1056
|
+
schema: Tpl_default,
|
|
1057
|
+
env
|
|
1058
|
+
});
|
|
1059
|
+
var getAIChatModelByTenant = ({ tenant, env, mongodb, dbConfigs }) => getModelByTenant({
|
|
1060
|
+
tenant,
|
|
1061
|
+
modelName: "aiChat",
|
|
1062
|
+
schema: AIChat_default,
|
|
704
1063
|
env
|
|
705
1064
|
});
|
|
706
1065
|
|
|
@@ -810,11 +1169,30 @@ var getAIConfigs = async ({
|
|
|
810
1169
|
}
|
|
811
1170
|
});
|
|
812
1171
|
};
|
|
1172
|
+
|
|
1173
|
+
// src/node.ts
|
|
1174
|
+
var import_WorkerManager = __toESM(require_WorkerManager());
|
|
1175
|
+
var import_BaseProducer = __toESM(require_BaseProducer());
|
|
1176
|
+
var import_BaseWorker = __toESM(require_BaseWorker());
|
|
1177
|
+
var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG());
|
|
1178
|
+
var export_BaseProducer = import_BaseProducer.BaseProducer;
|
|
1179
|
+
var export_BaseWorker = import_BaseWorker.BaseWorker;
|
|
1180
|
+
var export_GET_GLOBAL_BULLMQ_CONFIG = import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG;
|
|
1181
|
+
var export_WorkerManager = import_WorkerManager.WorkerManager;
|
|
813
1182
|
export {
|
|
1183
|
+
AIChat_default as AIChatSchema,
|
|
1184
|
+
Annotations_default as AnnotationSchema,
|
|
1185
|
+
export_BaseProducer as BaseProducer,
|
|
1186
|
+
export_BaseWorker as BaseWorker,
|
|
1187
|
+
export_GET_GLOBAL_BULLMQ_CONFIG as GET_GLOBAL_BULLMQ_CONFIG,
|
|
1188
|
+
PlatformConfigs_default as PlatformConfigsSchema,
|
|
1189
|
+
Tpl_default as TplSchema,
|
|
1190
|
+
export_WorkerManager as WorkerManager,
|
|
814
1191
|
connectToRedis,
|
|
815
1192
|
deleteVal,
|
|
816
1193
|
extractAllBlocksFromTpl,
|
|
817
1194
|
genTagId,
|
|
1195
|
+
getAIChatModelByTenant,
|
|
818
1196
|
getAIConfigs,
|
|
819
1197
|
getAnnotationsModelByTenant,
|
|
820
1198
|
getDbByTenant,
|