@tiledesk/tiledesk-tybot-connector 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/TiledeskExpression.js +3 -1
- package/engine/TiledeskChatbot.js +23 -1
- package/index.js +5 -2
- package/logs/app1.log +6267 -0
- package/package.json +22 -21
- package/tiledeskChatbotPlugs/DirectivesChatbotPlug.js +8 -2
- package/tiledeskChatbotPlugs/directives/DirAiCondition.js +14 -10
- package/tiledeskChatbotPlugs/directives/DirAiPrompt.js +5 -5
- package/tiledeskChatbotPlugs/directives/DirCaptureUserReply.js +36 -11
- package/tiledeskChatbotPlugs/directives/DirDocparseExtract.js +166 -48
- package/tiledeskChatbotPlugs/directives/DirInviteHuman.js +259 -0
- package/tiledeskChatbotPlugs/directives/DirIteration.js +3 -3
- package/tiledeskChatbotPlugs/directives/DirJSONConditionMulti.js +171 -0
- package/tiledeskChatbotPlugs/directives/DirRemoveCurrentBot.js +40 -2
- package/tiledeskChatbotPlugs/directives/DirRemoveHuman.js +129 -0
- package/tiledeskChatbotPlugs/directives/DirReplyV2.js +18 -3
- package/tiledeskChatbotPlugs/directives/Directives.js +3 -0
- package/utils/ConnectorsV3.js +3 -0
- package/utils/HttpUtils.js +15 -11
- package/utils/TiledeskChatbotUtil.js +4 -1
- package/utils/winston.js +80 -2
package/package.json
CHANGED
|
@@ -1,45 +1,46 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tiledesk/tiledesk-tybot-connector",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.2",
|
|
4
4
|
"description": "Tiledesk Tybot connector",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"scripts": {
|
|
7
7
|
"test": "mocha --timeout 60000 --exit",
|
|
8
8
|
"start": "node index.js"
|
|
9
9
|
},
|
|
10
|
+
"engines": {
|
|
11
|
+
"node": ">=24 <25"
|
|
12
|
+
},
|
|
10
13
|
"keywords": [],
|
|
11
14
|
"author": "",
|
|
12
15
|
"license": "ISC",
|
|
13
16
|
"dependencies": {
|
|
14
17
|
"@tiledesk/helpcenter-query-client": "^0.1.8",
|
|
15
|
-
"@tiledesk/tiledesk-chatbot-client": "^0.5.30",
|
|
16
18
|
"@tiledesk/tiledesk-chatbot-util": "^0.8.39",
|
|
17
19
|
"@tiledesk/tiledesk-client": "^0.10.13",
|
|
18
20
|
"@tiledesk/tiledesk-multi-worker": "^0.3.5",
|
|
19
21
|
"accept-language-parser": "^1.5.0",
|
|
20
22
|
"app-root-path": "^3.1.0",
|
|
21
|
-
"axios": "^1.
|
|
22
|
-
"body-parser": "^1.
|
|
23
|
-
"cors": "^2.8.
|
|
24
|
-
"dotenv": "^16.
|
|
25
|
-
"express": "^4.
|
|
26
|
-
"handlebars": "^4.7.
|
|
27
|
-
"jsonschema": "^1.
|
|
28
|
-
"jsonwebtoken": "^
|
|
29
|
-
"liquidjs": "^10.
|
|
23
|
+
"axios": "^1.21.0",
|
|
24
|
+
"body-parser": "^1.20.9",
|
|
25
|
+
"cors": "^2.8.6",
|
|
26
|
+
"dotenv": "^16.6.1",
|
|
27
|
+
"express": "^4.22.3",
|
|
28
|
+
"handlebars": "^4.7.10",
|
|
29
|
+
"jsonschema": "^1.5.0",
|
|
30
|
+
"jsonwebtoken": "^9.0.3",
|
|
31
|
+
"liquidjs": "^10.30.1",
|
|
32
|
+
"mime-types": "^2.1.35",
|
|
30
33
|
"minimist": "^1.2.8",
|
|
31
|
-
"mongoose": "^6.
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"nanoid": "^3.1.25",
|
|
35
|
-
"path": "^0.12.7",
|
|
36
|
-
"redis": "^4.7.0",
|
|
34
|
+
"mongoose": "^6.13.12",
|
|
35
|
+
"nanoid": "^3.3.20",
|
|
36
|
+
"redis": "^4.7.1",
|
|
37
37
|
"string-argv": "^0.3.2",
|
|
38
|
-
"uuid": "^
|
|
39
|
-
"vm2": "^3.
|
|
40
|
-
"winston": "^3.
|
|
38
|
+
"uuid": "^11.1.1",
|
|
39
|
+
"vm2": "^3.12.2",
|
|
40
|
+
"winston": "^3.19.0"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
|
-
"mocha": "^8.4.0"
|
|
43
|
+
"mocha": "^8.4.0",
|
|
44
|
+
"multer": "^1.4.5-lts.2"
|
|
44
45
|
}
|
|
45
46
|
}
|
|
@@ -23,6 +23,7 @@ const { DirAssignFromFunction } = require('./directives/DirAssignFromFunction');
|
|
|
23
23
|
const { DirCondition } = require('./directives/DirCondition');
|
|
24
24
|
const { DirJSONCondition } = require('./directives/DirJSONCondition');
|
|
25
25
|
const { DirJSONConditionV2 } = require('./directives/DirJSONConditionV2');
|
|
26
|
+
const { DirJSONConditionMulti } = require('./directives/DirJSONConditionMulti');
|
|
26
27
|
const { DirAssign } = require('./directives/DirAssign');
|
|
27
28
|
const { DirSetAttribute } = require('./directives/DirSetAttribute');
|
|
28
29
|
const { DirSetAttributeV2 } = require('./directives/DirSetAttributeV2');
|
|
@@ -50,6 +51,8 @@ const { DirDocparseExtract } = require('./directives/DirDocparseExtract');
|
|
|
50
51
|
const { DirAssistant } = require('./directives/DirAssistant');
|
|
51
52
|
const { DirReplyV2 } = require('./directives/DirReplyV2');
|
|
52
53
|
const { DirIfOnlineAgentsV2 } = require('./directives/DirIfOnlineAgentsV2');
|
|
54
|
+
const { DirInviteHuman } = require('./directives/DirInviteHuman');
|
|
55
|
+
const { DirRemoveHuman } = require('./directives/DirRemoveHuman');
|
|
53
56
|
const { DirContactUpdate } = require('./directives/DirContactUpdate');
|
|
54
57
|
const { DirClearTranscript } = require('./directives/DirClearTranscript');
|
|
55
58
|
const { DirMoveToUnassigned } = require('./directives/DirMoveToUnassigned');
|
|
@@ -239,8 +242,8 @@ class DirectivesChatbotPlug {
|
|
|
239
242
|
|
|
240
243
|
const directive_name = directive.name.toLowerCase();
|
|
241
244
|
|
|
242
|
-
// Controllo lock action
|
|
243
|
-
if (directive.action) {
|
|
245
|
+
// Controllo lock action (internal jumps to another block are not subject to it: the lock stays for the user's reply)
|
|
246
|
+
if (directive.action && this.message?.sender !== "_tdinternal") {
|
|
244
247
|
const action_id = directive.action["_tdActionId"];
|
|
245
248
|
const locked_action_id = await this.chatbot.currentLockedAction(this.supportRequest.request_id);
|
|
246
249
|
if (locked_action_id && locked_action_id !== action_id) {
|
|
@@ -270,6 +273,7 @@ class DirectivesChatbotPlug {
|
|
|
270
273
|
[Directives.FUNCTION_VALUE]: DirAssignFromFunction,
|
|
271
274
|
[Directives.JSON_CONDITION]: DirJSONCondition,
|
|
272
275
|
[Directives.JSON_CONDITION_2]: DirJSONConditionV2, // NEW: JSON Condition V2 (tipo dedicato dal DS); V1 dispatch invariato
|
|
276
|
+
[Directives.JSON_CONDITION_MULTI]: DirJSONConditionMulti, // NEW: condizione a piu' casi; V1 e V2 dispatch invariati
|
|
273
277
|
[Directives.ASSIGN]: DirAssign,
|
|
274
278
|
[Directives.SET_ATTRIBUTE]: DirSetAttribute,
|
|
275
279
|
[Directives.SET_ATTRIBUTE_V2]: DirSetAttributeV2,
|
|
@@ -317,6 +321,8 @@ class DirectivesChatbotPlug {
|
|
|
317
321
|
[Directives.FLOW_LOG]: DirFlowLog,
|
|
318
322
|
[Directives.ITERATION]: DirIteration,
|
|
319
323
|
[Directives.DATA_TABLES]: DirDataTables,
|
|
324
|
+
[Directives.INVITE_HUMAN]: DirInviteHuman,
|
|
325
|
+
[Directives.REMOVE_HUMAN]: DirRemoveHuman,
|
|
320
326
|
};
|
|
321
327
|
|
|
322
328
|
const HandlerClass = handlers[directive_name];
|
|
@@ -70,18 +70,20 @@ class DirAiCondition {
|
|
|
70
70
|
// ]
|
|
71
71
|
let fallbackIntent = action.fallbackIntent; // non condition met block
|
|
72
72
|
let errorIntent = action.errorIntent; // On error block
|
|
73
|
-
|
|
73
|
+
try {
|
|
74
|
+
await this.checkMandatoryParameters(action);
|
|
75
|
+
} catch (missing_param) {
|
|
74
76
|
const error = "AiCondition Error: '" + missing_param + "' attribute is undefined"
|
|
75
77
|
this.logger.error(error);
|
|
76
78
|
await this.chatbot.addParameter("flowError", error);
|
|
77
79
|
if (errorIntent) {
|
|
78
80
|
await this.#executeIntent(errorIntent);
|
|
79
81
|
callback(true);
|
|
80
|
-
return
|
|
82
|
+
return;
|
|
81
83
|
}
|
|
82
84
|
callback();
|
|
83
|
-
return
|
|
84
|
-
}
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
85
87
|
|
|
86
88
|
// fill attributes
|
|
87
89
|
let requestVariables = null;
|
|
@@ -342,7 +344,6 @@ class DirAiCondition {
|
|
|
342
344
|
api_key: vllm_server_config.apikey || vllm_server_config.token || null,
|
|
343
345
|
provider: 'vllm'
|
|
344
346
|
}
|
|
345
|
-
console.log("set json.model to: ", json.model);
|
|
346
347
|
}
|
|
347
348
|
|
|
348
349
|
if (action.llm === 'openrouter') {
|
|
@@ -367,19 +368,22 @@ class DirAiCondition {
|
|
|
367
368
|
httpUtils.request(
|
|
368
369
|
HTTPREQUEST, async (err, resbody) => {
|
|
369
370
|
if (err) {
|
|
370
|
-
winston.error("DirAiCondition openai err: ", err);
|
|
371
|
+
winston.error("DirAiCondition openai err: ", err.response?.data);
|
|
371
372
|
await this.#assignAttributes(action, answer);
|
|
372
373
|
let error;
|
|
373
|
-
if (err.response?.data?.detail[0]) {
|
|
374
|
+
if (err.response?.data?.detail && err.response?.data?.detail[0]) {
|
|
374
375
|
error = err.response.data.detail[0]?.msg;
|
|
375
|
-
} else if (err.response?.data?.detail?.answer) {
|
|
376
|
+
} else if (err.response?.data?.detail && err.response?.data?.detail?.answer) {
|
|
376
377
|
error = err.response.data.detail.answer;
|
|
377
|
-
} else {
|
|
378
|
+
} else if (err.response?.data) {
|
|
378
379
|
error = JSON.stringify(err.response.data);
|
|
380
|
+
} else {
|
|
381
|
+
error = err.message || "General error executing action";
|
|
379
382
|
}
|
|
383
|
+
winston.error("DirAiCondition error executing action: " + error);
|
|
380
384
|
this.logger.error("[AI Condition] error executing action: ", error);
|
|
381
385
|
if (errorIntent) {
|
|
382
|
-
await this.chatbot.addParameter("flowError", "
|
|
386
|
+
await this.chatbot.addParameter("flowError", "AiCondition Error: " + error);
|
|
383
387
|
await this.#executeIntent(errorIntent);
|
|
384
388
|
callback(true);
|
|
385
389
|
return;
|
|
@@ -342,7 +342,6 @@ class DirAiPrompt {
|
|
|
342
342
|
api_key: vllm_server_config.apikey || vllm_server_config.token || null,
|
|
343
343
|
provider: 'vllm'
|
|
344
344
|
}
|
|
345
|
-
console.log("set json.model to: ", json.model);
|
|
346
345
|
}
|
|
347
346
|
|
|
348
347
|
if (action.llm === 'openrouter') {
|
|
@@ -415,7 +414,6 @@ class DirAiPrompt {
|
|
|
415
414
|
callback();
|
|
416
415
|
return;
|
|
417
416
|
}
|
|
418
|
-
console.log('json.servers', json.servers);
|
|
419
417
|
}
|
|
420
418
|
|
|
421
419
|
|
|
@@ -438,7 +436,6 @@ class DirAiPrompt {
|
|
|
438
436
|
}
|
|
439
437
|
|
|
440
438
|
winston.debug("DirAiPrompt json: ", json);
|
|
441
|
-
console.log("DirAiPrompt json: ", json);
|
|
442
439
|
|
|
443
440
|
const HTTPREQUEST = {
|
|
444
441
|
url: AI_endpoint + apiEndpoint,
|
|
@@ -757,10 +754,13 @@ class DirAiPrompt {
|
|
|
757
754
|
}
|
|
758
755
|
|
|
759
756
|
buildEnabledTools(server) {
|
|
760
|
-
|
|
757
|
+
const source = Array.isArray(server?.tools) && server.tools.length > 0
|
|
758
|
+
? server.tools
|
|
759
|
+
: server?.enabled_tools;
|
|
760
|
+
if (!Array.isArray(source) || source.length === 0) {
|
|
761
761
|
return [];
|
|
762
762
|
}
|
|
763
|
-
return
|
|
763
|
+
return source
|
|
764
764
|
.map(t => (typeof t === 'string' ? t : t?.name))
|
|
765
765
|
.filter(name => typeof name === 'string' && name.length > 0);
|
|
766
766
|
}
|
|
@@ -33,22 +33,47 @@ class DirCaptureUserReply {
|
|
|
33
33
|
callback();
|
|
34
34
|
return;
|
|
35
35
|
}
|
|
36
|
-
this.go(action, () => {
|
|
37
|
-
callback();
|
|
36
|
+
this.go(action, (stop) => {
|
|
37
|
+
callback(stop);
|
|
38
38
|
});
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
#resolveIntentLockKey() {
|
|
42
|
+
const intent_info = this.reply?.attributes?.intent_info;
|
|
43
|
+
if (intent_info?.intent_id) {
|
|
44
|
+
return "#" + intent_info.intent_id;
|
|
45
|
+
}
|
|
46
|
+
if (intent_info?.intent_name) {
|
|
47
|
+
return intent_info.intent_name;
|
|
48
|
+
}
|
|
49
|
+
if (this.reply?.intent_id) {
|
|
50
|
+
return "#" + this.reply.intent_id;
|
|
51
|
+
}
|
|
52
|
+
if (this.reply?.intent_display_name) {
|
|
53
|
+
return this.reply.intent_display_name;
|
|
54
|
+
}
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
|
|
41
58
|
async go(action, callback) {
|
|
42
59
|
winston.debug("(DirCaptureUserReply) Action: ", action);
|
|
43
60
|
const goToIntent = action.goToIntent;
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
61
|
+
const lockedAction = await this.chatbot.currentLockedAction(this.requestId);
|
|
62
|
+
// a lock only counts as "waiting for the reply" when this very action set it
|
|
63
|
+
const own = lockedAction && lockedAction === action["_tdActionId"];
|
|
64
|
+
if (!own) {
|
|
65
|
+
const intent_name = this.#resolveIntentLockKey();
|
|
66
|
+
const actionId = action["_tdActionId"];
|
|
67
|
+
if (!intent_name) {
|
|
68
|
+
this.logger.error("[Capture User Reply] Cannot lock intent: missing intent info");
|
|
69
|
+
winston.error("(DirCaptureUserReply) Cannot lock intent: missing intent info");
|
|
70
|
+
callback(true);
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
48
73
|
await this.chatbot.lockIntent(this.requestId, intent_name);
|
|
49
74
|
await this.chatbot.lockAction(this.requestId, actionId);
|
|
50
75
|
this.logger.native("[Capture User Reply] Waiting for user reply...");
|
|
51
|
-
callback();
|
|
76
|
+
callback(true);
|
|
52
77
|
return;
|
|
53
78
|
} else {
|
|
54
79
|
try {
|
|
@@ -72,9 +97,9 @@ class DirCaptureUserReply {
|
|
|
72
97
|
|
|
73
98
|
if (callback) {
|
|
74
99
|
if (goToIntent) {
|
|
75
|
-
this.#executeGoTo(goToIntent, () => {
|
|
100
|
+
this.#executeGoTo(goToIntent, (stop) => {
|
|
76
101
|
this.logger.native("[Capture User Reply] Executed");
|
|
77
|
-
callback(); //
|
|
102
|
+
callback(stop); // a jump ends this block
|
|
78
103
|
});
|
|
79
104
|
}
|
|
80
105
|
else {
|
|
@@ -95,8 +120,8 @@ class DirCaptureUserReply {
|
|
|
95
120
|
if (intent) {
|
|
96
121
|
goToIntentDirective = DirIntent.intentDirectiveFor(intent);
|
|
97
122
|
}
|
|
98
|
-
this.intentDir.execute(goToIntentDirective, () => {
|
|
99
|
-
callback();
|
|
123
|
+
this.intentDir.execute(goToIntentDirective, (stop) => {
|
|
124
|
+
callback(stop);
|
|
100
125
|
});
|
|
101
126
|
}
|
|
102
127
|
|
|
@@ -7,6 +7,8 @@ const { Logger } = require("../../Logger");
|
|
|
7
7
|
const aiController = require("../../services/AIController");
|
|
8
8
|
|
|
9
9
|
const JOB_TIMEOUT_MS = 10 * 60 * 1000;
|
|
10
|
+
// Sync POST /v1/extract: docsparse runs gunicorn with a 300 s request timeout.
|
|
11
|
+
const SYNC_TIMEOUT_MS = 300 * 1000;
|
|
10
12
|
|
|
11
13
|
class DirDocparseExtract {
|
|
12
14
|
|
|
@@ -72,8 +74,8 @@ class DirDocparseExtract {
|
|
|
72
74
|
|
|
73
75
|
const requestVariables = await TiledeskChatbot.allParametersStatic(this.tdcache, this.requestId);
|
|
74
76
|
const filler = new Filler();
|
|
75
|
-
const filename = filler.fill(action.filename, requestVariables);
|
|
76
77
|
const url = filler.fill(action.url, requestVariables);
|
|
78
|
+
const filename = filler.fill(action.filename, requestVariables) || this.#filenameFromUrl(url);
|
|
77
79
|
const mode = filler.fill(action.mode, requestVariables) || "text";
|
|
78
80
|
|
|
79
81
|
if (!filename || !url) {
|
|
@@ -104,20 +106,19 @@ class DirDocparseExtract {
|
|
|
104
106
|
body.dpi = dpi;
|
|
105
107
|
}
|
|
106
108
|
|
|
107
|
-
const
|
|
108
|
-
if (
|
|
109
|
+
const confidence = this.#confidence(filler.fill(action.confidence, requestVariables));
|
|
110
|
+
if (confidence !== null) {
|
|
111
|
+
body.ocr_quality_threshold = confidence;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const visionLlm = await this.#resolveFallback(action, filler, requestVariables);
|
|
115
|
+
if (visionLlm) {
|
|
109
116
|
body.enable_vision_fallback = true;
|
|
110
|
-
body.
|
|
111
|
-
body.model = fallback.model;
|
|
112
|
-
if (fallback.api_key) {
|
|
113
|
-
body.api_key = fallback.api_key;
|
|
114
|
-
}
|
|
115
|
-
if (fallback.base_url) {
|
|
116
|
-
body.base_url = fallback.base_url;
|
|
117
|
-
}
|
|
117
|
+
body.vision_llm = visionLlm;
|
|
118
118
|
}
|
|
119
119
|
|
|
120
120
|
let created;
|
|
121
|
+
let outcome = null;
|
|
121
122
|
try {
|
|
122
123
|
created = await this.#request({
|
|
123
124
|
url: this.#endpoint(base, "/v1/jobs/extract"),
|
|
@@ -126,19 +127,28 @@ class DirDocparseExtract {
|
|
|
126
127
|
method: "POST"
|
|
127
128
|
});
|
|
128
129
|
} catch (err) {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
130
|
+
if (err && err.response && err.response.status === 404) {
|
|
131
|
+
// Async jobs are not mounted (docsparse < 1.3, or DOCPARSE_ENABLE_ASYNC_JOBS off):
|
|
132
|
+
// fall back to the synchronous endpoint.
|
|
133
|
+
winston.verbose("DirDocparseExtract /v1/jobs/extract not found, using sync /v1/extract");
|
|
134
|
+
outcome = await this.#extractSync(base, body);
|
|
135
|
+
} else {
|
|
136
|
+
winston.error("DirDocparseExtract submit error: ", this.#httpErrorLog(err));
|
|
137
|
+
await this.#fail(action, this.#httpErrorMessage(err), trueIntent, trueIntentAttributes, falseIntent, falseIntentAttributes, done);
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
132
140
|
}
|
|
133
141
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
142
|
+
if (!outcome) {
|
|
143
|
+
const jobId = created && created.job_id;
|
|
144
|
+
if (!jobId) {
|
|
145
|
+
await this.#fail(action, "docparse did not return a job_id", trueIntent, trueIntentAttributes, falseIntent, falseIntentAttributes, done);
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
139
148
|
|
|
140
|
-
|
|
141
|
-
|
|
149
|
+
winston.verbose("DirDocparseExtract job queued: " + jobId);
|
|
150
|
+
outcome = await this.#waitForJob(base, jobId);
|
|
151
|
+
}
|
|
142
152
|
if (!outcome.ok) {
|
|
143
153
|
await this.#fail(action, outcome.error, trueIntent, trueIntentAttributes, falseIntent, falseIntentAttributes, done);
|
|
144
154
|
return;
|
|
@@ -187,12 +197,72 @@ class DirDocparseExtract {
|
|
|
187
197
|
apiKey = process.env.GPTKEY;
|
|
188
198
|
}
|
|
189
199
|
|
|
190
|
-
|
|
191
|
-
provider: provider,
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
200
|
+
const modelConfig = {
|
|
201
|
+
provider: (resolved && resolved.provider) || provider,
|
|
202
|
+
name: (resolved && resolved.name) || model
|
|
203
|
+
};
|
|
204
|
+
if (apiKey) {
|
|
205
|
+
modelConfig.api_key = apiKey;
|
|
206
|
+
}
|
|
207
|
+
for (const field of ["url", "token", "project", "location", "provider_routing", "custom_headers"]) {
|
|
208
|
+
if (resolved && resolved[field]) {
|
|
209
|
+
modelConfig[field] = resolved[field];
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// llm stays populated because tiledesk-llm still branches on it.
|
|
214
|
+
// The credentials travel inside model; llm_key is only there for compatibility.
|
|
215
|
+
const visionLlm = {
|
|
216
|
+
llm: modelConfig.provider,
|
|
217
|
+
llm_key: "",
|
|
218
|
+
model: modelConfig
|
|
195
219
|
};
|
|
220
|
+
|
|
221
|
+
const temperature = this.#number(filler.fill(action.temperature, requestVariables));
|
|
222
|
+
if (temperature !== null) {
|
|
223
|
+
visionLlm.temperature = temperature;
|
|
224
|
+
}
|
|
225
|
+
const maxTokens = this.#integer(filler.fill(action.max_tokens, requestVariables));
|
|
226
|
+
visionLlm.max_tokens = maxTokens !== null ? maxTokens : 8192;
|
|
227
|
+
|
|
228
|
+
return visionLlm;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
async #extractSync(base, body) {
|
|
232
|
+
// Same request without the async-only fields: vision_llm is read only by the docsparse job
|
|
233
|
+
// worker (and carries the provider key), and the sync response is read in mode "full".
|
|
234
|
+
const syncBody = Object.assign({}, body, { mode: "full" });
|
|
235
|
+
delete syncBody.vision_llm;
|
|
236
|
+
const timeoutMs = this.#syncTimeoutMs();
|
|
237
|
+
let response;
|
|
238
|
+
try {
|
|
239
|
+
response = await this.#request({
|
|
240
|
+
url: this.#endpoint(base, "/v1/extract"),
|
|
241
|
+
headers: { 'Content-Type': 'application/json' },
|
|
242
|
+
json: syncBody,
|
|
243
|
+
method: "POST",
|
|
244
|
+
timeout: timeoutMs
|
|
245
|
+
});
|
|
246
|
+
} catch (err) {
|
|
247
|
+
if (err && (err.code === "ECONNABORTED" || err.code === "ETIMEDOUT") && !err.response) {
|
|
248
|
+
winston.error("DirDocparseExtract sync extract timed out after " + timeoutMs + " ms");
|
|
249
|
+
return { ok: false, error: "Docparse extract timed out after " + (timeoutMs / 1000) + " seconds" };
|
|
250
|
+
}
|
|
251
|
+
winston.error("DirDocparseExtract sync extract error: ", this.#httpErrorLog(err));
|
|
252
|
+
return { ok: false, error: this.#httpErrorMessage(err) };
|
|
253
|
+
}
|
|
254
|
+
if (typeof response === "string") {
|
|
255
|
+
return { ok: true, text: response };
|
|
256
|
+
}
|
|
257
|
+
if (response && typeof response === "object" && typeof response.text === "string") {
|
|
258
|
+
return { ok: true, text: response.text };
|
|
259
|
+
}
|
|
260
|
+
return { ok: false, error: "docparse did not return the extracted text" };
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
#syncTimeoutMs() {
|
|
264
|
+
const value = Number(process.env.DOCPARSE_SYNC_TIMEOUT_MS);
|
|
265
|
+
return Number.isFinite(value) && value > 0 ? value : SYNC_TIMEOUT_MS;
|
|
196
266
|
}
|
|
197
267
|
|
|
198
268
|
#waitForJob(base, jobId) {
|
|
@@ -217,24 +287,27 @@ class DirDocparseExtract {
|
|
|
217
287
|
});
|
|
218
288
|
};
|
|
219
289
|
|
|
220
|
-
const onMessage = (message) => {
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
290
|
+
const onMessage = async (message) => {
|
|
291
|
+
const received = typeof message === "string" ? message : JSON.stringify(message);
|
|
292
|
+
winston.info("(DirDocparseExtract) Redis message received on " + topic + ": " + received);
|
|
293
|
+
if (settled) {
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
try {
|
|
297
|
+
const outcome = await this.#readJob(base, jobId);
|
|
298
|
+
if (!outcome || outcome.pending || settled) {
|
|
227
299
|
return;
|
|
228
300
|
}
|
|
301
|
+
finish(outcome);
|
|
302
|
+
} catch (err) {
|
|
303
|
+
winston.error("DirDocparseExtract job fetch after redis event failed: ", this.#httpErrorLog(err));
|
|
304
|
+
finish({ ok: false, error: this.#httpErrorMessage(err) });
|
|
229
305
|
}
|
|
230
|
-
const outcome = this.#interpret(payload, false);
|
|
231
|
-
if (outcome.pending) {
|
|
232
|
-
return;
|
|
233
|
-
}
|
|
234
|
-
finish(outcome);
|
|
235
306
|
};
|
|
236
307
|
|
|
308
|
+
winston.info("(DirDocparseExtract) Subscribing to " + topic);
|
|
237
309
|
this.tdcache.subscribe(topic, onMessage).then(async () => {
|
|
310
|
+
winston.info("(DirDocparseExtract) Subscribed to " + topic);
|
|
238
311
|
if (settled) {
|
|
239
312
|
return;
|
|
240
313
|
}
|
|
@@ -243,18 +316,11 @@ class DirDocparseExtract {
|
|
|
243
316
|
}, this.jobTimeoutMs);
|
|
244
317
|
|
|
245
318
|
try {
|
|
246
|
-
const
|
|
247
|
-
|
|
248
|
-
headers: { 'Content-Type': 'application/json' },
|
|
249
|
-
method: "GET"
|
|
250
|
-
});
|
|
251
|
-
if (settled) {
|
|
319
|
+
const outcome = await this.#readJob(base, jobId);
|
|
320
|
+
if (!outcome || outcome.pending || settled) {
|
|
252
321
|
return;
|
|
253
322
|
}
|
|
254
|
-
|
|
255
|
-
if (!outcome.pending) {
|
|
256
|
-
finish(outcome);
|
|
257
|
-
}
|
|
323
|
+
finish(outcome);
|
|
258
324
|
} catch (err) {
|
|
259
325
|
winston.warn("DirDocparseExtract preventive job poll failed: ", this.#httpErrorMessage(err));
|
|
260
326
|
}
|
|
@@ -266,6 +332,15 @@ class DirDocparseExtract {
|
|
|
266
332
|
});
|
|
267
333
|
}
|
|
268
334
|
|
|
335
|
+
async #readJob(base, jobId) {
|
|
336
|
+
const job = await this.#request({
|
|
337
|
+
url: this.#endpoint(base, "/v1/jobs/" + encodeURIComponent(jobId)),
|
|
338
|
+
headers: { 'Content-Type': 'application/json' },
|
|
339
|
+
method: "GET"
|
|
340
|
+
});
|
|
341
|
+
return this.#interpret(job, true);
|
|
342
|
+
}
|
|
343
|
+
|
|
269
344
|
#interpret(payload, fromPoll) {
|
|
270
345
|
if (!payload || typeof payload !== "object") {
|
|
271
346
|
return fromPoll
|
|
@@ -336,6 +411,49 @@ class DirDocparseExtract {
|
|
|
336
411
|
return dpi;
|
|
337
412
|
}
|
|
338
413
|
|
|
414
|
+
#number(value) {
|
|
415
|
+
if (value === undefined || value === null || value === "") {
|
|
416
|
+
return null;
|
|
417
|
+
}
|
|
418
|
+
const number = Number(value);
|
|
419
|
+
if (!Number.isFinite(number)) {
|
|
420
|
+
return null;
|
|
421
|
+
}
|
|
422
|
+
return number;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
#integer(value) {
|
|
426
|
+
const number = this.#number(value);
|
|
427
|
+
if (number === null || !Number.isInteger(number)) {
|
|
428
|
+
return null;
|
|
429
|
+
}
|
|
430
|
+
return number;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
#confidence(value) {
|
|
434
|
+
if (value === undefined || value === null || value === "") {
|
|
435
|
+
return null;
|
|
436
|
+
}
|
|
437
|
+
const confidence = Number(value);
|
|
438
|
+
if (!Number.isInteger(confidence) || confidence < 1 || confidence > 100) {
|
|
439
|
+
return 0.86;
|
|
440
|
+
}
|
|
441
|
+
return confidence / 100;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
#filenameFromUrl(value) {
|
|
445
|
+
if (!value) {
|
|
446
|
+
return null;
|
|
447
|
+
}
|
|
448
|
+
try {
|
|
449
|
+
const pathname = new URL(value).pathname;
|
|
450
|
+
const filename = pathname.split("/").filter(Boolean).pop();
|
|
451
|
+
return filename ? decodeURIComponent(filename) : null;
|
|
452
|
+
} catch (err) {
|
|
453
|
+
return null;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
339
457
|
#endpoint(base, path) {
|
|
340
458
|
return String(base).replace(/\/+$/, "") + path;
|
|
341
459
|
}
|