@tiledesk/tiledesk-server 3.0.1 → 3.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.circleci/config.yml +1 -1
- package/CHANGELOG.md +47 -0
- package/Dockerfile +3 -2
- package/Dockerfile-en +3 -2
- package/Dockerfile-jobs +3 -2
- package/Dockerfile-profiler +3 -2
- package/README.md +15 -0
- package/app.js +12 -1
- package/bin/quota-reset +68 -0
- package/bin/quota-restore +86 -0
- package/channels/chat21/chat21Handler.js +50 -34
- package/channels/chat21/chat21JoinTracker.js +43 -0
- package/config/database.js +5 -5
- package/config/native_mcp/native_mcp_servers.json +1 -0
- package/controllers/chatbot.controller.js +80 -27
- package/controllers/intents.controller.js +42 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-chat21-ionic.md +468 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-design-studio.md +1349 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-server.md +1694 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-tybot.md +789 -0
- package/docs/superpowers/plans/2026-09-29-hitl-known-limits-server.md +83 -0
- package/docs/superpowers/plans/2026-09-29-hitl-known-limits-tybot.md +76 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-chat21-ionic.md +97 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-design-studio.md +72 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-server.md +223 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-tybot.md +121 -0
- package/docs/superpowers/plans/2026-09-29-hitl-pre-branch-ports.md +65 -0
- package/docs/superpowers/plans/2026-09-30-flow-start-points-design-studio.md +88 -0
- package/docs/superpowers/plans/2026-09-30-flow-start-points-server.md +103 -0
- package/docs/superpowers/plans/2026-09-30-flow-start-points-tybot.md +43 -0
- package/docs/superpowers/plans/2026-10-01-scheduled-start-point-design-studio.md +80 -0
- package/docs/superpowers/plans/2026-10-01-scheduled-start-point-runtime.md +39 -0
- package/docs/superpowers/plans/2026-10-01-scheduled-start-point-server.md +106 -0
- package/docs/superpowers/specs/2026-09-15-webhook-chatbot-hitl-design.md +715 -0
- package/docs/superpowers/specs/2026-09-30-flow-start-points-design.md +156 -0
- package/docs/superpowers/specs/2026-10-01-scheduled-start-point-design.md +303 -0
- package/docs/superpowers/todo/2026-09-14-webhook-chatbot-shared-code-fixes.md +191 -0
- package/event/botEvent.js +12 -1
- package/event/emailEvent.js +3 -0
- package/event/messageEvent.js +44 -0
- package/jobs.js +10 -0
- package/jobsManager.js +19 -0
- package/middleware/errorHandler.js +6 -2
- package/models/analyticResult.js +1 -1
- package/models/channelConstants.js +3 -2
- package/models/dataTable.js +16 -0
- package/models/location.js +6 -12
- package/models/quotaUsage.js +71 -0
- package/models/request.js +71 -61
- package/models/requestSnapshot.js +12 -0
- package/models/routerLogger.js +1 -1
- package/models/webhook.js +44 -0
- package/package.json +9 -7
- package/pubmodules/emailNotification/requestNotification.js +14 -22
- package/pubmodules/events/test/eventRoute.js +7 -6
- package/pubmodules/rules/conciergeBot.js +9 -4
- package/routes/auth.js +1 -12
- package/routes/campaigns.js +0 -13
- package/routes/dataTable.js +23 -3
- package/routes/faq_kb.js +12 -1
- package/routes/messagesRoot.js +0 -1
- package/routes/quotes.js +38 -1
- package/routes/request.js +81 -5
- package/routes/users.js +0 -1
- package/routes/webhook.js +160 -8
- package/routes/webhooks.js +276 -6
- package/routes/widgetLoader.js +6 -3
- package/services/QuoteManager.js +77 -16
- package/services/aiService.js +1 -1
- package/services/chatbotService.js +55 -1
- package/services/dataTableService.js +436 -42
- package/services/departmentService.js +60 -43
- package/services/dolphinSchedulerClient.js +116 -0
- package/services/emailService.js +6 -1
- package/services/hitlParticipantListener.js +260 -0
- package/services/participantInviteService.js +302 -0
- package/services/participantRemoveService.js +79 -0
- package/services/quotaSnapshotService.js +1167 -0
- package/services/requestService.js +102 -144
- package/services/scheduledStart/dolphinBackend.js +246 -0
- package/services/scheduledStart/schedulerBackend.js +101 -0
- package/services/scheduledStart/schedulerServiceClient.js +41 -0
- package/services/scheduledStartPublishListener.js +186 -0
- package/services/scheduledStartRules.js +173 -0
- package/services/scheduledStartService.js +130 -0
- package/services/startPointCleanupListener.js +30 -0
- package/services/startPointService.js +158 -0
- package/services/testWsService.js +1 -4
- package/services/updateRequestSnapshotQueued.js +8 -9
- package/services/webhookConversationService.js +185 -0
- package/services/webhookSettingsValidator.js +61 -0
- package/template/email/checkpointReachedEmail.html +5 -0
- package/test/agentPromptsRoute.test.js +3 -2
- package/test/agentTokensQuota.js +237 -0
- package/test/botEventHitlFallback.js +19 -0
- package/test/chat21HandlerJoinWiring.js +59 -0
- package/test/dataTablesRoute.test.js +649 -2
- package/test/dateUtils.test.js +0 -3
- package/test/departmentService.js +8 -12
- package/test/{fileRoute.js → deprecated/_deprecated_fileRoute} +2 -2
- package/test/{kbsettingsRoute.js → deprecated/_deprecated_kbsettingsRoute} +3 -3
- package/test/faqRoute.js +6 -1
- package/test/faqkbRoute.js +14 -10
- package/test/faqkbV3Route.js +1 -1
- package/test/filepRoute.js +4 -50
- package/test/hitlParticipantListener.js +446 -0
- package/test/hitlQueueWiring.js +191 -0
- package/test/hitlWaitingReset.js +124 -0
- package/test/inviteParticipantsRoute.js +492 -0
- package/test/kbRoute.js +1 -1
- package/test/messageRoute.js +33 -21
- package/test/nativemcp.test.js +7 -1
- package/test/openaiRoute.js +2 -8
- package/test/publishFailLoud.test.js +414 -0
- package/test/quotaSnapshots.js +1044 -0
- package/test/removeParticipantsRoute.js +186 -0
- package/test/requestService.js +24 -16
- package/test/requestServiceRemoveParticipant.js +144 -0
- package/test/requestVisibility.test.js +186 -0
- package/test/scheduledStartFacade.js +184 -0
- package/test/scheduledStartHandover.js +95 -0
- package/test/scheduledStartPublish.js +320 -0
- package/test/scheduledStartRoute.js +163 -0
- package/test/scheduledStartRules.js +177 -0
- package/test/scheduledStartRun.js +299 -0
- package/test/scheduledStartSchedulerBackend.js +116 -0
- package/test/scheduledStartService.js +290 -0
- package/test/startPointsCleanup.js +69 -0
- package/test/startPointsRegression.js +169 -0
- package/test/startPointsRoute.js +158 -0
- package/test/webhookConversationRoute.js +347 -0
- package/test/webhookConversationService.js +261 -0
- package/test/webhookConversationTestUtils.js +55 -0
- package/utils/aiUtils.js +95 -1
- package/utils/dataTableUtils.js +99 -9
- package/utils/intentFieldNames.js +192 -0
- package/utils/quotaCliUtil.js +43 -0
- package/utils/requestVisibility.js +355 -0
- package/websocket/webSocketServer.js +36 -31
- /package/test/{imageRoute.js → deprecated/_deprecated_imageRoute} +0 -0
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
//During the test the env variable is set to test
|
|
2
|
+
process.env.NODE_ENV = 'test';
|
|
3
|
+
process.env.LOG_LEVEL = 'error';
|
|
4
|
+
|
|
5
|
+
const chai = require('chai');
|
|
6
|
+
const expect = chai.expect;
|
|
7
|
+
const config = require('../config/database');
|
|
8
|
+
const mongoose = require('mongoose');
|
|
9
|
+
|
|
10
|
+
// The `users` collection has a unique (non-sparse) index on the optional `phone` field
|
|
11
|
+
// (see models/user.js). Multiple users without a phone number collide on the shared
|
|
12
|
+
// `null` key. Disable autoIndex for this run so pre-existing environment state doesn't
|
|
13
|
+
// break unrelated tests that sign up more than one user.
|
|
14
|
+
mongoose.set('autoIndex', false);
|
|
15
|
+
mongoose.connect(config.databasetest);
|
|
16
|
+
require('../services/mongoose-cache-fn')(mongoose);
|
|
17
|
+
|
|
18
|
+
const userService = require('../services/userService');
|
|
19
|
+
const projectService = require('../services/projectService');
|
|
20
|
+
const faqService = require('../services/faqService');
|
|
21
|
+
const departmentService = require('../services/departmentService');
|
|
22
|
+
const webhookConversationService = require('../services/webhookConversationService');
|
|
23
|
+
const Request = require('../models/request');
|
|
24
|
+
const Message = require('../models/message');
|
|
25
|
+
const Lead = require('../models/lead');
|
|
26
|
+
const leadEvent = require('../event/leadEvent');
|
|
27
|
+
const Department = require('../models/department');
|
|
28
|
+
const { Webhook } = require('../models/webhook');
|
|
29
|
+
const { waitFor, uniqueEmail, PWD } = require('./webhookConversationTestUtils');
|
|
30
|
+
|
|
31
|
+
async function setup(webhookFields = {}) {
|
|
32
|
+
const user = await userService.signup(uniqueEmail('test-webhook-conv-service'), PWD, 'Test', 'User');
|
|
33
|
+
const project = await projectService.create('test-webhook-conversation-service', user._id);
|
|
34
|
+
const chatbot = await faqService.create(project._id, user._id, { name: 'hitl-bot', type: 'tilebot', language: 'en', template: 'blank' });
|
|
35
|
+
const webhook = await new Webhook(Object.assign({
|
|
36
|
+
id_project: project._id.toString(),
|
|
37
|
+
chatbot_id: chatbot._id.toString(),
|
|
38
|
+
block_id: 'start-intent-id',
|
|
39
|
+
mode: 'conversation',
|
|
40
|
+
start_points: [{ type: 'webhook', block_id: 'webhook-box-id', enabled: true, mapping: { source_name: 'Grafana' } }]
|
|
41
|
+
}, webhookFields)).save();
|
|
42
|
+
return { user, project, chatbot, webhook };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
describe('WebhookConversationService', function () {
|
|
46
|
+
|
|
47
|
+
this.timeout(20000);
|
|
48
|
+
|
|
49
|
+
it('creates a lead, a bot-assigned request in the default department and a /start message', async () => {
|
|
50
|
+
const { project, chatbot, webhook } = await setup();
|
|
51
|
+
const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
|
|
52
|
+
|
|
53
|
+
const result = await webhookConversationService.start(webhook, chatbot, { alert: 'KubePodCrashLooping', webhook_http_method: 'POST' }, {});
|
|
54
|
+
|
|
55
|
+
expect(result.request_id.startsWith('support-group-' + project._id + '-')).to.equal(true);
|
|
56
|
+
|
|
57
|
+
const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
|
|
58
|
+
expect(request.participants).to.deep.equal(['bot_' + chatbot._id.toString()]);
|
|
59
|
+
expect(request.participantsBots).to.deep.equal([chatbot._id.toString()]);
|
|
60
|
+
expect(request.hasBot).to.equal(true);
|
|
61
|
+
expect(request.status).to.equal(200);
|
|
62
|
+
expect(request.channel.name).to.equal('webhook');
|
|
63
|
+
expect(request.department.toString()).to.equal(defaultDepartment.id);
|
|
64
|
+
expect(request.first_text).to.equal('/start');
|
|
65
|
+
expect(request.attributes.webhook_id).to.equal(webhook.webhook_id);
|
|
66
|
+
expect(request.attributes.payload.alert).to.equal('KubePodCrashLooping');
|
|
67
|
+
expect(request.draft).to.not.equal(true);
|
|
68
|
+
|
|
69
|
+
const lead = await Lead.findById(request.lead).lean().exec();
|
|
70
|
+
expect(lead.lead_id).to.equal('webhook_' + webhook.webhook_id);
|
|
71
|
+
expect(lead.fullname).to.equal('Grafana');
|
|
72
|
+
|
|
73
|
+
const startMessage = await waitFor(() => Message.findOne({ recipient: result.request_id, text: '/start' }).lean().exec());
|
|
74
|
+
expect(startMessage.sender).to.equal('system');
|
|
75
|
+
expect(startMessage.attributes.subtype).to.equal('info');
|
|
76
|
+
expect(startMessage.attributes.action).to.equal('#webhook-box-id');
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('names the lead after the chatbot when the start point mapping is empty', async () => {
|
|
80
|
+
const { project, chatbot, webhook } = await setup({
|
|
81
|
+
start_points: [{ type: 'webhook', block_id: 'webhook-box-id', enabled: true, mapping: {} }]
|
|
82
|
+
});
|
|
83
|
+
const result = await webhookConversationService.start(webhook, chatbot, { alert: 'x' }, {});
|
|
84
|
+
const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
|
|
85
|
+
const lead = await Lead.findById(request.lead).lean().exec();
|
|
86
|
+
expect(lead.fullname).to.equal(chatbot.name);
|
|
87
|
+
expect(chatbot.name).to.equal('hitl-bot');
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('renames the existing webhook lead when the mapping source name changes', async () => {
|
|
91
|
+
const { chatbot, webhook } = await setup();
|
|
92
|
+
await webhookConversationService.start(webhook, chatbot, {}, {});
|
|
93
|
+
webhook.start_points[0].mapping = { source_name: 'Prometheus' };
|
|
94
|
+
|
|
95
|
+
const emitted = [];
|
|
96
|
+
const originalEmit = leadEvent.emit;
|
|
97
|
+
leadEvent.emit = function (name) {
|
|
98
|
+
emitted.push(name);
|
|
99
|
+
return originalEmit.apply(this, arguments);
|
|
100
|
+
};
|
|
101
|
+
let result;
|
|
102
|
+
try {
|
|
103
|
+
result = await webhookConversationService.start(webhook, chatbot, {}, {});
|
|
104
|
+
} finally {
|
|
105
|
+
leadEvent.emit = originalEmit;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const request = await Request.findOne({ request_id: result.request_id }).lean().exec();
|
|
109
|
+
const lead = await Lead.findById(request.lead).lean().exec();
|
|
110
|
+
expect(lead.fullname).to.equal('Prometheus');
|
|
111
|
+
expect(request.snapshot.lead.fullname).to.equal('Prometheus');
|
|
112
|
+
// no lead.update: chat21Handler would retitle every past conversation of the shared lead
|
|
113
|
+
expect(emitted.filter((name) => name.startsWith('lead.'))).to.deep.equal([]);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('rejects a start without an enabled webhook start point with 422', async () => {
|
|
117
|
+
const { chatbot, webhook } = await setup({ start_points: [] });
|
|
118
|
+
let error;
|
|
119
|
+
try {
|
|
120
|
+
await webhookConversationService.start(webhook, chatbot, {}, {});
|
|
121
|
+
} catch (err) {
|
|
122
|
+
error = err;
|
|
123
|
+
}
|
|
124
|
+
expect(error.status).to.equal(422);
|
|
125
|
+
expect(error.message).to.equal('Webhook start is not enabled');
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
describe('resolveChatbotDepartment', () => {
|
|
129
|
+
it('picks the non-default department whose bot is the chatbot', async () => {
|
|
130
|
+
const { project, chatbot, webhook } = await setup();
|
|
131
|
+
const dep = await departmentService.create('Ops', project._id.toString(), 'assigned', 'system');
|
|
132
|
+
await Department.updateOne({ _id: dep._id }, { $set: { id_bot: chatbot._id.toString() } });
|
|
133
|
+
|
|
134
|
+
const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
|
|
135
|
+
expect(department._id.toString()).to.equal(dep._id.toString());
|
|
136
|
+
|
|
137
|
+
const result = await webhookConversationService.start(webhook, chatbot, {}, {});
|
|
138
|
+
const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
|
|
139
|
+
expect(request.department.toString()).to.equal(dep._id.toString());
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it('prefers the default department when the chatbot is attached to it and to another one', async () => {
|
|
143
|
+
const { project, chatbot } = await setup();
|
|
144
|
+
const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
|
|
145
|
+
const dep = await departmentService.create('Ops', project._id.toString(), 'assigned', 'system');
|
|
146
|
+
await Department.updateOne({ _id: dep._id }, { $set: { id_bot: chatbot._id.toString() } });
|
|
147
|
+
await Department.updateOne({ _id: defaultDepartment._id }, { $set: { id_bot: chatbot._id.toString() } });
|
|
148
|
+
|
|
149
|
+
const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
|
|
150
|
+
expect(department._id.toString()).to.equal(defaultDepartment._id.toString());
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it('picks the oldest attached department when several non-default ones have the chatbot', async () => {
|
|
154
|
+
const { project, chatbot } = await setup();
|
|
155
|
+
const first = await departmentService.create('Ops 1', project._id.toString(), 'assigned', 'system');
|
|
156
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
157
|
+
const second = await departmentService.create('Ops 2', project._id.toString(), 'assigned', 'system');
|
|
158
|
+
await Department.updateMany({ _id: { $in: [first._id, second._id] } }, { $set: { id_bot: chatbot._id.toString() } });
|
|
159
|
+
|
|
160
|
+
const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
|
|
161
|
+
expect(department._id.toString()).to.equal(first._id.toString());
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it('ignores inactive departments (hidden: status 0 or -1) that carry the chatbot', async () => {
|
|
165
|
+
const { project, chatbot } = await setup();
|
|
166
|
+
const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
|
|
167
|
+
const hidden = await departmentService.create('Hidden', project._id.toString(), 'assigned', 'system');
|
|
168
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
169
|
+
const widgetHidden = await departmentService.create('Widget hidden', project._id.toString(), 'assigned', 'system');
|
|
170
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
171
|
+
const active = await departmentService.create('Active', project._id.toString(), 'assigned', 'system');
|
|
172
|
+
await Department.updateMany({ _id: { $in: [hidden._id, widgetHidden._id, active._id] } }, { $set: { id_bot: chatbot._id.toString() } });
|
|
173
|
+
await Department.updateOne({ _id: hidden._id }, { $set: { status: -1 } });
|
|
174
|
+
await Department.updateOne({ _id: widgetHidden._id }, { $set: { status: 0 } });
|
|
175
|
+
|
|
176
|
+
let department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
|
|
177
|
+
expect(department._id.toString()).to.equal(active._id.toString());
|
|
178
|
+
|
|
179
|
+
// only inactive ones left: the project default
|
|
180
|
+
await Department.updateOne({ _id: active._id }, { $set: { status: -1 } });
|
|
181
|
+
department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
|
|
182
|
+
expect(department._id.toString()).to.equal(defaultDepartment._id.toString());
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it('skips a hidden default department carrying the chatbot in favour of an active attached one', async () => {
|
|
186
|
+
const { project, chatbot } = await setup();
|
|
187
|
+
const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
|
|
188
|
+
const dep = await departmentService.create('Ops', project._id.toString(), 'assigned', 'system');
|
|
189
|
+
await Department.updateOne({ _id: dep._id }, { $set: { id_bot: chatbot._id.toString() } });
|
|
190
|
+
await Department.updateOne({ _id: defaultDepartment._id }, { $set: { id_bot: chatbot._id.toString(), status: -1 } });
|
|
191
|
+
|
|
192
|
+
const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
|
|
193
|
+
expect(department._id.toString()).to.equal(dep._id.toString());
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it('falls back to the project default department when the chatbot is not attached', async () => {
|
|
197
|
+
const { project, chatbot } = await setup();
|
|
198
|
+
const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
|
|
199
|
+
const other = await departmentService.create('Other', project._id.toString(), 'assigned', 'system');
|
|
200
|
+
await Department.updateOne({ _id: other._id }, { $set: { id_bot: '5f0000000000000000000000' } });
|
|
201
|
+
|
|
202
|
+
const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
|
|
203
|
+
expect(department._id.toString()).to.equal(defaultDepartment._id.toString());
|
|
204
|
+
});
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it('marks the conversation start type as webhook', async () => {
|
|
208
|
+
const { project, chatbot, webhook } = await setup();
|
|
209
|
+
const result = await webhookConversationService.start(webhook, chatbot, { alert: 'x' }, {});
|
|
210
|
+
const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
|
|
211
|
+
expect(request.attributes.start_type).to.equal('webhook');
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
it('uses the given request id and marks the request as draft in dev mode', async () => {
|
|
215
|
+
const { user, project } = await setup();
|
|
216
|
+
const chatbotInProject = await faqService.create(project._id, user._id, { name: 'hitl-bot-2', type: 'tilebot', language: 'en', template: 'blank' });
|
|
217
|
+
const devWebhook = await new Webhook({
|
|
218
|
+
id_project: project._id.toString(),
|
|
219
|
+
chatbot_id: chatbotInProject._id.toString(),
|
|
220
|
+
block_id: 'start-intent-id',
|
|
221
|
+
mode: 'conversation',
|
|
222
|
+
start_points: [{ type: 'webhook', block_id: 'webhook-box-id' }]
|
|
223
|
+
}).save();
|
|
224
|
+
const request_id = webhookConversationService.newRequestId(project._id.toString());
|
|
225
|
+
|
|
226
|
+
const result = await webhookConversationService.start(devWebhook, chatbotInProject, { alert: 'dev' }, { dev: true, request_id: request_id });
|
|
227
|
+
|
|
228
|
+
expect(result.request_id).to.equal(request_id);
|
|
229
|
+
const request = await Request.findOne({ request_id: request_id }).lean().exec();
|
|
230
|
+
expect(request.draft).to.equal(true);
|
|
231
|
+
expect(request.attributes.sourcePage).to.include('&td_draft=true');
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
describe('sanitizePayloadKeys', () => {
|
|
235
|
+
it('rewrites dotted and leading-$ object keys recursively, keeping values, later key wins on collision', () => {
|
|
236
|
+
const result = webhookConversationService.sanitizePayloadKeys({
|
|
237
|
+
labels: { 'app.kubernetes.io/name': 'checkout-api' },
|
|
238
|
+
'$op': 1,
|
|
239
|
+
items: [{ 'x.y': 1 }],
|
|
240
|
+
keep: 'a.b'
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
expect(result).to.deep.equal({
|
|
244
|
+
labels: { 'app_kubernetes_io/name': 'checkout-api' },
|
|
245
|
+
_op: 1,
|
|
246
|
+
items: [{ x_y: 1 }],
|
|
247
|
+
keep: 'a.b'
|
|
248
|
+
});
|
|
249
|
+
});
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
it('sanitizes dotted payload keys (Kubernetes/Grafana labels) before storing the request', async () => {
|
|
253
|
+
const { chatbot, webhook } = await setup();
|
|
254
|
+
const payload = { commonLabels: { 'app.kubernetes.io/name': 'checkout-api' } };
|
|
255
|
+
|
|
256
|
+
const result = await webhookConversationService.start(webhook, chatbot, payload, {});
|
|
257
|
+
|
|
258
|
+
const request = await Request.findOne({ request_id: result.request_id, id_project: webhook.id_project.toString() }).lean().exec();
|
|
259
|
+
expect(request.attributes.payload.commonLabels['app_kubernetes_io/name']).to.equal('checkout-api');
|
|
260
|
+
});
|
|
261
|
+
});
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
const chai = require('chai');
|
|
2
|
+
const chaiHttp = require('chai-http');
|
|
3
|
+
chai.use(chaiHttp);
|
|
4
|
+
|
|
5
|
+
const userService = require('../services/userService');
|
|
6
|
+
const projectService = require('../services/projectService');
|
|
7
|
+
const Faq = require('../models/faq');
|
|
8
|
+
|
|
9
|
+
const PWD = 'pwd';
|
|
10
|
+
|
|
11
|
+
async function waitFor(fn, { timeout = 5000, interval = 100 } = {}) {
|
|
12
|
+
const start = Date.now();
|
|
13
|
+
while (Date.now() - start < timeout) {
|
|
14
|
+
const value = await fn();
|
|
15
|
+
if (value) {
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
await new Promise((resolve) => setTimeout(resolve, interval));
|
|
19
|
+
}
|
|
20
|
+
throw new Error('waitFor timeout after ' + timeout + 'ms');
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function uniqueEmail(prefix) {
|
|
24
|
+
return prefix + '-' + Date.now() + '-' + Math.floor(Math.random() * 100000) + '@email.com';
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Creates a user, a project and a tilebot chatbot through the API.
|
|
29
|
+
* The blank template's intents are saved asynchronously by faqService, so the
|
|
30
|
+
* start intent is awaited when waitStart is true.
|
|
31
|
+
*/
|
|
32
|
+
async function setupProjectWithBot(server, { subtype, waitStart = true } = {}) {
|
|
33
|
+
const email = uniqueEmail('test-webhook-conv');
|
|
34
|
+
const user = await userService.signup(email, PWD, 'Test Firstname', 'Test lastname');
|
|
35
|
+
const project = await projectService.create('test-webhook-conversation', user._id);
|
|
36
|
+
|
|
37
|
+
const body = { name: 'hitl-bot', type: 'tilebot', language: 'en', template: 'blank' };
|
|
38
|
+
if (subtype) {
|
|
39
|
+
body.subtype = subtype;
|
|
40
|
+
}
|
|
41
|
+
const res = await chai.request(server).post('/' + project._id + '/faq_kb').auth(email, PWD).send(body);
|
|
42
|
+
if (res.status !== 200) {
|
|
43
|
+
throw new Error('Chatbot creation failed with status ' + res.status);
|
|
44
|
+
}
|
|
45
|
+
const chatbot_id = res.body._id;
|
|
46
|
+
|
|
47
|
+
let startIntent = null;
|
|
48
|
+
if (waitStart) {
|
|
49
|
+
startIntent = await waitFor(() => Faq.findOne({ id_faq_kb: chatbot_id, intent_display_name: 'start' }).lean().exec());
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return { email, pwd: PWD, user, project, chatbot_id, startIntent };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
module.exports = { PWD, waitFor, uniqueEmail, setupProjectWithBot };
|
package/utils/aiUtils.js
CHANGED
|
@@ -40,4 +40,98 @@ function loadMultiplier() {
|
|
|
40
40
|
return models;
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
-
|
|
43
|
+
const DEFAULT_AGENT_TOKENS_PER_USD = 1000000;
|
|
44
|
+
const DEFAULT_AGENT_TOKENS_FALLBACK_MULTIPLIER = 1;
|
|
45
|
+
|
|
46
|
+
function loadPositiveNumber(name, defaultValue) {
|
|
47
|
+
let raw = process.env[name];
|
|
48
|
+
if (raw === undefined || raw === '') {
|
|
49
|
+
return defaultValue;
|
|
50
|
+
}
|
|
51
|
+
let value = Number(raw);
|
|
52
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
53
|
+
winston.warn("Invalid " + name + " value '" + raw + "': using default " + defaultValue);
|
|
54
|
+
return defaultValue;
|
|
55
|
+
}
|
|
56
|
+
return value;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function loadAgentTokensRates() {
|
|
60
|
+
let rates = {
|
|
61
|
+
per_usd: loadPositiveNumber('AGENT_TOKENS_PER_USD', DEFAULT_AGENT_TOKENS_PER_USD),
|
|
62
|
+
fallback_multiplier: loadPositiveNumber('AGENT_TOKENS_FALLBACK_MULTIPLIER', DEFAULT_AGENT_TOKENS_FALLBACK_MULTIPLIER)
|
|
63
|
+
}
|
|
64
|
+
winston.info("Agent tokens rates: ", rates);
|
|
65
|
+
return rates;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const AGENT_TOKENS_RATES = loadAgentTokensRates();
|
|
69
|
+
|
|
70
|
+
// Sanity caps for a single usage report
|
|
71
|
+
const AGENT_TOKENS_MAX_TOKENS = 1e9;
|
|
72
|
+
const AGENT_TOKENS_MAX_COST_USD = 1e4;
|
|
73
|
+
|
|
74
|
+
// ceil, ignoring float noise below 1/1000 of a token (e.g. 3e-6 * 1e6 = 3.0000000000000004)
|
|
75
|
+
function ceilCounted(value) {
|
|
76
|
+
return Math.ceil(Math.round(value * 1e3) / 1e3);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function finiteCounted(counted, mode) {
|
|
80
|
+
if (!Number.isFinite(counted)) {
|
|
81
|
+
return { error: "Invalid usage: the counted agent tokens are not a finite number" };
|
|
82
|
+
}
|
|
83
|
+
return { counted: counted, mode: mode };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Converts an agent runtime usage report into agent tokens.
|
|
88
|
+
* - cost {amount, currency: 'USD'} present and valid -> ceil(amount * per_usd)
|
|
89
|
+
* - otherwise -> ceil(tokens * fallback_multiplier)
|
|
90
|
+
* tokens > 1e9, cost.amount > 1e4 USD or a non finite result are rejected.
|
|
91
|
+
* Returns { counted, mode } or { error } for an invalid body.
|
|
92
|
+
*/
|
|
93
|
+
function computeAgentTokens(data, rates) {
|
|
94
|
+
rates = rates || AGENT_TOKENS_RATES;
|
|
95
|
+
if (!data || typeof data !== 'object') {
|
|
96
|
+
return { error: "Missing body" };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
let tokens = data.tokens;
|
|
100
|
+
let hasTokens = tokens !== undefined && tokens !== null;
|
|
101
|
+
if (hasTokens && (typeof tokens !== 'number' || !Number.isFinite(tokens) || tokens < 0)) {
|
|
102
|
+
return { error: "Invalid tokens: must be a finite number >= 0" };
|
|
103
|
+
}
|
|
104
|
+
if (hasTokens && tokens > AGENT_TOKENS_MAX_TOKENS) {
|
|
105
|
+
return { error: "Invalid tokens: must be <= " + AGENT_TOKENS_MAX_TOKENS };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
let cost = data.cost;
|
|
109
|
+
if (cost !== undefined && cost !== null) {
|
|
110
|
+
if (typeof cost !== 'object') {
|
|
111
|
+
return { error: "Invalid cost: must be an object {amount, currency}" };
|
|
112
|
+
}
|
|
113
|
+
let isUSD = typeof cost.currency === 'string' && cost.currency.toUpperCase() === 'USD';
|
|
114
|
+
let validAmount = typeof cost.amount === 'number' && Number.isFinite(cost.amount) && cost.amount >= 0;
|
|
115
|
+
if (validAmount && cost.amount > AGENT_TOKENS_MAX_COST_USD) {
|
|
116
|
+
return { error: "Invalid cost amount: must be <= " + AGENT_TOKENS_MAX_COST_USD };
|
|
117
|
+
}
|
|
118
|
+
if (isUSD && validAmount) {
|
|
119
|
+
return finiteCounted(ceilCounted(cost.amount * rates.per_usd), 'cost');
|
|
120
|
+
}
|
|
121
|
+
if (typeof cost.amount === 'number' && cost.amount < 0) {
|
|
122
|
+
return { error: "Invalid cost amount: must be >= 0" };
|
|
123
|
+
}
|
|
124
|
+
winston.verbose("(computeAgentTokens) cost not usable, falling back to tokens: ", cost);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
if (!hasTokens) {
|
|
128
|
+
return { error: "Missing tokens or a valid USD cost" };
|
|
129
|
+
}
|
|
130
|
+
return finiteCounted(ceilCounted(tokens * rates.fallback_multiplier), 'tokens');
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
module.exports = {
|
|
134
|
+
MODELS_MULTIPLIER: loadMultiplier(),
|
|
135
|
+
AGENT_TOKENS_RATES,
|
|
136
|
+
computeAgentTokens
|
|
137
|
+
}
|
package/utils/dataTableUtils.js
CHANGED
|
@@ -1,6 +1,23 @@
|
|
|
1
1
|
const crypto = require('crypto');
|
|
2
2
|
|
|
3
|
-
const COLUMN_TYPES =
|
|
3
|
+
const COLUMN_TYPES = Object.freeze({
|
|
4
|
+
STRING: 'string',
|
|
5
|
+
NUMBER: 'number',
|
|
6
|
+
BOOLEAN: 'boolean',
|
|
7
|
+
DATETIME: 'datetime',
|
|
8
|
+
TTL: 'ttl',
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
const COLUMN_TYPE_VALUES = Object.freeze(Object.values(COLUMN_TYPES));
|
|
12
|
+
|
|
13
|
+
// One year: well beyond any sensible retention, and far from the point where
|
|
14
|
+
// createdAt plus the ttl stops being a representable date.
|
|
15
|
+
const MAX_TTL_SECONDS = 365 * 24 * 60 * 60;
|
|
16
|
+
|
|
17
|
+
function isValidColumnType(type) {
|
|
18
|
+
return COLUMN_TYPE_VALUES.indexOf(type) > -1;
|
|
19
|
+
}
|
|
20
|
+
|
|
4
21
|
const MATCH_MODES = ['all', 'any'];
|
|
5
22
|
const FORBIDDEN_NAMES = ['__proto__', 'constructor', 'prototype', '$where'];
|
|
6
23
|
|
|
@@ -53,7 +70,7 @@ function normalizeSchemaForCreate(input) {
|
|
|
53
70
|
|
|
54
71
|
const name = typeof col.name === 'string' ? col.name.trim() : '';
|
|
55
72
|
assertValidColumnName(name, 'Invalid column at index ' + i);
|
|
56
|
-
if (!
|
|
73
|
+
if (!isValidColumnType(col.type)) {
|
|
57
74
|
throw new Error('Column type is required at index ' + i);
|
|
58
75
|
}
|
|
59
76
|
if (names[name]) throw new Error('Column name already exists: ' + name);
|
|
@@ -66,6 +83,7 @@ function normalizeSchemaForCreate(input) {
|
|
|
66
83
|
index: typeof col.index === 'number' ? col.index : i,
|
|
67
84
|
});
|
|
68
85
|
}
|
|
86
|
+
assertSingleTtlColumn(schema);
|
|
69
87
|
return schema;
|
|
70
88
|
}
|
|
71
89
|
|
|
@@ -100,7 +118,7 @@ function resolveSchemaUpdate(incoming, currentSchema) {
|
|
|
100
118
|
assertValidColumnName(name, 'Invalid column at index ' + i);
|
|
101
119
|
|
|
102
120
|
const type = col.type || prev.type;
|
|
103
|
-
if (!
|
|
121
|
+
if (!isValidColumnType(type)) {
|
|
104
122
|
throw new Error('Column type is required at index ' + i);
|
|
105
123
|
}
|
|
106
124
|
if (names[name]) throw new Error('Column name already exists: ' + name);
|
|
@@ -131,7 +149,15 @@ function resolveSchemaUpdate(incoming, currentSchema) {
|
|
|
131
149
|
deletes.push(remainingIds[id].name);
|
|
132
150
|
}
|
|
133
151
|
|
|
134
|
-
|
|
152
|
+
assertSingleTtlColumn(newSchema);
|
|
153
|
+
|
|
154
|
+
return {
|
|
155
|
+
schema: newSchema,
|
|
156
|
+
renames: renames,
|
|
157
|
+
deletes: deletes,
|
|
158
|
+
ttlBefore: findTtlColumn(currentSchema),
|
|
159
|
+
ttlAfter: findTtlColumn(newSchema),
|
|
160
|
+
};
|
|
135
161
|
}
|
|
136
162
|
|
|
137
163
|
function normalizeColumnToAdd(col, nextIndex, existingSchema) {
|
|
@@ -140,11 +166,14 @@ function normalizeColumnToAdd(col, nextIndex, existingSchema) {
|
|
|
140
166
|
|
|
141
167
|
const name = typeof col.name === 'string' ? col.name.trim() : '';
|
|
142
168
|
assertValidColumnName(name);
|
|
143
|
-
if (!
|
|
169
|
+
if (!isValidColumnType(col.type)) throw new Error('Column type is required');
|
|
144
170
|
|
|
145
171
|
for (let i = 0; i < existingSchema.length; i++) {
|
|
146
172
|
if (existingSchema[i].name === name) throw new Error('Column name already exists: ' + name);
|
|
147
173
|
}
|
|
174
|
+
if (col.type === COLUMN_TYPES.TTL && findTtlColumn(existingSchema)) {
|
|
175
|
+
throw new Error('Invalid column: only one ttl column is allowed per table');
|
|
176
|
+
}
|
|
148
177
|
|
|
149
178
|
return {
|
|
150
179
|
id: generateColumnId(),
|
|
@@ -168,15 +197,32 @@ function findColumnByName(schema, name) {
|
|
|
168
197
|
return null;
|
|
169
198
|
}
|
|
170
199
|
|
|
200
|
+
function findTtlColumn(schema) {
|
|
201
|
+
if (!schema) return null;
|
|
202
|
+
for (let i = 0; i < schema.length; i++) {
|
|
203
|
+
if (schema[i].type === COLUMN_TYPES.TTL) return schema[i];
|
|
204
|
+
}
|
|
205
|
+
return null;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function assertSingleTtlColumn(schema) {
|
|
209
|
+
let found = 0;
|
|
210
|
+
for (let i = 0; i < schema.length; i++) {
|
|
211
|
+
if (schema[i].type === COLUMN_TYPES.TTL && ++found > 1) {
|
|
212
|
+
throw new Error('Invalid schema: only one ttl column is allowed per table');
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
171
217
|
function coerceValue(value, column) {
|
|
172
218
|
if (value === null || value === undefined) return value;
|
|
173
219
|
|
|
174
220
|
switch (column.type) {
|
|
175
|
-
case
|
|
221
|
+
case COLUMN_TYPES.STRING:
|
|
176
222
|
if (typeof value !== 'string') throw new Error('Invalid value for column ' + column.name + ': expected string');
|
|
177
223
|
return value;
|
|
178
224
|
|
|
179
|
-
case
|
|
225
|
+
case COLUMN_TYPES.NUMBER:
|
|
180
226
|
if (typeof value === 'number' && !isNaN(value)) return value;
|
|
181
227
|
if (typeof value === 'string' && value.trim() !== '') {
|
|
182
228
|
const num = Number(value);
|
|
@@ -184,7 +230,7 @@ function coerceValue(value, column) {
|
|
|
184
230
|
}
|
|
185
231
|
throw new Error('Invalid value for column ' + column.name + ': expected number');
|
|
186
232
|
|
|
187
|
-
case
|
|
233
|
+
case COLUMN_TYPES.BOOLEAN:
|
|
188
234
|
if (typeof value === 'boolean') return value;
|
|
189
235
|
if (typeof value === 'string') {
|
|
190
236
|
const lower = value.trim().toLowerCase();
|
|
@@ -193,7 +239,7 @@ function coerceValue(value, column) {
|
|
|
193
239
|
}
|
|
194
240
|
throw new Error('Invalid value for column ' + column.name + ': expected boolean');
|
|
195
241
|
|
|
196
|
-
case
|
|
242
|
+
case COLUMN_TYPES.DATETIME:
|
|
197
243
|
if (value instanceof Date && !isNaN(value.getTime())) return value;
|
|
198
244
|
if (typeof value === 'string' && value.length > 0) {
|
|
199
245
|
const date = new Date(value);
|
|
@@ -201,6 +247,25 @@ function coerceValue(value, column) {
|
|
|
201
247
|
}
|
|
202
248
|
throw new Error('Invalid value for column ' + column.name + ': expected datetime');
|
|
203
249
|
|
|
250
|
+
case COLUMN_TYPES.TTL: {
|
|
251
|
+
let seconds;
|
|
252
|
+
if (typeof value === 'number') seconds = value;
|
|
253
|
+
else if (typeof value === 'string' && value.trim() !== '') seconds = Number(value);
|
|
254
|
+
if (seconds === undefined || isNaN(seconds)) {
|
|
255
|
+
throw new Error('Invalid value for column ' + column.name + ': expected a number of seconds');
|
|
256
|
+
}
|
|
257
|
+
// 0 and negatives would make MongoDB drop the row on the next TTL pass
|
|
258
|
+
if (!Number.isInteger(seconds) || seconds <= 0) {
|
|
259
|
+
throw new Error('Invalid value for column ' + column.name + ': ttl must be a positive integer number of seconds');
|
|
260
|
+
}
|
|
261
|
+
// Past this the resulting date overflows and the row is refused by the
|
|
262
|
+
// model with a cast error, which surfaces as a 500 instead of a 400
|
|
263
|
+
if (seconds > MAX_TTL_SECONDS) {
|
|
264
|
+
throw new Error('Invalid value for column ' + column.name + ': ttl must not exceed ' + MAX_TTL_SECONDS + ' seconds');
|
|
265
|
+
}
|
|
266
|
+
return seconds;
|
|
267
|
+
}
|
|
268
|
+
|
|
204
269
|
default:
|
|
205
270
|
throw new Error('Unsupported column type: ' + column.type);
|
|
206
271
|
}
|
|
@@ -309,6 +374,29 @@ function fillRowDataFromSchema(data, schema) {
|
|
|
309
374
|
return result;
|
|
310
375
|
}
|
|
311
376
|
|
|
377
|
+
/**
|
|
378
|
+
* Rebind conditions to a single row of a batch. In a bulk upsert the
|
|
379
|
+
* conditions name the columns that identify a row, and the value to match
|
|
380
|
+
* comes from the row itself: a fixed value would make every row of the batch
|
|
381
|
+
* compete for the same match.
|
|
382
|
+
*/
|
|
383
|
+
function bindConditionsToRow(conditions, rowData, schema) {
|
|
384
|
+
return conditions.map(function (item, i) {
|
|
385
|
+
if (!item || typeof item !== 'object') throw new Error('Invalid condition at index ' + i);
|
|
386
|
+
if (NO_VALUE_OPERATORS.indexOf(item.operator) !== -1) return item;
|
|
387
|
+
|
|
388
|
+
const columnName = typeof item.column === 'string' ? item.column.trim() : '';
|
|
389
|
+
const colMeta = findColumnByName(schema, columnName);
|
|
390
|
+
if (!colMeta) throw new Error('Column ' + item.column + ' does not exist');
|
|
391
|
+
|
|
392
|
+
const value = rowData ? rowData[colMeta.name] : undefined;
|
|
393
|
+
if (value === undefined || value === null) {
|
|
394
|
+
throw new Error('a value for column ' + colMeta.name + ' is required to match an existing row');
|
|
395
|
+
}
|
|
396
|
+
return Object.assign({}, item, { value: value });
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
|
|
312
400
|
function buildRowQueryFilter(body, schema) {
|
|
313
401
|
if (body.id_row) {
|
|
314
402
|
return { _id: body.id_row };
|
|
@@ -339,6 +427,7 @@ function buildUpdateSet(data, schema) {
|
|
|
339
427
|
|
|
340
428
|
module.exports = {
|
|
341
429
|
COLUMN_TYPES: COLUMN_TYPES,
|
|
430
|
+
findTtlColumn: findTtlColumn,
|
|
342
431
|
generateColumnId: generateColumnId,
|
|
343
432
|
normalizeSchemaForCreate: normalizeSchemaForCreate,
|
|
344
433
|
normalizeColumnToAdd: normalizeColumnToAdd,
|
|
@@ -347,6 +436,7 @@ module.exports = {
|
|
|
347
436
|
findColumnByName: findColumnByName,
|
|
348
437
|
validateRowData: validateRowData,
|
|
349
438
|
buildSearchFilter: buildSearchFilter,
|
|
439
|
+
bindConditionsToRow: bindConditionsToRow,
|
|
350
440
|
buildRowQueryFilter: buildRowQueryFilter,
|
|
351
441
|
buildUpdateSet: buildUpdateSet,
|
|
352
442
|
fillRowDataFromSchema: fillRowDataFromSchema,
|