@tiledesk/tiledesk-server 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/.circleci/config.yml +1 -1
  2. package/CHANGELOG.md +37 -0
  3. package/Dockerfile +3 -2
  4. package/Dockerfile-en +3 -2
  5. package/Dockerfile-jobs +3 -2
  6. package/Dockerfile-profiler +3 -2
  7. package/README.md +15 -0
  8. package/app.js +14 -1
  9. package/bin/quota-reset +68 -0
  10. package/bin/quota-restore +86 -0
  11. package/channels/chat21/chat21Handler.js +50 -34
  12. package/channels/chat21/chat21JoinTracker.js +43 -0
  13. package/config/database.js +5 -5
  14. package/config/native_mcp/native_mcp_servers.json +1 -0
  15. package/controllers/chatbot.controller.js +80 -27
  16. package/controllers/intents.controller.js +42 -0
  17. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-chat21-ionic.md +468 -0
  18. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-design-studio.md +1349 -0
  19. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-server.md +1694 -0
  20. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-tybot.md +789 -0
  21. package/docs/superpowers/plans/2026-09-29-hitl-known-limits-server.md +83 -0
  22. package/docs/superpowers/plans/2026-09-29-hitl-known-limits-tybot.md +76 -0
  23. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-chat21-ionic.md +97 -0
  24. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-design-studio.md +72 -0
  25. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-server.md +223 -0
  26. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-tybot.md +121 -0
  27. package/docs/superpowers/plans/2026-09-29-hitl-pre-branch-ports.md +65 -0
  28. package/docs/superpowers/plans/2026-09-30-flow-start-points-design-studio.md +88 -0
  29. package/docs/superpowers/plans/2026-09-30-flow-start-points-server.md +103 -0
  30. package/docs/superpowers/plans/2026-09-30-flow-start-points-tybot.md +43 -0
  31. package/docs/superpowers/plans/2026-10-01-scheduled-start-point-design-studio.md +80 -0
  32. package/docs/superpowers/plans/2026-10-01-scheduled-start-point-runtime.md +39 -0
  33. package/docs/superpowers/plans/2026-10-01-scheduled-start-point-server.md +106 -0
  34. package/docs/superpowers/specs/2026-09-15-webhook-chatbot-hitl-design.md +715 -0
  35. package/docs/superpowers/specs/2026-09-30-flow-start-points-design.md +156 -0
  36. package/docs/superpowers/specs/2026-10-01-scheduled-start-point-design.md +303 -0
  37. package/docs/superpowers/todo/2026-09-14-webhook-chatbot-shared-code-fixes.md +191 -0
  38. package/event/botEvent.js +12 -1
  39. package/event/emailEvent.js +3 -0
  40. package/event/messageEvent.js +44 -0
  41. package/jobs.js +10 -0
  42. package/jobsManager.js +19 -0
  43. package/middleware/errorHandler.js +6 -2
  44. package/models/agentPrompt.js +104 -0
  45. package/models/analyticResult.js +1 -1
  46. package/models/channelConstants.js +3 -2
  47. package/models/quotaUsage.js +71 -0
  48. package/models/request.js +6 -0
  49. package/models/requestSnapshot.js +12 -0
  50. package/models/routerLogger.js +1 -1
  51. package/models/webhook.js +44 -0
  52. package/package.json +7 -5
  53. package/pubmodules/emailNotification/requestNotification.js +14 -22
  54. package/pubmodules/events/test/eventRoute.js +7 -6
  55. package/pubmodules/rules/conciergeBot.js +9 -4
  56. package/routes/agentPrompts.js +132 -0
  57. package/routes/auth.js +1 -12
  58. package/routes/campaigns.js +0 -13
  59. package/routes/dataTable.js +0 -3
  60. package/routes/faq_kb.js +12 -1
  61. package/routes/messagesRoot.js +0 -1
  62. package/routes/quotes.js +38 -1
  63. package/routes/request.js +81 -5
  64. package/routes/users.js +0 -1
  65. package/routes/webhook.js +160 -8
  66. package/routes/webhooks.js +276 -6
  67. package/routes/widgetLoader.js +6 -3
  68. package/services/QuoteManager.js +77 -16
  69. package/services/aiService.js +1 -1
  70. package/services/chatbotService.js +55 -1
  71. package/services/departmentService.js +60 -43
  72. package/services/dolphinSchedulerClient.js +116 -0
  73. package/services/emailService.js +6 -1
  74. package/services/hitlParticipantListener.js +260 -0
  75. package/services/participantInviteService.js +302 -0
  76. package/services/participantRemoveService.js +79 -0
  77. package/services/quotaSnapshotService.js +1167 -0
  78. package/services/requestService.js +102 -144
  79. package/services/scheduledStart/dolphinBackend.js +246 -0
  80. package/services/scheduledStart/schedulerBackend.js +101 -0
  81. package/services/scheduledStart/schedulerServiceClient.js +41 -0
  82. package/services/scheduledStartPublishListener.js +186 -0
  83. package/services/scheduledStartRules.js +173 -0
  84. package/services/scheduledStartService.js +130 -0
  85. package/services/startPointCleanupListener.js +30 -0
  86. package/services/startPointService.js +158 -0
  87. package/services/testWsService.js +1 -4
  88. package/services/updateRequestSnapshotQueued.js +8 -9
  89. package/services/webhookConversationService.js +185 -0
  90. package/services/webhookSettingsValidator.js +61 -0
  91. package/template/email/checkpointReachedEmail.html +5 -0
  92. package/test/agentPromptsRoute.test.js +154 -0
  93. package/test/agentTokensQuota.js +237 -0
  94. package/test/botEventHitlFallback.js +19 -0
  95. package/test/chat21HandlerJoinWiring.js +59 -0
  96. package/test/dataTablesRoute.test.js +10 -2
  97. package/test/dateUtils.test.js +0 -3
  98. package/test/departmentService.js +8 -12
  99. package/test/{fileRoute.js → deprecated/_deprecated_fileRoute} +2 -2
  100. package/test/{kbsettingsRoute.js → deprecated/_deprecated_kbsettingsRoute} +3 -3
  101. package/test/faqRoute.js +6 -1
  102. package/test/faqkbRoute.js +14 -10
  103. package/test/faqkbV3Route.js +1 -1
  104. package/test/filepRoute.js +4 -50
  105. package/test/hitlParticipantListener.js +446 -0
  106. package/test/hitlQueueWiring.js +191 -0
  107. package/test/hitlWaitingReset.js +124 -0
  108. package/test/inviteParticipantsRoute.js +492 -0
  109. package/test/kbRoute.js +1 -1
  110. package/test/messageRoute.js +33 -21
  111. package/test/nativemcp.test.js +7 -1
  112. package/test/openaiRoute.js +2 -8
  113. package/test/publishFailLoud.test.js +414 -0
  114. package/test/quotaSnapshots.js +1044 -0
  115. package/test/removeParticipantsRoute.js +186 -0
  116. package/test/requestService.js +24 -16
  117. package/test/requestServiceRemoveParticipant.js +144 -0
  118. package/test/requestVisibility.test.js +186 -0
  119. package/test/scheduledStartFacade.js +184 -0
  120. package/test/scheduledStartHandover.js +95 -0
  121. package/test/scheduledStartPublish.js +320 -0
  122. package/test/scheduledStartRoute.js +163 -0
  123. package/test/scheduledStartRules.js +177 -0
  124. package/test/scheduledStartRun.js +299 -0
  125. package/test/scheduledStartSchedulerBackend.js +116 -0
  126. package/test/scheduledStartService.js +290 -0
  127. package/test/startPointsCleanup.js +69 -0
  128. package/test/startPointsRegression.js +169 -0
  129. package/test/startPointsRoute.js +158 -0
  130. package/test/webhookConversationRoute.js +347 -0
  131. package/test/webhookConversationService.js +261 -0
  132. package/test/webhookConversationTestUtils.js +55 -0
  133. package/utils/aiUtils.js +95 -1
  134. package/utils/intentFieldNames.js +192 -0
  135. package/utils/quotaCliUtil.js +43 -0
  136. package/utils/requestVisibility.js +355 -0
  137. package/websocket/webSocketServer.js +36 -31
  138. /package/test/{imageRoute.js → deprecated/_deprecated_imageRoute} +0 -0
@@ -0,0 +1,261 @@
1
+ //During the test the env variable is set to test
2
+ process.env.NODE_ENV = 'test';
3
+ process.env.LOG_LEVEL = 'error';
4
+
5
+ const chai = require('chai');
6
+ const expect = chai.expect;
7
+ const config = require('../config/database');
8
+ const mongoose = require('mongoose');
9
+
10
+ // The `users` collection has a unique (non-sparse) index on the optional `phone` field
11
+ // (see models/user.js). Multiple users without a phone number collide on the shared
12
+ // `null` key. Disable autoIndex for this run so pre-existing environment state doesn't
13
+ // break unrelated tests that sign up more than one user.
14
+ mongoose.set('autoIndex', false);
15
+ mongoose.connect(config.databasetest);
16
+ require('../services/mongoose-cache-fn')(mongoose);
17
+
18
+ const userService = require('../services/userService');
19
+ const projectService = require('../services/projectService');
20
+ const faqService = require('../services/faqService');
21
+ const departmentService = require('../services/departmentService');
22
+ const webhookConversationService = require('../services/webhookConversationService');
23
+ const Request = require('../models/request');
24
+ const Message = require('../models/message');
25
+ const Lead = require('../models/lead');
26
+ const leadEvent = require('../event/leadEvent');
27
+ const Department = require('../models/department');
28
+ const { Webhook } = require('../models/webhook');
29
+ const { waitFor, uniqueEmail, PWD } = require('./webhookConversationTestUtils');
30
+
31
+ async function setup(webhookFields = {}) {
32
+ const user = await userService.signup(uniqueEmail('test-webhook-conv-service'), PWD, 'Test', 'User');
33
+ const project = await projectService.create('test-webhook-conversation-service', user._id);
34
+ const chatbot = await faqService.create(project._id, user._id, { name: 'hitl-bot', type: 'tilebot', language: 'en', template: 'blank' });
35
+ const webhook = await new Webhook(Object.assign({
36
+ id_project: project._id.toString(),
37
+ chatbot_id: chatbot._id.toString(),
38
+ block_id: 'start-intent-id',
39
+ mode: 'conversation',
40
+ start_points: [{ type: 'webhook', block_id: 'webhook-box-id', enabled: true, mapping: { source_name: 'Grafana' } }]
41
+ }, webhookFields)).save();
42
+ return { user, project, chatbot, webhook };
43
+ }
44
+
45
+ describe('WebhookConversationService', function () {
46
+
47
+ this.timeout(20000);
48
+
49
+ it('creates a lead, a bot-assigned request in the default department and a /start message', async () => {
50
+ const { project, chatbot, webhook } = await setup();
51
+ const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
52
+
53
+ const result = await webhookConversationService.start(webhook, chatbot, { alert: 'KubePodCrashLooping', webhook_http_method: 'POST' }, {});
54
+
55
+ expect(result.request_id.startsWith('support-group-' + project._id + '-')).to.equal(true);
56
+
57
+ const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
58
+ expect(request.participants).to.deep.equal(['bot_' + chatbot._id.toString()]);
59
+ expect(request.participantsBots).to.deep.equal([chatbot._id.toString()]);
60
+ expect(request.hasBot).to.equal(true);
61
+ expect(request.status).to.equal(200);
62
+ expect(request.channel.name).to.equal('webhook');
63
+ expect(request.department.toString()).to.equal(defaultDepartment.id);
64
+ expect(request.first_text).to.equal('/start');
65
+ expect(request.attributes.webhook_id).to.equal(webhook.webhook_id);
66
+ expect(request.attributes.payload.alert).to.equal('KubePodCrashLooping');
67
+ expect(request.draft).to.not.equal(true);
68
+
69
+ const lead = await Lead.findById(request.lead).lean().exec();
70
+ expect(lead.lead_id).to.equal('webhook_' + webhook.webhook_id);
71
+ expect(lead.fullname).to.equal('Grafana');
72
+
73
+ const startMessage = await waitFor(() => Message.findOne({ recipient: result.request_id, text: '/start' }).lean().exec());
74
+ expect(startMessage.sender).to.equal('system');
75
+ expect(startMessage.attributes.subtype).to.equal('info');
76
+ expect(startMessage.attributes.action).to.equal('#webhook-box-id');
77
+ });
78
+
79
+ it('names the lead after the chatbot when the start point mapping is empty', async () => {
80
+ const { project, chatbot, webhook } = await setup({
81
+ start_points: [{ type: 'webhook', block_id: 'webhook-box-id', enabled: true, mapping: {} }]
82
+ });
83
+ const result = await webhookConversationService.start(webhook, chatbot, { alert: 'x' }, {});
84
+ const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
85
+ const lead = await Lead.findById(request.lead).lean().exec();
86
+ expect(lead.fullname).to.equal(chatbot.name);
87
+ expect(chatbot.name).to.equal('hitl-bot');
88
+ });
89
+
90
+ it('renames the existing webhook lead when the mapping source name changes', async () => {
91
+ const { chatbot, webhook } = await setup();
92
+ await webhookConversationService.start(webhook, chatbot, {}, {});
93
+ webhook.start_points[0].mapping = { source_name: 'Prometheus' };
94
+
95
+ const emitted = [];
96
+ const originalEmit = leadEvent.emit;
97
+ leadEvent.emit = function (name) {
98
+ emitted.push(name);
99
+ return originalEmit.apply(this, arguments);
100
+ };
101
+ let result;
102
+ try {
103
+ result = await webhookConversationService.start(webhook, chatbot, {}, {});
104
+ } finally {
105
+ leadEvent.emit = originalEmit;
106
+ }
107
+
108
+ const request = await Request.findOne({ request_id: result.request_id }).lean().exec();
109
+ const lead = await Lead.findById(request.lead).lean().exec();
110
+ expect(lead.fullname).to.equal('Prometheus');
111
+ expect(request.snapshot.lead.fullname).to.equal('Prometheus');
112
+ // no lead.update: chat21Handler would retitle every past conversation of the shared lead
113
+ expect(emitted.filter((name) => name.startsWith('lead.'))).to.deep.equal([]);
114
+ });
115
+
116
+ it('rejects a start without an enabled webhook start point with 422', async () => {
117
+ const { chatbot, webhook } = await setup({ start_points: [] });
118
+ let error;
119
+ try {
120
+ await webhookConversationService.start(webhook, chatbot, {}, {});
121
+ } catch (err) {
122
+ error = err;
123
+ }
124
+ expect(error.status).to.equal(422);
125
+ expect(error.message).to.equal('Webhook start is not enabled');
126
+ });
127
+
128
+ describe('resolveChatbotDepartment', () => {
129
+ it('picks the non-default department whose bot is the chatbot', async () => {
130
+ const { project, chatbot, webhook } = await setup();
131
+ const dep = await departmentService.create('Ops', project._id.toString(), 'assigned', 'system');
132
+ await Department.updateOne({ _id: dep._id }, { $set: { id_bot: chatbot._id.toString() } });
133
+
134
+ const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
135
+ expect(department._id.toString()).to.equal(dep._id.toString());
136
+
137
+ const result = await webhookConversationService.start(webhook, chatbot, {}, {});
138
+ const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
139
+ expect(request.department.toString()).to.equal(dep._id.toString());
140
+ });
141
+
142
+ it('prefers the default department when the chatbot is attached to it and to another one', async () => {
143
+ const { project, chatbot } = await setup();
144
+ const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
145
+ const dep = await departmentService.create('Ops', project._id.toString(), 'assigned', 'system');
146
+ await Department.updateOne({ _id: dep._id }, { $set: { id_bot: chatbot._id.toString() } });
147
+ await Department.updateOne({ _id: defaultDepartment._id }, { $set: { id_bot: chatbot._id.toString() } });
148
+
149
+ const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
150
+ expect(department._id.toString()).to.equal(defaultDepartment._id.toString());
151
+ });
152
+
153
+ it('picks the oldest attached department when several non-default ones have the chatbot', async () => {
154
+ const { project, chatbot } = await setup();
155
+ const first = await departmentService.create('Ops 1', project._id.toString(), 'assigned', 'system');
156
+ await new Promise((resolve) => setTimeout(resolve, 20));
157
+ const second = await departmentService.create('Ops 2', project._id.toString(), 'assigned', 'system');
158
+ await Department.updateMany({ _id: { $in: [first._id, second._id] } }, { $set: { id_bot: chatbot._id.toString() } });
159
+
160
+ const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
161
+ expect(department._id.toString()).to.equal(first._id.toString());
162
+ });
163
+
164
+ it('ignores inactive departments (hidden: status 0 or -1) that carry the chatbot', async () => {
165
+ const { project, chatbot } = await setup();
166
+ const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
167
+ const hidden = await departmentService.create('Hidden', project._id.toString(), 'assigned', 'system');
168
+ await new Promise((resolve) => setTimeout(resolve, 20));
169
+ const widgetHidden = await departmentService.create('Widget hidden', project._id.toString(), 'assigned', 'system');
170
+ await new Promise((resolve) => setTimeout(resolve, 20));
171
+ const active = await departmentService.create('Active', project._id.toString(), 'assigned', 'system');
172
+ await Department.updateMany({ _id: { $in: [hidden._id, widgetHidden._id, active._id] } }, { $set: { id_bot: chatbot._id.toString() } });
173
+ await Department.updateOne({ _id: hidden._id }, { $set: { status: -1 } });
174
+ await Department.updateOne({ _id: widgetHidden._id }, { $set: { status: 0 } });
175
+
176
+ let department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
177
+ expect(department._id.toString()).to.equal(active._id.toString());
178
+
179
+ // only inactive ones left: the project default
180
+ await Department.updateOne({ _id: active._id }, { $set: { status: -1 } });
181
+ department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
182
+ expect(department._id.toString()).to.equal(defaultDepartment._id.toString());
183
+ });
184
+
185
+ it('skips a hidden default department carrying the chatbot in favour of an active attached one', async () => {
186
+ const { project, chatbot } = await setup();
187
+ const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
188
+ const dep = await departmentService.create('Ops', project._id.toString(), 'assigned', 'system');
189
+ await Department.updateOne({ _id: dep._id }, { $set: { id_bot: chatbot._id.toString() } });
190
+ await Department.updateOne({ _id: defaultDepartment._id }, { $set: { id_bot: chatbot._id.toString(), status: -1 } });
191
+
192
+ const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
193
+ expect(department._id.toString()).to.equal(dep._id.toString());
194
+ });
195
+
196
+ it('falls back to the project default department when the chatbot is not attached', async () => {
197
+ const { project, chatbot } = await setup();
198
+ const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
199
+ const other = await departmentService.create('Other', project._id.toString(), 'assigned', 'system');
200
+ await Department.updateOne({ _id: other._id }, { $set: { id_bot: '5f0000000000000000000000' } });
201
+
202
+ const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
203
+ expect(department._id.toString()).to.equal(defaultDepartment._id.toString());
204
+ });
205
+ });
206
+
207
+ it('marks the conversation start type as webhook', async () => {
208
+ const { project, chatbot, webhook } = await setup();
209
+ const result = await webhookConversationService.start(webhook, chatbot, { alert: 'x' }, {});
210
+ const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
211
+ expect(request.attributes.start_type).to.equal('webhook');
212
+ });
213
+
214
+ it('uses the given request id and marks the request as draft in dev mode', async () => {
215
+ const { user, project } = await setup();
216
+ const chatbotInProject = await faqService.create(project._id, user._id, { name: 'hitl-bot-2', type: 'tilebot', language: 'en', template: 'blank' });
217
+ const devWebhook = await new Webhook({
218
+ id_project: project._id.toString(),
219
+ chatbot_id: chatbotInProject._id.toString(),
220
+ block_id: 'start-intent-id',
221
+ mode: 'conversation',
222
+ start_points: [{ type: 'webhook', block_id: 'webhook-box-id' }]
223
+ }).save();
224
+ const request_id = webhookConversationService.newRequestId(project._id.toString());
225
+
226
+ const result = await webhookConversationService.start(devWebhook, chatbotInProject, { alert: 'dev' }, { dev: true, request_id: request_id });
227
+
228
+ expect(result.request_id).to.equal(request_id);
229
+ const request = await Request.findOne({ request_id: request_id }).lean().exec();
230
+ expect(request.draft).to.equal(true);
231
+ expect(request.attributes.sourcePage).to.include('&td_draft=true');
232
+ });
233
+
234
+ describe('sanitizePayloadKeys', () => {
235
+ it('rewrites dotted and leading-$ object keys recursively, keeping values, later key wins on collision', () => {
236
+ const result = webhookConversationService.sanitizePayloadKeys({
237
+ labels: { 'app.kubernetes.io/name': 'checkout-api' },
238
+ '$op': 1,
239
+ items: [{ 'x.y': 1 }],
240
+ keep: 'a.b'
241
+ });
242
+
243
+ expect(result).to.deep.equal({
244
+ labels: { 'app_kubernetes_io/name': 'checkout-api' },
245
+ _op: 1,
246
+ items: [{ x_y: 1 }],
247
+ keep: 'a.b'
248
+ });
249
+ });
250
+ });
251
+
252
+ it('sanitizes dotted payload keys (Kubernetes/Grafana labels) before storing the request', async () => {
253
+ const { chatbot, webhook } = await setup();
254
+ const payload = { commonLabels: { 'app.kubernetes.io/name': 'checkout-api' } };
255
+
256
+ const result = await webhookConversationService.start(webhook, chatbot, payload, {});
257
+
258
+ const request = await Request.findOne({ request_id: result.request_id, id_project: webhook.id_project.toString() }).lean().exec();
259
+ expect(request.attributes.payload.commonLabels['app_kubernetes_io/name']).to.equal('checkout-api');
260
+ });
261
+ });
@@ -0,0 +1,55 @@
1
+ const chai = require('chai');
2
+ const chaiHttp = require('chai-http');
3
+ chai.use(chaiHttp);
4
+
5
+ const userService = require('../services/userService');
6
+ const projectService = require('../services/projectService');
7
+ const Faq = require('../models/faq');
8
+
9
+ const PWD = 'pwd';
10
+
11
+ async function waitFor(fn, { timeout = 5000, interval = 100 } = {}) {
12
+ const start = Date.now();
13
+ while (Date.now() - start < timeout) {
14
+ const value = await fn();
15
+ if (value) {
16
+ return value;
17
+ }
18
+ await new Promise((resolve) => setTimeout(resolve, interval));
19
+ }
20
+ throw new Error('waitFor timeout after ' + timeout + 'ms');
21
+ }
22
+
23
+ function uniqueEmail(prefix) {
24
+ return prefix + '-' + Date.now() + '-' + Math.floor(Math.random() * 100000) + '@email.com';
25
+ }
26
+
27
+ /**
28
+ * Creates a user, a project and a tilebot chatbot through the API.
29
+ * The blank template's intents are saved asynchronously by faqService, so the
30
+ * start intent is awaited when waitStart is true.
31
+ */
32
+ async function setupProjectWithBot(server, { subtype, waitStart = true } = {}) {
33
+ const email = uniqueEmail('test-webhook-conv');
34
+ const user = await userService.signup(email, PWD, 'Test Firstname', 'Test lastname');
35
+ const project = await projectService.create('test-webhook-conversation', user._id);
36
+
37
+ const body = { name: 'hitl-bot', type: 'tilebot', language: 'en', template: 'blank' };
38
+ if (subtype) {
39
+ body.subtype = subtype;
40
+ }
41
+ const res = await chai.request(server).post('/' + project._id + '/faq_kb').auth(email, PWD).send(body);
42
+ if (res.status !== 200) {
43
+ throw new Error('Chatbot creation failed with status ' + res.status);
44
+ }
45
+ const chatbot_id = res.body._id;
46
+
47
+ let startIntent = null;
48
+ if (waitStart) {
49
+ startIntent = await waitFor(() => Faq.findOne({ id_faq_kb: chatbot_id, intent_display_name: 'start' }).lean().exec());
50
+ }
51
+
52
+ return { email, pwd: PWD, user, project, chatbot_id, startIntent };
53
+ }
54
+
55
+ module.exports = { PWD, waitFor, uniqueEmail, setupProjectWithBot };
package/utils/aiUtils.js CHANGED
@@ -40,4 +40,98 @@ function loadMultiplier() {
40
40
  return models;
41
41
  }
42
42
 
43
- module.exports = { MODELS_MULTIPLIER: loadMultiplier() }
43
+ const DEFAULT_AGENT_TOKENS_PER_USD = 1000000;
44
+ const DEFAULT_AGENT_TOKENS_FALLBACK_MULTIPLIER = 1;
45
+
46
+ function loadPositiveNumber(name, defaultValue) {
47
+ let raw = process.env[name];
48
+ if (raw === undefined || raw === '') {
49
+ return defaultValue;
50
+ }
51
+ let value = Number(raw);
52
+ if (!Number.isFinite(value) || value <= 0) {
53
+ winston.warn("Invalid " + name + " value '" + raw + "': using default " + defaultValue);
54
+ return defaultValue;
55
+ }
56
+ return value;
57
+ }
58
+
59
+ function loadAgentTokensRates() {
60
+ let rates = {
61
+ per_usd: loadPositiveNumber('AGENT_TOKENS_PER_USD', DEFAULT_AGENT_TOKENS_PER_USD),
62
+ fallback_multiplier: loadPositiveNumber('AGENT_TOKENS_FALLBACK_MULTIPLIER', DEFAULT_AGENT_TOKENS_FALLBACK_MULTIPLIER)
63
+ }
64
+ winston.info("Agent tokens rates: ", rates);
65
+ return rates;
66
+ }
67
+
68
+ const AGENT_TOKENS_RATES = loadAgentTokensRates();
69
+
70
+ // Sanity caps for a single usage report
71
+ const AGENT_TOKENS_MAX_TOKENS = 1e9;
72
+ const AGENT_TOKENS_MAX_COST_USD = 1e4;
73
+
74
+ // ceil, ignoring float noise below 1/1000 of a token (e.g. 3e-6 * 1e6 = 3.0000000000000004)
75
+ function ceilCounted(value) {
76
+ return Math.ceil(Math.round(value * 1e3) / 1e3);
77
+ }
78
+
79
+ function finiteCounted(counted, mode) {
80
+ if (!Number.isFinite(counted)) {
81
+ return { error: "Invalid usage: the counted agent tokens are not a finite number" };
82
+ }
83
+ return { counted: counted, mode: mode };
84
+ }
85
+
86
+ /**
87
+ * Converts an agent runtime usage report into agent tokens.
88
+ * - cost {amount, currency: 'USD'} present and valid -> ceil(amount * per_usd)
89
+ * - otherwise -> ceil(tokens * fallback_multiplier)
90
+ * tokens > 1e9, cost.amount > 1e4 USD or a non finite result are rejected.
91
+ * Returns { counted, mode } or { error } for an invalid body.
92
+ */
93
+ function computeAgentTokens(data, rates) {
94
+ rates = rates || AGENT_TOKENS_RATES;
95
+ if (!data || typeof data !== 'object') {
96
+ return { error: "Missing body" };
97
+ }
98
+
99
+ let tokens = data.tokens;
100
+ let hasTokens = tokens !== undefined && tokens !== null;
101
+ if (hasTokens && (typeof tokens !== 'number' || !Number.isFinite(tokens) || tokens < 0)) {
102
+ return { error: "Invalid tokens: must be a finite number >= 0" };
103
+ }
104
+ if (hasTokens && tokens > AGENT_TOKENS_MAX_TOKENS) {
105
+ return { error: "Invalid tokens: must be <= " + AGENT_TOKENS_MAX_TOKENS };
106
+ }
107
+
108
+ let cost = data.cost;
109
+ if (cost !== undefined && cost !== null) {
110
+ if (typeof cost !== 'object') {
111
+ return { error: "Invalid cost: must be an object {amount, currency}" };
112
+ }
113
+ let isUSD = typeof cost.currency === 'string' && cost.currency.toUpperCase() === 'USD';
114
+ let validAmount = typeof cost.amount === 'number' && Number.isFinite(cost.amount) && cost.amount >= 0;
115
+ if (validAmount && cost.amount > AGENT_TOKENS_MAX_COST_USD) {
116
+ return { error: "Invalid cost amount: must be <= " + AGENT_TOKENS_MAX_COST_USD };
117
+ }
118
+ if (isUSD && validAmount) {
119
+ return finiteCounted(ceilCounted(cost.amount * rates.per_usd), 'cost');
120
+ }
121
+ if (typeof cost.amount === 'number' && cost.amount < 0) {
122
+ return { error: "Invalid cost amount: must be >= 0" };
123
+ }
124
+ winston.verbose("(computeAgentTokens) cost not usable, falling back to tokens: ", cost);
125
+ }
126
+
127
+ if (!hasTokens) {
128
+ return { error: "Missing tokens or a valid USD cost" };
129
+ }
130
+ return finiteCounted(ceilCounted(tokens * rates.fallback_multiplier), 'tokens');
131
+ }
132
+
133
+ module.exports = {
134
+ MODELS_MULTIPLIER: loadMultiplier(),
135
+ AGENT_TOKENS_RATES,
136
+ computeAgentTokens
137
+ }
@@ -0,0 +1,192 @@
1
+ /**
2
+ * Field names MongoDB cannot store in a block (intent): a name containing '.' or starting with '$'.
3
+ *
4
+ * An update stores such a name without complaint (block saves keep doing so), but every later
5
+ * insert of the same block (publish, fork, import, restore) is refused by the driver. These helpers
6
+ * find the name before such an insert and say, in plain words, which block and which field are wrong.
7
+ */
8
+ const AppError = require("./AppError");
9
+
10
+ // The block fields that hold free-form objects written by the editors.
11
+ const CHECKED_FIELDS = ["actions", "attributes", "reply", "form", "question", "answer"];
12
+
13
+ function isPlainObject(value) {
14
+ if (value === null || typeof value !== "object") {
15
+ return false;
16
+ }
17
+ const proto = Object.getPrototypeOf(value);
18
+ return proto === Object.prototype || proto === null;
19
+ }
20
+
21
+ function badName(key) {
22
+ if (key.includes(".")) {
23
+ return "contains '.'";
24
+ }
25
+ if (key.startsWith("$")) {
26
+ return "starts with '$'";
27
+ }
28
+ return null;
29
+ }
30
+
31
+ /**
32
+ * First invalid field name inside `value`, depth first: `{ key, problem, path }` where `path`
33
+ * is the list of keys/indexes from `value` to the object holding the name. `null` when clean.
34
+ */
35
+ function findInvalidFieldName(value, path = [], seen = new Set()) {
36
+ if (Array.isArray(value)) {
37
+ if (seen.has(value)) { return null; }
38
+ seen.add(value);
39
+ for (let i = 0; i < value.length; i++) {
40
+ const found = findInvalidFieldName(value[i], path.concat(i), seen);
41
+ if (found) { return found; }
42
+ }
43
+ return null;
44
+ }
45
+ if (!isPlainObject(value)) {
46
+ return null;
47
+ }
48
+ if (seen.has(value)) { return null; }
49
+ seen.add(value);
50
+ for (const key of Object.keys(value)) {
51
+ const problem = badName(key);
52
+ if (problem) {
53
+ return { key, problem, path };
54
+ }
55
+ const found = findInvalidFieldName(value[key], path.concat(key), seen);
56
+ if (found) { return found; }
57
+ }
58
+ return null;
59
+ }
60
+
61
+ /** "in action 3 (setattribute-v2)", "in the block attributes", ... */
62
+ function describeLocation(fields, path) {
63
+ if (!path.length) {
64
+ return "";
65
+ }
66
+ const [field, index] = path;
67
+ if (field === "actions" && typeof index === "number") {
68
+ const action = Array.isArray(fields.actions) ? fields.actions[index] : null;
69
+ const type = action && typeof action._tdActionType === "string" ? " (" + action._tdActionType + ")" : "";
70
+ return " in action " + (index + 1) + type;
71
+ }
72
+ return " in the block " + field;
73
+ }
74
+
75
+ /**
76
+ * Checks the block fields of `fields` (a block, or the fields an update writes).
77
+ * Returns `{ key, problem, path, reason }` for the first invalid name, or `null`.
78
+ */
79
+ function checkIntentFields(fields) {
80
+ if (!fields || typeof fields !== "object") {
81
+ return null;
82
+ }
83
+ for (const field of CHECKED_FIELDS) {
84
+ if (fields[field] === undefined || fields[field] === null) {
85
+ continue;
86
+ }
87
+ const found = findInvalidFieldName(fields[field], [field]);
88
+ if (found) {
89
+ found.reason = 'field name "' + found.key + '"' + describeLocation(fields, found.path) +
90
+ " " + found.problem + ", which is not allowed";
91
+ return found;
92
+ }
93
+ }
94
+ return null;
95
+ }
96
+
97
+ function blockLabel(name, intent_id) {
98
+ if (name) {
99
+ return '"' + name + '"';
100
+ }
101
+ if (intent_id) {
102
+ return "with id " + intent_id;
103
+ }
104
+ return "";
105
+ }
106
+
107
+ /** Every block of `intents` with an invalid name: `[{ intent_id, intent_display_name, field, reason }]`. */
108
+ function collectIntentErrors(intents) {
109
+ const errors = [];
110
+ (intents || []).forEach((intent) => {
111
+ const found = checkIntentFields(intent);
112
+ if (found) {
113
+ errors.push({
114
+ intent_id: intent.intent_id,
115
+ intent_display_name: intent.intent_display_name,
116
+ field: found.key,
117
+ reason: found.reason
118
+ });
119
+ }
120
+ });
121
+ return errors;
122
+ }
123
+
124
+ /** The agent's own attributes (faq_kb.attributes): `[{ agent, field, reason }]` or `[]`. */
125
+ function collectAgentErrors(agentName, attributes) {
126
+ const found = findInvalidFieldName(attributes, ["attributes"]);
127
+ if (!found) {
128
+ return [];
129
+ }
130
+ return [{
131
+ agent: agentName,
132
+ field: found.key,
133
+ reason: 'field name "' + found.key + '" in the agent attributes ' + found.problem + ", which is not allowed"
134
+ }];
135
+ }
136
+
137
+ function itemLabel(e) {
138
+ if (e.agent !== undefined) {
139
+ return "the agent settings" + (e.agent ? ' of "' + e.agent + '"' : "");
140
+ }
141
+ return blockLabel(e.intent_display_name, e.intent_id) || "a block";
142
+ }
143
+
144
+ /** "1 block could not be published — "X": field name ..." */
145
+ function summarizeErrors(errors, verb = "saved") {
146
+ const blocks = errors.filter((e) => e.agent === undefined).length;
147
+ const subjects = [];
148
+ if (blocks > 0) {
149
+ subjects.push(blocks + (blocks === 1 ? " block" : " blocks"));
150
+ }
151
+ if (blocks < errors.length) {
152
+ subjects.push("the agent settings");
153
+ }
154
+ const details = errors.map((e) => itemLabel(e) + ": " + e.reason).join("; ");
155
+ return subjects.join(" and ") + " could not be " + verb + " — " + details;
156
+ }
157
+
158
+ function whatToFix(errors) {
159
+ const blocks = errors.filter((e) => e.agent === undefined).length;
160
+ if (blocks === errors.length) {
161
+ return blocks === 1 ? "the block" : "the blocks";
162
+ }
163
+ return blocks === 0 ? "the agent settings" : "them";
164
+ }
165
+
166
+ /** Refused import, fork or restore: nothing was written. */
167
+ function invalidFlowError(errors, status = 422) {
168
+ const err = new AppError(status, "The flow could not be imported: " + summarizeErrors(errors, "stored") +
169
+ ". Fix " + whatToFix(errors) + " and try again.");
170
+ err.code = "INVALID_FIELD_NAME";
171
+ err.errors = errors;
172
+ return err;
173
+ }
174
+
175
+ /** Refused publish: nothing was written. */
176
+ function publishError(errors, status = 422) {
177
+ const err = new AppError(status, "The flow was not published: " + summarizeErrors(errors, "published") +
178
+ ". Fix " + whatToFix(errors) + " and publish again.");
179
+ err.code = "PUBLISH_BLOCKS_INVALID";
180
+ err.errors = errors;
181
+ return err;
182
+ }
183
+
184
+ module.exports = {
185
+ findInvalidFieldName,
186
+ checkIntentFields,
187
+ collectIntentErrors,
188
+ collectAgentErrors,
189
+ invalidFlowError,
190
+ publishError,
191
+ summarizeErrors
192
+ };
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Bootstrap shared by bin/quota-restore and bin/quota-reset: loads the env like app.js and connects
3
+ * to the server's mongo (DATABASE_URI / MONGODB_URI, test db when NODE_ENV=test) and Redis (CACHE_REDIS_*).
4
+ */
5
+ let dotenvPath = undefined;
6
+ if (process.env.DOTENV_PATH) {
7
+ dotenvPath = process.env.DOTENV_PATH;
8
+ }
9
+ if (process.env.LOAD_DOTENV_SUBFOLDER) {
10
+ dotenvPath = __dirname + '/../confenv/.env';
11
+ }
12
+ require('dotenv').config({ path: dotenvPath });
13
+
14
+ const mongoose = require('mongoose');
15
+ const config = require('../config/database');
16
+ const { TdCache } = require('./TdCache');
17
+
18
+ async function connect() {
19
+ let databaseUri = process.env.DATABASE_URI || process.env.MONGODB_URI || config.database;
20
+ if (process.env.NODE_ENV == 'test') {
21
+ databaseUri = config.databasetest;
22
+ }
23
+ mongoose.set('useCreateIndex', true);
24
+ mongoose.set('useFindAndModify', false);
25
+ await mongoose.connect(databaseUri, { useNewUrlParser: true, useUnifiedTopology: process.env.MONGOOSE_UNIFIED_TOPOLOGY === 'true' });
26
+
27
+ const tdCache = new TdCache({
28
+ host: process.env.CACHE_REDIS_HOST,
29
+ port: process.env.CACHE_REDIS_PORT,
30
+ password: process.env.CACHE_REDIS_PASSWORD
31
+ });
32
+ await tdCache.connect();
33
+
34
+ return {
35
+ tdCache: tdCache,
36
+ close: async () => {
37
+ tdCache.getClient().quit();
38
+ await mongoose.disconnect();
39
+ }
40
+ };
41
+ }
42
+
43
+ module.exports = { connect };