@tiledesk/tiledesk-server 3.0.1 → 3.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/.circleci/config.yml +1 -1
  2. package/CHANGELOG.md +47 -0
  3. package/Dockerfile +3 -2
  4. package/Dockerfile-en +3 -2
  5. package/Dockerfile-jobs +3 -2
  6. package/Dockerfile-profiler +3 -2
  7. package/README.md +15 -0
  8. package/app.js +12 -1
  9. package/bin/quota-reset +68 -0
  10. package/bin/quota-restore +86 -0
  11. package/channels/chat21/chat21Handler.js +50 -34
  12. package/channels/chat21/chat21JoinTracker.js +43 -0
  13. package/config/database.js +5 -5
  14. package/config/native_mcp/native_mcp_servers.json +1 -0
  15. package/controllers/chatbot.controller.js +80 -27
  16. package/controllers/intents.controller.js +42 -0
  17. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-chat21-ionic.md +468 -0
  18. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-design-studio.md +1349 -0
  19. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-server.md +1694 -0
  20. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-tybot.md +789 -0
  21. package/docs/superpowers/plans/2026-09-29-hitl-known-limits-server.md +83 -0
  22. package/docs/superpowers/plans/2026-09-29-hitl-known-limits-tybot.md +76 -0
  23. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-chat21-ionic.md +97 -0
  24. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-design-studio.md +72 -0
  25. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-server.md +223 -0
  26. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-tybot.md +121 -0
  27. package/docs/superpowers/plans/2026-09-29-hitl-pre-branch-ports.md +65 -0
  28. package/docs/superpowers/plans/2026-09-30-flow-start-points-design-studio.md +88 -0
  29. package/docs/superpowers/plans/2026-09-30-flow-start-points-server.md +103 -0
  30. package/docs/superpowers/plans/2026-09-30-flow-start-points-tybot.md +43 -0
  31. package/docs/superpowers/plans/2026-10-01-scheduled-start-point-design-studio.md +80 -0
  32. package/docs/superpowers/plans/2026-10-01-scheduled-start-point-runtime.md +39 -0
  33. package/docs/superpowers/plans/2026-10-01-scheduled-start-point-server.md +106 -0
  34. package/docs/superpowers/specs/2026-09-15-webhook-chatbot-hitl-design.md +715 -0
  35. package/docs/superpowers/specs/2026-09-30-flow-start-points-design.md +156 -0
  36. package/docs/superpowers/specs/2026-10-01-scheduled-start-point-design.md +303 -0
  37. package/docs/superpowers/todo/2026-09-14-webhook-chatbot-shared-code-fixes.md +191 -0
  38. package/event/botEvent.js +12 -1
  39. package/event/emailEvent.js +3 -0
  40. package/event/messageEvent.js +44 -0
  41. package/jobs.js +10 -0
  42. package/jobsManager.js +19 -0
  43. package/middleware/errorHandler.js +6 -2
  44. package/models/analyticResult.js +1 -1
  45. package/models/channelConstants.js +3 -2
  46. package/models/dataTable.js +16 -0
  47. package/models/location.js +6 -12
  48. package/models/quotaUsage.js +71 -0
  49. package/models/request.js +71 -61
  50. package/models/requestSnapshot.js +12 -0
  51. package/models/routerLogger.js +1 -1
  52. package/models/webhook.js +44 -0
  53. package/package.json +9 -7
  54. package/pubmodules/emailNotification/requestNotification.js +14 -22
  55. package/pubmodules/events/test/eventRoute.js +7 -6
  56. package/pubmodules/rules/conciergeBot.js +9 -4
  57. package/routes/auth.js +1 -12
  58. package/routes/campaigns.js +0 -13
  59. package/routes/dataTable.js +23 -3
  60. package/routes/faq_kb.js +12 -1
  61. package/routes/messagesRoot.js +0 -1
  62. package/routes/quotes.js +38 -1
  63. package/routes/request.js +81 -5
  64. package/routes/users.js +0 -1
  65. package/routes/webhook.js +160 -8
  66. package/routes/webhooks.js +276 -6
  67. package/routes/widgetLoader.js +6 -3
  68. package/services/QuoteManager.js +77 -16
  69. package/services/aiService.js +1 -1
  70. package/services/chatbotService.js +55 -1
  71. package/services/dataTableService.js +436 -42
  72. package/services/departmentService.js +60 -43
  73. package/services/dolphinSchedulerClient.js +116 -0
  74. package/services/emailService.js +6 -1
  75. package/services/hitlParticipantListener.js +260 -0
  76. package/services/participantInviteService.js +302 -0
  77. package/services/participantRemoveService.js +79 -0
  78. package/services/quotaSnapshotService.js +1167 -0
  79. package/services/requestService.js +102 -144
  80. package/services/scheduledStart/dolphinBackend.js +246 -0
  81. package/services/scheduledStart/schedulerBackend.js +101 -0
  82. package/services/scheduledStart/schedulerServiceClient.js +41 -0
  83. package/services/scheduledStartPublishListener.js +186 -0
  84. package/services/scheduledStartRules.js +173 -0
  85. package/services/scheduledStartService.js +130 -0
  86. package/services/startPointCleanupListener.js +30 -0
  87. package/services/startPointService.js +158 -0
  88. package/services/testWsService.js +1 -4
  89. package/services/updateRequestSnapshotQueued.js +8 -9
  90. package/services/webhookConversationService.js +185 -0
  91. package/services/webhookSettingsValidator.js +61 -0
  92. package/template/email/checkpointReachedEmail.html +5 -0
  93. package/test/agentPromptsRoute.test.js +3 -2
  94. package/test/agentTokensQuota.js +237 -0
  95. package/test/botEventHitlFallback.js +19 -0
  96. package/test/chat21HandlerJoinWiring.js +59 -0
  97. package/test/dataTablesRoute.test.js +649 -2
  98. package/test/dateUtils.test.js +0 -3
  99. package/test/departmentService.js +8 -12
  100. package/test/{fileRoute.js → deprecated/_deprecated_fileRoute} +2 -2
  101. package/test/{kbsettingsRoute.js → deprecated/_deprecated_kbsettingsRoute} +3 -3
  102. package/test/faqRoute.js +6 -1
  103. package/test/faqkbRoute.js +14 -10
  104. package/test/faqkbV3Route.js +1 -1
  105. package/test/filepRoute.js +4 -50
  106. package/test/hitlParticipantListener.js +446 -0
  107. package/test/hitlQueueWiring.js +191 -0
  108. package/test/hitlWaitingReset.js +124 -0
  109. package/test/inviteParticipantsRoute.js +492 -0
  110. package/test/kbRoute.js +1 -1
  111. package/test/messageRoute.js +33 -21
  112. package/test/nativemcp.test.js +7 -1
  113. package/test/openaiRoute.js +2 -8
  114. package/test/publishFailLoud.test.js +414 -0
  115. package/test/quotaSnapshots.js +1044 -0
  116. package/test/removeParticipantsRoute.js +186 -0
  117. package/test/requestService.js +24 -16
  118. package/test/requestServiceRemoveParticipant.js +144 -0
  119. package/test/requestVisibility.test.js +186 -0
  120. package/test/scheduledStartFacade.js +184 -0
  121. package/test/scheduledStartHandover.js +95 -0
  122. package/test/scheduledStartPublish.js +320 -0
  123. package/test/scheduledStartRoute.js +163 -0
  124. package/test/scheduledStartRules.js +177 -0
  125. package/test/scheduledStartRun.js +299 -0
  126. package/test/scheduledStartSchedulerBackend.js +116 -0
  127. package/test/scheduledStartService.js +290 -0
  128. package/test/startPointsCleanup.js +69 -0
  129. package/test/startPointsRegression.js +169 -0
  130. package/test/startPointsRoute.js +158 -0
  131. package/test/webhookConversationRoute.js +347 -0
  132. package/test/webhookConversationService.js +261 -0
  133. package/test/webhookConversationTestUtils.js +55 -0
  134. package/utils/aiUtils.js +95 -1
  135. package/utils/dataTableUtils.js +99 -9
  136. package/utils/intentFieldNames.js +192 -0
  137. package/utils/quotaCliUtil.js +43 -0
  138. package/utils/requestVisibility.js +355 -0
  139. package/websocket/webSocketServer.js +36 -31
  140. /package/test/{imageRoute.js → deprecated/_deprecated_imageRoute} +0 -0
@@ -0,0 +1,261 @@
1
+ //During the test the env variable is set to test
2
+ process.env.NODE_ENV = 'test';
3
+ process.env.LOG_LEVEL = 'error';
4
+
5
+ const chai = require('chai');
6
+ const expect = chai.expect;
7
+ const config = require('../config/database');
8
+ const mongoose = require('mongoose');
9
+
10
+ // The `users` collection has a unique (non-sparse) index on the optional `phone` field
11
+ // (see models/user.js). Multiple users without a phone number collide on the shared
12
+ // `null` key. Disable autoIndex for this run so pre-existing environment state doesn't
13
+ // break unrelated tests that sign up more than one user.
14
+ mongoose.set('autoIndex', false);
15
+ mongoose.connect(config.databasetest);
16
+ require('../services/mongoose-cache-fn')(mongoose);
17
+
18
+ const userService = require('../services/userService');
19
+ const projectService = require('../services/projectService');
20
+ const faqService = require('../services/faqService');
21
+ const departmentService = require('../services/departmentService');
22
+ const webhookConversationService = require('../services/webhookConversationService');
23
+ const Request = require('../models/request');
24
+ const Message = require('../models/message');
25
+ const Lead = require('../models/lead');
26
+ const leadEvent = require('../event/leadEvent');
27
+ const Department = require('../models/department');
28
+ const { Webhook } = require('../models/webhook');
29
+ const { waitFor, uniqueEmail, PWD } = require('./webhookConversationTestUtils');
30
+
31
+ async function setup(webhookFields = {}) {
32
+ const user = await userService.signup(uniqueEmail('test-webhook-conv-service'), PWD, 'Test', 'User');
33
+ const project = await projectService.create('test-webhook-conversation-service', user._id);
34
+ const chatbot = await faqService.create(project._id, user._id, { name: 'hitl-bot', type: 'tilebot', language: 'en', template: 'blank' });
35
+ const webhook = await new Webhook(Object.assign({
36
+ id_project: project._id.toString(),
37
+ chatbot_id: chatbot._id.toString(),
38
+ block_id: 'start-intent-id',
39
+ mode: 'conversation',
40
+ start_points: [{ type: 'webhook', block_id: 'webhook-box-id', enabled: true, mapping: { source_name: 'Grafana' } }]
41
+ }, webhookFields)).save();
42
+ return { user, project, chatbot, webhook };
43
+ }
44
+
45
+ describe('WebhookConversationService', function () {
46
+
47
+ this.timeout(20000);
48
+
49
+ it('creates a lead, a bot-assigned request in the default department and a /start message', async () => {
50
+ const { project, chatbot, webhook } = await setup();
51
+ const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
52
+
53
+ const result = await webhookConversationService.start(webhook, chatbot, { alert: 'KubePodCrashLooping', webhook_http_method: 'POST' }, {});
54
+
55
+ expect(result.request_id.startsWith('support-group-' + project._id + '-')).to.equal(true);
56
+
57
+ const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
58
+ expect(request.participants).to.deep.equal(['bot_' + chatbot._id.toString()]);
59
+ expect(request.participantsBots).to.deep.equal([chatbot._id.toString()]);
60
+ expect(request.hasBot).to.equal(true);
61
+ expect(request.status).to.equal(200);
62
+ expect(request.channel.name).to.equal('webhook');
63
+ expect(request.department.toString()).to.equal(defaultDepartment.id);
64
+ expect(request.first_text).to.equal('/start');
65
+ expect(request.attributes.webhook_id).to.equal(webhook.webhook_id);
66
+ expect(request.attributes.payload.alert).to.equal('KubePodCrashLooping');
67
+ expect(request.draft).to.not.equal(true);
68
+
69
+ const lead = await Lead.findById(request.lead).lean().exec();
70
+ expect(lead.lead_id).to.equal('webhook_' + webhook.webhook_id);
71
+ expect(lead.fullname).to.equal('Grafana');
72
+
73
+ const startMessage = await waitFor(() => Message.findOne({ recipient: result.request_id, text: '/start' }).lean().exec());
74
+ expect(startMessage.sender).to.equal('system');
75
+ expect(startMessage.attributes.subtype).to.equal('info');
76
+ expect(startMessage.attributes.action).to.equal('#webhook-box-id');
77
+ });
78
+
79
+ it('names the lead after the chatbot when the start point mapping is empty', async () => {
80
+ const { project, chatbot, webhook } = await setup({
81
+ start_points: [{ type: 'webhook', block_id: 'webhook-box-id', enabled: true, mapping: {} }]
82
+ });
83
+ const result = await webhookConversationService.start(webhook, chatbot, { alert: 'x' }, {});
84
+ const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
85
+ const lead = await Lead.findById(request.lead).lean().exec();
86
+ expect(lead.fullname).to.equal(chatbot.name);
87
+ expect(chatbot.name).to.equal('hitl-bot');
88
+ });
89
+
90
+ it('renames the existing webhook lead when the mapping source name changes', async () => {
91
+ const { chatbot, webhook } = await setup();
92
+ await webhookConversationService.start(webhook, chatbot, {}, {});
93
+ webhook.start_points[0].mapping = { source_name: 'Prometheus' };
94
+
95
+ const emitted = [];
96
+ const originalEmit = leadEvent.emit;
97
+ leadEvent.emit = function (name) {
98
+ emitted.push(name);
99
+ return originalEmit.apply(this, arguments);
100
+ };
101
+ let result;
102
+ try {
103
+ result = await webhookConversationService.start(webhook, chatbot, {}, {});
104
+ } finally {
105
+ leadEvent.emit = originalEmit;
106
+ }
107
+
108
+ const request = await Request.findOne({ request_id: result.request_id }).lean().exec();
109
+ const lead = await Lead.findById(request.lead).lean().exec();
110
+ expect(lead.fullname).to.equal('Prometheus');
111
+ expect(request.snapshot.lead.fullname).to.equal('Prometheus');
112
+ // no lead.update: chat21Handler would retitle every past conversation of the shared lead
113
+ expect(emitted.filter((name) => name.startsWith('lead.'))).to.deep.equal([]);
114
+ });
115
+
116
+ it('rejects a start without an enabled webhook start point with 422', async () => {
117
+ const { chatbot, webhook } = await setup({ start_points: [] });
118
+ let error;
119
+ try {
120
+ await webhookConversationService.start(webhook, chatbot, {}, {});
121
+ } catch (err) {
122
+ error = err;
123
+ }
124
+ expect(error.status).to.equal(422);
125
+ expect(error.message).to.equal('Webhook start is not enabled');
126
+ });
127
+
128
+ describe('resolveChatbotDepartment', () => {
129
+ it('picks the non-default department whose bot is the chatbot', async () => {
130
+ const { project, chatbot, webhook } = await setup();
131
+ const dep = await departmentService.create('Ops', project._id.toString(), 'assigned', 'system');
132
+ await Department.updateOne({ _id: dep._id }, { $set: { id_bot: chatbot._id.toString() } });
133
+
134
+ const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
135
+ expect(department._id.toString()).to.equal(dep._id.toString());
136
+
137
+ const result = await webhookConversationService.start(webhook, chatbot, {}, {});
138
+ const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
139
+ expect(request.department.toString()).to.equal(dep._id.toString());
140
+ });
141
+
142
+ it('prefers the default department when the chatbot is attached to it and to another one', async () => {
143
+ const { project, chatbot } = await setup();
144
+ const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
145
+ const dep = await departmentService.create('Ops', project._id.toString(), 'assigned', 'system');
146
+ await Department.updateOne({ _id: dep._id }, { $set: { id_bot: chatbot._id.toString() } });
147
+ await Department.updateOne({ _id: defaultDepartment._id }, { $set: { id_bot: chatbot._id.toString() } });
148
+
149
+ const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
150
+ expect(department._id.toString()).to.equal(defaultDepartment._id.toString());
151
+ });
152
+
153
+ it('picks the oldest attached department when several non-default ones have the chatbot', async () => {
154
+ const { project, chatbot } = await setup();
155
+ const first = await departmentService.create('Ops 1', project._id.toString(), 'assigned', 'system');
156
+ await new Promise((resolve) => setTimeout(resolve, 20));
157
+ const second = await departmentService.create('Ops 2', project._id.toString(), 'assigned', 'system');
158
+ await Department.updateMany({ _id: { $in: [first._id, second._id] } }, { $set: { id_bot: chatbot._id.toString() } });
159
+
160
+ const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
161
+ expect(department._id.toString()).to.equal(first._id.toString());
162
+ });
163
+
164
+ it('ignores inactive departments (hidden: status 0 or -1) that carry the chatbot', async () => {
165
+ const { project, chatbot } = await setup();
166
+ const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
167
+ const hidden = await departmentService.create('Hidden', project._id.toString(), 'assigned', 'system');
168
+ await new Promise((resolve) => setTimeout(resolve, 20));
169
+ const widgetHidden = await departmentService.create('Widget hidden', project._id.toString(), 'assigned', 'system');
170
+ await new Promise((resolve) => setTimeout(resolve, 20));
171
+ const active = await departmentService.create('Active', project._id.toString(), 'assigned', 'system');
172
+ await Department.updateMany({ _id: { $in: [hidden._id, widgetHidden._id, active._id] } }, { $set: { id_bot: chatbot._id.toString() } });
173
+ await Department.updateOne({ _id: hidden._id }, { $set: { status: -1 } });
174
+ await Department.updateOne({ _id: widgetHidden._id }, { $set: { status: 0 } });
175
+
176
+ let department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
177
+ expect(department._id.toString()).to.equal(active._id.toString());
178
+
179
+ // only inactive ones left: the project default
180
+ await Department.updateOne({ _id: active._id }, { $set: { status: -1 } });
181
+ department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
182
+ expect(department._id.toString()).to.equal(defaultDepartment._id.toString());
183
+ });
184
+
185
+ it('skips a hidden default department carrying the chatbot in favour of an active attached one', async () => {
186
+ const { project, chatbot } = await setup();
187
+ const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
188
+ const dep = await departmentService.create('Ops', project._id.toString(), 'assigned', 'system');
189
+ await Department.updateOne({ _id: dep._id }, { $set: { id_bot: chatbot._id.toString() } });
190
+ await Department.updateOne({ _id: defaultDepartment._id }, { $set: { id_bot: chatbot._id.toString(), status: -1 } });
191
+
192
+ const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
193
+ expect(department._id.toString()).to.equal(dep._id.toString());
194
+ });
195
+
196
+ it('falls back to the project default department when the chatbot is not attached', async () => {
197
+ const { project, chatbot } = await setup();
198
+ const defaultDepartment = await departmentService.getDefaultDepartment(project._id);
199
+ const other = await departmentService.create('Other', project._id.toString(), 'assigned', 'system');
200
+ await Department.updateOne({ _id: other._id }, { $set: { id_bot: '5f0000000000000000000000' } });
201
+
202
+ const department = await webhookConversationService.resolveChatbotDepartment(project._id.toString(), chatbot._id.toString());
203
+ expect(department._id.toString()).to.equal(defaultDepartment._id.toString());
204
+ });
205
+ });
206
+
207
+ it('marks the conversation start type as webhook', async () => {
208
+ const { project, chatbot, webhook } = await setup();
209
+ const result = await webhookConversationService.start(webhook, chatbot, { alert: 'x' }, {});
210
+ const request = await Request.findOne({ request_id: result.request_id, id_project: project._id.toString() }).lean().exec();
211
+ expect(request.attributes.start_type).to.equal('webhook');
212
+ });
213
+
214
+ it('uses the given request id and marks the request as draft in dev mode', async () => {
215
+ const { user, project } = await setup();
216
+ const chatbotInProject = await faqService.create(project._id, user._id, { name: 'hitl-bot-2', type: 'tilebot', language: 'en', template: 'blank' });
217
+ const devWebhook = await new Webhook({
218
+ id_project: project._id.toString(),
219
+ chatbot_id: chatbotInProject._id.toString(),
220
+ block_id: 'start-intent-id',
221
+ mode: 'conversation',
222
+ start_points: [{ type: 'webhook', block_id: 'webhook-box-id' }]
223
+ }).save();
224
+ const request_id = webhookConversationService.newRequestId(project._id.toString());
225
+
226
+ const result = await webhookConversationService.start(devWebhook, chatbotInProject, { alert: 'dev' }, { dev: true, request_id: request_id });
227
+
228
+ expect(result.request_id).to.equal(request_id);
229
+ const request = await Request.findOne({ request_id: request_id }).lean().exec();
230
+ expect(request.draft).to.equal(true);
231
+ expect(request.attributes.sourcePage).to.include('&td_draft=true');
232
+ });
233
+
234
+ describe('sanitizePayloadKeys', () => {
235
+ it('rewrites dotted and leading-$ object keys recursively, keeping values, later key wins on collision', () => {
236
+ const result = webhookConversationService.sanitizePayloadKeys({
237
+ labels: { 'app.kubernetes.io/name': 'checkout-api' },
238
+ '$op': 1,
239
+ items: [{ 'x.y': 1 }],
240
+ keep: 'a.b'
241
+ });
242
+
243
+ expect(result).to.deep.equal({
244
+ labels: { 'app_kubernetes_io/name': 'checkout-api' },
245
+ _op: 1,
246
+ items: [{ x_y: 1 }],
247
+ keep: 'a.b'
248
+ });
249
+ });
250
+ });
251
+
252
+ it('sanitizes dotted payload keys (Kubernetes/Grafana labels) before storing the request', async () => {
253
+ const { chatbot, webhook } = await setup();
254
+ const payload = { commonLabels: { 'app.kubernetes.io/name': 'checkout-api' } };
255
+
256
+ const result = await webhookConversationService.start(webhook, chatbot, payload, {});
257
+
258
+ const request = await Request.findOne({ request_id: result.request_id, id_project: webhook.id_project.toString() }).lean().exec();
259
+ expect(request.attributes.payload.commonLabels['app_kubernetes_io/name']).to.equal('checkout-api');
260
+ });
261
+ });
@@ -0,0 +1,55 @@
1
+ const chai = require('chai');
2
+ const chaiHttp = require('chai-http');
3
+ chai.use(chaiHttp);
4
+
5
+ const userService = require('../services/userService');
6
+ const projectService = require('../services/projectService');
7
+ const Faq = require('../models/faq');
8
+
9
+ const PWD = 'pwd';
10
+
11
+ async function waitFor(fn, { timeout = 5000, interval = 100 } = {}) {
12
+ const start = Date.now();
13
+ while (Date.now() - start < timeout) {
14
+ const value = await fn();
15
+ if (value) {
16
+ return value;
17
+ }
18
+ await new Promise((resolve) => setTimeout(resolve, interval));
19
+ }
20
+ throw new Error('waitFor timeout after ' + timeout + 'ms');
21
+ }
22
+
23
+ function uniqueEmail(prefix) {
24
+ return prefix + '-' + Date.now() + '-' + Math.floor(Math.random() * 100000) + '@email.com';
25
+ }
26
+
27
+ /**
28
+ * Creates a user, a project and a tilebot chatbot through the API.
29
+ * The blank template's intents are saved asynchronously by faqService, so the
30
+ * start intent is awaited when waitStart is true.
31
+ */
32
+ async function setupProjectWithBot(server, { subtype, waitStart = true } = {}) {
33
+ const email = uniqueEmail('test-webhook-conv');
34
+ const user = await userService.signup(email, PWD, 'Test Firstname', 'Test lastname');
35
+ const project = await projectService.create('test-webhook-conversation', user._id);
36
+
37
+ const body = { name: 'hitl-bot', type: 'tilebot', language: 'en', template: 'blank' };
38
+ if (subtype) {
39
+ body.subtype = subtype;
40
+ }
41
+ const res = await chai.request(server).post('/' + project._id + '/faq_kb').auth(email, PWD).send(body);
42
+ if (res.status !== 200) {
43
+ throw new Error('Chatbot creation failed with status ' + res.status);
44
+ }
45
+ const chatbot_id = res.body._id;
46
+
47
+ let startIntent = null;
48
+ if (waitStart) {
49
+ startIntent = await waitFor(() => Faq.findOne({ id_faq_kb: chatbot_id, intent_display_name: 'start' }).lean().exec());
50
+ }
51
+
52
+ return { email, pwd: PWD, user, project, chatbot_id, startIntent };
53
+ }
54
+
55
+ module.exports = { PWD, waitFor, uniqueEmail, setupProjectWithBot };
package/utils/aiUtils.js CHANGED
@@ -40,4 +40,98 @@ function loadMultiplier() {
40
40
  return models;
41
41
  }
42
42
 
43
- module.exports = { MODELS_MULTIPLIER: loadMultiplier() }
43
+ const DEFAULT_AGENT_TOKENS_PER_USD = 1000000;
44
+ const DEFAULT_AGENT_TOKENS_FALLBACK_MULTIPLIER = 1;
45
+
46
+ function loadPositiveNumber(name, defaultValue) {
47
+ let raw = process.env[name];
48
+ if (raw === undefined || raw === '') {
49
+ return defaultValue;
50
+ }
51
+ let value = Number(raw);
52
+ if (!Number.isFinite(value) || value <= 0) {
53
+ winston.warn("Invalid " + name + " value '" + raw + "': using default " + defaultValue);
54
+ return defaultValue;
55
+ }
56
+ return value;
57
+ }
58
+
59
+ function loadAgentTokensRates() {
60
+ let rates = {
61
+ per_usd: loadPositiveNumber('AGENT_TOKENS_PER_USD', DEFAULT_AGENT_TOKENS_PER_USD),
62
+ fallback_multiplier: loadPositiveNumber('AGENT_TOKENS_FALLBACK_MULTIPLIER', DEFAULT_AGENT_TOKENS_FALLBACK_MULTIPLIER)
63
+ }
64
+ winston.info("Agent tokens rates: ", rates);
65
+ return rates;
66
+ }
67
+
68
+ const AGENT_TOKENS_RATES = loadAgentTokensRates();
69
+
70
+ // Sanity caps for a single usage report
71
+ const AGENT_TOKENS_MAX_TOKENS = 1e9;
72
+ const AGENT_TOKENS_MAX_COST_USD = 1e4;
73
+
74
+ // ceil, ignoring float noise below 1/1000 of a token (e.g. 3e-6 * 1e6 = 3.0000000000000004)
75
+ function ceilCounted(value) {
76
+ return Math.ceil(Math.round(value * 1e3) / 1e3);
77
+ }
78
+
79
+ function finiteCounted(counted, mode) {
80
+ if (!Number.isFinite(counted)) {
81
+ return { error: "Invalid usage: the counted agent tokens are not a finite number" };
82
+ }
83
+ return { counted: counted, mode: mode };
84
+ }
85
+
86
+ /**
87
+ * Converts an agent runtime usage report into agent tokens.
88
+ * - cost {amount, currency: 'USD'} present and valid -> ceil(amount * per_usd)
89
+ * - otherwise -> ceil(tokens * fallback_multiplier)
90
+ * tokens > 1e9, cost.amount > 1e4 USD or a non finite result are rejected.
91
+ * Returns { counted, mode } or { error } for an invalid body.
92
+ */
93
+ function computeAgentTokens(data, rates) {
94
+ rates = rates || AGENT_TOKENS_RATES;
95
+ if (!data || typeof data !== 'object') {
96
+ return { error: "Missing body" };
97
+ }
98
+
99
+ let tokens = data.tokens;
100
+ let hasTokens = tokens !== undefined && tokens !== null;
101
+ if (hasTokens && (typeof tokens !== 'number' || !Number.isFinite(tokens) || tokens < 0)) {
102
+ return { error: "Invalid tokens: must be a finite number >= 0" };
103
+ }
104
+ if (hasTokens && tokens > AGENT_TOKENS_MAX_TOKENS) {
105
+ return { error: "Invalid tokens: must be <= " + AGENT_TOKENS_MAX_TOKENS };
106
+ }
107
+
108
+ let cost = data.cost;
109
+ if (cost !== undefined && cost !== null) {
110
+ if (typeof cost !== 'object') {
111
+ return { error: "Invalid cost: must be an object {amount, currency}" };
112
+ }
113
+ let isUSD = typeof cost.currency === 'string' && cost.currency.toUpperCase() === 'USD';
114
+ let validAmount = typeof cost.amount === 'number' && Number.isFinite(cost.amount) && cost.amount >= 0;
115
+ if (validAmount && cost.amount > AGENT_TOKENS_MAX_COST_USD) {
116
+ return { error: "Invalid cost amount: must be <= " + AGENT_TOKENS_MAX_COST_USD };
117
+ }
118
+ if (isUSD && validAmount) {
119
+ return finiteCounted(ceilCounted(cost.amount * rates.per_usd), 'cost');
120
+ }
121
+ if (typeof cost.amount === 'number' && cost.amount < 0) {
122
+ return { error: "Invalid cost amount: must be >= 0" };
123
+ }
124
+ winston.verbose("(computeAgentTokens) cost not usable, falling back to tokens: ", cost);
125
+ }
126
+
127
+ if (!hasTokens) {
128
+ return { error: "Missing tokens or a valid USD cost" };
129
+ }
130
+ return finiteCounted(ceilCounted(tokens * rates.fallback_multiplier), 'tokens');
131
+ }
132
+
133
+ module.exports = {
134
+ MODELS_MULTIPLIER: loadMultiplier(),
135
+ AGENT_TOKENS_RATES,
136
+ computeAgentTokens
137
+ }
@@ -1,6 +1,23 @@
1
1
  const crypto = require('crypto');
2
2
 
3
- const COLUMN_TYPES = ['string', 'number', 'boolean', 'datetime'];
3
+ const COLUMN_TYPES = Object.freeze({
4
+ STRING: 'string',
5
+ NUMBER: 'number',
6
+ BOOLEAN: 'boolean',
7
+ DATETIME: 'datetime',
8
+ TTL: 'ttl',
9
+ });
10
+
11
+ const COLUMN_TYPE_VALUES = Object.freeze(Object.values(COLUMN_TYPES));
12
+
13
+ // One year: well beyond any sensible retention, and far from the point where
14
+ // createdAt plus the ttl stops being a representable date.
15
+ const MAX_TTL_SECONDS = 365 * 24 * 60 * 60;
16
+
17
+ function isValidColumnType(type) {
18
+ return COLUMN_TYPE_VALUES.indexOf(type) > -1;
19
+ }
20
+
4
21
  const MATCH_MODES = ['all', 'any'];
5
22
  const FORBIDDEN_NAMES = ['__proto__', 'constructor', 'prototype', '$where'];
6
23
 
@@ -53,7 +70,7 @@ function normalizeSchemaForCreate(input) {
53
70
 
54
71
  const name = typeof col.name === 'string' ? col.name.trim() : '';
55
72
  assertValidColumnName(name, 'Invalid column at index ' + i);
56
- if (!col.type || COLUMN_TYPES.indexOf(col.type) === -1) {
73
+ if (!isValidColumnType(col.type)) {
57
74
  throw new Error('Column type is required at index ' + i);
58
75
  }
59
76
  if (names[name]) throw new Error('Column name already exists: ' + name);
@@ -66,6 +83,7 @@ function normalizeSchemaForCreate(input) {
66
83
  index: typeof col.index === 'number' ? col.index : i,
67
84
  });
68
85
  }
86
+ assertSingleTtlColumn(schema);
69
87
  return schema;
70
88
  }
71
89
 
@@ -100,7 +118,7 @@ function resolveSchemaUpdate(incoming, currentSchema) {
100
118
  assertValidColumnName(name, 'Invalid column at index ' + i);
101
119
 
102
120
  const type = col.type || prev.type;
103
- if (!type || COLUMN_TYPES.indexOf(type) === -1) {
121
+ if (!isValidColumnType(type)) {
104
122
  throw new Error('Column type is required at index ' + i);
105
123
  }
106
124
  if (names[name]) throw new Error('Column name already exists: ' + name);
@@ -131,7 +149,15 @@ function resolveSchemaUpdate(incoming, currentSchema) {
131
149
  deletes.push(remainingIds[id].name);
132
150
  }
133
151
 
134
- return { schema: newSchema, renames: renames, deletes: deletes };
152
+ assertSingleTtlColumn(newSchema);
153
+
154
+ return {
155
+ schema: newSchema,
156
+ renames: renames,
157
+ deletes: deletes,
158
+ ttlBefore: findTtlColumn(currentSchema),
159
+ ttlAfter: findTtlColumn(newSchema),
160
+ };
135
161
  }
136
162
 
137
163
  function normalizeColumnToAdd(col, nextIndex, existingSchema) {
@@ -140,11 +166,14 @@ function normalizeColumnToAdd(col, nextIndex, existingSchema) {
140
166
 
141
167
  const name = typeof col.name === 'string' ? col.name.trim() : '';
142
168
  assertValidColumnName(name);
143
- if (!col.type || COLUMN_TYPES.indexOf(col.type) === -1) throw new Error('Column type is required');
169
+ if (!isValidColumnType(col.type)) throw new Error('Column type is required');
144
170
 
145
171
  for (let i = 0; i < existingSchema.length; i++) {
146
172
  if (existingSchema[i].name === name) throw new Error('Column name already exists: ' + name);
147
173
  }
174
+ if (col.type === COLUMN_TYPES.TTL && findTtlColumn(existingSchema)) {
175
+ throw new Error('Invalid column: only one ttl column is allowed per table');
176
+ }
148
177
 
149
178
  return {
150
179
  id: generateColumnId(),
@@ -168,15 +197,32 @@ function findColumnByName(schema, name) {
168
197
  return null;
169
198
  }
170
199
 
200
+ function findTtlColumn(schema) {
201
+ if (!schema) return null;
202
+ for (let i = 0; i < schema.length; i++) {
203
+ if (schema[i].type === COLUMN_TYPES.TTL) return schema[i];
204
+ }
205
+ return null;
206
+ }
207
+
208
+ function assertSingleTtlColumn(schema) {
209
+ let found = 0;
210
+ for (let i = 0; i < schema.length; i++) {
211
+ if (schema[i].type === COLUMN_TYPES.TTL && ++found > 1) {
212
+ throw new Error('Invalid schema: only one ttl column is allowed per table');
213
+ }
214
+ }
215
+ }
216
+
171
217
  function coerceValue(value, column) {
172
218
  if (value === null || value === undefined) return value;
173
219
 
174
220
  switch (column.type) {
175
- case 'string':
221
+ case COLUMN_TYPES.STRING:
176
222
  if (typeof value !== 'string') throw new Error('Invalid value for column ' + column.name + ': expected string');
177
223
  return value;
178
224
 
179
- case 'number':
225
+ case COLUMN_TYPES.NUMBER:
180
226
  if (typeof value === 'number' && !isNaN(value)) return value;
181
227
  if (typeof value === 'string' && value.trim() !== '') {
182
228
  const num = Number(value);
@@ -184,7 +230,7 @@ function coerceValue(value, column) {
184
230
  }
185
231
  throw new Error('Invalid value for column ' + column.name + ': expected number');
186
232
 
187
- case 'boolean':
233
+ case COLUMN_TYPES.BOOLEAN:
188
234
  if (typeof value === 'boolean') return value;
189
235
  if (typeof value === 'string') {
190
236
  const lower = value.trim().toLowerCase();
@@ -193,7 +239,7 @@ function coerceValue(value, column) {
193
239
  }
194
240
  throw new Error('Invalid value for column ' + column.name + ': expected boolean');
195
241
 
196
- case 'datetime':
242
+ case COLUMN_TYPES.DATETIME:
197
243
  if (value instanceof Date && !isNaN(value.getTime())) return value;
198
244
  if (typeof value === 'string' && value.length > 0) {
199
245
  const date = new Date(value);
@@ -201,6 +247,25 @@ function coerceValue(value, column) {
201
247
  }
202
248
  throw new Error('Invalid value for column ' + column.name + ': expected datetime');
203
249
 
250
+ case COLUMN_TYPES.TTL: {
251
+ let seconds;
252
+ if (typeof value === 'number') seconds = value;
253
+ else if (typeof value === 'string' && value.trim() !== '') seconds = Number(value);
254
+ if (seconds === undefined || isNaN(seconds)) {
255
+ throw new Error('Invalid value for column ' + column.name + ': expected a number of seconds');
256
+ }
257
+ // 0 and negatives would make MongoDB drop the row on the next TTL pass
258
+ if (!Number.isInteger(seconds) || seconds <= 0) {
259
+ throw new Error('Invalid value for column ' + column.name + ': ttl must be a positive integer number of seconds');
260
+ }
261
+ // Past this the resulting date overflows and the row is refused by the
262
+ // model with a cast error, which surfaces as a 500 instead of a 400
263
+ if (seconds > MAX_TTL_SECONDS) {
264
+ throw new Error('Invalid value for column ' + column.name + ': ttl must not exceed ' + MAX_TTL_SECONDS + ' seconds');
265
+ }
266
+ return seconds;
267
+ }
268
+
204
269
  default:
205
270
  throw new Error('Unsupported column type: ' + column.type);
206
271
  }
@@ -309,6 +374,29 @@ function fillRowDataFromSchema(data, schema) {
309
374
  return result;
310
375
  }
311
376
 
377
+ /**
378
+ * Rebind conditions to a single row of a batch. In a bulk upsert the
379
+ * conditions name the columns that identify a row, and the value to match
380
+ * comes from the row itself: a fixed value would make every row of the batch
381
+ * compete for the same match.
382
+ */
383
+ function bindConditionsToRow(conditions, rowData, schema) {
384
+ return conditions.map(function (item, i) {
385
+ if (!item || typeof item !== 'object') throw new Error('Invalid condition at index ' + i);
386
+ if (NO_VALUE_OPERATORS.indexOf(item.operator) !== -1) return item;
387
+
388
+ const columnName = typeof item.column === 'string' ? item.column.trim() : '';
389
+ const colMeta = findColumnByName(schema, columnName);
390
+ if (!colMeta) throw new Error('Column ' + item.column + ' does not exist');
391
+
392
+ const value = rowData ? rowData[colMeta.name] : undefined;
393
+ if (value === undefined || value === null) {
394
+ throw new Error('a value for column ' + colMeta.name + ' is required to match an existing row');
395
+ }
396
+ return Object.assign({}, item, { value: value });
397
+ });
398
+ }
399
+
312
400
  function buildRowQueryFilter(body, schema) {
313
401
  if (body.id_row) {
314
402
  return { _id: body.id_row };
@@ -339,6 +427,7 @@ function buildUpdateSet(data, schema) {
339
427
 
340
428
  module.exports = {
341
429
  COLUMN_TYPES: COLUMN_TYPES,
430
+ findTtlColumn: findTtlColumn,
342
431
  generateColumnId: generateColumnId,
343
432
  normalizeSchemaForCreate: normalizeSchemaForCreate,
344
433
  normalizeColumnToAdd: normalizeColumnToAdd,
@@ -347,6 +436,7 @@ module.exports = {
347
436
  findColumnByName: findColumnByName,
348
437
  validateRowData: validateRowData,
349
438
  buildSearchFilter: buildSearchFilter,
439
+ bindConditionsToRow: bindConditionsToRow,
350
440
  buildRowQueryFilter: buildRowQueryFilter,
351
441
  buildUpdateSet: buildUpdateSet,
352
442
  fillRowDataFromSchema: fillRowDataFromSchema,