@tiledesk/tiledesk-server 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/.circleci/config.yml +1 -1
  2. package/CHANGELOG.md +37 -0
  3. package/Dockerfile +3 -2
  4. package/Dockerfile-en +3 -2
  5. package/Dockerfile-jobs +3 -2
  6. package/Dockerfile-profiler +3 -2
  7. package/README.md +15 -0
  8. package/app.js +14 -1
  9. package/bin/quota-reset +68 -0
  10. package/bin/quota-restore +86 -0
  11. package/channels/chat21/chat21Handler.js +50 -34
  12. package/channels/chat21/chat21JoinTracker.js +43 -0
  13. package/config/database.js +5 -5
  14. package/config/native_mcp/native_mcp_servers.json +1 -0
  15. package/controllers/chatbot.controller.js +80 -27
  16. package/controllers/intents.controller.js +42 -0
  17. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-chat21-ionic.md +468 -0
  18. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-design-studio.md +1349 -0
  19. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-server.md +1694 -0
  20. package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-tybot.md +789 -0
  21. package/docs/superpowers/plans/2026-09-29-hitl-known-limits-server.md +83 -0
  22. package/docs/superpowers/plans/2026-09-29-hitl-known-limits-tybot.md +76 -0
  23. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-chat21-ionic.md +97 -0
  24. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-design-studio.md +72 -0
  25. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-server.md +223 -0
  26. package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-tybot.md +121 -0
  27. package/docs/superpowers/plans/2026-09-29-hitl-pre-branch-ports.md +65 -0
  28. package/docs/superpowers/plans/2026-09-30-flow-start-points-design-studio.md +88 -0
  29. package/docs/superpowers/plans/2026-09-30-flow-start-points-server.md +103 -0
  30. package/docs/superpowers/plans/2026-09-30-flow-start-points-tybot.md +43 -0
  31. package/docs/superpowers/plans/2026-10-01-scheduled-start-point-design-studio.md +80 -0
  32. package/docs/superpowers/plans/2026-10-01-scheduled-start-point-runtime.md +39 -0
  33. package/docs/superpowers/plans/2026-10-01-scheduled-start-point-server.md +106 -0
  34. package/docs/superpowers/specs/2026-09-15-webhook-chatbot-hitl-design.md +715 -0
  35. package/docs/superpowers/specs/2026-09-30-flow-start-points-design.md +156 -0
  36. package/docs/superpowers/specs/2026-10-01-scheduled-start-point-design.md +303 -0
  37. package/docs/superpowers/todo/2026-09-14-webhook-chatbot-shared-code-fixes.md +191 -0
  38. package/event/botEvent.js +12 -1
  39. package/event/emailEvent.js +3 -0
  40. package/event/messageEvent.js +44 -0
  41. package/jobs.js +10 -0
  42. package/jobsManager.js +19 -0
  43. package/middleware/errorHandler.js +6 -2
  44. package/models/agentPrompt.js +104 -0
  45. package/models/analyticResult.js +1 -1
  46. package/models/channelConstants.js +3 -2
  47. package/models/quotaUsage.js +71 -0
  48. package/models/request.js +6 -0
  49. package/models/requestSnapshot.js +12 -0
  50. package/models/routerLogger.js +1 -1
  51. package/models/webhook.js +44 -0
  52. package/package.json +7 -5
  53. package/pubmodules/emailNotification/requestNotification.js +14 -22
  54. package/pubmodules/events/test/eventRoute.js +7 -6
  55. package/pubmodules/rules/conciergeBot.js +9 -4
  56. package/routes/agentPrompts.js +132 -0
  57. package/routes/auth.js +1 -12
  58. package/routes/campaigns.js +0 -13
  59. package/routes/dataTable.js +0 -3
  60. package/routes/faq_kb.js +12 -1
  61. package/routes/messagesRoot.js +0 -1
  62. package/routes/quotes.js +38 -1
  63. package/routes/request.js +81 -5
  64. package/routes/users.js +0 -1
  65. package/routes/webhook.js +160 -8
  66. package/routes/webhooks.js +276 -6
  67. package/routes/widgetLoader.js +6 -3
  68. package/services/QuoteManager.js +77 -16
  69. package/services/aiService.js +1 -1
  70. package/services/chatbotService.js +55 -1
  71. package/services/departmentService.js +60 -43
  72. package/services/dolphinSchedulerClient.js +116 -0
  73. package/services/emailService.js +6 -1
  74. package/services/hitlParticipantListener.js +260 -0
  75. package/services/participantInviteService.js +302 -0
  76. package/services/participantRemoveService.js +79 -0
  77. package/services/quotaSnapshotService.js +1167 -0
  78. package/services/requestService.js +102 -144
  79. package/services/scheduledStart/dolphinBackend.js +246 -0
  80. package/services/scheduledStart/schedulerBackend.js +101 -0
  81. package/services/scheduledStart/schedulerServiceClient.js +41 -0
  82. package/services/scheduledStartPublishListener.js +186 -0
  83. package/services/scheduledStartRules.js +173 -0
  84. package/services/scheduledStartService.js +130 -0
  85. package/services/startPointCleanupListener.js +30 -0
  86. package/services/startPointService.js +158 -0
  87. package/services/testWsService.js +1 -4
  88. package/services/updateRequestSnapshotQueued.js +8 -9
  89. package/services/webhookConversationService.js +185 -0
  90. package/services/webhookSettingsValidator.js +61 -0
  91. package/template/email/checkpointReachedEmail.html +5 -0
  92. package/test/agentPromptsRoute.test.js +154 -0
  93. package/test/agentTokensQuota.js +237 -0
  94. package/test/botEventHitlFallback.js +19 -0
  95. package/test/chat21HandlerJoinWiring.js +59 -0
  96. package/test/dataTablesRoute.test.js +10 -2
  97. package/test/dateUtils.test.js +0 -3
  98. package/test/departmentService.js +8 -12
  99. package/test/{fileRoute.js → deprecated/_deprecated_fileRoute} +2 -2
  100. package/test/{kbsettingsRoute.js → deprecated/_deprecated_kbsettingsRoute} +3 -3
  101. package/test/faqRoute.js +6 -1
  102. package/test/faqkbRoute.js +14 -10
  103. package/test/faqkbV3Route.js +1 -1
  104. package/test/filepRoute.js +4 -50
  105. package/test/hitlParticipantListener.js +446 -0
  106. package/test/hitlQueueWiring.js +191 -0
  107. package/test/hitlWaitingReset.js +124 -0
  108. package/test/inviteParticipantsRoute.js +492 -0
  109. package/test/kbRoute.js +1 -1
  110. package/test/messageRoute.js +33 -21
  111. package/test/nativemcp.test.js +7 -1
  112. package/test/openaiRoute.js +2 -8
  113. package/test/publishFailLoud.test.js +414 -0
  114. package/test/quotaSnapshots.js +1044 -0
  115. package/test/removeParticipantsRoute.js +186 -0
  116. package/test/requestService.js +24 -16
  117. package/test/requestServiceRemoveParticipant.js +144 -0
  118. package/test/requestVisibility.test.js +186 -0
  119. package/test/scheduledStartFacade.js +184 -0
  120. package/test/scheduledStartHandover.js +95 -0
  121. package/test/scheduledStartPublish.js +320 -0
  122. package/test/scheduledStartRoute.js +163 -0
  123. package/test/scheduledStartRules.js +177 -0
  124. package/test/scheduledStartRun.js +299 -0
  125. package/test/scheduledStartSchedulerBackend.js +116 -0
  126. package/test/scheduledStartService.js +290 -0
  127. package/test/startPointsCleanup.js +69 -0
  128. package/test/startPointsRegression.js +169 -0
  129. package/test/startPointsRoute.js +158 -0
  130. package/test/webhookConversationRoute.js +347 -0
  131. package/test/webhookConversationService.js +261 -0
  132. package/test/webhookConversationTestUtils.js +55 -0
  133. package/utils/aiUtils.js +95 -1
  134. package/utils/intentFieldNames.js +192 -0
  135. package/utils/quotaCliUtil.js +43 -0
  136. package/utils/requestVisibility.js +355 -0
  137. package/websocket/webSocketServer.js +36 -31
  138. /package/test/{imageRoute.js → deprecated/_deprecated_imageRoute} +0 -0
@@ -0,0 +1,191 @@
1
+ # TODO — shared-code fixes found while designing webhook-started chatbot conversations
2
+
3
+ Status: **not scheduled** — optional follow-up. The webhook → chatbot conversation + "Invite human"
4
+ design (v1) must work without these; they are recorded here in case the work is wanted.
5
+
6
+ Context: design for starting conversational (`subtype: chatbot`) flows from a webhook and pausing for
7
+ human-in-the-loop answers from platform users in chat21-ionic.
8
+
9
+ ---
10
+
11
+ ## 1. `requestService.create` drops the department when `participants` are passed
12
+
13
+ - **Where:** `services/requestService.js`, `async create(request)` — the "Assignment" block.
14
+ - **Current behavior:** `dep_id = result.department._id` is only set inside
15
+ `if (participants.length === 0)`. When a caller passes explicit participants
16
+ (e.g. `participants: ['bot_<id>']`), the request is saved with `department: undefined`
17
+ and `snapshot.department` is not set, even though `departmentid` was provided and resolved
18
+ by `departmentService.getOperators`.
19
+ - **Effect:** department-based filters in the dashboard, `department_id`/`department_name` flow
20
+ variables in tybot (`TiledeskChatbotUtil` reads `message.request.department`), department
21
+ analytics and chat21 group attributes are missing for such requests.
22
+ - **Proposed fix:** when `status !== TEMP`, always set `dep_id = result.department._id`
23
+ (and `snapshot.department`), independently of whether participants were passed.
24
+ - **Callers affected:** every caller of `requestService.create` that passes `participants`
25
+ (`routes/request.js` POST `/`, `routes/message.js` POST `/` on first message, and the new
26
+ webhook conversation starter). Behavior change = requests gain a department they previously lacked.
27
+ - **v1 workaround (without this fix):** the webhook conversation starter sets `department`
28
+ with a targeted update on the created request right after `create`, before sending `/start`.
29
+ - **Tests:** create request with `participants` + `departmentid` → `department` and
30
+ `snapshot.department` populated; existing request/message route tests still pass.
31
+
32
+ ## 2. `addParticipantByRequestId` forces `hasBot = false` when a human is added
33
+
34
+ - **Where:** `services/requestService.js`, `addParticipantByRequestId(request_id, id_project, member, options)`.
35
+ - **Current behavior:** adding a non-bot member runs `request.hasBot = false; //???` even when a bot
36
+ is still in `participants` / `participantsBots`.
37
+ - **Effect:** a conversation where the bot keeps running next to an invited human is flagged as
38
+ not having a bot. Consumers of `hasBot` (indexes/jobs for closing unresponsive requests,
39
+ analytics, any `hasBot` filter) see the wrong value. Bot message delivery is NOT affected
40
+ (`event/botEvent.js` resolves the bot from `participants`).
41
+ - **Proposed fix:** recompute `request.hasBot = request.participantsBots.length > 0` after pushing
42
+ the member (same in `removeParticipantByRequestId` / `setParticipantsByRequestId` for consistency).
43
+ - **Callers affected:** `POST /:projectid/requests/:requestid/participants`, dashboard "join",
44
+ and the new invite endpoint.
45
+ - **Tests:** request with bot participant → add agent → `hasBot === true`,
46
+ `participantsBots` unchanged; request without bot → add agent → `hasBot === false`.
47
+
48
+ ## 3. (tiledesk-chatbot `main`) block jumps are swallowed by an active lock; Capture User Reply does not stop
49
+
50
+ - **Repo / base:** `tiledesk-chatbot`, branch `origin/main` (2.1.13). Already fixed on `main_pre`, not on `main`.
51
+ - **Where:**
52
+ - `tybotRoute/engine/TiledeskChatbot.js` — locked-intent branch: `if (locked_intent)` on `main`
53
+ vs `if (locked_intent && message.sender !== "_tdinternal")` on `main_pre`.
54
+ - `tybotRoute/tiledeskChatbotPlugs/directives/DirIntent.js` — on `main` block jumps are sent through
55
+ `/ext` as a message from `_tdinternal` (`sendMessageToBot`), not `/exec`.
56
+ - `tybotRoute/tiledeskChatbotPlugs/directives/DirCaptureUserReply.js` — on `main`, after locking it calls
57
+ `callback()` (flow continues) instead of `callback(true)` (stop); lock key read from
58
+ `this.reply.attributes.intent_info.intent_name` without guards.
59
+ - **Effect:** while a conversation is locked (waiting for a reply), any block jump in the same run is
60
+ treated as the user's reply. With Capture User Reply not stopping, actions placed after it in the same
61
+ block keep running and a jump among them is captured as the "answer".
62
+ - **v1 guidance (without this fix):** in HITL flows, put Capture User Reply as the LAST action of its block
63
+ and prefer Reply V2 with buttons (which stops correctly); invite the human before asking.
64
+ - **Proposed fix:** backport the `main_pre` changes (the `_tdinternal` lock bypass, Capture User Reply
65
+ `callback(true)` + guarded lock key) to `main` in a dedicated PR.
66
+ - **Tests:** locked conversation + internal jump → jump executes, lock untouched; Capture User Reply
67
+ followed by another action in the same block → second action not executed until reply.
68
+
69
+ - **Status:** Done on tiledesk-chatbot feature/webhook-chatbot-hitl 5ee755dd (engine lock bypass for `_tdinternal`, action-lock bypass for internal jumps in `DirectivesChatbotPlug`, Capture User Reply stops the block); tests in `test/conversation-lock_internal_jump_test.js`. The fix goes beyond `main_pre`: (a) `DirectivesChatbotPlug.process()` also lets `_tdinternal` messages bypass the locked action (on `main_pre` only the engine check exists, so an internal jump made while locked would run nothing); (b) Capture User Reply now ends the block after its goToIntent jump — actions placed after a Capture with goToIntent in the same block no longer run after the reply (they used to run alongside the jump). Coordinate with the `main_pre` owner so these reach upstream instead of being reverted on the next merge. Follow-up: an internal jump into a block that contains a Capture or Reply V2 while another question is locked makes that action take its "reply" path, because locks are checked by existence; Capture and `DirReplyV2.lockUnlock` should compare the locked action id with their own `_tdActionId` and treat a different id as a first pass.
70
+
71
+ ## 4. Pre-existing webhook issues found during the master re-verification (2026-09-15)
72
+
73
+ Not required by the design (the new conversation path does not reuse this code), recorded for a separate cleanup.
74
+ Lines refer to `tiledesk-server` master `5d240683` (2.24.2).
75
+
76
+ - **Bot secret leaked in the automation webhook token** — `services/webhookService.js:85-88`
77
+ `generateChatbotToken` signs `chatbot.toObject()` including `secret` (JWT payload is readable, and tybot stores
78
+ the token in flow variables). `services/BotSubscriptionNotifier.js:59-64` already deletes `secret`,
79
+ `description`, `attributes` before signing — do the same. **Security-relevant; consider prioritising.**
80
+ - **`webhookService.run` crashes when the chatbot is missing** — `services/webhookService.js:31`: `reject(...)`
81
+ without `return`, then `chatbot.url` is read on `undefined`.
82
+ - **Webhook model pre-save hook never resolves for existing documents** — `models/webhook.js:51-66`: `next()` is only
83
+ called inside `if (this.isNew)`; any `.save()` on an existing webhook hangs. Also `winston` is used at `:73`
84
+ (MONGOOSE_SYNCINDEX) without being required.
85
+ - **`/webhook/:webhook_id/dev` skips the `enabled` check and the rate limit** — `routes/webhook.js:324-357`
86
+ (production route applies both at `:287` and `:291-294`).
87
+ - **Small route bugs** — `routes/webhooks.js:73` `name: "webhook-" + this.webhook_id` (overwritten by pre-save);
88
+ `routes/webhooks.js:235` error message references undefined `chatbot_id`; `routes/webhooks.js:105`
89
+ `redis_client.set` not awaited.
90
+ - **tiledesk-chatbot `main`** — `tybotRoute/tiledeskChatbotPlugs/directives/DirIfOnlineAgentsV2.js:140` references an
91
+ undefined `intentDirective` (ReferenceError on the "no falseIntent" path); `tybotRoute/index.js` sync `/block`
92
+ has no timeout and ignores send errors (`:633-658`).
93
+
94
+ ## 5. Found during implementation (2026-09-15)
95
+
96
+ - **`models/user.js:69-75` `phone` is `unique: true` without `sparse: true`** — users saved
97
+ without a `phone` collide on the shared `null` key (`E11000 duplicate key error ... phone: null`)
98
+ once two or more such users exist and the index is built. `test/webhookConversationService.js`
99
+ and `test/hitlWaitingReset.js` use `mongoose.set('autoIndex', false)` as a test-only stopgap.
100
+ Suggested fix: add `sparse: true` to the `phone` field and migrate the existing index.
101
+ - **Circular require `event/messageEvent.js` → `models/faq_kb.js` → `event/botEvent.js` →
102
+ `event/messageEvent.js`** — `npx mocha test/requestRoute.js --exit` run alone crashes before any
103
+ test runs, on `origin/master` too; it stays latent only because of file-list/require ordering in
104
+ the suites that currently pass. Suggested fix: make one edge of the cycle a lazy `require(...)`
105
+ call inside the function that needs it instead of a top-level require.
106
+ - **Request cache read-through race** — `addParticipantByRequestId` emits `request.update` with the
107
+ pre-marker document (its own cache write is deferred via `setImmediate`), and
108
+ `populateMessageWithRequest`'s `.cache()` can then store a read taken before a later update.
109
+ This affects cache readers such as the requester `pending → open` rule; the HITL reset listener
110
+ (Ruling 9) avoids it by deciding on an atomic DB filter instead of trusting the cached request.
111
+ - **Email opt-out checks duplicated** — `services/participantInviteService.js`'s `notifyInvited`
112
+ and `pubmodules/emailNotification/requestNotification.js`'s `sendAgentEmail` each re-implement the
113
+ same project/user email opt-out logic (spec §5.4.7 asked for a shared helper); kept deferred by
114
+ Ruling 14. Suggested fix: extract a shared `isOptedOutOfAssignedEmail(project, project_user)` helper.
115
+ - **Partial invite is not atomic** — in `participantInviteService.invite()`, if
116
+ `addParticipantByRequestId` fails partway through a multi-member invite, the members added before
117
+ the failure stay participants without the `hitl_waiting` marker or notification email, and the
118
+ caller gets a 500. Deferred by Ruling 14. Suggested fix: either make the loop best-effort and
119
+ report per-member outcomes, or roll back the participants added so far on failure.
120
+ - **24 pre-existing test failures on origin/master in this local environment** (FileRoute/
121
+ ImagesRoute/KbSettingsRoute storage config, a DepartmentService missing import) — see
122
+ `.superpowers/sdd/2026-09-15-webhook-chatbot-hitl-server/task-6-report.md` for the list;
123
+ reproduce before relying on a plain `npm test` run to judge this branch.
124
+ - **Concurrent participant removals can lose one removal** — `services/requestService.js`
125
+ `removeParticipantByRequestId` loads the request, splices `participants`, and saves
126
+ (read-modify-save), so two simultaneous removals on the same conversation can each save a list
127
+ still containing the other member. Found while testing the HITL listener
128
+ (`test/hitlParticipantListener.js` feeds `onLeave` directly for the back-to-back case).
129
+ Suggested fix: an atomic `$pull` on `participants`/`participantsAgents`/`participantsBots` (or
130
+ optimistic concurrency with a version check) and re-reading the updated document for events.
131
+ The same save also writes back the whole `attributes` object read before an `await`, so a concurrent
132
+ DELETE can resurrect `hitl_invited`/`hitl_waiting` or erase `hitl_left` written by the HITL listener; the fix
133
+ should `$set` `attributes.abandoned_by_project_users.<id>` atomically instead of saving `attributes`.
134
+ Done on feature/webhook-chatbot-hitl (commit "fix(requests): remove participants atomically"): `$pull` + `$set` of the abandoned key via `findOneAndUpdate`, conditional status updates; tests in `test/requestServiceRemoveParticipant.js`.
135
+
136
+ ## 6. (tiledesk-chatbot `main`) HttpUtils logs the raw axios error, including the bot JWT
137
+
138
+ - **Where:** `tybotRoute/utils/HttpUtils.js`, `request(options, callback)` — the `.catch` handler
139
+ (`winston.error("Axios error response: ", error.response)` when the response has no body, and
140
+ `winston.error("Axios error: ", error)` when there is no response at all: timeouts, connection errors).
141
+ - **Effect:** both objects carry the request config, including `headers.Authorization` (`JWT <bot token>`),
142
+ so the bot token is written to the tybot logs. This affects every directive that calls `HttpUtils.request`
143
+ with the bot token. Responses that do have a body (e.g. JSON 4xx/5xx) only log `error.response.data` and are safe.
144
+ The `invite_human` directive no longer logs the raw error itself (tybot branch `feature/webhook-chatbot-hitl`,
145
+ `DirInviteHuman.errorSummary`), but an invite timeout (30 s) or network failure still reaches this handler.
146
+ - **Proposed fix:** log only `{ message: error.message, code: error.code, status: error.response?.status,
147
+ data: error.response?.data }` in both branches; never log `error`, `error.response`, `error.config` or
148
+ `error.request` as a whole.
149
+ - **Tests:** a request to a closed port and a request that times out — the captured log output does not
150
+ contain the Authorization header value; the callback still receives the error.
151
+ - **Security-relevant; consider prioritising.**
152
+ - **Status:** HttpUtils handler fixed on tiledesk-chatbot feature/webhook-chatbot-hitl; logger-level redaction added (see item 7). `HttpUtils.errorSummary` logs `{ message, code, status, url (no query), data }`; the callback still receives the original error; tests in `tybotRoute/test/http_utils_test.js`.
153
+
154
+ ## 7. (tiledesk-chatbot) raw error objects and secrets in logs
155
+
156
+ - **Where (found in the final review):** many directives and services still pass the raw axios/error object or a
157
+ secret-bearing value to the logger: `DirReplyV2:255`, `DirClose:27`, `DirFireTiledeskEvent:31`,
158
+ `DirMoveToUnassigned:34`, `DirAddTags:180/215/245`, `DirSetAttributeV2:207`, `DirReply:171`, `index.js:448`,
159
+ `DirMoveToAgent:36`, `DirDepartment:118/134`, `DirIfOnlineAgents:64`, `DirReplaceBotV2:81`,
160
+ `DirReplaceBotV3:80`, `DirSendEmail:74`, `DirSetConversationTags:100`, `QuotasService:25/56`, `DirAddKbContent`,
161
+ `DirAskGPTV2`, `DirAiPrompt:711`, `DirAiCondition:633`, `DirAskGPT:153` (`gptkey`), and debug-level token logs
162
+ (`TiledeskChatbot.js:330`, HTTPREQUEST dumps).
163
+ - **Effect:** an axios error carries `config.headers.Authorization` (`JWT <bot token>`) and request bodies with
164
+ provider keys.
165
+ - **Done:** `tybotRoute/utils/winston.js` now applies a redaction format to every transport: values of keys
166
+ `authorization|token|x-api-key|apikey|api_key|gptkey|password|secret` become `[REDACTED]`, and `JWT <token>` /
167
+ `Bearer <token>` in any string (messages included) become `JWT [REDACTED]` / `Bearer [REDACTED]` (copies only,
168
+ depth-limited, cycle-safe). A secret under an unlisted key name would still be logged.
169
+ - **Follow-up:** log summaries at the source (`HttpUtils.errorSummary` style) at each call site above instead of
170
+ relying on the filter.
171
+ - **Server side (also follow-up):** `routes/webhook.js:360-361` and `services/webhookService.js:66-71` forward and
172
+ log tybot's error body verbatim. That is safe now that tybot's async `/block` error response is
173
+ `{ success: false, error: <message> }` and its log is a summary, but a future tybot change could put a secret
174
+ back into that body.
175
+
176
+ ## 8. (tiledesk-chatbot main_pre only) Invite/Remove human inside a sub-agent
177
+
178
+ - **Where (found in the pre-branch ports final review):** main_pre adds sub-agents, which run in their own
179
+ `automation-request-<project>-<runId>` request while replies go to the parent conversation.
180
+ `DirInviteHuman.js` and `DirRemoveHuman.js` still use the running request id.
181
+ - **Effect:** Invite human inside a sub-agent calls `/requests/automation-request-…/participants/invite`, gets 404
182
+ and takes the "No human available" branch instead of inviting into the parent conversation. Nothing is corrupted.
183
+ - **Proposed fix:** when main_pre's sub-agent context names a parent request, invite/remove against the parent;
184
+ decide in the spec whether a sub-agent may invite at all.
185
+ - **Status:** open; exists only on `feature/webhook-chatbot-hitl-PRE` (tybot) until sub-agents reach main.
186
+
187
+ ## 9. (tiledesk-server) HITL listener in queue mode
188
+
189
+ - The listener subscribes to the `.queue` request events in the API process (`services/hitlParticipantListener.js:46-51`).
190
+ If a deployment emits those events only in a worker that does not load `app.js`, leave/reassignment and the
191
+ fallback never fire. Unverified on master and master-COLLAUDO; check the deployment's worker topology before release.
package/event/botEvent.js CHANGED
@@ -48,7 +48,7 @@ class BotEvent extends EventEmitter {
48
48
  // }
49
49
 
50
50
  if (message.sender === "system") {
51
- if (message.text && (message.text == "\\start" || message.text == "/start")) {
51
+ if (shouldForwardSystemMessage(message)) {
52
52
  winston.debug("it s a start message");
53
53
  } else {
54
54
  winston.debug("it s a message sent from system, exit");
@@ -288,4 +288,15 @@ function getBotId(message) {
288
288
 
289
289
 
290
290
 
291
+ // System messages are not forwarded to the bot, except /start and the HITL fallback (an info message
292
+ // whose attributes.action makes the flow jump to its "No human available" block).
293
+ function shouldForwardSystemMessage(message) {
294
+ if (message.text && (message.text == "\\start" || message.text == "/start")) {
295
+ return true;
296
+ }
297
+ return !!(message.attributes && message.attributes.hitl_fallback === true && message.attributes.action);
298
+ }
299
+
300
+ botEvent.shouldForwardSystemMessage = shouldForwardSystemMessage;
301
+
291
302
  module.exports = botEvent;
@@ -51,6 +51,9 @@ class EmailEvent extends EventEmitter {
51
51
  if (data.type == 'email') {
52
52
  resource_name = 'Chatbot Email'
53
53
  }
54
+ if (data.type == 'agent_tokens') {
55
+ resource_name = 'AI Agent Tokens'
56
+ }
54
57
 
55
58
  const emailService = require('../services/emailService'); // imported here to ensure that the emailService instance was already created
56
59
 
@@ -4,6 +4,7 @@ var Request = require("../models/request");
4
4
  var Message = require("../models/message");
5
5
  var Faq_kb = require("../models/faq_kb");
6
6
  var MessageConstants = require("../models/messageConstants");
7
+ var ChannelConstants = require("../models/channelConstants");
7
8
  var message2Event = require("../event/message2Event");
8
9
  var requestEvent = require("../event/requestEvent");
9
10
 
@@ -202,6 +203,49 @@ messageEvent.on('message.create.from.requester', function (messageJson) {
202
203
  );
203
204
  });
204
205
 
206
+ // HITL: when an invited human participant answers a conversation that is waiting for them,
207
+ // reopen it so the normal auto-close rules apply again (the requester rule above does not
208
+ // fire because the requester of webhook conversations is a source lead that never writes).
209
+ // messageJson.request comes from populateMessageWithRequest, which reads through the request
210
+ // cache that request.update events refresh - it can still be a moment stale right after an
211
+ // invite (participantInviteService.invite's own request.participants.join side effects are not
212
+ // awaited). So this only pre-filters on data the cache can't get stale for a *different* reason
213
+ // (the channel and the sender of the very message being processed) and lets the atomic
214
+ // findOneAndUpdate - matched directly against the DB, not the cached copy - decide whether the
215
+ // sender is really an invited agent and the request is really still waiting.
216
+ messageEvent.on('message.create', function (messageJson) {
217
+ var request = messageJson.request;
218
+ if (!request || !request.channel || request.channel.name !== ChannelConstants.WEBHOOK) return;
219
+
220
+ var sender = messageJson.sender;
221
+ if (!sender || sender === 'system' || sender.indexOf('bot_') === 0) return;
222
+
223
+ var attributes = messageJson.attributes || {};
224
+ if (attributes.subtype === 'info' && !attributes.action) return;
225
+
226
+ Request.findOneAndUpdate(
227
+ { request_id: request.request_id, id_project: request.id_project, 'attributes.hitl_waiting': true, participantsAgents: sender },
228
+ { $set: { workingStatus: 'open' }, $unset: { 'attributes.hitl_waiting': 1 } },
229
+ { new: true }
230
+ )
231
+ .populate('lead')
232
+ .populate('department')
233
+ .populate('participatingBots')
234
+ .populate('participatingAgents')
235
+ .populate({ path: 'requester', populate: { path: 'id_user' } })
236
+ .exec(function (err, updatedRequest) {
237
+ if (err) {
238
+ winston.error("Error resetting HITL waiting status to open", err);
239
+ return;
240
+ }
241
+ if (updatedRequest) {
242
+ winston.debug("HITL request workingStatus set to open", { request_id: request.request_id });
243
+ requestEvent.emit('request.workingStatus.update', { request: updatedRequest });
244
+ requestEvent.emit('request.update', updatedRequest);
245
+ }
246
+ });
247
+ });
248
+
205
249
 
206
250
 
207
251
  // // riattiva commentato per performance
package/jobs.js CHANGED
@@ -71,6 +71,13 @@ function startCoreModules() {
71
71
  // Mongoose query cache.
72
72
  require('./pubmodules/cache').cachegoose(mongoose);
73
73
 
74
+ // Forwards to the chatbots the messages created by this process (e.g. the HITL fallback message),
75
+ // as app.js does for its own: botEvent must listen to the plain message.create, so it starts
76
+ // before pubmodules/queue sets queueEnabled (message.create.queue also carries the server's
77
+ // messages, already forwarded by app.js).
78
+ botSubscriptionNotifier.start();
79
+ botEvent.listen();
80
+
74
81
  // Concierge bot.
75
82
  require('./pubmodules/rules/appRules').start();
76
83
 
@@ -103,6 +110,9 @@ function startJobListeners(tdCache) {
103
110
 
104
111
  jobsManager.listenWhatsappQueue(require('@tiledesk/tiledesk-whatsapp-jobworker'));
105
112
 
113
+ // HITL reassignment and fallback on request.participants.leave.queue / .update.queue.
114
+ jobsManager.listenHitlParticipants(require('./services/hitlParticipantListener'));
115
+
106
116
  // Cron tasks, disabled by TASK_SCHEDULER_ENABLED=false.
107
117
  jobsManager.listenScheduler(require('./pubmodules/scheduler'));
108
118
 
package/jobsManager.js CHANGED
@@ -1,4 +1,5 @@
1
1
  var winston = require('./config/winston');
2
+ var requestEvent = require('./event/requestEvent');
2
3
 
3
4
  /**
4
5
  * Starts the queued job listeners, shared by the two processes:
@@ -79,6 +80,24 @@ class JobsManager {
79
80
  });
80
81
  }
81
82
 
83
+ /**
84
+ * HITL participant listener (services/hitlParticipantListener.js).
85
+ * It must run in the process that receives its events:
86
+ * - queue disabled: the plain request.participants.* events are emitted in the server, so the server always listens,
87
+ * even with JOB_WORKER_ENABLED=true.
88
+ * - queue enabled: it listens to the *.queue events, emitted only by the process consuming RabbitMQ,
89
+ * i.e. the server without a job worker, or the worker (jobs.js).
90
+ * Call it after pubmodules/queue is loaded, so requestEvent.queueEnabled is already set.
91
+ */
92
+ listenHitlParticipants(hitlParticipantListener) {
93
+ if (this.jobWorkerEnabled == true && requestEvent.queueEnabled) {
94
+ return winston.info("JobsManager jobWorkerEnabled is enabled. Skipping HITL participants");
95
+ }
96
+ winston.info("JobsManager starting HITL participants");
97
+ this.hitlParticipantListener = hitlParticipantListener;
98
+ this.hitlParticipantListener.listen();
99
+ }
100
+
82
101
  listenMultiWorker(multiWorkerQueue) {
83
102
  this.start("MultiWorker Queue", () => {
84
103
  if (process.env.MULTI_WORKER_ENABLED === "false") {
@@ -4,10 +4,14 @@ const winston = require('../config/winston');
4
4
  function errorHandler(err, req, res, next) {
5
5
  if (err instanceof AppError) {
6
6
  winston.error(err.message, err);
7
- return res.status(err.status).json({
7
+ const body = {
8
8
  success: false,
9
9
  message: err.message
10
- });
10
+ };
11
+ if (Array.isArray(err.errors)) {
12
+ body.errors = err.errors;
13
+ }
14
+ return res.status(err.status).json(body);
11
15
  }
12
16
 
13
17
  winston.debug("err.name", err.name);
@@ -0,0 +1,104 @@
1
+ const mongoose = require('mongoose');
2
+ const Schema = mongoose.Schema;
3
+ const winston = require('../config/winston');
4
+
5
+ // The prompt an AI agent was created from, kept with the agent, the project and its author.
6
+ // Written by the dashboard right after the agent is created (Home, agents list, website landing).
7
+
8
+ // `source`: where in the dashboard the prompt was sent from. `origin`: where it was written, the
9
+ // dashboard itself or a web page (website, landing page, ...) that sent it to the dashboard.
10
+ const AGENT_PROMPT_SOURCES = ['home', 'website', 'bots_list', 'onboarding'];
11
+ const AGENT_PROMPT_ORIGIN_KINDS = ['dashboard', 'web'];
12
+ const AGENT_PROMPT_ORIGIN_URL_MAX_LENGTH = 2048;
13
+ const AGENT_PROMPT_MAX_LENGTH = 2000;
14
+ const AGENT_PROMPT_MAX_ATTACHMENTS = 5;
15
+
16
+ // Only the link of a file is kept here: the file itself lives in the file storage.
17
+ const AttachmentSchema = new Schema({
18
+ name: {
19
+ type: String,
20
+ required: true
21
+ },
22
+ type: {
23
+ type: String
24
+ },
25
+ size: {
26
+ type: Number
27
+ },
28
+ url: {
29
+ type: String,
30
+ required: true
31
+ }
32
+ }, { _id: false });
33
+
34
+ const OriginSchema = new Schema({
35
+ kind: {
36
+ type: String,
37
+ enum: AGENT_PROMPT_ORIGIN_KINDS,
38
+ default: 'dashboard'
39
+ },
40
+ // The address of the web page; only for kind 'web'.
41
+ url: {
42
+ type: String,
43
+ maxlength: AGENT_PROMPT_ORIGIN_URL_MAX_LENGTH
44
+ }
45
+ }, { _id: false });
46
+
47
+ const AgentPromptSchema = new Schema({
48
+ id_project: {
49
+ type: String,
50
+ required: true,
51
+ index: true
52
+ },
53
+ id_faq_kb: {
54
+ type: String,
55
+ required: true
56
+ },
57
+ createdBy: {
58
+ type: String,
59
+ required: true
60
+ },
61
+ text: {
62
+ type: String,
63
+ required: true,
64
+ maxlength: AGENT_PROMPT_MAX_LENGTH
65
+ },
66
+ language: {
67
+ type: String
68
+ },
69
+ source: {
70
+ type: String,
71
+ enum: AGENT_PROMPT_SOURCES,
72
+ required: true
73
+ },
74
+ origin: {
75
+ type: OriginSchema,
76
+ default: () => ({ kind: 'dashboard' })
77
+ },
78
+ attachments: {
79
+ type: [AttachmentSchema],
80
+ default: []
81
+ }
82
+ }, {
83
+ timestamps: true
84
+ });
85
+
86
+ AgentPromptSchema.index({ id_project: 1, createdAt: -1 });
87
+ AgentPromptSchema.index({ id_project: 1, id_faq_kb: 1 });
88
+ // Read across every project by the Super Admin Console: newest first, and by author.
89
+ AgentPromptSchema.index({ createdAt: -1 });
90
+ AgentPromptSchema.index({ createdBy: 1, createdAt: -1 });
91
+
92
+ const AgentPrompt = mongoose.model('agent_prompt', AgentPromptSchema);
93
+
94
+ if (process.env.MONGOOSE_SYNCINDEX) {
95
+ AgentPrompt.syncIndexes();
96
+ winston.verbose('AgentPrompt syncIndexes');
97
+ }
98
+
99
+ module.exports = AgentPrompt;
100
+ module.exports.AGENT_PROMPT_SOURCES = AGENT_PROMPT_SOURCES;
101
+ module.exports.AGENT_PROMPT_MAX_LENGTH = AGENT_PROMPT_MAX_LENGTH;
102
+ module.exports.AGENT_PROMPT_MAX_ATTACHMENTS = AGENT_PROMPT_MAX_ATTACHMENTS;
103
+ module.exports.AGENT_PROMPT_ORIGIN_KINDS = AGENT_PROMPT_ORIGIN_KINDS;
104
+ module.exports.AGENT_PROMPT_ORIGIN_URL_MAX_LENGTH = AGENT_PROMPT_ORIGIN_URL_MAX_LENGTH;
@@ -9,7 +9,7 @@ var databaseUri = process.env.DATABASE_URI || process.env.MONGODB_URI || config.
9
9
  var readPreference = process.env.ANALYTICS_READ_PREFERENCE || "primary";
10
10
  winston.info("Annalytics readPreference: " + readPreference);
11
11
 
12
- var conn = mongoose.createConnection(databaseUri, { "autoIndex": true, readPreference: readPreference});
12
+ var conn = mongoose.createConnection(databaseUri, { "useNewUrlParser": true, "autoIndex": true, readPreference: readPreference });
13
13
 
14
14
 
15
15
  var AnalyticResultSchema = new Schema({
@@ -3,6 +3,7 @@ module.exports = {
3
3
  FACEBOOK : 'facebook',
4
4
  TELEGRAM : 'telegram',
5
5
  WHATSAPP : 'whatsapp',
6
- FORM : 'form',
7
- EMAIL : 'email',
6
+ FORM : 'form',
7
+ EMAIL : 'email',
8
+ WEBHOOK : 'webhook',
8
9
  }
@@ -0,0 +1,71 @@
1
+ const mongoose = require('mongoose');
2
+ const Schema = mongoose.Schema;
3
+
4
+ /**
5
+ * "M/D/YYYY" (the date part of a quota key) -> Date at 00:00 UTC, null when malformed
6
+ */
7
+ function periodToDate(period) {
8
+ const match = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/.exec(period || '');
9
+ if (!match) {
10
+ return null;
11
+ }
12
+ return new Date(Date.UTC(Number(match[3]), Number(match[1]) - 1, Number(match[2])));
13
+ }
14
+
15
+ /**
16
+ * Durable copy of the quota usage counters kept in Redis
17
+ * (keys "quotes:<type>:<id_project>:<period>", see QuoteManager.generateKey).
18
+ * Written by the periodic snapshot (services/quotaSnapshotService.js) with $max on value,
19
+ * read to restore a counter when its Redis key is missing or Redis was wiped.
20
+ */
21
+ const QuotaUsageSchema = new Schema(
22
+ {
23
+ id_project: {
24
+ type: String,
25
+ required: true
26
+ },
27
+ type: {
28
+ type: String,
29
+ required: true
30
+ },
31
+ // the date part of the Redis key, exactly as written there (e.g. "10/31/2026")
32
+ period: {
33
+ type: String,
34
+ required: true
35
+ },
36
+ // period parsed as a date, to query the recent periods
37
+ periodDate: {
38
+ type: Date
39
+ },
40
+ value: {
41
+ type: Number,
42
+ default: 0
43
+ },
44
+ // ":notify:<pct>" flags already set for this counter: { "<pct>": expiresAt }
45
+ notified: {
46
+ type: Schema.Types.Mixed
47
+ },
48
+ // last time the backup changed
49
+ updatedAt: {
50
+ type: Date
51
+ }
52
+ },
53
+ {
54
+ collection: 'quota_usages'
55
+ }
56
+ );
57
+
58
+ QuotaUsageSchema.pre('validate', function (next) {
59
+ if (this.period && !this.periodDate) {
60
+ this.periodDate = periodToDate(this.period);
61
+ }
62
+ next();
63
+ });
64
+
65
+ QuotaUsageSchema.index({ id_project: 1, type: 1, period: 1 }, { unique: true });
66
+ QuotaUsageSchema.index({ periodDate: 1 });
67
+
68
+ const QuotaUsage = mongoose.model('QuotaUsage', QuotaUsageSchema);
69
+ QuotaUsage.periodToDate = periodToDate;
70
+
71
+ module.exports = QuotaUsage;
package/models/request.js CHANGED
@@ -482,6 +482,9 @@ RequestSchema.index({ lead: 1, id_project: 1, preflight: 1, createdAt: -1 });
482
482
 
483
483
  // suggested by atlas
484
484
  RequestSchema.index({ lead: 1, "snapshot.agents.id_user": 1, id_project: 1, preflight: 1, createdAt: -1 });
485
+ // Same lead query for the new visibility shape. One key per request, not per agent.
486
+ RequestSchema.index({ lead: 1, "snapshot.agentIds": 1, id_project: 1, preflight: 1, createdAt: -1 });
487
+ RequestSchema.index({ lead: 1, "snapshot.visibility": 1, "snapshot.visibilitySince": 1, id_project: 1, preflight: 1, createdAt: -1 });
485
488
 
486
489
 
487
490
  // suggested by atlas
@@ -496,6 +499,9 @@ RequestSchema.index({ hasBot: 1, status: 1, createdAt: 1});
496
499
  // suggested by atlas
497
500
  RequestSchema.index({ "channel.name": 1, id_project: 1, preflight: 1, "snapshot.requester.uuid_user": 1, createdAt: - 1, status: 1 })
498
501
  RequestSchema.index({ id_project: 1, preflight: 1, "snapshot.agents.id_user": 1, updatedAt: -1, draft: 1, status: 1 })
502
+ // Inbox and websocket for the new visibility shape. One key per request, not per agent.
503
+ RequestSchema.index({ id_project: 1, preflight: 1, "snapshot.agentIds": 1, updatedAt: -1, draft: 1, status: 1 })
504
+ RequestSchema.index({ id_project: 1, preflight: 1, "snapshot.visibility": 1, "snapshot.visibilitySince": 1, updatedAt: -1, draft: 1, status: 1 })
499
505
  RequestSchema.index({ id_project: 1, participants: 1, preflight: 1, updatedAt: -1, draft: 1, status: 1 })
500
506
  RequestSchema.index({ id_project: 1, preflight: 1, updatedAt: -1, draft: 1, status: 1 })
501
507
  RequestSchema.index({ id_project: 1, preflight: 1, "snapshot.requester.uuid_user": 1, createdAt: -1, status: 1 })
@@ -21,6 +21,18 @@ var RequestSnapshotSchema = new Schema({
21
21
  // select: true
22
22
  select: false
23
23
  },
24
+ // 'project': every teammate who had already joined at visibilitySince.
25
+ // 'agents': only snapshot.agentIds (the department group at routing time).
26
+ // Legacy documents have neither and use snapshot.agents instead.
27
+ visibility: {
28
+ type: String
29
+ },
30
+ visibilitySince: {
31
+ type: Date
32
+ },
33
+ agentIds: [{
34
+ type: Schema.Types.ObjectId
35
+ }],
24
36
  // participatingAgents
25
37
  availableAgentsCount: {
26
38
  type: Number,
@@ -12,7 +12,7 @@ winston.info('VisitorCounterSchema dbUrl: '+dbUrl);
12
12
  // mongoose.set('useCreateIndex', true);
13
13
  // mongoose.set('useUnifiedTopology', true);
14
14
 
15
- var conn = mongoose.createConnection(dbUrl, { "autoIndex": true });
15
+ var conn = mongoose.createConnection(dbUrl, { "useNewUrlParser": true, "autoIndex": true });
16
16
  // var conn = mongoose.connect(dbUrl, { "useNewUrlParser": true, "autoIndex": true });
17
17
 
18
18
  // db.getCollection('reqlogs').aggregate([ {$group:{_id:{id_project:"$id_project"}, "count":{$sum:1}}},{$sort:{"count":-1}}])