@futdevpro/nts-dynamo 1.15.127 → 1.15.129

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (389) hide show
  1. package/.c8rc.json +26 -26
  2. package/.copilot/patterns.json +7 -7
  3. package/.cursor/rules/__assistant_guide.mdc +30 -30
  4. package/.cursor/rules/_ag_backend-structure.mdc +85 -85
  5. package/.cursor/rules/_ag_backend.mdc +16 -16
  6. package/.cursor/rules/_ag_frontend-structure.mdc +86 -86
  7. package/.cursor/rules/_ag_frontend.mdc +39 -39
  8. package/.cursor/rules/_ag_import-rules.mdc +44 -44
  9. package/.cursor/rules/_ag_naming.mdc +115 -115
  10. package/.cursor/rules/_ag_should-be.mdc +6 -6
  11. package/.cursor/rules/ai_development_guide.md +60 -60
  12. package/.cursor/rules/cursor-rules.md +160 -160
  13. package/.cursor/rules/default-command.mdc +464 -464
  14. package/.cursor/rules/error_code_pattern.md +39 -39
  15. package/.cursor/rules/saved rule mcp server use.md +15 -15
  16. package/.dynamo/logs/cicd-pipeline/output.log +2852 -0
  17. package/.dynamo/logs/cicd-pipeline/status.json +74 -0
  18. package/.dynamo/pipeline.cicd.config.json +3 -1
  19. package/.vscode/settings.json +10 -10
  20. package/HOWTO.md +15 -15
  21. package/LICENSE +21 -21
  22. package/__documentations/nts-integration-tests-2026-03-17.md +26 -26
  23. package/__documentations/plans/BEDROCK-HYPERPLAN.md +95 -95
  24. package/_specifications/BACKLOG.md +92 -92
  25. package/_specifications/TODO.md +15 -15
  26. package/_specifications/agent.md +138 -138
  27. package/build/_models/control-models/endpoint-params.control-model.d.ts.map +1 -1
  28. package/build/_models/control-models/endpoint-params.control-model.js +9 -0
  29. package/build/_models/control-models/endpoint-params.control-model.js.map +1 -1
  30. package/build/_modules/server/errors/errors.data-service.d.ts +6 -0
  31. package/build/_modules/server/errors/errors.data-service.d.ts.map +1 -1
  32. package/build/_modules/server/errors/errors.data-service.js +29 -3
  33. package/build/_modules/server/errors/errors.data-service.js.map +1 -1
  34. package/build/_services/core/event-loop-diag.service.d.ts +6 -0
  35. package/build/_services/core/event-loop-diag.service.d.ts.map +1 -1
  36. package/build/_services/core/event-loop-diag.service.js +19 -1
  37. package/build/_services/core/event-loop-diag.service.js.map +1 -1
  38. package/build/_services/core/request-hunter.service.d.ts +74 -0
  39. package/build/_services/core/request-hunter.service.d.ts.map +1 -0
  40. package/build/_services/core/request-hunter.service.js +123 -0
  41. package/build/_services/core/request-hunter.service.js.map +1 -0
  42. package/build/index.d.ts +1 -0
  43. package/build/index.d.ts.map +1 -1
  44. package/build/index.js +1 -0
  45. package/build/index.js.map +1 -1
  46. package/eslint.config.js +3 -3
  47. package/nodemon.json +24 -24
  48. package/package.json +2 -2
  49. package/pnpm-workspace.yaml +5 -5
  50. package/scripts/run-coverage-tests.js +28 -28
  51. package/spec/support/helpers/spec-reporter-loader.js +359 -359
  52. package/spec/support/helpers/ts-node-helper.js +93 -93
  53. package/spec/support/jasmine.coverage.json +24 -24
  54. package/spec/support/jasmine.json +24 -24
  55. package/src/_collections/archive.util.spec.ts +57 -57
  56. package/src/_collections/archive.util.ts +18 -18
  57. package/src/_collections/atlas-default-db-options.const.ts +9 -9
  58. package/src/_collections/default-fallback-cache-max-age.const.spec.ts +11 -11
  59. package/src/_collections/default-fallback-cache-max-age.const.ts +2 -2
  60. package/src/_collections/default-not-found-page.const.spec.ts +19 -19
  61. package/src/_collections/default-not-found-page.const.ts +22 -22
  62. package/src/_collections/default-socket-path.const.spec.ts +12 -12
  63. package/src/_collections/default-socket-path.const.ts +2 -2
  64. package/src/_collections/get-environment-settings.util.spec.ts +210 -210
  65. package/src/_collections/get-environment-settings.util.ts +48 -48
  66. package/src/_collections/global-settings.const.ts +126 -126
  67. package/src/_collections/sample.env +21 -21
  68. package/src/_collections/signed-url.util.spec.ts +198 -198
  69. package/src/_collections/signed-url.util.ts +170 -170
  70. package/src/_collections/star.controller.spec.ts +224 -224
  71. package/src/_collections/star.controller.ts +129 -129
  72. package/src/_enums/data-model-type.enum.ts +14 -14
  73. package/src/_enums/data-service-function.enum.ts +24 -24
  74. package/src/_enums/predefined-data-types.enum.ts +16 -16
  75. package/src/_enums/route-security.enum.ts +12 -12
  76. package/src/_models/control-models/api-call-params.control-model.spec.ts +152 -152
  77. package/src/_models/control-models/api-call-params.control-model.ts +142 -142
  78. package/src/_models/control-models/app-ext-system-controls.control-model.spec.ts +52 -52
  79. package/src/_models/control-models/app-ext-system-controls.control-model.ts +9 -9
  80. package/src/_models/control-models/app-params.control-model.spec.ts +225 -225
  81. package/src/_models/control-models/app-params.control-model.ts +136 -136
  82. package/src/_models/control-models/app-system-controls.control-model.spec.ts +31 -31
  83. package/src/_models/control-models/app-system-controls.control-model.ts +9 -9
  84. package/src/_models/control-models/endpoint-params.control-model.spec.ts +669 -669
  85. package/src/_models/control-models/endpoint-params.control-model.ts +664 -656
  86. package/src/_models/control-models/http-settings.control-model.spec.ts +77 -77
  87. package/src/_models/control-models/http-settings.control-model.ts +37 -37
  88. package/src/_models/control-models/system-control.control-model.spec.ts +27 -27
  89. package/src/_models/control-models/system-control.control-model.ts +12 -12
  90. package/src/_models/interfaces/certification-settings.interface.ts +7 -7
  91. package/src/_models/interfaces/environment-settings.interface.ts +59 -59
  92. package/src/_models/interfaces/global-log-settings.interface.ts +171 -171
  93. package/src/_models/interfaces/global-service-settings.interface.ts +47 -47
  94. package/src/_models/interfaces/global-settings.interface.ts +286 -286
  95. package/src/_models/interfaces/routing-module-settings.interface.ts +21 -21
  96. package/src/_models/interfaces/static-client-settings.interface.spec.ts +29 -29
  97. package/src/_models/interfaces/static-client-settings.interface.ts +36 -36
  98. package/src/_models/types/db-update.type.ts +100 -100
  99. package/src/_modules/ai/_models/ai-input-interfaces.ts +117 -117
  100. package/src/_modules/ai/_models/ai-test-generation-result.interface.ts +16 -16
  101. package/src/_modules/ai/_modules/ai-mock/_services/ai-embedding-mock.service.spec.ts +115 -115
  102. package/src/_modules/ai/_modules/ai-mock/_services/ai-embedding-mock.service.ts +233 -233
  103. package/src/_modules/ai/_modules/anthropic/_services/aai-user-key.control-service.ts +138 -138
  104. package/src/_modules/ai/_modules/anthropic/index.ts +5 -5
  105. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.spec.ts +242 -242
  106. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.ts +639 -639
  107. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.spec.ts +295 -295
  108. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.ts +518 -518
  109. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.spec.ts +209 -209
  110. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.ts +85 -85
  111. package/src/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.ts +7 -7
  112. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.ts +146 -146
  113. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model.ts +162 -162
  114. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-document.data-model.ts +99 -99
  115. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-code-chunk.interface.ts +68 -68
  116. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface.ts +18 -18
  117. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-page-compare-result.interface.ts +19 -19
  118. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-document-compare-result.interface.ts +25 -25
  119. package/src/_modules/ai/_modules/document-ai/index.ts +30 -30
  120. package/src/_modules/ai/_modules/fdp-ai/_services/fdpai-user-key.control-service.ts +189 -189
  121. package/src/_modules/ai/_modules/fdp-ai/index.ts +5 -5
  122. package/src/_modules/ai/_modules/open-ai/_collections/oai-global-settings.const.ts +9 -9
  123. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests-hu.conts.ts +82 -82
  124. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests.conts.ts +75 -75
  125. package/src/_modules/ai/_modules/open-ai/_enums/oai-gpt-message-role.enum.ts +45 -45
  126. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-global-settings.interface.ts +7 -7
  127. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-gpt-message.interface.ts +7 -7
  128. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-llm-predefined-requests.interface.ts +57 -57
  129. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.ts +292 -292
  130. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.spec.ts +342 -342
  131. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.spec.ts +550 -550
  132. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.ts +630 -630
  133. package/src/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.spec.ts +332 -332
  134. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.spec.ts +462 -462
  135. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.ts +634 -634
  136. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.spec.ts +489 -489
  137. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.tools.spec.ts +173 -173
  138. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.ts +1085 -1085
  139. package/src/_modules/ai/_modules/open-ai/_services/oai-user-key.control-service.ts +157 -157
  140. package/src/_modules/ai/_services/ai-embedding-provider.registry.spec.ts +110 -110
  141. package/src/_modules/ai/_services/ai-embedding-provider.registry.ts +114 -114
  142. package/src/_modules/ai/_services/ai-embedding.service-base.spec.ts +98 -98
  143. package/src/_modules/ai/_services/ai-embedding.service-base.ts +48 -48
  144. package/src/_modules/ai/_services/ai-llm-chat.service-base.spec.ts +229 -229
  145. package/src/_modules/ai/_services/ai-llm-chat.service-base.ts +68 -68
  146. package/src/_modules/ai/_services/ai-llm.service-base.spec.ts +250 -250
  147. package/src/_modules/ai/_services/ai-llm.service-base.ts +599 -599
  148. package/src/_modules/ai/_services/ai-provider.service-base.spec.ts +158 -158
  149. package/src/_modules/ai/_services/ai-user-key.service-base.ts +59 -59
  150. package/src/_modules/ai/_services/lmstudio-embedding.control-service.spec.ts +197 -197
  151. package/src/_modules/ai/_services/lmstudio-embedding.control-service.ts +371 -371
  152. package/src/_modules/ai/index.ts +33 -33
  153. package/src/_modules/assistant/_collections/ass-global-settings.const.ts +13 -13
  154. package/src/_modules/assistant/_collections/ass.util.spec.ts +176 -176
  155. package/src/_modules/assistant/_collections/ass.util.ts +50 -50
  156. package/src/_modules/assistant/_models/ass-global-settings.interface.ts +15 -15
  157. package/src/_modules/assistant/_services/ass-io.control-service.spec.ts +140 -140
  158. package/src/_modules/assistant/_services/ass-main.control-service.spec.ts +192 -192
  159. package/src/_modules/assistant/_services/ass-main.control-service.ts +107 -107
  160. package/src/_modules/bot/_collections/bot-default-commands.const.ts +12 -12
  161. package/src/_modules/bot/_collections/bot-global-settings.const.ts +39 -39
  162. package/src/_modules/bot/_models/bot-channel-wrapper.interface.ts +62 -62
  163. package/src/_modules/bot/_models/bot-command.interface.ts +8 -8
  164. package/src/_modules/bot/_models/bot-global-settings.interface.ts +96 -96
  165. package/src/_modules/bot/_models/bot-last-mention-date.interface.ts +6 -6
  166. package/src/_modules/bot/_models/bot-last-message-date.interface.ts +5 -5
  167. package/src/_modules/bot/_models/bot-user-wrapper.interface.ts +41 -41
  168. package/src/_modules/bot/_modules/discord-bot/_models/dib-platform.types.ts +9 -9
  169. package/src/_modules/bot/_modules/discord-bot/_services/dib-messaging-provider.control-service.spec.ts +431 -431
  170. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.spec.ts +160 -160
  171. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.ts +55 -55
  172. package/src/_modules/bot/_modules/dynamo-bot/_models/dyb-platform.types.ts +15 -15
  173. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.spec.ts +374 -374
  174. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.ts +447 -447
  175. package/src/_modules/bot/_modules/dynamo-bot/index.ts +15 -15
  176. package/src/_modules/bot/_modules/slack-bot/_models/slb-platform.types.ts +9 -9
  177. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.spec.ts +344 -344
  178. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.ts +197 -197
  179. package/src/_modules/bot/_modules/teams-bot/_models/teb-platform.types.ts +9 -9
  180. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.spec.ts +345 -345
  181. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.ts +197 -197
  182. package/src/_modules/bot/_services/bot-commands.control-service.spec.ts +116 -116
  183. package/src/_modules/bot/_services/bot-io.control-service.spec.ts +285 -285
  184. package/src/_modules/bot/_services/bot-main.control-service.spec.ts +208 -208
  185. package/src/_modules/bot/_services/bot-messaging-provider.service-base.spec.ts +349 -349
  186. package/src/_modules/bot/_services/bot-routines.control-service.spec.ts +111 -111
  187. package/src/_modules/custom-data/custom-data.controller.spec.ts +49 -49
  188. package/src/_modules/custom-data/custom-data.controller.ts +67 -67
  189. package/src/_modules/custom-data/custom-data.data-service.spec.ts +54 -54
  190. package/src/_modules/custom-data/custom-data.data-service.ts +21 -21
  191. package/src/_modules/custom-data/get-custom-data-routing-module.util.spec.ts +28 -28
  192. package/src/_modules/custom-data/get-custom-data-routing-module.util.ts +24 -24
  193. package/src/_modules/custom-data/index.ts +9 -9
  194. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.spec.ts +161 -161
  195. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.ts +203 -203
  196. package/src/_modules/data-readers/_models/interfaces/dynts-sqlite-reader.interface.ts +33 -33
  197. package/src/_modules/data-readers/index.ts +11 -11
  198. package/src/_modules/defaults/_collections/default-endpoints.util.ts +487 -487
  199. package/src/_modules/defaults/_models/default-user.data-model.ts +72 -72
  200. package/src/_modules/defaults/_services/default-auth.service.spec.ts +269 -269
  201. package/src/_modules/defaults/_services/default-auth.service.ts +177 -177
  202. package/src/_modules/defaults/_services/default-socket-events.service.spec.ts +42 -42
  203. package/src/_modules/defaults/_services/default-socket-events.service.ts +61 -61
  204. package/src/_modules/defaults/_services/default-user.data-service.spec.ts +187 -187
  205. package/src/_modules/defaults/_services/default-user.data-service.ts +98 -98
  206. package/src/_modules/defaults/index.ts +17 -17
  207. package/src/_modules/discord-assistant/_collections/dias-global-settings.const.ts +19 -19
  208. package/src/_modules/discord-assistant/_collections/dias.util.spec.ts +366 -366
  209. package/src/_modules/discord-assistant/_collections/dias.util.ts +132 -132
  210. package/src/_modules/discord-assistant/_models/dias-global-settings.interface.ts +19 -19
  211. package/src/_modules/discord-assistant/_models/dias-knowledge.data-model.ts +52 -52
  212. package/src/_modules/discord-assistant/_services/dias-chunk.data-service.ts +177 -177
  213. package/src/_modules/discord-assistant/_services/dias-io.control-service.spec.ts +108 -108
  214. package/src/_modules/discord-assistant/_services/dias-io.control-service.ts +69 -69
  215. package/src/_modules/discord-assistant/_services/dias-main.control-service.spec.ts +22 -22
  216. package/src/_modules/discord-assistant/_services/dias-main.control-service.ts +27 -27
  217. package/src/_modules/discord-assistant/_services/dias.service-base.spec.ts +195 -195
  218. package/src/_modules/discord-assistant/_services/dias.service-base.ts +76 -76
  219. package/src/_modules/discord-assistant/index.ts +38 -38
  220. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.spec.ts +34 -34
  221. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.ts +11 -11
  222. package/src/_modules/discord-assistant-voiced/index.ts +36 -36
  223. package/src/_modules/discord-bot/_collections/dibo-default-commands.const.ts +16 -16
  224. package/src/_modules/discord-bot/_collections/dibo-global-settings.conts.ts +55 -55
  225. package/src/_modules/discord-bot/_collections/dibo-operations.util.spec.ts +214 -214
  226. package/src/_modules/discord-bot/_collections/dibo-operations.util.ts +387 -387
  227. package/src/_modules/discord-bot/_models/dibo-command.interface.ts +12 -12
  228. package/src/_modules/discord-bot/_models/dibo-global-settings.interface.ts +98 -98
  229. package/src/_modules/discord-bot/_models/dibo-last-mention-date.inteface.ts +7 -7
  230. package/src/_modules/discord-bot/_models/dibo-last-message-date.interface.ts +6 -6
  231. package/src/_modules/discord-bot/_services/dibo-commands.control-service.spec.ts +154 -154
  232. package/src/_modules/discord-bot/_services/dibo-commands.control-service.ts +153 -153
  233. package/src/_modules/discord-bot/_services/dibo-io.control-service.spec.ts +264 -264
  234. package/src/_modules/discord-bot/_services/dibo-io.control-service.ts +306 -306
  235. package/src/_modules/discord-bot/_services/dibo-main.control-service.spec.ts +408 -408
  236. package/src/_modules/discord-bot/_services/dibo-main.control-service.ts +487 -487
  237. package/src/_modules/discord-bot/_services/dibo-routines.control-service.spec.ts +105 -105
  238. package/src/_modules/discord-bot/index.ts +36 -36
  239. package/src/_modules/local-vector-search/_enums/lvs-search-mode.enum.ts +35 -35
  240. package/src/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.ts +59 -59
  241. package/src/_modules/local-vector-search/_models/lvs-search-result.interface.ts +17 -17
  242. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.spec.ts +418 -418
  243. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.ts +276 -276
  244. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.spec.ts +480 -480
  245. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.ts +416 -416
  246. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.spec.ts +214 -214
  247. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.ts +186 -186
  248. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.spec.ts +167 -167
  249. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.ts +130 -130
  250. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.spec.ts +509 -509
  251. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.ts +323 -323
  252. package/src/_modules/local-vector-search/index.ts +16 -16
  253. package/src/_modules/logs/index.ts +11 -11
  254. package/src/_modules/mcp/_models/interfaces/dynts-mcp.interface.ts +111 -111
  255. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.spec.ts +142 -142
  256. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.ts +120 -120
  257. package/src/_modules/mcp/_services/dynts-mcp.adapter.ts +168 -168
  258. package/src/_modules/mcp/index.ts +13 -13
  259. package/src/_modules/messaging/README.md +354 -354
  260. package/src/_modules/messaging/_collections/get-messaging-routing-module.util.ts +26 -26
  261. package/src/_modules/messaging/_collections/msg-global-settings.const.ts +22 -22
  262. package/src/_modules/messaging/_collections/msg.util.spec.ts +226 -226
  263. package/src/_modules/messaging/_models/msg-global-settings.interface.ts +37 -37
  264. package/src/_modules/messaging/_services/msg-conversation.data-service.ts +146 -146
  265. package/src/_modules/messaging/_services/msg-events.service.spec.ts +219 -219
  266. package/src/_modules/messaging/_services/msg-events.service.ts +267 -267
  267. package/src/_modules/messaging/_services/msg-integration.control-service.ts +179 -179
  268. package/src/_modules/messaging/_services/msg-main.control-service.spec.ts +147 -147
  269. package/src/_modules/messaging/_services/msg-main.control-service.ts +571 -571
  270. package/src/_modules/messaging/_services/msg-message.data-service.ts +129 -129
  271. package/src/_modules/messaging/_services/msg.controller.spec.ts +201 -201
  272. package/src/_modules/messaging/index.ts +30 -30
  273. package/src/_modules/mock/app-extended-server.mock.ts +201 -201
  274. package/src/_modules/mock/app-integration-test.mock.ts +51 -51
  275. package/src/_modules/mock/app-params.mock.spec.ts +21 -21
  276. package/src/_modules/mock/app-params.mock.ts +9 -9
  277. package/src/_modules/mock/app-server.mock.ts +188 -188
  278. package/src/_modules/mock/auth-service.mock.spec.ts +47 -47
  279. package/src/_modules/mock/auth-service.mock.ts +28 -28
  280. package/src/_modules/mock/controller.mock.spec.ts +26 -26
  281. package/src/_modules/mock/controller.mock.ts +16 -16
  282. package/src/_modules/mock/data-model.mock.spec.ts +111 -111
  283. package/src/_modules/mock/data-model.mock.ts +82 -82
  284. package/src/_modules/mock/email-service-collection.mock.spec.ts +24 -24
  285. package/src/_modules/mock/email-service-collection.mock.ts +15 -15
  286. package/src/_modules/mock/email-service.mock.spec.ts +17 -17
  287. package/src/_modules/mock/email-service.mock.ts +20 -20
  288. package/src/_modules/mock/email-template.mock.html +14 -14
  289. package/src/_modules/mock/endpoint.mock.ts +91 -91
  290. package/src/_modules/mock/socket-client.mock.spec.ts +40 -40
  291. package/src/_modules/mock/socket-client.mock.ts +45 -45
  292. package/src/_modules/mock/socket-server.mock.spec.ts +44 -44
  293. package/src/_modules/mock/socket-server.mock.ts +46 -46
  294. package/src/_modules/oauth2/_routes/oauth2.controller.spec.ts +107 -107
  295. package/src/_modules/oauth2/_routes/oauth2.controller.ts +98 -98
  296. package/src/_modules/oauth2/_services/oauth2.auth-service.spec.ts +254 -254
  297. package/src/_modules/oauth2/_services/oauth2.auth-service.ts +232 -232
  298. package/src/_modules/oauth2/_services/oauth2.control-service.spec.ts +585 -585
  299. package/src/_modules/oauth2/_services/oauth2.control-service.ts +653 -653
  300. package/src/_modules/oauth2/index.ts +17 -17
  301. package/src/_modules/scheduler/_models/scheduler-job.interface.ts +35 -35
  302. package/src/_modules/scheduler/get-scheduler-routing-module.util.ts +33 -33
  303. package/src/_modules/scheduler/index.ts +8 -8
  304. package/src/_modules/scheduler/scheduler.controller.spec.ts +42 -42
  305. package/src/_modules/scheduler/scheduler.controller.ts +69 -69
  306. package/src/_modules/scheduler/scheduler.service.spec.ts +141 -141
  307. package/src/_modules/scheduler/scheduler.service.ts +176 -176
  308. package/src/_modules/scoped-config/_enums/dynts-scoped-config-level.enum.ts +22 -22
  309. package/src/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.ts +81 -81
  310. package/src/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.ts +107 -107
  311. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.spec.ts +306 -306
  312. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.ts +295 -295
  313. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.spec.ts +118 -118
  314. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.ts +105 -105
  315. package/src/_modules/scoped-config/index.ts +17 -17
  316. package/src/_modules/server/errors/errors.control-service.spec.ts +238 -238
  317. package/src/_modules/server/errors/errors.control-service.ts +100 -100
  318. package/src/_modules/server/errors/errors.controller.spec.ts +268 -268
  319. package/src/_modules/server/errors/errors.controller.ts +515 -515
  320. package/src/_modules/server/errors/errors.data-service.spec.ts +480 -480
  321. package/src/_modules/server/errors/errors.data-service.ts +30 -3
  322. package/src/_modules/server/index.ts +30 -30
  323. package/src/_modules/server/server-status/server-status-snapshot.control-service.spec.ts +70 -70
  324. package/src/_modules/server/server-status/server-status-snapshot.control-service.ts +17 -17
  325. package/src/_modules/server/server-status/server-status-snapshot.data-service.spec.ts +77 -77
  326. package/src/_modules/server/server-status/server-status-snapshot.data-service.ts +37 -37
  327. package/src/_modules/server/server-status/server-status.control-service.spec.ts +672 -672
  328. package/src/_modules/server/server-status/server-status.control-service.ts +490 -490
  329. package/src/_modules/server/server-status/server-status.controller.spec.ts +276 -276
  330. package/src/_modules/server/server-status/server-status.controller.ts +336 -336
  331. package/src/_modules/socket/_enums/socket-security.enum.ts +11 -11
  332. package/src/_modules/socket/_models/socket-client-service-params.control-model.spec.ts +32 -32
  333. package/src/_modules/socket/_models/socket-client-service-params.control-model.ts +22 -22
  334. package/src/_modules/socket/_models/socket-presence.control-model.spec.ts +164 -164
  335. package/src/_modules/socket/_models/socket-presence.control-model.ts +210 -210
  336. package/src/_modules/socket/_models/socket-server-service-params.control-model.spec.ts +69 -69
  337. package/src/_modules/socket/_services/socket-client.service.spec.ts +15 -15
  338. package/src/_modules/socket/_services/socket-client.service.ts +260 -260
  339. package/src/_modules/socket/_services/socket-server.service.spec.ts +11 -11
  340. package/src/_modules/socket/app-extended.integration.spec.ts +85 -85
  341. package/src/_modules/socket/app-extended.server.ts +648 -648
  342. package/src/_modules/socket/index.ts +42 -42
  343. package/src/_modules/test/get-test-routing-module.util.spec.ts +28 -28
  344. package/src/_modules/test/get-test-routing-module.util.ts +23 -23
  345. package/src/_modules/test/index.ts +11 -11
  346. package/src/_modules/test/test.controller.spec.ts +72 -72
  347. package/src/_modules/test/test.controller.ts +115 -115
  348. package/src/_modules/usage/get-usage-routing-module.util.ts +22 -22
  349. package/src/_modules/usage/index.ts +15 -15
  350. package/src/_modules/usage/usage.controller.spec.ts +81 -81
  351. package/src/_modules/usage/usage.controller.ts +126 -126
  352. package/src/_modules/usage/usage.data-service.spec.ts +332 -332
  353. package/src/_modules/usage/usage.data-service.ts +185 -185
  354. package/src/_services/base/api.service-base.spec.ts +125 -125
  355. package/src/_services/base/api.service-base.ts +74 -74
  356. package/src/_services/base/archive-data.service.spec.ts +209 -209
  357. package/src/_services/base/archive-data.service.ts +223 -223
  358. package/src/_services/base/data.service.spec.ts +729 -729
  359. package/src/_services/base/data.service.ts +2758 -2758
  360. package/src/_services/base/db.service.spec.ts +160 -160
  361. package/src/_services/base/db.service.ts +1698 -1698
  362. package/src/_services/base/singleton.service-base.spec.ts +28 -28
  363. package/src/_services/base/singleton.service-base.ts +24 -24
  364. package/src/_services/base/singleton.service.spec.ts +114 -114
  365. package/src/_services/base/singleton.service.ts +38 -38
  366. package/src/_services/core/api.service.spec.ts +140 -140
  367. package/src/_services/core/auth.service.spec.ts +159 -159
  368. package/src/_services/core/auth.service.ts +174 -174
  369. package/src/_services/core/email.service.spec.ts +85 -85
  370. package/src/_services/core/email.service.ts +757 -757
  371. package/src/_services/core/event-loop-diag.service.ts +23 -1
  372. package/src/_services/core/global.service.spec.ts +292 -292
  373. package/src/_services/core/global.service.ts +487 -487
  374. package/src/_services/core/memory-guard.service.spec.ts +245 -245
  375. package/src/_services/core/memory-guard.service.ts +481 -481
  376. package/src/_services/core/request-hunter.service.spec.ts +124 -0
  377. package/src/_services/core/request-hunter.service.ts +157 -0
  378. package/src/_services/core/service-collection.service.spec.ts +46 -46
  379. package/src/_services/core/service-collection.service.ts +6 -6
  380. package/src/_services/route/controller.service.spec.ts +53 -53
  381. package/src/_services/route/controller.service.ts +148 -148
  382. package/src/_services/route/routing-module.service.spec.ts +98 -98
  383. package/src/_services/route/routing-module.service.ts +330 -330
  384. package/src/_services/server/app.server.ts +1985 -1985
  385. package/src/_services/shared.static-service.spec.ts +99 -99
  386. package/src/_services/shared.static-service.ts +78 -78
  387. package/src/index.ts +113 -112
  388. package/tsconfig.app.json +12 -12
  389. package/tsconfig.json +42 -42
@@ -1,416 +1,416 @@
1
- import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
- import {
3
- DyFM_DataModel_Params,
4
- DyFM_DataProperty_Params,
5
- DyFM_DBFilter,
6
- DyFM_Metadata,
7
- } from '@futdevpro/fsm-dynamo';
8
- import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
-
10
- import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
- import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
- import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
- import {
14
- DyNTS_LVS_BM25_Corpus,
15
- DyNTS_LVS_BM25_DocScore,
16
- dyNTS_LVS_BM25_minMaxNormalize,
17
- } from './lvs-bm25.util';
18
- import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
- import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
-
21
- /**
22
- * Local Vector Data Service
23
- * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
- * instead of MongoDB vector search
25
- *
26
- * This service loads data from the database, builds an in-memory vector pool,
27
- * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
- *
29
- * @example
30
- * ```typescript
31
- * const service = new DyNTS_OAI_LocalVectorDataService(
32
- * data,
33
- * dataParams,
34
- * openAISettings,
35
- * issuer
36
- * );
37
- *
38
- * const results = await service.vectorSearch({
39
- * input: 'search query',
40
- * searchInKey: 'contentVectorized',
41
- * limit: 10,
42
- * filterBy: { category: 'docs' }
43
- * });
44
- * ```
45
- */
46
- export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
-
48
- /**
49
- * Default search mode
50
- * Cosine similarity is typically preferred for text embeddings
51
- */
52
- defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
-
54
- /**
55
- * Whether to use L2 normalization when storing vectors
56
- * Normalized vectors are more efficient for cosine similarity calculations
57
- */
58
- useL2Normalization: boolean = true;
59
-
60
- /**
61
- * Instance-based vector pool utility
62
- * Minden service instance saját vektor pool-t tart fenn
63
- */
64
- private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
-
66
- constructor(
67
- /**
68
- * Initial data, this will be used by functions on default
69
- */
70
- data: T,
71
- /**
72
- * DB data params will be used to connect to usable dbService on GlobalService
73
- */
74
- dataParams: DyFM_DataModel_Params<T>,
75
- /**
76
- * OpenAI settings
77
- */
78
- openAISettings: DyFM_OAI_Settings,
79
- /**
80
- * Initial set for issuer to be able to follow the issuer's activity
81
- */
82
- issuer: string
83
- ) {
84
- super(data, dataParams, openAISettings, issuer);
85
- }
86
-
87
- /**
88
- * Overrides the parent vectorSearch method to perform local in-memory vector search
89
- *
90
- * Process:
91
- * 1. Load data from DB using filterBy (if provided) or getAll()
92
- * 2. Extract vectorized properties from loaded data
93
- * 3. Build in-memory vector pool
94
- * 4. Vectorize input query
95
- * 5. Perform local vector search
96
- * 6. Map search results back to full data objects
97
- *
98
- * @param set Search parameters
99
- * @returns Array of data objects sorted by similarity score
100
- */
101
- override async vectorSearch(
102
- set: {
103
- input: string;
104
- /** this should be the vectorized property key */
105
- searchInKey: string;
106
- limit?: number;
107
- /**
108
- * Number of candidates that are used to find the best match
109
- * (Not used in local search, but kept for compatibility)
110
- */
111
- numberOfCandidates?: number;
112
- /**
113
- * Filter to narrow down candidates from DB before vector search
114
- */
115
- filterBy?: DyFM_DBFilter<T>;
116
- /**
117
- * Search mode (cosine similarity, L2 distance, or hybrid).
118
- * Defaults to this.defaultSearchMode.
119
- *
120
- * `hybrid` mode combines cosine similarity (vector half) with BM25
121
- * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
- */
123
- searchMode?: LVS_Search_Mode;
124
- /**
125
- * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
- * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
- */
128
- hybridWeight?: { vector: number; text: number };
129
- /**
130
- * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
- * text-search. NEM kell hogy a vectorized property legyen.
132
- */
133
- textSearchKey?: keyof T;
134
- },
135
- ): Promise<T[]> {
136
- try {
137
- if (!set.input) {
138
- throw new DyFM_Error({
139
- ...this.getDefaultErrorSettings(
140
- 'vectorSearch',
141
- new Error('input is required')
142
- ),
143
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
- });
145
- }
146
-
147
- set.limit ??= 3;
148
- const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
- const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
-
151
- // Hybrid mode korai validacio
152
- if (effectiveMode === LVS_Search_Mode.hybrid) {
153
- if (!textSearchKey) {
154
- throw new DyFM_Error({
155
- ...this.getDefaultErrorSettings(
156
- 'vectorSearch',
157
- new Error('textSearchKey is required when searchMode is hybrid'),
158
- ),
159
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
- });
161
- }
162
- if (hybridWeight) {
163
- const w: { vector: number; text: number } = hybridWeight;
164
- if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
- throw new DyFM_Error({
166
- ...this.getDefaultErrorSettings(
167
- 'vectorSearch',
168
- new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
- ),
170
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
- });
172
- }
173
- }
174
- }
175
-
176
- // Validáljuk, hogy a searchInKey létezik-e
177
- const property: DyFM_DataProperty_Params<any, T> =
178
- this.dataParams.properties[searchInKey];
179
-
180
- if (!property) {
181
- throw new DyFM_Error({
182
- ...this.getDefaultErrorSettings(
183
- 'vectorSearch',
184
- new Error(
185
- `Property "${searchInKey}" not found! ` +
186
- `while searching "${this.dataParams.dataName}" ` +
187
- `(The searchable properties are: ` +
188
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
- )
190
- ),
191
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
- __localStack: this.dataParams.stackLocation,
193
- });
194
- }
195
-
196
- // Ellenőrizzük, hogy a property vectorized-e
197
- if (!this.vectorizedProperties.some(
198
- (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
- )) {
200
- throw new DyFM_Error({
201
- ...this.getDefaultErrorSettings(
202
- 'vectorSearch',
203
- new Error(
204
- `Property "${searchInKey}" is not a vectorized property! ` +
205
- `(The vectorized properties are: ` +
206
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
- )
208
- ),
209
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
- __localStack: this.dataParams.stackLocation,
211
- });
212
- }
213
-
214
- // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
- let dataList: T[];
216
- if (filterBy) {
217
- // Ha van filterBy, akkor először szűrünk a DB-ben
218
- if (this.debugLog) {
219
- DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
- }
221
- dataList = await this.findDataList(filterBy, true);
222
- } else {
223
- // Ha nincs filterBy, akkor minden adatot betöltünk
224
- if (this.debugLog) {
225
- DyFM_Log.log(`Local vector search: loading all data...`);
226
- }
227
- dataList = await this.getAll(true);
228
- }
229
-
230
- if (dataList.length === 0) {
231
- if (this.debugLog) {
232
- DyFM_Log.log(`Local vector search: no data found`);
233
- }
234
- return [];
235
- }
236
-
237
- // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
- if (this.debugLog) {
239
- DyFM_Log.log(
240
- `Local vector search: building vector pool from ${dataList.length} items...`
241
- );
242
- }
243
-
244
- // Töröljük a korábbi pool-t
245
- this.vectorPool.clearPool();
246
-
247
- // Hozzáadjuk a vektorokat a pool-hoz
248
- const dataMap: Map<string, T> = new Map<string, T>();
249
- for (const dataItem of dataList) {
250
- if (!dataItem._id) {
251
- // Ha nincs _id, akkor kihagyjuk
252
- if (this.debugLog) {
253
- DyFM_Log.warn(
254
- `Local vector search: skipping item without _id`
255
- );
256
- }
257
- continue;
258
- }
259
-
260
- const vectorizedValue: number[] | undefined =
261
- dataItem[searchInKey] as number[] | undefined;
262
-
263
- if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
- // Ha nincs vectorized érték, akkor kihagyjuk
265
- if (this.debugLog) {
266
- DyFM_Log.warn(
267
- `Local vector search: skipping item "${dataItem._id}" ` +
268
- `without vectorized value for "${searchInKey}"`
269
- );
270
- }
271
- continue;
272
- }
273
-
274
- // Hozzáadjuk a vektort a pool-hoz
275
- this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
- dataMap.set(dataItem._id, dataItem);
277
- }
278
-
279
- if (dataMap.size === 0) {
280
- if (this.debugLog) {
281
- DyFM_Log.log(`Local vector search: no valid vectors found`);
282
- }
283
- return [];
284
- }
285
-
286
- // 3. Vectorizáljuk a query input-ot
287
- if (this.debugLog) {
288
- DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
- }
290
-
291
- const queryVector: number[] = await this.vectorize(
292
- input,
293
- property.embeddingModel
294
- );
295
-
296
- // 4. Végrehajtjuk a local vector search-t
297
- const mode: LVS_Search_Mode = effectiveMode;
298
-
299
- if (this.debugLog) {
300
- DyFM_Log.log(
301
- `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
- );
303
- }
304
-
305
- let searchResults: LVS_SearchResult[];
306
-
307
- if (mode === LVS_Search_Mode.hybrid) {
308
- // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
- const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
- queryVector,
311
- dataMap.size,
312
- LVS_Search_Mode.cosineSimilarity,
313
- );
314
-
315
- // BM25 corpus epitese a `textSearchKey` property-bol
316
- const docs: { id: string; text: string }[] = [];
317
- for (const [docId, dataItem] of dataMap) {
318
- const textValue: unknown = dataItem[textSearchKey as keyof T];
319
- docs.push({
320
- id: docId,
321
- text: typeof textValue === 'string' ? textValue : '',
322
- });
323
- }
324
- const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
- const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
- const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
-
328
- const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
- for (const s of bm25Normalized) {
330
- bm25ScoreById.set(s.id, s.score);
331
- }
332
-
333
- const wVector: number = hybridWeight?.vector ?? 0.5;
334
- const wText: number = hybridWeight?.text ?? 0.5;
335
-
336
- const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
- const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
- return {
339
- id: c.id,
340
- score: wVector * c.score + wText * bm25Score,
341
- };
342
- });
343
- merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
- searchResults = merged.slice(0, limit);
345
- } else {
346
- searchResults = this.vectorPool.search(
347
- queryVector,
348
- limit,
349
- mode,
350
- );
351
- }
352
-
353
- if (this.debugLog) {
354
- DyFM_Log.log(
355
- `Local vector search: found ${searchResults.length} results`
356
- );
357
- }
358
-
359
- // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
- const results: T[] = [];
361
- for (const searchResult of searchResults) {
362
- const dataItem: T | undefined = dataMap.get(searchResult.id);
363
- if (dataItem) {
364
- results.push(dataItem);
365
- }
366
- }
367
-
368
- // Töröljük a pool-t a memória felszabadítása érdekében
369
- this.vectorPool.clearPool();
370
-
371
- return results;
372
- } catch (error) {
373
- // Töröljük a pool-t hiba esetén is
374
- this.vectorPool.clearPool();
375
-
376
- throw new DyFM_Error({
377
- ...this.getDefaultErrorSettings('vectorSearch', error),
378
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
- });
380
- }
381
- }
382
- }
383
-
384
- /**
385
- * @example
386
- * ```typescript
387
- * // Create a local vector data service
388
- * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
- * new CodeChunk(),
390
- * codeChunk_dataParams,
391
- * openAISettings,
392
- * 'example-issuer'
393
- * );
394
- *
395
- * // Perform vector search with filterBy
396
- * const results = await localVectorService.vectorSearch({
397
- * input: 'How to implement authentication?',
398
- * searchInKey: 'chunkParentedContentVectorized',
399
- * limit: 5,
400
- * filterBy: {
401
- * // Filter by category before vector search
402
- * category: 'documentation'
403
- * },
404
- * searchMode: LVS_Search_Mode.cosineSimilarity
405
- * });
406
- *
407
- * // Results are sorted by similarity score (highest first for cosine)
408
- * console.log(`Found ${results.length} similar chunks`);
409
- *
410
- * // Access the results
411
- * for (const result of results) {
412
- * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
- * }
414
- * ```
415
- */
416
-
1
+ import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
+ import {
3
+ DyFM_DataModel_Params,
4
+ DyFM_DataProperty_Params,
5
+ DyFM_DBFilter,
6
+ DyFM_Metadata,
7
+ } from '@futdevpro/fsm-dynamo';
8
+ import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
+
10
+ import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
+ import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
+ import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
+ import {
14
+ DyNTS_LVS_BM25_Corpus,
15
+ DyNTS_LVS_BM25_DocScore,
16
+ dyNTS_LVS_BM25_minMaxNormalize,
17
+ } from './lvs-bm25.util';
18
+ import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
+ import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
+
21
+ /**
22
+ * Local Vector Data Service
23
+ * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
+ * instead of MongoDB vector search
25
+ *
26
+ * This service loads data from the database, builds an in-memory vector pool,
27
+ * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const service = new DyNTS_OAI_LocalVectorDataService(
32
+ * data,
33
+ * dataParams,
34
+ * openAISettings,
35
+ * issuer
36
+ * );
37
+ *
38
+ * const results = await service.vectorSearch({
39
+ * input: 'search query',
40
+ * searchInKey: 'contentVectorized',
41
+ * limit: 10,
42
+ * filterBy: { category: 'docs' }
43
+ * });
44
+ * ```
45
+ */
46
+ export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
+
48
+ /**
49
+ * Default search mode
50
+ * Cosine similarity is typically preferred for text embeddings
51
+ */
52
+ defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
+
54
+ /**
55
+ * Whether to use L2 normalization when storing vectors
56
+ * Normalized vectors are more efficient for cosine similarity calculations
57
+ */
58
+ useL2Normalization: boolean = true;
59
+
60
+ /**
61
+ * Instance-based vector pool utility
62
+ * Minden service instance saját vektor pool-t tart fenn
63
+ */
64
+ private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
+
66
+ constructor(
67
+ /**
68
+ * Initial data, this will be used by functions on default
69
+ */
70
+ data: T,
71
+ /**
72
+ * DB data params will be used to connect to usable dbService on GlobalService
73
+ */
74
+ dataParams: DyFM_DataModel_Params<T>,
75
+ /**
76
+ * OpenAI settings
77
+ */
78
+ openAISettings: DyFM_OAI_Settings,
79
+ /**
80
+ * Initial set for issuer to be able to follow the issuer's activity
81
+ */
82
+ issuer: string
83
+ ) {
84
+ super(data, dataParams, openAISettings, issuer);
85
+ }
86
+
87
+ /**
88
+ * Overrides the parent vectorSearch method to perform local in-memory vector search
89
+ *
90
+ * Process:
91
+ * 1. Load data from DB using filterBy (if provided) or getAll()
92
+ * 2. Extract vectorized properties from loaded data
93
+ * 3. Build in-memory vector pool
94
+ * 4. Vectorize input query
95
+ * 5. Perform local vector search
96
+ * 6. Map search results back to full data objects
97
+ *
98
+ * @param set Search parameters
99
+ * @returns Array of data objects sorted by similarity score
100
+ */
101
+ override async vectorSearch(
102
+ set: {
103
+ input: string;
104
+ /** this should be the vectorized property key */
105
+ searchInKey: string;
106
+ limit?: number;
107
+ /**
108
+ * Number of candidates that are used to find the best match
109
+ * (Not used in local search, but kept for compatibility)
110
+ */
111
+ numberOfCandidates?: number;
112
+ /**
113
+ * Filter to narrow down candidates from DB before vector search
114
+ */
115
+ filterBy?: DyFM_DBFilter<T>;
116
+ /**
117
+ * Search mode (cosine similarity, L2 distance, or hybrid).
118
+ * Defaults to this.defaultSearchMode.
119
+ *
120
+ * `hybrid` mode combines cosine similarity (vector half) with BM25
121
+ * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
+ */
123
+ searchMode?: LVS_Search_Mode;
124
+ /**
125
+ * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
+ * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
+ */
128
+ hybridWeight?: { vector: number; text: number };
129
+ /**
130
+ * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
+ * text-search. NEM kell hogy a vectorized property legyen.
132
+ */
133
+ textSearchKey?: keyof T;
134
+ },
135
+ ): Promise<T[]> {
136
+ try {
137
+ if (!set.input) {
138
+ throw new DyFM_Error({
139
+ ...this.getDefaultErrorSettings(
140
+ 'vectorSearch',
141
+ new Error('input is required')
142
+ ),
143
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
+ });
145
+ }
146
+
147
+ set.limit ??= 3;
148
+ const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
+ const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
+
151
+ // Hybrid mode korai validacio
152
+ if (effectiveMode === LVS_Search_Mode.hybrid) {
153
+ if (!textSearchKey) {
154
+ throw new DyFM_Error({
155
+ ...this.getDefaultErrorSettings(
156
+ 'vectorSearch',
157
+ new Error('textSearchKey is required when searchMode is hybrid'),
158
+ ),
159
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
+ });
161
+ }
162
+ if (hybridWeight) {
163
+ const w: { vector: number; text: number } = hybridWeight;
164
+ if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
+ throw new DyFM_Error({
166
+ ...this.getDefaultErrorSettings(
167
+ 'vectorSearch',
168
+ new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
+ ),
170
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
+ });
172
+ }
173
+ }
174
+ }
175
+
176
+ // Validáljuk, hogy a searchInKey létezik-e
177
+ const property: DyFM_DataProperty_Params<any, T> =
178
+ this.dataParams.properties[searchInKey];
179
+
180
+ if (!property) {
181
+ throw new DyFM_Error({
182
+ ...this.getDefaultErrorSettings(
183
+ 'vectorSearch',
184
+ new Error(
185
+ `Property "${searchInKey}" not found! ` +
186
+ `while searching "${this.dataParams.dataName}" ` +
187
+ `(The searchable properties are: ` +
188
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
+ )
190
+ ),
191
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
+ __localStack: this.dataParams.stackLocation,
193
+ });
194
+ }
195
+
196
+ // Ellenőrizzük, hogy a property vectorized-e
197
+ if (!this.vectorizedProperties.some(
198
+ (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
+ )) {
200
+ throw new DyFM_Error({
201
+ ...this.getDefaultErrorSettings(
202
+ 'vectorSearch',
203
+ new Error(
204
+ `Property "${searchInKey}" is not a vectorized property! ` +
205
+ `(The vectorized properties are: ` +
206
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
+ )
208
+ ),
209
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
+ __localStack: this.dataParams.stackLocation,
211
+ });
212
+ }
213
+
214
+ // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
+ let dataList: T[];
216
+ if (filterBy) {
217
+ // Ha van filterBy, akkor először szűrünk a DB-ben
218
+ if (this.debugLog) {
219
+ DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
+ }
221
+ dataList = await this.findDataList(filterBy, true);
222
+ } else {
223
+ // Ha nincs filterBy, akkor minden adatot betöltünk
224
+ if (this.debugLog) {
225
+ DyFM_Log.log(`Local vector search: loading all data...`);
226
+ }
227
+ dataList = await this.getAll(true);
228
+ }
229
+
230
+ if (dataList.length === 0) {
231
+ if (this.debugLog) {
232
+ DyFM_Log.log(`Local vector search: no data found`);
233
+ }
234
+ return [];
235
+ }
236
+
237
+ // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
+ if (this.debugLog) {
239
+ DyFM_Log.log(
240
+ `Local vector search: building vector pool from ${dataList.length} items...`
241
+ );
242
+ }
243
+
244
+ // Töröljük a korábbi pool-t
245
+ this.vectorPool.clearPool();
246
+
247
+ // Hozzáadjuk a vektorokat a pool-hoz
248
+ const dataMap: Map<string, T> = new Map<string, T>();
249
+ for (const dataItem of dataList) {
250
+ if (!dataItem._id) {
251
+ // Ha nincs _id, akkor kihagyjuk
252
+ if (this.debugLog) {
253
+ DyFM_Log.warn(
254
+ `Local vector search: skipping item without _id`
255
+ );
256
+ }
257
+ continue;
258
+ }
259
+
260
+ const vectorizedValue: number[] | undefined =
261
+ dataItem[searchInKey] as number[] | undefined;
262
+
263
+ if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
+ // Ha nincs vectorized érték, akkor kihagyjuk
265
+ if (this.debugLog) {
266
+ DyFM_Log.warn(
267
+ `Local vector search: skipping item "${dataItem._id}" ` +
268
+ `without vectorized value for "${searchInKey}"`
269
+ );
270
+ }
271
+ continue;
272
+ }
273
+
274
+ // Hozzáadjuk a vektort a pool-hoz
275
+ this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
+ dataMap.set(dataItem._id, dataItem);
277
+ }
278
+
279
+ if (dataMap.size === 0) {
280
+ if (this.debugLog) {
281
+ DyFM_Log.log(`Local vector search: no valid vectors found`);
282
+ }
283
+ return [];
284
+ }
285
+
286
+ // 3. Vectorizáljuk a query input-ot
287
+ if (this.debugLog) {
288
+ DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
+ }
290
+
291
+ const queryVector: number[] = await this.vectorize(
292
+ input,
293
+ property.embeddingModel
294
+ );
295
+
296
+ // 4. Végrehajtjuk a local vector search-t
297
+ const mode: LVS_Search_Mode = effectiveMode;
298
+
299
+ if (this.debugLog) {
300
+ DyFM_Log.log(
301
+ `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
+ );
303
+ }
304
+
305
+ let searchResults: LVS_SearchResult[];
306
+
307
+ if (mode === LVS_Search_Mode.hybrid) {
308
+ // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
+ const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
+ queryVector,
311
+ dataMap.size,
312
+ LVS_Search_Mode.cosineSimilarity,
313
+ );
314
+
315
+ // BM25 corpus epitese a `textSearchKey` property-bol
316
+ const docs: { id: string; text: string }[] = [];
317
+ for (const [docId, dataItem] of dataMap) {
318
+ const textValue: unknown = dataItem[textSearchKey as keyof T];
319
+ docs.push({
320
+ id: docId,
321
+ text: typeof textValue === 'string' ? textValue : '',
322
+ });
323
+ }
324
+ const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
+ const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
+ const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
+
328
+ const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
+ for (const s of bm25Normalized) {
330
+ bm25ScoreById.set(s.id, s.score);
331
+ }
332
+
333
+ const wVector: number = hybridWeight?.vector ?? 0.5;
334
+ const wText: number = hybridWeight?.text ?? 0.5;
335
+
336
+ const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
+ const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
+ return {
339
+ id: c.id,
340
+ score: wVector * c.score + wText * bm25Score,
341
+ };
342
+ });
343
+ merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
+ searchResults = merged.slice(0, limit);
345
+ } else {
346
+ searchResults = this.vectorPool.search(
347
+ queryVector,
348
+ limit,
349
+ mode,
350
+ );
351
+ }
352
+
353
+ if (this.debugLog) {
354
+ DyFM_Log.log(
355
+ `Local vector search: found ${searchResults.length} results`
356
+ );
357
+ }
358
+
359
+ // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
+ const results: T[] = [];
361
+ for (const searchResult of searchResults) {
362
+ const dataItem: T | undefined = dataMap.get(searchResult.id);
363
+ if (dataItem) {
364
+ results.push(dataItem);
365
+ }
366
+ }
367
+
368
+ // Töröljük a pool-t a memória felszabadítása érdekében
369
+ this.vectorPool.clearPool();
370
+
371
+ return results;
372
+ } catch (error) {
373
+ // Töröljük a pool-t hiba esetén is
374
+ this.vectorPool.clearPool();
375
+
376
+ throw new DyFM_Error({
377
+ ...this.getDefaultErrorSettings('vectorSearch', error),
378
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
+ });
380
+ }
381
+ }
382
+ }
383
+
384
+ /**
385
+ * @example
386
+ * ```typescript
387
+ * // Create a local vector data service
388
+ * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
+ * new CodeChunk(),
390
+ * codeChunk_dataParams,
391
+ * openAISettings,
392
+ * 'example-issuer'
393
+ * );
394
+ *
395
+ * // Perform vector search with filterBy
396
+ * const results = await localVectorService.vectorSearch({
397
+ * input: 'How to implement authentication?',
398
+ * searchInKey: 'chunkParentedContentVectorized',
399
+ * limit: 5,
400
+ * filterBy: {
401
+ * // Filter by category before vector search
402
+ * category: 'documentation'
403
+ * },
404
+ * searchMode: LVS_Search_Mode.cosineSimilarity
405
+ * });
406
+ *
407
+ * // Results are sorted by similarity score (highest first for cosine)
408
+ * console.log(`Found ${results.length} similar chunks`);
409
+ *
410
+ * // Access the results
411
+ * for (const result of results) {
412
+ * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
+ * }
414
+ * ```
415
+ */
416
+