@futdevpro/nts-dynamo 1.15.89 → 1.15.90

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (384) hide show
  1. package/.c8rc.json +26 -26
  2. package/.copilot/patterns.json +7 -7
  3. package/.cursor/rules/__assistant_guide.mdc +30 -30
  4. package/.cursor/rules/_ag_backend-structure.mdc +85 -85
  5. package/.cursor/rules/_ag_backend.mdc +16 -16
  6. package/.cursor/rules/_ag_frontend-structure.mdc +86 -86
  7. package/.cursor/rules/_ag_frontend.mdc +39 -39
  8. package/.cursor/rules/_ag_import-rules.mdc +44 -44
  9. package/.cursor/rules/_ag_naming.mdc +115 -115
  10. package/.cursor/rules/_ag_should-be.mdc +6 -6
  11. package/.cursor/rules/ai_development_guide.md +60 -60
  12. package/.cursor/rules/cursor-rules.md +160 -160
  13. package/.cursor/rules/default-command.mdc +464 -464
  14. package/.cursor/rules/error_code_pattern.md +39 -39
  15. package/.cursor/rules/saved rule mcp server use.md +15 -15
  16. package/.dynamo/logs/cicd-pipeline/output.log +2816 -0
  17. package/.dynamo/logs/cicd-pipeline/status.json +94 -0
  18. package/.vscode/settings.json +10 -10
  19. package/HOWTO.md +15 -15
  20. package/LICENSE +21 -21
  21. package/__documentations/nts-integration-tests-2026-03-17.md +26 -26
  22. package/__documentations/plans/BEDROCK-HYPERPLAN.md +95 -95
  23. package/_specifications/BACKLOG.md +92 -92
  24. package/_specifications/TODO.md +15 -15
  25. package/_specifications/agent.md +138 -138
  26. package/build/_modules/scheduler/_models/scheduler-job.interface.d.ts +34 -0
  27. package/build/_modules/scheduler/_models/scheduler-job.interface.d.ts.map +1 -0
  28. package/build/_modules/scheduler/_models/scheduler-job.interface.js +3 -0
  29. package/build/_modules/scheduler/_models/scheduler-job.interface.js.map +1 -0
  30. package/build/_modules/scheduler/get-scheduler-routing-module.util.d.ts +18 -0
  31. package/build/_modules/scheduler/get-scheduler-routing-module.util.d.ts.map +1 -0
  32. package/build/_modules/scheduler/get-scheduler-routing-module.util.js +31 -0
  33. package/build/_modules/scheduler/get-scheduler-routing-module.util.js.map +1 -0
  34. package/build/_modules/scheduler/index.d.ts +5 -0
  35. package/build/_modules/scheduler/index.d.ts.map +1 -0
  36. package/build/_modules/scheduler/index.js +10 -0
  37. package/build/_modules/scheduler/index.js.map +1 -0
  38. package/build/_modules/scheduler/scheduler.controller.d.ts +25 -0
  39. package/build/_modules/scheduler/scheduler.controller.d.ts.map +1 -0
  40. package/build/_modules/scheduler/scheduler.controller.js +54 -0
  41. package/build/_modules/scheduler/scheduler.controller.js.map +1 -0
  42. package/build/_modules/scheduler/scheduler.service.d.ts +54 -0
  43. package/build/_modules/scheduler/scheduler.service.d.ts.map +1 -0
  44. package/build/_modules/scheduler/scheduler.service.js +164 -0
  45. package/build/_modules/scheduler/scheduler.service.js.map +1 -0
  46. package/eslint.config.js +3 -3
  47. package/nodemon.json +24 -24
  48. package/package.json +10 -1
  49. package/pnpm-workspace.yaml +5 -5
  50. package/scripts/run-coverage-tests.js +28 -28
  51. package/spec/support/helpers/spec-reporter-loader.js +359 -359
  52. package/spec/support/helpers/ts-node-helper.js +93 -93
  53. package/spec/support/jasmine.coverage.json +24 -24
  54. package/spec/support/jasmine.json +24 -24
  55. package/src/_collections/archive.util.spec.ts +57 -57
  56. package/src/_collections/archive.util.ts +18 -18
  57. package/src/_collections/atlas-default-db-options.const.ts +9 -9
  58. package/src/_collections/default-fallback-cache-max-age.const.spec.ts +11 -11
  59. package/src/_collections/default-fallback-cache-max-age.const.ts +2 -2
  60. package/src/_collections/default-not-found-page.const.spec.ts +19 -19
  61. package/src/_collections/default-not-found-page.const.ts +22 -22
  62. package/src/_collections/default-socket-path.const.spec.ts +12 -12
  63. package/src/_collections/default-socket-path.const.ts +2 -2
  64. package/src/_collections/get-environment-settings.util.spec.ts +210 -210
  65. package/src/_collections/get-environment-settings.util.ts +48 -48
  66. package/src/_collections/global-settings.const.ts +109 -109
  67. package/src/_collections/sample.env +21 -21
  68. package/src/_collections/star.controller.spec.ts +224 -224
  69. package/src/_collections/star.controller.ts +129 -129
  70. package/src/_enums/data-model-type.enum.ts +14 -14
  71. package/src/_enums/data-service-function.enum.ts +24 -24
  72. package/src/_enums/predefined-data-types.enum.ts +16 -16
  73. package/src/_enums/route-security.enum.ts +12 -12
  74. package/src/_models/control-models/api-call-params.control-model.spec.ts +152 -152
  75. package/src/_models/control-models/api-call-params.control-model.ts +142 -142
  76. package/src/_models/control-models/app-ext-system-controls.control-model.spec.ts +52 -52
  77. package/src/_models/control-models/app-ext-system-controls.control-model.ts +9 -9
  78. package/src/_models/control-models/app-params.control-model.spec.ts +225 -225
  79. package/src/_models/control-models/app-params.control-model.ts +136 -136
  80. package/src/_models/control-models/app-system-controls.control-model.spec.ts +31 -31
  81. package/src/_models/control-models/app-system-controls.control-model.ts +9 -9
  82. package/src/_models/control-models/endpoint-params.control-model.spec.ts +627 -627
  83. package/src/_models/control-models/endpoint-params.control-model.ts +627 -627
  84. package/src/_models/control-models/http-settings.control-model.spec.ts +77 -77
  85. package/src/_models/control-models/http-settings.control-model.ts +37 -37
  86. package/src/_models/control-models/system-control.control-model.spec.ts +27 -27
  87. package/src/_models/control-models/system-control.control-model.ts +12 -12
  88. package/src/_models/interfaces/certification-settings.interface.ts +7 -7
  89. package/src/_models/interfaces/environment-settings.interface.ts +59 -59
  90. package/src/_models/interfaces/global-log-settings.interface.ts +171 -171
  91. package/src/_models/interfaces/global-service-settings.interface.ts +47 -47
  92. package/src/_models/interfaces/global-settings.interface.ts +244 -244
  93. package/src/_models/interfaces/routing-module-settings.interface.ts +21 -21
  94. package/src/_models/interfaces/static-client-settings.interface.spec.ts +29 -29
  95. package/src/_models/interfaces/static-client-settings.interface.ts +28 -28
  96. package/src/_models/types/db-update.type.ts +100 -100
  97. package/src/_modules/ai/_models/ai-input-interfaces.ts +117 -117
  98. package/src/_modules/ai/_models/ai-test-generation-result.interface.ts +16 -16
  99. package/src/_modules/ai/_modules/anthropic/_services/aai-user-key.control-service.ts +138 -138
  100. package/src/_modules/ai/_modules/anthropic/index.ts +5 -5
  101. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.spec.ts +242 -242
  102. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.ts +639 -639
  103. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.spec.ts +295 -295
  104. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.ts +518 -518
  105. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.spec.ts +209 -209
  106. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.ts +85 -85
  107. package/src/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.ts +7 -7
  108. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.ts +146 -146
  109. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model.ts +162 -162
  110. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-document.data-model.ts +99 -99
  111. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-code-chunk.interface.ts +68 -68
  112. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface.ts +18 -18
  113. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-page-compare-result.interface.ts +19 -19
  114. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-document-compare-result.interface.ts +25 -25
  115. package/src/_modules/ai/_modules/document-ai/index.ts +30 -30
  116. package/src/_modules/ai/_modules/fdp-ai/_services/fdpai-user-key.control-service.ts +189 -189
  117. package/src/_modules/ai/_modules/fdp-ai/index.ts +5 -5
  118. package/src/_modules/ai/_modules/open-ai/_collections/oai-global-settings.const.ts +9 -9
  119. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests-hu.conts.ts +82 -82
  120. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests.conts.ts +75 -75
  121. package/src/_modules/ai/_modules/open-ai/_enums/oai-gpt-message-role.enum.ts +45 -45
  122. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-global-settings.interface.ts +7 -7
  123. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-gpt-message.interface.ts +7 -7
  124. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-llm-predefined-requests.interface.ts +57 -57
  125. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.ts +292 -292
  126. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.spec.ts +342 -342
  127. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.spec.ts +550 -550
  128. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.ts +630 -630
  129. package/src/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.spec.ts +332 -332
  130. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.spec.ts +462 -462
  131. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.ts +634 -634
  132. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.spec.ts +489 -489
  133. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.tools.spec.ts +173 -173
  134. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.ts +1033 -1033
  135. package/src/_modules/ai/_modules/open-ai/_services/oai-user-key.control-service.ts +157 -157
  136. package/src/_modules/ai/_services/ai-embedding-mock.service.spec.ts +115 -115
  137. package/src/_modules/ai/_services/ai-embedding-mock.service.ts +212 -212
  138. package/src/_modules/ai/_services/ai-embedding-provider.registry.spec.ts +110 -110
  139. package/src/_modules/ai/_services/ai-embedding-provider.registry.ts +110 -110
  140. package/src/_modules/ai/_services/ai-embedding.service-base.spec.ts +98 -98
  141. package/src/_modules/ai/_services/ai-embedding.service-base.ts +48 -48
  142. package/src/_modules/ai/_services/ai-llm-chat.service-base.spec.ts +229 -229
  143. package/src/_modules/ai/_services/ai-llm-chat.service-base.ts +68 -68
  144. package/src/_modules/ai/_services/ai-llm.service-base.spec.ts +250 -250
  145. package/src/_modules/ai/_services/ai-llm.service-base.ts +519 -519
  146. package/src/_modules/ai/_services/ai-provider.service-base.spec.ts +158 -158
  147. package/src/_modules/ai/_services/ai-user-key.service-base.ts +59 -59
  148. package/src/_modules/ai/_services/lmstudio-embedding.control-service.spec.ts +197 -197
  149. package/src/_modules/ai/_services/lmstudio-embedding.control-service.ts +371 -371
  150. package/src/_modules/ai/index.ts +23 -23
  151. package/src/_modules/assistant/_collections/ass-global-settings.const.ts +13 -13
  152. package/src/_modules/assistant/_collections/ass.util.spec.ts +176 -176
  153. package/src/_modules/assistant/_collections/ass.util.ts +50 -50
  154. package/src/_modules/assistant/_models/ass-global-settings.interface.ts +15 -15
  155. package/src/_modules/assistant/_services/ass-io.control-service.spec.ts +140 -140
  156. package/src/_modules/assistant/_services/ass-main.control-service.spec.ts +192 -192
  157. package/src/_modules/assistant/_services/ass-main.control-service.ts +107 -107
  158. package/src/_modules/bot/_collections/bot-default-commands.const.ts +12 -12
  159. package/src/_modules/bot/_collections/bot-global-settings.const.ts +39 -39
  160. package/src/_modules/bot/_models/bot-channel-wrapper.interface.ts +62 -62
  161. package/src/_modules/bot/_models/bot-command.interface.ts +8 -8
  162. package/src/_modules/bot/_models/bot-global-settings.interface.ts +96 -96
  163. package/src/_modules/bot/_models/bot-last-mention-date.interface.ts +6 -6
  164. package/src/_modules/bot/_models/bot-last-message-date.interface.ts +5 -5
  165. package/src/_modules/bot/_models/bot-user-wrapper.interface.ts +41 -41
  166. package/src/_modules/bot/_modules/discord-bot/_models/dib-platform.types.ts +9 -9
  167. package/src/_modules/bot/_modules/discord-bot/_services/dib-messaging-provider.control-service.spec.ts +431 -431
  168. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.spec.ts +160 -160
  169. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.ts +55 -55
  170. package/src/_modules/bot/_modules/dynamo-bot/_models/dyb-platform.types.ts +15 -15
  171. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.spec.ts +374 -374
  172. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.ts +447 -447
  173. package/src/_modules/bot/_modules/dynamo-bot/index.ts +15 -15
  174. package/src/_modules/bot/_modules/slack-bot/_models/slb-platform.types.ts +9 -9
  175. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.spec.ts +344 -344
  176. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.ts +197 -197
  177. package/src/_modules/bot/_modules/teams-bot/_models/teb-platform.types.ts +9 -9
  178. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.spec.ts +345 -345
  179. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.ts +197 -197
  180. package/src/_modules/bot/_services/bot-commands.control-service.spec.ts +116 -116
  181. package/src/_modules/bot/_services/bot-io.control-service.spec.ts +285 -285
  182. package/src/_modules/bot/_services/bot-main.control-service.spec.ts +208 -208
  183. package/src/_modules/bot/_services/bot-messaging-provider.service-base.spec.ts +349 -349
  184. package/src/_modules/bot/_services/bot-routines.control-service.spec.ts +111 -111
  185. package/src/_modules/custom-data/custom-data.controller.spec.ts +49 -49
  186. package/src/_modules/custom-data/custom-data.controller.ts +67 -67
  187. package/src/_modules/custom-data/custom-data.data-service.spec.ts +54 -54
  188. package/src/_modules/custom-data/custom-data.data-service.ts +21 -21
  189. package/src/_modules/custom-data/get-custom-data-routing-module.util.spec.ts +28 -28
  190. package/src/_modules/custom-data/get-custom-data-routing-module.util.ts +24 -24
  191. package/src/_modules/custom-data/index.ts +9 -9
  192. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.spec.ts +161 -161
  193. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.ts +203 -203
  194. package/src/_modules/data-readers/_models/interfaces/dynts-sqlite-reader.interface.ts +33 -33
  195. package/src/_modules/data-readers/index.ts +11 -11
  196. package/src/_modules/defaults/_collections/default-endpoints.util.ts +487 -487
  197. package/src/_modules/defaults/_models/default-user.data-model.ts +72 -72
  198. package/src/_modules/defaults/_services/default-auth.service.spec.ts +269 -269
  199. package/src/_modules/defaults/_services/default-auth.service.ts +177 -177
  200. package/src/_modules/defaults/_services/default-socket-events.service.spec.ts +42 -42
  201. package/src/_modules/defaults/_services/default-socket-events.service.ts +61 -61
  202. package/src/_modules/defaults/_services/default-user.data-service.spec.ts +187 -187
  203. package/src/_modules/defaults/_services/default-user.data-service.ts +98 -98
  204. package/src/_modules/defaults/index.ts +17 -17
  205. package/src/_modules/discord-assistant/_collections/dias-global-settings.const.ts +19 -19
  206. package/src/_modules/discord-assistant/_collections/dias.util.spec.ts +366 -366
  207. package/src/_modules/discord-assistant/_collections/dias.util.ts +132 -132
  208. package/src/_modules/discord-assistant/_models/dias-global-settings.interface.ts +19 -19
  209. package/src/_modules/discord-assistant/_models/dias-knowledge.data-model.ts +52 -52
  210. package/src/_modules/discord-assistant/_services/dias-chunk.data-service.ts +177 -177
  211. package/src/_modules/discord-assistant/_services/dias-io.control-service.spec.ts +108 -108
  212. package/src/_modules/discord-assistant/_services/dias-io.control-service.ts +69 -69
  213. package/src/_modules/discord-assistant/_services/dias-main.control-service.spec.ts +22 -22
  214. package/src/_modules/discord-assistant/_services/dias-main.control-service.ts +27 -27
  215. package/src/_modules/discord-assistant/_services/dias.service-base.spec.ts +195 -195
  216. package/src/_modules/discord-assistant/_services/dias.service-base.ts +76 -76
  217. package/src/_modules/discord-assistant/index.ts +38 -38
  218. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.spec.ts +34 -34
  219. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.ts +11 -11
  220. package/src/_modules/discord-assistant-voiced/index.ts +36 -36
  221. package/src/_modules/discord-bot/_collections/dibo-default-commands.const.ts +16 -16
  222. package/src/_modules/discord-bot/_collections/dibo-global-settings.conts.ts +55 -55
  223. package/src/_modules/discord-bot/_collections/dibo-operations.util.spec.ts +214 -214
  224. package/src/_modules/discord-bot/_collections/dibo-operations.util.ts +387 -387
  225. package/src/_modules/discord-bot/_models/dibo-command.interface.ts +12 -12
  226. package/src/_modules/discord-bot/_models/dibo-global-settings.interface.ts +98 -98
  227. package/src/_modules/discord-bot/_models/dibo-last-mention-date.inteface.ts +7 -7
  228. package/src/_modules/discord-bot/_models/dibo-last-message-date.interface.ts +6 -6
  229. package/src/_modules/discord-bot/_services/dibo-commands.control-service.spec.ts +154 -154
  230. package/src/_modules/discord-bot/_services/dibo-commands.control-service.ts +153 -153
  231. package/src/_modules/discord-bot/_services/dibo-io.control-service.spec.ts +264 -264
  232. package/src/_modules/discord-bot/_services/dibo-io.control-service.ts +306 -306
  233. package/src/_modules/discord-bot/_services/dibo-main.control-service.spec.ts +408 -408
  234. package/src/_modules/discord-bot/_services/dibo-main.control-service.ts +487 -487
  235. package/src/_modules/discord-bot/_services/dibo-routines.control-service.spec.ts +105 -105
  236. package/src/_modules/discord-bot/index.ts +36 -36
  237. package/src/_modules/local-vector-search/_enums/lvs-search-mode.enum.ts +35 -35
  238. package/src/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.ts +59 -59
  239. package/src/_modules/local-vector-search/_models/lvs-search-result.interface.ts +17 -17
  240. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.spec.ts +418 -418
  241. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.ts +276 -276
  242. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.spec.ts +480 -480
  243. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.ts +416 -416
  244. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.spec.ts +198 -198
  245. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.ts +146 -146
  246. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.spec.ts +167 -167
  247. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.ts +106 -106
  248. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.spec.ts +507 -507
  249. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.ts +272 -272
  250. package/src/_modules/local-vector-search/index.ts +16 -16
  251. package/src/_modules/logs/index.ts +11 -11
  252. package/src/_modules/mcp/_models/interfaces/dynts-mcp.interface.ts +111 -111
  253. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.spec.ts +142 -142
  254. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.ts +120 -120
  255. package/src/_modules/mcp/_services/dynts-mcp.adapter.ts +168 -168
  256. package/src/_modules/mcp/index.ts +13 -13
  257. package/src/_modules/messaging/README.md +354 -354
  258. package/src/_modules/messaging/_collections/get-messaging-routing-module.util.ts +26 -26
  259. package/src/_modules/messaging/_collections/msg-global-settings.const.ts +22 -22
  260. package/src/_modules/messaging/_collections/msg.util.spec.ts +226 -226
  261. package/src/_modules/messaging/_models/msg-global-settings.interface.ts +37 -37
  262. package/src/_modules/messaging/_services/msg-conversation.data-service.ts +146 -146
  263. package/src/_modules/messaging/_services/msg-events.service.spec.ts +219 -219
  264. package/src/_modules/messaging/_services/msg-events.service.ts +267 -267
  265. package/src/_modules/messaging/_services/msg-integration.control-service.ts +179 -179
  266. package/src/_modules/messaging/_services/msg-main.control-service.spec.ts +147 -147
  267. package/src/_modules/messaging/_services/msg-main.control-service.ts +571 -571
  268. package/src/_modules/messaging/_services/msg-message.data-service.ts +129 -129
  269. package/src/_modules/messaging/_services/msg.controller.spec.ts +201 -201
  270. package/src/_modules/messaging/index.ts +30 -30
  271. package/src/_modules/mock/app-extended-server.mock.ts +201 -201
  272. package/src/_modules/mock/app-integration-test.mock.ts +51 -51
  273. package/src/_modules/mock/app-params.mock.spec.ts +21 -21
  274. package/src/_modules/mock/app-params.mock.ts +9 -9
  275. package/src/_modules/mock/app-server.mock.ts +188 -188
  276. package/src/_modules/mock/auth-service.mock.spec.ts +47 -47
  277. package/src/_modules/mock/auth-service.mock.ts +28 -28
  278. package/src/_modules/mock/controller.mock.spec.ts +26 -26
  279. package/src/_modules/mock/controller.mock.ts +16 -16
  280. package/src/_modules/mock/data-model.mock.spec.ts +111 -111
  281. package/src/_modules/mock/data-model.mock.ts +82 -82
  282. package/src/_modules/mock/email-service-collection.mock.spec.ts +24 -24
  283. package/src/_modules/mock/email-service-collection.mock.ts +15 -15
  284. package/src/_modules/mock/email-service.mock.spec.ts +17 -17
  285. package/src/_modules/mock/email-service.mock.ts +20 -20
  286. package/src/_modules/mock/email-template.mock.html +14 -14
  287. package/src/_modules/mock/endpoint.mock.ts +91 -91
  288. package/src/_modules/mock/socket-client.mock.spec.ts +40 -40
  289. package/src/_modules/mock/socket-client.mock.ts +45 -45
  290. package/src/_modules/mock/socket-server.mock.spec.ts +44 -44
  291. package/src/_modules/mock/socket-server.mock.ts +46 -46
  292. package/src/_modules/oauth2/_routes/oauth2.controller.spec.ts +107 -107
  293. package/src/_modules/oauth2/_routes/oauth2.controller.ts +98 -98
  294. package/src/_modules/oauth2/_services/oauth2.auth-service.spec.ts +254 -254
  295. package/src/_modules/oauth2/_services/oauth2.auth-service.ts +232 -232
  296. package/src/_modules/oauth2/_services/oauth2.control-service.spec.ts +585 -585
  297. package/src/_modules/oauth2/_services/oauth2.control-service.ts +653 -653
  298. package/src/_modules/oauth2/index.ts +17 -17
  299. package/src/_modules/scheduler/_models/scheduler-job.interface.ts +35 -0
  300. package/src/_modules/scheduler/get-scheduler-routing-module.util.ts +33 -0
  301. package/src/_modules/scheduler/index.ts +8 -0
  302. package/src/_modules/scheduler/scheduler.controller.spec.ts +42 -0
  303. package/src/_modules/scheduler/scheduler.controller.ts +69 -0
  304. package/src/_modules/scheduler/scheduler.service.spec.ts +141 -0
  305. package/src/_modules/scheduler/scheduler.service.ts +176 -0
  306. package/src/_modules/scoped-config/_enums/dynts-scoped-config-level.enum.ts +22 -22
  307. package/src/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.ts +81 -81
  308. package/src/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.ts +107 -107
  309. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.spec.ts +306 -306
  310. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.ts +295 -295
  311. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.spec.ts +118 -118
  312. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.ts +105 -105
  313. package/src/_modules/scoped-config/index.ts +17 -17
  314. package/src/_modules/server/errors/errors.control-service.spec.ts +238 -238
  315. package/src/_modules/server/errors/errors.control-service.ts +100 -100
  316. package/src/_modules/server/errors/errors.controller.spec.ts +268 -268
  317. package/src/_modules/server/errors/errors.controller.ts +515 -515
  318. package/src/_modules/server/errors/errors.data-service.spec.ts +480 -480
  319. package/src/_modules/server/index.ts +30 -30
  320. package/src/_modules/server/server-status/server-status-snapshot.control-service.spec.ts +70 -70
  321. package/src/_modules/server/server-status/server-status-snapshot.control-service.ts +17 -17
  322. package/src/_modules/server/server-status/server-status-snapshot.data-service.spec.ts +77 -77
  323. package/src/_modules/server/server-status/server-status-snapshot.data-service.ts +37 -37
  324. package/src/_modules/server/server-status/server-status.control-service.spec.ts +576 -576
  325. package/src/_modules/server/server-status/server-status.control-service.ts +396 -396
  326. package/src/_modules/server/server-status/server-status.controller.spec.ts +255 -255
  327. package/src/_modules/server/server-status/server-status.controller.ts +272 -272
  328. package/src/_modules/socket/_enums/socket-security.enum.ts +11 -11
  329. package/src/_modules/socket/_models/socket-client-service-params.control-model.spec.ts +32 -32
  330. package/src/_modules/socket/_models/socket-client-service-params.control-model.ts +22 -22
  331. package/src/_modules/socket/_models/socket-presence.control-model.spec.ts +164 -164
  332. package/src/_modules/socket/_models/socket-presence.control-model.ts +210 -210
  333. package/src/_modules/socket/_models/socket-server-service-params.control-model.spec.ts +46 -46
  334. package/src/_modules/socket/_models/socket-server-service-params.control-model.ts +22 -22
  335. package/src/_modules/socket/_services/socket-client.service.spec.ts +15 -15
  336. package/src/_modules/socket/_services/socket-client.service.ts +260 -260
  337. package/src/_modules/socket/_services/socket-server.service.spec.ts +11 -11
  338. package/src/_modules/socket/app-extended.integration.spec.ts +85 -85
  339. package/src/_modules/socket/app-extended.server.ts +630 -630
  340. package/src/_modules/socket/index.ts +42 -42
  341. package/src/_modules/test/get-test-routing-module.util.spec.ts +28 -28
  342. package/src/_modules/test/get-test-routing-module.util.ts +23 -23
  343. package/src/_modules/test/index.ts +11 -11
  344. package/src/_modules/test/test.controller.spec.ts +72 -72
  345. package/src/_modules/test/test.controller.ts +115 -115
  346. package/src/_modules/usage/get-usage-routing-module.util.ts +22 -22
  347. package/src/_modules/usage/index.ts +15 -15
  348. package/src/_modules/usage/usage.controller.spec.ts +81 -81
  349. package/src/_modules/usage/usage.controller.ts +126 -126
  350. package/src/_modules/usage/usage.data-service.spec.ts +332 -332
  351. package/src/_modules/usage/usage.data-service.ts +185 -185
  352. package/src/_services/base/api.service-base.spec.ts +125 -125
  353. package/src/_services/base/api.service-base.ts +74 -74
  354. package/src/_services/base/archive-data.service.spec.ts +209 -209
  355. package/src/_services/base/archive-data.service.ts +224 -224
  356. package/src/_services/base/data.service.spec.ts +729 -729
  357. package/src/_services/base/data.service.ts +2740 -2740
  358. package/src/_services/base/db.service.spec.ts +73 -73
  359. package/src/_services/base/db.service.ts +1575 -1575
  360. package/src/_services/base/singleton.service-base.spec.ts +28 -28
  361. package/src/_services/base/singleton.service-base.ts +24 -24
  362. package/src/_services/base/singleton.service.spec.ts +114 -114
  363. package/src/_services/base/singleton.service.ts +38 -38
  364. package/src/_services/core/api.service.spec.ts +140 -140
  365. package/src/_services/core/auth.service.spec.ts +159 -159
  366. package/src/_services/core/auth.service.ts +174 -174
  367. package/src/_services/core/email.service.spec.ts +85 -85
  368. package/src/_services/core/email.service.ts +742 -742
  369. package/src/_services/core/global.service.spec.ts +292 -292
  370. package/src/_services/core/global.service.ts +487 -487
  371. package/src/_services/core/memory-guard.service.spec.ts +245 -245
  372. package/src/_services/core/memory-guard.service.ts +481 -481
  373. package/src/_services/core/service-collection.service.spec.ts +46 -46
  374. package/src/_services/core/service-collection.service.ts +6 -6
  375. package/src/_services/route/controller.service.spec.ts +53 -53
  376. package/src/_services/route/controller.service.ts +148 -148
  377. package/src/_services/route/routing-module.service.spec.ts +98 -98
  378. package/src/_services/route/routing-module.service.ts +330 -330
  379. package/src/_services/server/app.server.ts +1941 -1941
  380. package/src/_services/shared.static-service.spec.ts +99 -99
  381. package/src/_services/shared.static-service.ts +78 -78
  382. package/src/index.ts +97 -97
  383. package/tsconfig.app.json +12 -12
  384. package/tsconfig.json +42 -42
@@ -1,416 +1,416 @@
1
- import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
- import {
3
- DyFM_DataModel_Params,
4
- DyFM_DataProperty_Params,
5
- DyFM_DBFilter,
6
- DyFM_Metadata,
7
- } from '@futdevpro/fsm-dynamo';
8
- import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
-
10
- import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
- import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
- import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
- import {
14
- DyNTS_LVS_BM25_Corpus,
15
- DyNTS_LVS_BM25_DocScore,
16
- dyNTS_LVS_BM25_minMaxNormalize,
17
- } from './lvs-bm25.util';
18
- import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
- import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
-
21
- /**
22
- * Local Vector Data Service
23
- * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
- * instead of MongoDB vector search
25
- *
26
- * This service loads data from the database, builds an in-memory vector pool,
27
- * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
- *
29
- * @example
30
- * ```typescript
31
- * const service = new DyNTS_OAI_LocalVectorDataService(
32
- * data,
33
- * dataParams,
34
- * openAISettings,
35
- * issuer
36
- * );
37
- *
38
- * const results = await service.vectorSearch({
39
- * input: 'search query',
40
- * searchInKey: 'contentVectorized',
41
- * limit: 10,
42
- * filterBy: { category: 'docs' }
43
- * });
44
- * ```
45
- */
46
- export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
-
48
- /**
49
- * Default search mode
50
- * Cosine similarity is typically preferred for text embeddings
51
- */
52
- defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
-
54
- /**
55
- * Whether to use L2 normalization when storing vectors
56
- * Normalized vectors are more efficient for cosine similarity calculations
57
- */
58
- useL2Normalization: boolean = true;
59
-
60
- /**
61
- * Instance-based vector pool utility
62
- * Minden service instance saját vektor pool-t tart fenn
63
- */
64
- private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
-
66
- constructor(
67
- /**
68
- * Initial data, this will be used by functions on default
69
- */
70
- data: T,
71
- /**
72
- * DB data params will be used to connect to usable dbService on GlobalService
73
- */
74
- dataParams: DyFM_DataModel_Params<T>,
75
- /**
76
- * OpenAI settings
77
- */
78
- openAISettings: DyFM_OAI_Settings,
79
- /**
80
- * Initial set for issuer to be able to follow the issuer's activity
81
- */
82
- issuer: string
83
- ) {
84
- super(data, dataParams, openAISettings, issuer);
85
- }
86
-
87
- /**
88
- * Overrides the parent vectorSearch method to perform local in-memory vector search
89
- *
90
- * Process:
91
- * 1. Load data from DB using filterBy (if provided) or getAll()
92
- * 2. Extract vectorized properties from loaded data
93
- * 3. Build in-memory vector pool
94
- * 4. Vectorize input query
95
- * 5. Perform local vector search
96
- * 6. Map search results back to full data objects
97
- *
98
- * @param set Search parameters
99
- * @returns Array of data objects sorted by similarity score
100
- */
101
- override async vectorSearch(
102
- set: {
103
- input: string;
104
- /** this should be the vectorized property key */
105
- searchInKey: string;
106
- limit?: number;
107
- /**
108
- * Number of candidates that are used to find the best match
109
- * (Not used in local search, but kept for compatibility)
110
- */
111
- numberOfCandidates?: number;
112
- /**
113
- * Filter to narrow down candidates from DB before vector search
114
- */
115
- filterBy?: DyFM_DBFilter<T>;
116
- /**
117
- * Search mode (cosine similarity, L2 distance, or hybrid).
118
- * Defaults to this.defaultSearchMode.
119
- *
120
- * `hybrid` mode combines cosine similarity (vector half) with BM25
121
- * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
- */
123
- searchMode?: LVS_Search_Mode;
124
- /**
125
- * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
- * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
- */
128
- hybridWeight?: { vector: number; text: number };
129
- /**
130
- * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
- * text-search. NEM kell hogy a vectorized property legyen.
132
- */
133
- textSearchKey?: keyof T;
134
- },
135
- ): Promise<T[]> {
136
- try {
137
- if (!set.input) {
138
- throw new DyFM_Error({
139
- ...this.getDefaultErrorSettings(
140
- 'vectorSearch',
141
- new Error('input is required')
142
- ),
143
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
- });
145
- }
146
-
147
- set.limit ??= 3;
148
- const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
- const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
-
151
- // Hybrid mode korai validacio
152
- if (effectiveMode === LVS_Search_Mode.hybrid) {
153
- if (!textSearchKey) {
154
- throw new DyFM_Error({
155
- ...this.getDefaultErrorSettings(
156
- 'vectorSearch',
157
- new Error('textSearchKey is required when searchMode is hybrid'),
158
- ),
159
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
- });
161
- }
162
- if (hybridWeight) {
163
- const w: { vector: number; text: number } = hybridWeight;
164
- if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
- throw new DyFM_Error({
166
- ...this.getDefaultErrorSettings(
167
- 'vectorSearch',
168
- new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
- ),
170
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
- });
172
- }
173
- }
174
- }
175
-
176
- // Validáljuk, hogy a searchInKey létezik-e
177
- const property: DyFM_DataProperty_Params<any, T> =
178
- this.dataParams.properties[searchInKey];
179
-
180
- if (!property) {
181
- throw new DyFM_Error({
182
- ...this.getDefaultErrorSettings(
183
- 'vectorSearch',
184
- new Error(
185
- `Property "${searchInKey}" not found! ` +
186
- `while searching "${this.dataParams.dataName}" ` +
187
- `(The searchable properties are: ` +
188
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
- )
190
- ),
191
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
- __localStack: this.dataParams.stackLocation,
193
- });
194
- }
195
-
196
- // Ellenőrizzük, hogy a property vectorized-e
197
- if (!this.vectorizedProperties.some(
198
- (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
- )) {
200
- throw new DyFM_Error({
201
- ...this.getDefaultErrorSettings(
202
- 'vectorSearch',
203
- new Error(
204
- `Property "${searchInKey}" is not a vectorized property! ` +
205
- `(The vectorized properties are: ` +
206
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
- )
208
- ),
209
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
- __localStack: this.dataParams.stackLocation,
211
- });
212
- }
213
-
214
- // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
- let dataList: T[];
216
- if (filterBy) {
217
- // Ha van filterBy, akkor először szűrünk a DB-ben
218
- if (this.debugLog) {
219
- DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
- }
221
- dataList = await this.findDataList(filterBy, true);
222
- } else {
223
- // Ha nincs filterBy, akkor minden adatot betöltünk
224
- if (this.debugLog) {
225
- DyFM_Log.log(`Local vector search: loading all data...`);
226
- }
227
- dataList = await this.getAll(true);
228
- }
229
-
230
- if (dataList.length === 0) {
231
- if (this.debugLog) {
232
- DyFM_Log.log(`Local vector search: no data found`);
233
- }
234
- return [];
235
- }
236
-
237
- // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
- if (this.debugLog) {
239
- DyFM_Log.log(
240
- `Local vector search: building vector pool from ${dataList.length} items...`
241
- );
242
- }
243
-
244
- // Töröljük a korábbi pool-t
245
- this.vectorPool.clearPool();
246
-
247
- // Hozzáadjuk a vektorokat a pool-hoz
248
- const dataMap: Map<string, T> = new Map<string, T>();
249
- for (const dataItem of dataList) {
250
- if (!dataItem._id) {
251
- // Ha nincs _id, akkor kihagyjuk
252
- if (this.debugLog) {
253
- DyFM_Log.warn(
254
- `Local vector search: skipping item without _id`
255
- );
256
- }
257
- continue;
258
- }
259
-
260
- const vectorizedValue: number[] | undefined =
261
- dataItem[searchInKey] as number[] | undefined;
262
-
263
- if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
- // Ha nincs vectorized érték, akkor kihagyjuk
265
- if (this.debugLog) {
266
- DyFM_Log.warn(
267
- `Local vector search: skipping item "${dataItem._id}" ` +
268
- `without vectorized value for "${searchInKey}"`
269
- );
270
- }
271
- continue;
272
- }
273
-
274
- // Hozzáadjuk a vektort a pool-hoz
275
- this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
- dataMap.set(dataItem._id, dataItem);
277
- }
278
-
279
- if (dataMap.size === 0) {
280
- if (this.debugLog) {
281
- DyFM_Log.log(`Local vector search: no valid vectors found`);
282
- }
283
- return [];
284
- }
285
-
286
- // 3. Vectorizáljuk a query input-ot
287
- if (this.debugLog) {
288
- DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
- }
290
-
291
- const queryVector: number[] = await this.vectorize(
292
- input,
293
- property.embeddingModel
294
- );
295
-
296
- // 4. Végrehajtjuk a local vector search-t
297
- const mode: LVS_Search_Mode = effectiveMode;
298
-
299
- if (this.debugLog) {
300
- DyFM_Log.log(
301
- `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
- );
303
- }
304
-
305
- let searchResults: LVS_SearchResult[];
306
-
307
- if (mode === LVS_Search_Mode.hybrid) {
308
- // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
- const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
- queryVector,
311
- dataMap.size,
312
- LVS_Search_Mode.cosineSimilarity,
313
- );
314
-
315
- // BM25 corpus epitese a `textSearchKey` property-bol
316
- const docs: { id: string; text: string }[] = [];
317
- for (const [docId, dataItem] of dataMap) {
318
- const textValue: unknown = dataItem[textSearchKey as keyof T];
319
- docs.push({
320
- id: docId,
321
- text: typeof textValue === 'string' ? textValue : '',
322
- });
323
- }
324
- const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
- const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
- const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
-
328
- const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
- for (const s of bm25Normalized) {
330
- bm25ScoreById.set(s.id, s.score);
331
- }
332
-
333
- const wVector: number = hybridWeight?.vector ?? 0.5;
334
- const wText: number = hybridWeight?.text ?? 0.5;
335
-
336
- const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
- const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
- return {
339
- id: c.id,
340
- score: wVector * c.score + wText * bm25Score,
341
- };
342
- });
343
- merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
- searchResults = merged.slice(0, limit);
345
- } else {
346
- searchResults = this.vectorPool.search(
347
- queryVector,
348
- limit,
349
- mode,
350
- );
351
- }
352
-
353
- if (this.debugLog) {
354
- DyFM_Log.log(
355
- `Local vector search: found ${searchResults.length} results`
356
- );
357
- }
358
-
359
- // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
- const results: T[] = [];
361
- for (const searchResult of searchResults) {
362
- const dataItem: T | undefined = dataMap.get(searchResult.id);
363
- if (dataItem) {
364
- results.push(dataItem);
365
- }
366
- }
367
-
368
- // Töröljük a pool-t a memória felszabadítása érdekében
369
- this.vectorPool.clearPool();
370
-
371
- return results;
372
- } catch (error) {
373
- // Töröljük a pool-t hiba esetén is
374
- this.vectorPool.clearPool();
375
-
376
- throw new DyFM_Error({
377
- ...this.getDefaultErrorSettings('vectorSearch', error),
378
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
- });
380
- }
381
- }
382
- }
383
-
384
- /**
385
- * @example
386
- * ```typescript
387
- * // Create a local vector data service
388
- * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
- * new CodeChunk(),
390
- * codeChunk_dataParams,
391
- * openAISettings,
392
- * 'example-issuer'
393
- * );
394
- *
395
- * // Perform vector search with filterBy
396
- * const results = await localVectorService.vectorSearch({
397
- * input: 'How to implement authentication?',
398
- * searchInKey: 'chunkParentedContentVectorized',
399
- * limit: 5,
400
- * filterBy: {
401
- * // Filter by category before vector search
402
- * category: 'documentation'
403
- * },
404
- * searchMode: LVS_Search_Mode.cosineSimilarity
405
- * });
406
- *
407
- * // Results are sorted by similarity score (highest first for cosine)
408
- * console.log(`Found ${results.length} similar chunks`);
409
- *
410
- * // Access the results
411
- * for (const result of results) {
412
- * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
- * }
414
- * ```
415
- */
416
-
1
+ import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
+ import {
3
+ DyFM_DataModel_Params,
4
+ DyFM_DataProperty_Params,
5
+ DyFM_DBFilter,
6
+ DyFM_Metadata,
7
+ } from '@futdevpro/fsm-dynamo';
8
+ import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
+
10
+ import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
+ import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
+ import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
+ import {
14
+ DyNTS_LVS_BM25_Corpus,
15
+ DyNTS_LVS_BM25_DocScore,
16
+ dyNTS_LVS_BM25_minMaxNormalize,
17
+ } from './lvs-bm25.util';
18
+ import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
+ import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
+
21
+ /**
22
+ * Local Vector Data Service
23
+ * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
+ * instead of MongoDB vector search
25
+ *
26
+ * This service loads data from the database, builds an in-memory vector pool,
27
+ * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const service = new DyNTS_OAI_LocalVectorDataService(
32
+ * data,
33
+ * dataParams,
34
+ * openAISettings,
35
+ * issuer
36
+ * );
37
+ *
38
+ * const results = await service.vectorSearch({
39
+ * input: 'search query',
40
+ * searchInKey: 'contentVectorized',
41
+ * limit: 10,
42
+ * filterBy: { category: 'docs' }
43
+ * });
44
+ * ```
45
+ */
46
+ export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
+
48
+ /**
49
+ * Default search mode
50
+ * Cosine similarity is typically preferred for text embeddings
51
+ */
52
+ defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
+
54
+ /**
55
+ * Whether to use L2 normalization when storing vectors
56
+ * Normalized vectors are more efficient for cosine similarity calculations
57
+ */
58
+ useL2Normalization: boolean = true;
59
+
60
+ /**
61
+ * Instance-based vector pool utility
62
+ * Minden service instance saját vektor pool-t tart fenn
63
+ */
64
+ private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
+
66
+ constructor(
67
+ /**
68
+ * Initial data, this will be used by functions on default
69
+ */
70
+ data: T,
71
+ /**
72
+ * DB data params will be used to connect to usable dbService on GlobalService
73
+ */
74
+ dataParams: DyFM_DataModel_Params<T>,
75
+ /**
76
+ * OpenAI settings
77
+ */
78
+ openAISettings: DyFM_OAI_Settings,
79
+ /**
80
+ * Initial set for issuer to be able to follow the issuer's activity
81
+ */
82
+ issuer: string
83
+ ) {
84
+ super(data, dataParams, openAISettings, issuer);
85
+ }
86
+
87
+ /**
88
+ * Overrides the parent vectorSearch method to perform local in-memory vector search
89
+ *
90
+ * Process:
91
+ * 1. Load data from DB using filterBy (if provided) or getAll()
92
+ * 2. Extract vectorized properties from loaded data
93
+ * 3. Build in-memory vector pool
94
+ * 4. Vectorize input query
95
+ * 5. Perform local vector search
96
+ * 6. Map search results back to full data objects
97
+ *
98
+ * @param set Search parameters
99
+ * @returns Array of data objects sorted by similarity score
100
+ */
101
+ override async vectorSearch(
102
+ set: {
103
+ input: string;
104
+ /** this should be the vectorized property key */
105
+ searchInKey: string;
106
+ limit?: number;
107
+ /**
108
+ * Number of candidates that are used to find the best match
109
+ * (Not used in local search, but kept for compatibility)
110
+ */
111
+ numberOfCandidates?: number;
112
+ /**
113
+ * Filter to narrow down candidates from DB before vector search
114
+ */
115
+ filterBy?: DyFM_DBFilter<T>;
116
+ /**
117
+ * Search mode (cosine similarity, L2 distance, or hybrid).
118
+ * Defaults to this.defaultSearchMode.
119
+ *
120
+ * `hybrid` mode combines cosine similarity (vector half) with BM25
121
+ * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
+ */
123
+ searchMode?: LVS_Search_Mode;
124
+ /**
125
+ * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
+ * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
+ */
128
+ hybridWeight?: { vector: number; text: number };
129
+ /**
130
+ * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
+ * text-search. NEM kell hogy a vectorized property legyen.
132
+ */
133
+ textSearchKey?: keyof T;
134
+ },
135
+ ): Promise<T[]> {
136
+ try {
137
+ if (!set.input) {
138
+ throw new DyFM_Error({
139
+ ...this.getDefaultErrorSettings(
140
+ 'vectorSearch',
141
+ new Error('input is required')
142
+ ),
143
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
+ });
145
+ }
146
+
147
+ set.limit ??= 3;
148
+ const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
+ const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
+
151
+ // Hybrid mode korai validacio
152
+ if (effectiveMode === LVS_Search_Mode.hybrid) {
153
+ if (!textSearchKey) {
154
+ throw new DyFM_Error({
155
+ ...this.getDefaultErrorSettings(
156
+ 'vectorSearch',
157
+ new Error('textSearchKey is required when searchMode is hybrid'),
158
+ ),
159
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
+ });
161
+ }
162
+ if (hybridWeight) {
163
+ const w: { vector: number; text: number } = hybridWeight;
164
+ if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
+ throw new DyFM_Error({
166
+ ...this.getDefaultErrorSettings(
167
+ 'vectorSearch',
168
+ new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
+ ),
170
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
+ });
172
+ }
173
+ }
174
+ }
175
+
176
+ // Validáljuk, hogy a searchInKey létezik-e
177
+ const property: DyFM_DataProperty_Params<any, T> =
178
+ this.dataParams.properties[searchInKey];
179
+
180
+ if (!property) {
181
+ throw new DyFM_Error({
182
+ ...this.getDefaultErrorSettings(
183
+ 'vectorSearch',
184
+ new Error(
185
+ `Property "${searchInKey}" not found! ` +
186
+ `while searching "${this.dataParams.dataName}" ` +
187
+ `(The searchable properties are: ` +
188
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
+ )
190
+ ),
191
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
+ __localStack: this.dataParams.stackLocation,
193
+ });
194
+ }
195
+
196
+ // Ellenőrizzük, hogy a property vectorized-e
197
+ if (!this.vectorizedProperties.some(
198
+ (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
+ )) {
200
+ throw new DyFM_Error({
201
+ ...this.getDefaultErrorSettings(
202
+ 'vectorSearch',
203
+ new Error(
204
+ `Property "${searchInKey}" is not a vectorized property! ` +
205
+ `(The vectorized properties are: ` +
206
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
+ )
208
+ ),
209
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
+ __localStack: this.dataParams.stackLocation,
211
+ });
212
+ }
213
+
214
+ // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
+ let dataList: T[];
216
+ if (filterBy) {
217
+ // Ha van filterBy, akkor először szűrünk a DB-ben
218
+ if (this.debugLog) {
219
+ DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
+ }
221
+ dataList = await this.findDataList(filterBy, true);
222
+ } else {
223
+ // Ha nincs filterBy, akkor minden adatot betöltünk
224
+ if (this.debugLog) {
225
+ DyFM_Log.log(`Local vector search: loading all data...`);
226
+ }
227
+ dataList = await this.getAll(true);
228
+ }
229
+
230
+ if (dataList.length === 0) {
231
+ if (this.debugLog) {
232
+ DyFM_Log.log(`Local vector search: no data found`);
233
+ }
234
+ return [];
235
+ }
236
+
237
+ // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
+ if (this.debugLog) {
239
+ DyFM_Log.log(
240
+ `Local vector search: building vector pool from ${dataList.length} items...`
241
+ );
242
+ }
243
+
244
+ // Töröljük a korábbi pool-t
245
+ this.vectorPool.clearPool();
246
+
247
+ // Hozzáadjuk a vektorokat a pool-hoz
248
+ const dataMap: Map<string, T> = new Map<string, T>();
249
+ for (const dataItem of dataList) {
250
+ if (!dataItem._id) {
251
+ // Ha nincs _id, akkor kihagyjuk
252
+ if (this.debugLog) {
253
+ DyFM_Log.warn(
254
+ `Local vector search: skipping item without _id`
255
+ );
256
+ }
257
+ continue;
258
+ }
259
+
260
+ const vectorizedValue: number[] | undefined =
261
+ dataItem[searchInKey] as number[] | undefined;
262
+
263
+ if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
+ // Ha nincs vectorized érték, akkor kihagyjuk
265
+ if (this.debugLog) {
266
+ DyFM_Log.warn(
267
+ `Local vector search: skipping item "${dataItem._id}" ` +
268
+ `without vectorized value for "${searchInKey}"`
269
+ );
270
+ }
271
+ continue;
272
+ }
273
+
274
+ // Hozzáadjuk a vektort a pool-hoz
275
+ this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
+ dataMap.set(dataItem._id, dataItem);
277
+ }
278
+
279
+ if (dataMap.size === 0) {
280
+ if (this.debugLog) {
281
+ DyFM_Log.log(`Local vector search: no valid vectors found`);
282
+ }
283
+ return [];
284
+ }
285
+
286
+ // 3. Vectorizáljuk a query input-ot
287
+ if (this.debugLog) {
288
+ DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
+ }
290
+
291
+ const queryVector: number[] = await this.vectorize(
292
+ input,
293
+ property.embeddingModel
294
+ );
295
+
296
+ // 4. Végrehajtjuk a local vector search-t
297
+ const mode: LVS_Search_Mode = effectiveMode;
298
+
299
+ if (this.debugLog) {
300
+ DyFM_Log.log(
301
+ `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
+ );
303
+ }
304
+
305
+ let searchResults: LVS_SearchResult[];
306
+
307
+ if (mode === LVS_Search_Mode.hybrid) {
308
+ // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
+ const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
+ queryVector,
311
+ dataMap.size,
312
+ LVS_Search_Mode.cosineSimilarity,
313
+ );
314
+
315
+ // BM25 corpus epitese a `textSearchKey` property-bol
316
+ const docs: { id: string; text: string }[] = [];
317
+ for (const [docId, dataItem] of dataMap) {
318
+ const textValue: unknown = dataItem[textSearchKey as keyof T];
319
+ docs.push({
320
+ id: docId,
321
+ text: typeof textValue === 'string' ? textValue : '',
322
+ });
323
+ }
324
+ const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
+ const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
+ const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
+
328
+ const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
+ for (const s of bm25Normalized) {
330
+ bm25ScoreById.set(s.id, s.score);
331
+ }
332
+
333
+ const wVector: number = hybridWeight?.vector ?? 0.5;
334
+ const wText: number = hybridWeight?.text ?? 0.5;
335
+
336
+ const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
+ const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
+ return {
339
+ id: c.id,
340
+ score: wVector * c.score + wText * bm25Score,
341
+ };
342
+ });
343
+ merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
+ searchResults = merged.slice(0, limit);
345
+ } else {
346
+ searchResults = this.vectorPool.search(
347
+ queryVector,
348
+ limit,
349
+ mode,
350
+ );
351
+ }
352
+
353
+ if (this.debugLog) {
354
+ DyFM_Log.log(
355
+ `Local vector search: found ${searchResults.length} results`
356
+ );
357
+ }
358
+
359
+ // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
+ const results: T[] = [];
361
+ for (const searchResult of searchResults) {
362
+ const dataItem: T | undefined = dataMap.get(searchResult.id);
363
+ if (dataItem) {
364
+ results.push(dataItem);
365
+ }
366
+ }
367
+
368
+ // Töröljük a pool-t a memória felszabadítása érdekében
369
+ this.vectorPool.clearPool();
370
+
371
+ return results;
372
+ } catch (error) {
373
+ // Töröljük a pool-t hiba esetén is
374
+ this.vectorPool.clearPool();
375
+
376
+ throw new DyFM_Error({
377
+ ...this.getDefaultErrorSettings('vectorSearch', error),
378
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
+ });
380
+ }
381
+ }
382
+ }
383
+
384
+ /**
385
+ * @example
386
+ * ```typescript
387
+ * // Create a local vector data service
388
+ * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
+ * new CodeChunk(),
390
+ * codeChunk_dataParams,
391
+ * openAISettings,
392
+ * 'example-issuer'
393
+ * );
394
+ *
395
+ * // Perform vector search with filterBy
396
+ * const results = await localVectorService.vectorSearch({
397
+ * input: 'How to implement authentication?',
398
+ * searchInKey: 'chunkParentedContentVectorized',
399
+ * limit: 5,
400
+ * filterBy: {
401
+ * // Filter by category before vector search
402
+ * category: 'documentation'
403
+ * },
404
+ * searchMode: LVS_Search_Mode.cosineSimilarity
405
+ * });
406
+ *
407
+ * // Results are sorted by similarity score (highest first for cosine)
408
+ * console.log(`Found ${results.length} similar chunks`);
409
+ *
410
+ * // Access the results
411
+ * for (const result of results) {
412
+ * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
+ * }
414
+ * ```
415
+ */
416
+