@futdevpro/nts-dynamo 1.15.89 → 1.15.91

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (385) hide show
  1. package/.c8rc.json +26 -26
  2. package/.copilot/patterns.json +7 -7
  3. package/.cursor/rules/__assistant_guide.mdc +30 -30
  4. package/.cursor/rules/_ag_backend-structure.mdc +85 -85
  5. package/.cursor/rules/_ag_backend.mdc +16 -16
  6. package/.cursor/rules/_ag_frontend-structure.mdc +86 -86
  7. package/.cursor/rules/_ag_frontend.mdc +39 -39
  8. package/.cursor/rules/_ag_import-rules.mdc +44 -44
  9. package/.cursor/rules/_ag_naming.mdc +115 -115
  10. package/.cursor/rules/_ag_should-be.mdc +6 -6
  11. package/.cursor/rules/ai_development_guide.md +60 -60
  12. package/.cursor/rules/cursor-rules.md +160 -160
  13. package/.cursor/rules/default-command.mdc +464 -464
  14. package/.cursor/rules/error_code_pattern.md +39 -39
  15. package/.cursor/rules/saved rule mcp server use.md +15 -15
  16. package/.dynamo/logs/cicd-pipeline/output.log +2856 -0
  17. package/.dynamo/logs/cicd-pipeline/status.json +74 -0
  18. package/.dynamo/pipeline.cicd.config.json +5 -1
  19. package/.vscode/settings.json +10 -10
  20. package/HOWTO.md +15 -15
  21. package/LICENSE +21 -21
  22. package/__documentations/nts-integration-tests-2026-03-17.md +26 -26
  23. package/__documentations/plans/BEDROCK-HYPERPLAN.md +95 -95
  24. package/_specifications/BACKLOG.md +92 -92
  25. package/_specifications/TODO.md +15 -15
  26. package/_specifications/agent.md +138 -138
  27. package/build/_modules/scheduler/_models/scheduler-job.interface.d.ts +34 -0
  28. package/build/_modules/scheduler/_models/scheduler-job.interface.d.ts.map +1 -0
  29. package/build/_modules/scheduler/_models/scheduler-job.interface.js +3 -0
  30. package/build/_modules/scheduler/_models/scheduler-job.interface.js.map +1 -0
  31. package/build/_modules/scheduler/get-scheduler-routing-module.util.d.ts +18 -0
  32. package/build/_modules/scheduler/get-scheduler-routing-module.util.d.ts.map +1 -0
  33. package/build/_modules/scheduler/get-scheduler-routing-module.util.js +31 -0
  34. package/build/_modules/scheduler/get-scheduler-routing-module.util.js.map +1 -0
  35. package/build/_modules/scheduler/index.d.ts +5 -0
  36. package/build/_modules/scheduler/index.d.ts.map +1 -0
  37. package/build/_modules/scheduler/index.js +10 -0
  38. package/build/_modules/scheduler/index.js.map +1 -0
  39. package/build/_modules/scheduler/scheduler.controller.d.ts +25 -0
  40. package/build/_modules/scheduler/scheduler.controller.d.ts.map +1 -0
  41. package/build/_modules/scheduler/scheduler.controller.js +54 -0
  42. package/build/_modules/scheduler/scheduler.controller.js.map +1 -0
  43. package/build/_modules/scheduler/scheduler.service.d.ts +54 -0
  44. package/build/_modules/scheduler/scheduler.service.d.ts.map +1 -0
  45. package/build/_modules/scheduler/scheduler.service.js +164 -0
  46. package/build/_modules/scheduler/scheduler.service.js.map +1 -0
  47. package/eslint.config.js +3 -3
  48. package/nodemon.json +24 -24
  49. package/package.json +10 -1
  50. package/pnpm-workspace.yaml +5 -5
  51. package/scripts/run-coverage-tests.js +28 -28
  52. package/spec/support/helpers/spec-reporter-loader.js +359 -359
  53. package/spec/support/helpers/ts-node-helper.js +93 -93
  54. package/spec/support/jasmine.coverage.json +24 -24
  55. package/spec/support/jasmine.json +24 -24
  56. package/src/_collections/archive.util.spec.ts +57 -57
  57. package/src/_collections/archive.util.ts +18 -18
  58. package/src/_collections/atlas-default-db-options.const.ts +9 -9
  59. package/src/_collections/default-fallback-cache-max-age.const.spec.ts +11 -11
  60. package/src/_collections/default-fallback-cache-max-age.const.ts +2 -2
  61. package/src/_collections/default-not-found-page.const.spec.ts +19 -19
  62. package/src/_collections/default-not-found-page.const.ts +22 -22
  63. package/src/_collections/default-socket-path.const.spec.ts +12 -12
  64. package/src/_collections/default-socket-path.const.ts +2 -2
  65. package/src/_collections/get-environment-settings.util.spec.ts +210 -210
  66. package/src/_collections/get-environment-settings.util.ts +48 -48
  67. package/src/_collections/global-settings.const.ts +109 -109
  68. package/src/_collections/sample.env +21 -21
  69. package/src/_collections/star.controller.spec.ts +224 -224
  70. package/src/_collections/star.controller.ts +129 -129
  71. package/src/_enums/data-model-type.enum.ts +14 -14
  72. package/src/_enums/data-service-function.enum.ts +24 -24
  73. package/src/_enums/predefined-data-types.enum.ts +16 -16
  74. package/src/_enums/route-security.enum.ts +12 -12
  75. package/src/_models/control-models/api-call-params.control-model.spec.ts +152 -152
  76. package/src/_models/control-models/api-call-params.control-model.ts +142 -142
  77. package/src/_models/control-models/app-ext-system-controls.control-model.spec.ts +52 -52
  78. package/src/_models/control-models/app-ext-system-controls.control-model.ts +9 -9
  79. package/src/_models/control-models/app-params.control-model.spec.ts +225 -225
  80. package/src/_models/control-models/app-params.control-model.ts +136 -136
  81. package/src/_models/control-models/app-system-controls.control-model.spec.ts +31 -31
  82. package/src/_models/control-models/app-system-controls.control-model.ts +9 -9
  83. package/src/_models/control-models/endpoint-params.control-model.spec.ts +627 -627
  84. package/src/_models/control-models/endpoint-params.control-model.ts +627 -627
  85. package/src/_models/control-models/http-settings.control-model.spec.ts +77 -77
  86. package/src/_models/control-models/http-settings.control-model.ts +37 -37
  87. package/src/_models/control-models/system-control.control-model.spec.ts +27 -27
  88. package/src/_models/control-models/system-control.control-model.ts +12 -12
  89. package/src/_models/interfaces/certification-settings.interface.ts +7 -7
  90. package/src/_models/interfaces/environment-settings.interface.ts +59 -59
  91. package/src/_models/interfaces/global-log-settings.interface.ts +171 -171
  92. package/src/_models/interfaces/global-service-settings.interface.ts +47 -47
  93. package/src/_models/interfaces/global-settings.interface.ts +244 -244
  94. package/src/_models/interfaces/routing-module-settings.interface.ts +21 -21
  95. package/src/_models/interfaces/static-client-settings.interface.spec.ts +29 -29
  96. package/src/_models/interfaces/static-client-settings.interface.ts +28 -28
  97. package/src/_models/types/db-update.type.ts +100 -100
  98. package/src/_modules/ai/_models/ai-input-interfaces.ts +117 -117
  99. package/src/_modules/ai/_models/ai-test-generation-result.interface.ts +16 -16
  100. package/src/_modules/ai/_modules/anthropic/_services/aai-user-key.control-service.ts +138 -138
  101. package/src/_modules/ai/_modules/anthropic/index.ts +5 -5
  102. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.spec.ts +242 -242
  103. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.ts +639 -639
  104. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.spec.ts +295 -295
  105. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.ts +518 -518
  106. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.spec.ts +209 -209
  107. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.ts +85 -85
  108. package/src/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.ts +7 -7
  109. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.ts +146 -146
  110. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model.ts +162 -162
  111. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-document.data-model.ts +99 -99
  112. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-code-chunk.interface.ts +68 -68
  113. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface.ts +18 -18
  114. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-page-compare-result.interface.ts +19 -19
  115. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-document-compare-result.interface.ts +25 -25
  116. package/src/_modules/ai/_modules/document-ai/index.ts +30 -30
  117. package/src/_modules/ai/_modules/fdp-ai/_services/fdpai-user-key.control-service.ts +189 -189
  118. package/src/_modules/ai/_modules/fdp-ai/index.ts +5 -5
  119. package/src/_modules/ai/_modules/open-ai/_collections/oai-global-settings.const.ts +9 -9
  120. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests-hu.conts.ts +82 -82
  121. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests.conts.ts +75 -75
  122. package/src/_modules/ai/_modules/open-ai/_enums/oai-gpt-message-role.enum.ts +45 -45
  123. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-global-settings.interface.ts +7 -7
  124. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-gpt-message.interface.ts +7 -7
  125. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-llm-predefined-requests.interface.ts +57 -57
  126. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.ts +292 -292
  127. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.spec.ts +342 -342
  128. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.spec.ts +550 -550
  129. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.ts +630 -630
  130. package/src/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.spec.ts +332 -332
  131. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.spec.ts +462 -462
  132. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.ts +634 -634
  133. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.spec.ts +489 -489
  134. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.tools.spec.ts +173 -173
  135. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.ts +1033 -1033
  136. package/src/_modules/ai/_modules/open-ai/_services/oai-user-key.control-service.ts +157 -157
  137. package/src/_modules/ai/_services/ai-embedding-mock.service.spec.ts +115 -115
  138. package/src/_modules/ai/_services/ai-embedding-mock.service.ts +212 -212
  139. package/src/_modules/ai/_services/ai-embedding-provider.registry.spec.ts +110 -110
  140. package/src/_modules/ai/_services/ai-embedding-provider.registry.ts +110 -110
  141. package/src/_modules/ai/_services/ai-embedding.service-base.spec.ts +98 -98
  142. package/src/_modules/ai/_services/ai-embedding.service-base.ts +48 -48
  143. package/src/_modules/ai/_services/ai-llm-chat.service-base.spec.ts +229 -229
  144. package/src/_modules/ai/_services/ai-llm-chat.service-base.ts +68 -68
  145. package/src/_modules/ai/_services/ai-llm.service-base.spec.ts +250 -250
  146. package/src/_modules/ai/_services/ai-llm.service-base.ts +519 -519
  147. package/src/_modules/ai/_services/ai-provider.service-base.spec.ts +158 -158
  148. package/src/_modules/ai/_services/ai-user-key.service-base.ts +59 -59
  149. package/src/_modules/ai/_services/lmstudio-embedding.control-service.spec.ts +197 -197
  150. package/src/_modules/ai/_services/lmstudio-embedding.control-service.ts +371 -371
  151. package/src/_modules/ai/index.ts +23 -23
  152. package/src/_modules/assistant/_collections/ass-global-settings.const.ts +13 -13
  153. package/src/_modules/assistant/_collections/ass.util.spec.ts +176 -176
  154. package/src/_modules/assistant/_collections/ass.util.ts +50 -50
  155. package/src/_modules/assistant/_models/ass-global-settings.interface.ts +15 -15
  156. package/src/_modules/assistant/_services/ass-io.control-service.spec.ts +140 -140
  157. package/src/_modules/assistant/_services/ass-main.control-service.spec.ts +192 -192
  158. package/src/_modules/assistant/_services/ass-main.control-service.ts +107 -107
  159. package/src/_modules/bot/_collections/bot-default-commands.const.ts +12 -12
  160. package/src/_modules/bot/_collections/bot-global-settings.const.ts +39 -39
  161. package/src/_modules/bot/_models/bot-channel-wrapper.interface.ts +62 -62
  162. package/src/_modules/bot/_models/bot-command.interface.ts +8 -8
  163. package/src/_modules/bot/_models/bot-global-settings.interface.ts +96 -96
  164. package/src/_modules/bot/_models/bot-last-mention-date.interface.ts +6 -6
  165. package/src/_modules/bot/_models/bot-last-message-date.interface.ts +5 -5
  166. package/src/_modules/bot/_models/bot-user-wrapper.interface.ts +41 -41
  167. package/src/_modules/bot/_modules/discord-bot/_models/dib-platform.types.ts +9 -9
  168. package/src/_modules/bot/_modules/discord-bot/_services/dib-messaging-provider.control-service.spec.ts +431 -431
  169. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.spec.ts +160 -160
  170. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.ts +55 -55
  171. package/src/_modules/bot/_modules/dynamo-bot/_models/dyb-platform.types.ts +15 -15
  172. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.spec.ts +374 -374
  173. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.ts +447 -447
  174. package/src/_modules/bot/_modules/dynamo-bot/index.ts +15 -15
  175. package/src/_modules/bot/_modules/slack-bot/_models/slb-platform.types.ts +9 -9
  176. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.spec.ts +344 -344
  177. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.ts +197 -197
  178. package/src/_modules/bot/_modules/teams-bot/_models/teb-platform.types.ts +9 -9
  179. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.spec.ts +345 -345
  180. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.ts +197 -197
  181. package/src/_modules/bot/_services/bot-commands.control-service.spec.ts +116 -116
  182. package/src/_modules/bot/_services/bot-io.control-service.spec.ts +285 -285
  183. package/src/_modules/bot/_services/bot-main.control-service.spec.ts +208 -208
  184. package/src/_modules/bot/_services/bot-messaging-provider.service-base.spec.ts +349 -349
  185. package/src/_modules/bot/_services/bot-routines.control-service.spec.ts +111 -111
  186. package/src/_modules/custom-data/custom-data.controller.spec.ts +49 -49
  187. package/src/_modules/custom-data/custom-data.controller.ts +67 -67
  188. package/src/_modules/custom-data/custom-data.data-service.spec.ts +54 -54
  189. package/src/_modules/custom-data/custom-data.data-service.ts +21 -21
  190. package/src/_modules/custom-data/get-custom-data-routing-module.util.spec.ts +28 -28
  191. package/src/_modules/custom-data/get-custom-data-routing-module.util.ts +24 -24
  192. package/src/_modules/custom-data/index.ts +9 -9
  193. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.spec.ts +161 -161
  194. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.ts +203 -203
  195. package/src/_modules/data-readers/_models/interfaces/dynts-sqlite-reader.interface.ts +33 -33
  196. package/src/_modules/data-readers/index.ts +11 -11
  197. package/src/_modules/defaults/_collections/default-endpoints.util.ts +487 -487
  198. package/src/_modules/defaults/_models/default-user.data-model.ts +72 -72
  199. package/src/_modules/defaults/_services/default-auth.service.spec.ts +269 -269
  200. package/src/_modules/defaults/_services/default-auth.service.ts +177 -177
  201. package/src/_modules/defaults/_services/default-socket-events.service.spec.ts +42 -42
  202. package/src/_modules/defaults/_services/default-socket-events.service.ts +61 -61
  203. package/src/_modules/defaults/_services/default-user.data-service.spec.ts +187 -187
  204. package/src/_modules/defaults/_services/default-user.data-service.ts +98 -98
  205. package/src/_modules/defaults/index.ts +17 -17
  206. package/src/_modules/discord-assistant/_collections/dias-global-settings.const.ts +19 -19
  207. package/src/_modules/discord-assistant/_collections/dias.util.spec.ts +366 -366
  208. package/src/_modules/discord-assistant/_collections/dias.util.ts +132 -132
  209. package/src/_modules/discord-assistant/_models/dias-global-settings.interface.ts +19 -19
  210. package/src/_modules/discord-assistant/_models/dias-knowledge.data-model.ts +52 -52
  211. package/src/_modules/discord-assistant/_services/dias-chunk.data-service.ts +177 -177
  212. package/src/_modules/discord-assistant/_services/dias-io.control-service.spec.ts +108 -108
  213. package/src/_modules/discord-assistant/_services/dias-io.control-service.ts +69 -69
  214. package/src/_modules/discord-assistant/_services/dias-main.control-service.spec.ts +22 -22
  215. package/src/_modules/discord-assistant/_services/dias-main.control-service.ts +27 -27
  216. package/src/_modules/discord-assistant/_services/dias.service-base.spec.ts +195 -195
  217. package/src/_modules/discord-assistant/_services/dias.service-base.ts +76 -76
  218. package/src/_modules/discord-assistant/index.ts +38 -38
  219. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.spec.ts +34 -34
  220. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.ts +11 -11
  221. package/src/_modules/discord-assistant-voiced/index.ts +36 -36
  222. package/src/_modules/discord-bot/_collections/dibo-default-commands.const.ts +16 -16
  223. package/src/_modules/discord-bot/_collections/dibo-global-settings.conts.ts +55 -55
  224. package/src/_modules/discord-bot/_collections/dibo-operations.util.spec.ts +214 -214
  225. package/src/_modules/discord-bot/_collections/dibo-operations.util.ts +387 -387
  226. package/src/_modules/discord-bot/_models/dibo-command.interface.ts +12 -12
  227. package/src/_modules/discord-bot/_models/dibo-global-settings.interface.ts +98 -98
  228. package/src/_modules/discord-bot/_models/dibo-last-mention-date.inteface.ts +7 -7
  229. package/src/_modules/discord-bot/_models/dibo-last-message-date.interface.ts +6 -6
  230. package/src/_modules/discord-bot/_services/dibo-commands.control-service.spec.ts +154 -154
  231. package/src/_modules/discord-bot/_services/dibo-commands.control-service.ts +153 -153
  232. package/src/_modules/discord-bot/_services/dibo-io.control-service.spec.ts +264 -264
  233. package/src/_modules/discord-bot/_services/dibo-io.control-service.ts +306 -306
  234. package/src/_modules/discord-bot/_services/dibo-main.control-service.spec.ts +408 -408
  235. package/src/_modules/discord-bot/_services/dibo-main.control-service.ts +487 -487
  236. package/src/_modules/discord-bot/_services/dibo-routines.control-service.spec.ts +105 -105
  237. package/src/_modules/discord-bot/index.ts +36 -36
  238. package/src/_modules/local-vector-search/_enums/lvs-search-mode.enum.ts +35 -35
  239. package/src/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.ts +59 -59
  240. package/src/_modules/local-vector-search/_models/lvs-search-result.interface.ts +17 -17
  241. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.spec.ts +418 -418
  242. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.ts +276 -276
  243. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.spec.ts +480 -480
  244. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.ts +416 -416
  245. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.spec.ts +198 -198
  246. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.ts +146 -146
  247. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.spec.ts +167 -167
  248. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.ts +106 -106
  249. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.spec.ts +507 -507
  250. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.ts +272 -272
  251. package/src/_modules/local-vector-search/index.ts +16 -16
  252. package/src/_modules/logs/index.ts +11 -11
  253. package/src/_modules/mcp/_models/interfaces/dynts-mcp.interface.ts +111 -111
  254. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.spec.ts +142 -142
  255. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.ts +120 -120
  256. package/src/_modules/mcp/_services/dynts-mcp.adapter.ts +168 -168
  257. package/src/_modules/mcp/index.ts +13 -13
  258. package/src/_modules/messaging/README.md +354 -354
  259. package/src/_modules/messaging/_collections/get-messaging-routing-module.util.ts +26 -26
  260. package/src/_modules/messaging/_collections/msg-global-settings.const.ts +22 -22
  261. package/src/_modules/messaging/_collections/msg.util.spec.ts +226 -226
  262. package/src/_modules/messaging/_models/msg-global-settings.interface.ts +37 -37
  263. package/src/_modules/messaging/_services/msg-conversation.data-service.ts +146 -146
  264. package/src/_modules/messaging/_services/msg-events.service.spec.ts +219 -219
  265. package/src/_modules/messaging/_services/msg-events.service.ts +267 -267
  266. package/src/_modules/messaging/_services/msg-integration.control-service.ts +179 -179
  267. package/src/_modules/messaging/_services/msg-main.control-service.spec.ts +147 -147
  268. package/src/_modules/messaging/_services/msg-main.control-service.ts +571 -571
  269. package/src/_modules/messaging/_services/msg-message.data-service.ts +129 -129
  270. package/src/_modules/messaging/_services/msg.controller.spec.ts +201 -201
  271. package/src/_modules/messaging/index.ts +30 -30
  272. package/src/_modules/mock/app-extended-server.mock.ts +201 -201
  273. package/src/_modules/mock/app-integration-test.mock.ts +51 -51
  274. package/src/_modules/mock/app-params.mock.spec.ts +21 -21
  275. package/src/_modules/mock/app-params.mock.ts +9 -9
  276. package/src/_modules/mock/app-server.mock.ts +188 -188
  277. package/src/_modules/mock/auth-service.mock.spec.ts +47 -47
  278. package/src/_modules/mock/auth-service.mock.ts +28 -28
  279. package/src/_modules/mock/controller.mock.spec.ts +26 -26
  280. package/src/_modules/mock/controller.mock.ts +16 -16
  281. package/src/_modules/mock/data-model.mock.spec.ts +111 -111
  282. package/src/_modules/mock/data-model.mock.ts +82 -82
  283. package/src/_modules/mock/email-service-collection.mock.spec.ts +24 -24
  284. package/src/_modules/mock/email-service-collection.mock.ts +15 -15
  285. package/src/_modules/mock/email-service.mock.spec.ts +17 -17
  286. package/src/_modules/mock/email-service.mock.ts +20 -20
  287. package/src/_modules/mock/email-template.mock.html +14 -14
  288. package/src/_modules/mock/endpoint.mock.ts +91 -91
  289. package/src/_modules/mock/socket-client.mock.spec.ts +40 -40
  290. package/src/_modules/mock/socket-client.mock.ts +45 -45
  291. package/src/_modules/mock/socket-server.mock.spec.ts +44 -44
  292. package/src/_modules/mock/socket-server.mock.ts +46 -46
  293. package/src/_modules/oauth2/_routes/oauth2.controller.spec.ts +107 -107
  294. package/src/_modules/oauth2/_routes/oauth2.controller.ts +98 -98
  295. package/src/_modules/oauth2/_services/oauth2.auth-service.spec.ts +254 -254
  296. package/src/_modules/oauth2/_services/oauth2.auth-service.ts +232 -232
  297. package/src/_modules/oauth2/_services/oauth2.control-service.spec.ts +585 -585
  298. package/src/_modules/oauth2/_services/oauth2.control-service.ts +653 -653
  299. package/src/_modules/oauth2/index.ts +17 -17
  300. package/src/_modules/scheduler/_models/scheduler-job.interface.ts +35 -0
  301. package/src/_modules/scheduler/get-scheduler-routing-module.util.ts +33 -0
  302. package/src/_modules/scheduler/index.ts +8 -0
  303. package/src/_modules/scheduler/scheduler.controller.spec.ts +42 -0
  304. package/src/_modules/scheduler/scheduler.controller.ts +69 -0
  305. package/src/_modules/scheduler/scheduler.service.spec.ts +141 -0
  306. package/src/_modules/scheduler/scheduler.service.ts +176 -0
  307. package/src/_modules/scoped-config/_enums/dynts-scoped-config-level.enum.ts +22 -22
  308. package/src/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.ts +81 -81
  309. package/src/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.ts +107 -107
  310. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.spec.ts +306 -306
  311. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.ts +295 -295
  312. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.spec.ts +118 -118
  313. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.ts +105 -105
  314. package/src/_modules/scoped-config/index.ts +17 -17
  315. package/src/_modules/server/errors/errors.control-service.spec.ts +238 -238
  316. package/src/_modules/server/errors/errors.control-service.ts +100 -100
  317. package/src/_modules/server/errors/errors.controller.spec.ts +268 -268
  318. package/src/_modules/server/errors/errors.controller.ts +515 -515
  319. package/src/_modules/server/errors/errors.data-service.spec.ts +480 -480
  320. package/src/_modules/server/index.ts +30 -30
  321. package/src/_modules/server/server-status/server-status-snapshot.control-service.spec.ts +70 -70
  322. package/src/_modules/server/server-status/server-status-snapshot.control-service.ts +17 -17
  323. package/src/_modules/server/server-status/server-status-snapshot.data-service.spec.ts +77 -77
  324. package/src/_modules/server/server-status/server-status-snapshot.data-service.ts +37 -37
  325. package/src/_modules/server/server-status/server-status.control-service.spec.ts +576 -576
  326. package/src/_modules/server/server-status/server-status.control-service.ts +396 -396
  327. package/src/_modules/server/server-status/server-status.controller.spec.ts +255 -255
  328. package/src/_modules/server/server-status/server-status.controller.ts +272 -272
  329. package/src/_modules/socket/_enums/socket-security.enum.ts +11 -11
  330. package/src/_modules/socket/_models/socket-client-service-params.control-model.spec.ts +32 -32
  331. package/src/_modules/socket/_models/socket-client-service-params.control-model.ts +22 -22
  332. package/src/_modules/socket/_models/socket-presence.control-model.spec.ts +164 -164
  333. package/src/_modules/socket/_models/socket-presence.control-model.ts +210 -210
  334. package/src/_modules/socket/_models/socket-server-service-params.control-model.spec.ts +46 -46
  335. package/src/_modules/socket/_models/socket-server-service-params.control-model.ts +22 -22
  336. package/src/_modules/socket/_services/socket-client.service.spec.ts +15 -15
  337. package/src/_modules/socket/_services/socket-client.service.ts +260 -260
  338. package/src/_modules/socket/_services/socket-server.service.spec.ts +11 -11
  339. package/src/_modules/socket/app-extended.integration.spec.ts +85 -85
  340. package/src/_modules/socket/app-extended.server.ts +630 -630
  341. package/src/_modules/socket/index.ts +42 -42
  342. package/src/_modules/test/get-test-routing-module.util.spec.ts +28 -28
  343. package/src/_modules/test/get-test-routing-module.util.ts +23 -23
  344. package/src/_modules/test/index.ts +11 -11
  345. package/src/_modules/test/test.controller.spec.ts +72 -72
  346. package/src/_modules/test/test.controller.ts +115 -115
  347. package/src/_modules/usage/get-usage-routing-module.util.ts +22 -22
  348. package/src/_modules/usage/index.ts +15 -15
  349. package/src/_modules/usage/usage.controller.spec.ts +81 -81
  350. package/src/_modules/usage/usage.controller.ts +126 -126
  351. package/src/_modules/usage/usage.data-service.spec.ts +332 -332
  352. package/src/_modules/usage/usage.data-service.ts +185 -185
  353. package/src/_services/base/api.service-base.spec.ts +125 -125
  354. package/src/_services/base/api.service-base.ts +74 -74
  355. package/src/_services/base/archive-data.service.spec.ts +209 -209
  356. package/src/_services/base/archive-data.service.ts +224 -224
  357. package/src/_services/base/data.service.spec.ts +729 -729
  358. package/src/_services/base/data.service.ts +2740 -2740
  359. package/src/_services/base/db.service.spec.ts +73 -73
  360. package/src/_services/base/db.service.ts +1575 -1575
  361. package/src/_services/base/singleton.service-base.spec.ts +28 -28
  362. package/src/_services/base/singleton.service-base.ts +24 -24
  363. package/src/_services/base/singleton.service.spec.ts +114 -114
  364. package/src/_services/base/singleton.service.ts +38 -38
  365. package/src/_services/core/api.service.spec.ts +140 -140
  366. package/src/_services/core/auth.service.spec.ts +159 -159
  367. package/src/_services/core/auth.service.ts +174 -174
  368. package/src/_services/core/email.service.spec.ts +85 -85
  369. package/src/_services/core/email.service.ts +742 -742
  370. package/src/_services/core/global.service.spec.ts +292 -292
  371. package/src/_services/core/global.service.ts +487 -487
  372. package/src/_services/core/memory-guard.service.spec.ts +245 -245
  373. package/src/_services/core/memory-guard.service.ts +481 -481
  374. package/src/_services/core/service-collection.service.spec.ts +46 -46
  375. package/src/_services/core/service-collection.service.ts +6 -6
  376. package/src/_services/route/controller.service.spec.ts +53 -53
  377. package/src/_services/route/controller.service.ts +148 -148
  378. package/src/_services/route/routing-module.service.spec.ts +98 -98
  379. package/src/_services/route/routing-module.service.ts +330 -330
  380. package/src/_services/server/app.server.ts +1941 -1941
  381. package/src/_services/shared.static-service.spec.ts +99 -99
  382. package/src/_services/shared.static-service.ts +78 -78
  383. package/src/index.ts +97 -97
  384. package/tsconfig.app.json +12 -12
  385. package/tsconfig.json +42 -42
@@ -1,416 +1,416 @@
1
- import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
- import {
3
- DyFM_DataModel_Params,
4
- DyFM_DataProperty_Params,
5
- DyFM_DBFilter,
6
- DyFM_Metadata,
7
- } from '@futdevpro/fsm-dynamo';
8
- import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
-
10
- import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
- import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
- import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
- import {
14
- DyNTS_LVS_BM25_Corpus,
15
- DyNTS_LVS_BM25_DocScore,
16
- dyNTS_LVS_BM25_minMaxNormalize,
17
- } from './lvs-bm25.util';
18
- import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
- import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
-
21
- /**
22
- * Local Vector Data Service
23
- * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
- * instead of MongoDB vector search
25
- *
26
- * This service loads data from the database, builds an in-memory vector pool,
27
- * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
- *
29
- * @example
30
- * ```typescript
31
- * const service = new DyNTS_OAI_LocalVectorDataService(
32
- * data,
33
- * dataParams,
34
- * openAISettings,
35
- * issuer
36
- * );
37
- *
38
- * const results = await service.vectorSearch({
39
- * input: 'search query',
40
- * searchInKey: 'contentVectorized',
41
- * limit: 10,
42
- * filterBy: { category: 'docs' }
43
- * });
44
- * ```
45
- */
46
- export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
-
48
- /**
49
- * Default search mode
50
- * Cosine similarity is typically preferred for text embeddings
51
- */
52
- defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
-
54
- /**
55
- * Whether to use L2 normalization when storing vectors
56
- * Normalized vectors are more efficient for cosine similarity calculations
57
- */
58
- useL2Normalization: boolean = true;
59
-
60
- /**
61
- * Instance-based vector pool utility
62
- * Minden service instance saját vektor pool-t tart fenn
63
- */
64
- private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
-
66
- constructor(
67
- /**
68
- * Initial data, this will be used by functions on default
69
- */
70
- data: T,
71
- /**
72
- * DB data params will be used to connect to usable dbService on GlobalService
73
- */
74
- dataParams: DyFM_DataModel_Params<T>,
75
- /**
76
- * OpenAI settings
77
- */
78
- openAISettings: DyFM_OAI_Settings,
79
- /**
80
- * Initial set for issuer to be able to follow the issuer's activity
81
- */
82
- issuer: string
83
- ) {
84
- super(data, dataParams, openAISettings, issuer);
85
- }
86
-
87
- /**
88
- * Overrides the parent vectorSearch method to perform local in-memory vector search
89
- *
90
- * Process:
91
- * 1. Load data from DB using filterBy (if provided) or getAll()
92
- * 2. Extract vectorized properties from loaded data
93
- * 3. Build in-memory vector pool
94
- * 4. Vectorize input query
95
- * 5. Perform local vector search
96
- * 6. Map search results back to full data objects
97
- *
98
- * @param set Search parameters
99
- * @returns Array of data objects sorted by similarity score
100
- */
101
- override async vectorSearch(
102
- set: {
103
- input: string;
104
- /** this should be the vectorized property key */
105
- searchInKey: string;
106
- limit?: number;
107
- /**
108
- * Number of candidates that are used to find the best match
109
- * (Not used in local search, but kept for compatibility)
110
- */
111
- numberOfCandidates?: number;
112
- /**
113
- * Filter to narrow down candidates from DB before vector search
114
- */
115
- filterBy?: DyFM_DBFilter<T>;
116
- /**
117
- * Search mode (cosine similarity, L2 distance, or hybrid).
118
- * Defaults to this.defaultSearchMode.
119
- *
120
- * `hybrid` mode combines cosine similarity (vector half) with BM25
121
- * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
- */
123
- searchMode?: LVS_Search_Mode;
124
- /**
125
- * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
- * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
- */
128
- hybridWeight?: { vector: number; text: number };
129
- /**
130
- * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
- * text-search. NEM kell hogy a vectorized property legyen.
132
- */
133
- textSearchKey?: keyof T;
134
- },
135
- ): Promise<T[]> {
136
- try {
137
- if (!set.input) {
138
- throw new DyFM_Error({
139
- ...this.getDefaultErrorSettings(
140
- 'vectorSearch',
141
- new Error('input is required')
142
- ),
143
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
- });
145
- }
146
-
147
- set.limit ??= 3;
148
- const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
- const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
-
151
- // Hybrid mode korai validacio
152
- if (effectiveMode === LVS_Search_Mode.hybrid) {
153
- if (!textSearchKey) {
154
- throw new DyFM_Error({
155
- ...this.getDefaultErrorSettings(
156
- 'vectorSearch',
157
- new Error('textSearchKey is required when searchMode is hybrid'),
158
- ),
159
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
- });
161
- }
162
- if (hybridWeight) {
163
- const w: { vector: number; text: number } = hybridWeight;
164
- if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
- throw new DyFM_Error({
166
- ...this.getDefaultErrorSettings(
167
- 'vectorSearch',
168
- new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
- ),
170
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
- });
172
- }
173
- }
174
- }
175
-
176
- // Validáljuk, hogy a searchInKey létezik-e
177
- const property: DyFM_DataProperty_Params<any, T> =
178
- this.dataParams.properties[searchInKey];
179
-
180
- if (!property) {
181
- throw new DyFM_Error({
182
- ...this.getDefaultErrorSettings(
183
- 'vectorSearch',
184
- new Error(
185
- `Property "${searchInKey}" not found! ` +
186
- `while searching "${this.dataParams.dataName}" ` +
187
- `(The searchable properties are: ` +
188
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
- )
190
- ),
191
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
- __localStack: this.dataParams.stackLocation,
193
- });
194
- }
195
-
196
- // Ellenőrizzük, hogy a property vectorized-e
197
- if (!this.vectorizedProperties.some(
198
- (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
- )) {
200
- throw new DyFM_Error({
201
- ...this.getDefaultErrorSettings(
202
- 'vectorSearch',
203
- new Error(
204
- `Property "${searchInKey}" is not a vectorized property! ` +
205
- `(The vectorized properties are: ` +
206
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
- )
208
- ),
209
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
- __localStack: this.dataParams.stackLocation,
211
- });
212
- }
213
-
214
- // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
- let dataList: T[];
216
- if (filterBy) {
217
- // Ha van filterBy, akkor először szűrünk a DB-ben
218
- if (this.debugLog) {
219
- DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
- }
221
- dataList = await this.findDataList(filterBy, true);
222
- } else {
223
- // Ha nincs filterBy, akkor minden adatot betöltünk
224
- if (this.debugLog) {
225
- DyFM_Log.log(`Local vector search: loading all data...`);
226
- }
227
- dataList = await this.getAll(true);
228
- }
229
-
230
- if (dataList.length === 0) {
231
- if (this.debugLog) {
232
- DyFM_Log.log(`Local vector search: no data found`);
233
- }
234
- return [];
235
- }
236
-
237
- // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
- if (this.debugLog) {
239
- DyFM_Log.log(
240
- `Local vector search: building vector pool from ${dataList.length} items...`
241
- );
242
- }
243
-
244
- // Töröljük a korábbi pool-t
245
- this.vectorPool.clearPool();
246
-
247
- // Hozzáadjuk a vektorokat a pool-hoz
248
- const dataMap: Map<string, T> = new Map<string, T>();
249
- for (const dataItem of dataList) {
250
- if (!dataItem._id) {
251
- // Ha nincs _id, akkor kihagyjuk
252
- if (this.debugLog) {
253
- DyFM_Log.warn(
254
- `Local vector search: skipping item without _id`
255
- );
256
- }
257
- continue;
258
- }
259
-
260
- const vectorizedValue: number[] | undefined =
261
- dataItem[searchInKey] as number[] | undefined;
262
-
263
- if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
- // Ha nincs vectorized érték, akkor kihagyjuk
265
- if (this.debugLog) {
266
- DyFM_Log.warn(
267
- `Local vector search: skipping item "${dataItem._id}" ` +
268
- `without vectorized value for "${searchInKey}"`
269
- );
270
- }
271
- continue;
272
- }
273
-
274
- // Hozzáadjuk a vektort a pool-hoz
275
- this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
- dataMap.set(dataItem._id, dataItem);
277
- }
278
-
279
- if (dataMap.size === 0) {
280
- if (this.debugLog) {
281
- DyFM_Log.log(`Local vector search: no valid vectors found`);
282
- }
283
- return [];
284
- }
285
-
286
- // 3. Vectorizáljuk a query input-ot
287
- if (this.debugLog) {
288
- DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
- }
290
-
291
- const queryVector: number[] = await this.vectorize(
292
- input,
293
- property.embeddingModel
294
- );
295
-
296
- // 4. Végrehajtjuk a local vector search-t
297
- const mode: LVS_Search_Mode = effectiveMode;
298
-
299
- if (this.debugLog) {
300
- DyFM_Log.log(
301
- `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
- );
303
- }
304
-
305
- let searchResults: LVS_SearchResult[];
306
-
307
- if (mode === LVS_Search_Mode.hybrid) {
308
- // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
- const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
- queryVector,
311
- dataMap.size,
312
- LVS_Search_Mode.cosineSimilarity,
313
- );
314
-
315
- // BM25 corpus epitese a `textSearchKey` property-bol
316
- const docs: { id: string; text: string }[] = [];
317
- for (const [docId, dataItem] of dataMap) {
318
- const textValue: unknown = dataItem[textSearchKey as keyof T];
319
- docs.push({
320
- id: docId,
321
- text: typeof textValue === 'string' ? textValue : '',
322
- });
323
- }
324
- const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
- const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
- const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
-
328
- const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
- for (const s of bm25Normalized) {
330
- bm25ScoreById.set(s.id, s.score);
331
- }
332
-
333
- const wVector: number = hybridWeight?.vector ?? 0.5;
334
- const wText: number = hybridWeight?.text ?? 0.5;
335
-
336
- const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
- const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
- return {
339
- id: c.id,
340
- score: wVector * c.score + wText * bm25Score,
341
- };
342
- });
343
- merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
- searchResults = merged.slice(0, limit);
345
- } else {
346
- searchResults = this.vectorPool.search(
347
- queryVector,
348
- limit,
349
- mode,
350
- );
351
- }
352
-
353
- if (this.debugLog) {
354
- DyFM_Log.log(
355
- `Local vector search: found ${searchResults.length} results`
356
- );
357
- }
358
-
359
- // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
- const results: T[] = [];
361
- for (const searchResult of searchResults) {
362
- const dataItem: T | undefined = dataMap.get(searchResult.id);
363
- if (dataItem) {
364
- results.push(dataItem);
365
- }
366
- }
367
-
368
- // Töröljük a pool-t a memória felszabadítása érdekében
369
- this.vectorPool.clearPool();
370
-
371
- return results;
372
- } catch (error) {
373
- // Töröljük a pool-t hiba esetén is
374
- this.vectorPool.clearPool();
375
-
376
- throw new DyFM_Error({
377
- ...this.getDefaultErrorSettings('vectorSearch', error),
378
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
- });
380
- }
381
- }
382
- }
383
-
384
- /**
385
- * @example
386
- * ```typescript
387
- * // Create a local vector data service
388
- * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
- * new CodeChunk(),
390
- * codeChunk_dataParams,
391
- * openAISettings,
392
- * 'example-issuer'
393
- * );
394
- *
395
- * // Perform vector search with filterBy
396
- * const results = await localVectorService.vectorSearch({
397
- * input: 'How to implement authentication?',
398
- * searchInKey: 'chunkParentedContentVectorized',
399
- * limit: 5,
400
- * filterBy: {
401
- * // Filter by category before vector search
402
- * category: 'documentation'
403
- * },
404
- * searchMode: LVS_Search_Mode.cosineSimilarity
405
- * });
406
- *
407
- * // Results are sorted by similarity score (highest first for cosine)
408
- * console.log(`Found ${results.length} similar chunks`);
409
- *
410
- * // Access the results
411
- * for (const result of results) {
412
- * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
- * }
414
- * ```
415
- */
416
-
1
+ import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
+ import {
3
+ DyFM_DataModel_Params,
4
+ DyFM_DataProperty_Params,
5
+ DyFM_DBFilter,
6
+ DyFM_Metadata,
7
+ } from '@futdevpro/fsm-dynamo';
8
+ import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
+
10
+ import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
+ import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
+ import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
+ import {
14
+ DyNTS_LVS_BM25_Corpus,
15
+ DyNTS_LVS_BM25_DocScore,
16
+ dyNTS_LVS_BM25_minMaxNormalize,
17
+ } from './lvs-bm25.util';
18
+ import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
+ import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
+
21
+ /**
22
+ * Local Vector Data Service
23
+ * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
+ * instead of MongoDB vector search
25
+ *
26
+ * This service loads data from the database, builds an in-memory vector pool,
27
+ * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const service = new DyNTS_OAI_LocalVectorDataService(
32
+ * data,
33
+ * dataParams,
34
+ * openAISettings,
35
+ * issuer
36
+ * );
37
+ *
38
+ * const results = await service.vectorSearch({
39
+ * input: 'search query',
40
+ * searchInKey: 'contentVectorized',
41
+ * limit: 10,
42
+ * filterBy: { category: 'docs' }
43
+ * });
44
+ * ```
45
+ */
46
+ export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
+
48
+ /**
49
+ * Default search mode
50
+ * Cosine similarity is typically preferred for text embeddings
51
+ */
52
+ defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
+
54
+ /**
55
+ * Whether to use L2 normalization when storing vectors
56
+ * Normalized vectors are more efficient for cosine similarity calculations
57
+ */
58
+ useL2Normalization: boolean = true;
59
+
60
+ /**
61
+ * Instance-based vector pool utility
62
+ * Minden service instance saját vektor pool-t tart fenn
63
+ */
64
+ private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
+
66
+ constructor(
67
+ /**
68
+ * Initial data, this will be used by functions on default
69
+ */
70
+ data: T,
71
+ /**
72
+ * DB data params will be used to connect to usable dbService on GlobalService
73
+ */
74
+ dataParams: DyFM_DataModel_Params<T>,
75
+ /**
76
+ * OpenAI settings
77
+ */
78
+ openAISettings: DyFM_OAI_Settings,
79
+ /**
80
+ * Initial set for issuer to be able to follow the issuer's activity
81
+ */
82
+ issuer: string
83
+ ) {
84
+ super(data, dataParams, openAISettings, issuer);
85
+ }
86
+
87
+ /**
88
+ * Overrides the parent vectorSearch method to perform local in-memory vector search
89
+ *
90
+ * Process:
91
+ * 1. Load data from DB using filterBy (if provided) or getAll()
92
+ * 2. Extract vectorized properties from loaded data
93
+ * 3. Build in-memory vector pool
94
+ * 4. Vectorize input query
95
+ * 5. Perform local vector search
96
+ * 6. Map search results back to full data objects
97
+ *
98
+ * @param set Search parameters
99
+ * @returns Array of data objects sorted by similarity score
100
+ */
101
+ override async vectorSearch(
102
+ set: {
103
+ input: string;
104
+ /** this should be the vectorized property key */
105
+ searchInKey: string;
106
+ limit?: number;
107
+ /**
108
+ * Number of candidates that are used to find the best match
109
+ * (Not used in local search, but kept for compatibility)
110
+ */
111
+ numberOfCandidates?: number;
112
+ /**
113
+ * Filter to narrow down candidates from DB before vector search
114
+ */
115
+ filterBy?: DyFM_DBFilter<T>;
116
+ /**
117
+ * Search mode (cosine similarity, L2 distance, or hybrid).
118
+ * Defaults to this.defaultSearchMode.
119
+ *
120
+ * `hybrid` mode combines cosine similarity (vector half) with BM25
121
+ * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
+ */
123
+ searchMode?: LVS_Search_Mode;
124
+ /**
125
+ * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
+ * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
+ */
128
+ hybridWeight?: { vector: number; text: number };
129
+ /**
130
+ * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
+ * text-search. NEM kell hogy a vectorized property legyen.
132
+ */
133
+ textSearchKey?: keyof T;
134
+ },
135
+ ): Promise<T[]> {
136
+ try {
137
+ if (!set.input) {
138
+ throw new DyFM_Error({
139
+ ...this.getDefaultErrorSettings(
140
+ 'vectorSearch',
141
+ new Error('input is required')
142
+ ),
143
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
+ });
145
+ }
146
+
147
+ set.limit ??= 3;
148
+ const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
+ const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
+
151
+ // Hybrid mode korai validacio
152
+ if (effectiveMode === LVS_Search_Mode.hybrid) {
153
+ if (!textSearchKey) {
154
+ throw new DyFM_Error({
155
+ ...this.getDefaultErrorSettings(
156
+ 'vectorSearch',
157
+ new Error('textSearchKey is required when searchMode is hybrid'),
158
+ ),
159
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
+ });
161
+ }
162
+ if (hybridWeight) {
163
+ const w: { vector: number; text: number } = hybridWeight;
164
+ if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
+ throw new DyFM_Error({
166
+ ...this.getDefaultErrorSettings(
167
+ 'vectorSearch',
168
+ new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
+ ),
170
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
+ });
172
+ }
173
+ }
174
+ }
175
+
176
+ // Validáljuk, hogy a searchInKey létezik-e
177
+ const property: DyFM_DataProperty_Params<any, T> =
178
+ this.dataParams.properties[searchInKey];
179
+
180
+ if (!property) {
181
+ throw new DyFM_Error({
182
+ ...this.getDefaultErrorSettings(
183
+ 'vectorSearch',
184
+ new Error(
185
+ `Property "${searchInKey}" not found! ` +
186
+ `while searching "${this.dataParams.dataName}" ` +
187
+ `(The searchable properties are: ` +
188
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
+ )
190
+ ),
191
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
+ __localStack: this.dataParams.stackLocation,
193
+ });
194
+ }
195
+
196
+ // Ellenőrizzük, hogy a property vectorized-e
197
+ if (!this.vectorizedProperties.some(
198
+ (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
+ )) {
200
+ throw new DyFM_Error({
201
+ ...this.getDefaultErrorSettings(
202
+ 'vectorSearch',
203
+ new Error(
204
+ `Property "${searchInKey}" is not a vectorized property! ` +
205
+ `(The vectorized properties are: ` +
206
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
+ )
208
+ ),
209
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
+ __localStack: this.dataParams.stackLocation,
211
+ });
212
+ }
213
+
214
+ // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
+ let dataList: T[];
216
+ if (filterBy) {
217
+ // Ha van filterBy, akkor először szűrünk a DB-ben
218
+ if (this.debugLog) {
219
+ DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
+ }
221
+ dataList = await this.findDataList(filterBy, true);
222
+ } else {
223
+ // Ha nincs filterBy, akkor minden adatot betöltünk
224
+ if (this.debugLog) {
225
+ DyFM_Log.log(`Local vector search: loading all data...`);
226
+ }
227
+ dataList = await this.getAll(true);
228
+ }
229
+
230
+ if (dataList.length === 0) {
231
+ if (this.debugLog) {
232
+ DyFM_Log.log(`Local vector search: no data found`);
233
+ }
234
+ return [];
235
+ }
236
+
237
+ // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
+ if (this.debugLog) {
239
+ DyFM_Log.log(
240
+ `Local vector search: building vector pool from ${dataList.length} items...`
241
+ );
242
+ }
243
+
244
+ // Töröljük a korábbi pool-t
245
+ this.vectorPool.clearPool();
246
+
247
+ // Hozzáadjuk a vektorokat a pool-hoz
248
+ const dataMap: Map<string, T> = new Map<string, T>();
249
+ for (const dataItem of dataList) {
250
+ if (!dataItem._id) {
251
+ // Ha nincs _id, akkor kihagyjuk
252
+ if (this.debugLog) {
253
+ DyFM_Log.warn(
254
+ `Local vector search: skipping item without _id`
255
+ );
256
+ }
257
+ continue;
258
+ }
259
+
260
+ const vectorizedValue: number[] | undefined =
261
+ dataItem[searchInKey] as number[] | undefined;
262
+
263
+ if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
+ // Ha nincs vectorized érték, akkor kihagyjuk
265
+ if (this.debugLog) {
266
+ DyFM_Log.warn(
267
+ `Local vector search: skipping item "${dataItem._id}" ` +
268
+ `without vectorized value for "${searchInKey}"`
269
+ );
270
+ }
271
+ continue;
272
+ }
273
+
274
+ // Hozzáadjuk a vektort a pool-hoz
275
+ this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
+ dataMap.set(dataItem._id, dataItem);
277
+ }
278
+
279
+ if (dataMap.size === 0) {
280
+ if (this.debugLog) {
281
+ DyFM_Log.log(`Local vector search: no valid vectors found`);
282
+ }
283
+ return [];
284
+ }
285
+
286
+ // 3. Vectorizáljuk a query input-ot
287
+ if (this.debugLog) {
288
+ DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
+ }
290
+
291
+ const queryVector: number[] = await this.vectorize(
292
+ input,
293
+ property.embeddingModel
294
+ );
295
+
296
+ // 4. Végrehajtjuk a local vector search-t
297
+ const mode: LVS_Search_Mode = effectiveMode;
298
+
299
+ if (this.debugLog) {
300
+ DyFM_Log.log(
301
+ `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
+ );
303
+ }
304
+
305
+ let searchResults: LVS_SearchResult[];
306
+
307
+ if (mode === LVS_Search_Mode.hybrid) {
308
+ // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
+ const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
+ queryVector,
311
+ dataMap.size,
312
+ LVS_Search_Mode.cosineSimilarity,
313
+ );
314
+
315
+ // BM25 corpus epitese a `textSearchKey` property-bol
316
+ const docs: { id: string; text: string }[] = [];
317
+ for (const [docId, dataItem] of dataMap) {
318
+ const textValue: unknown = dataItem[textSearchKey as keyof T];
319
+ docs.push({
320
+ id: docId,
321
+ text: typeof textValue === 'string' ? textValue : '',
322
+ });
323
+ }
324
+ const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
+ const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
+ const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
+
328
+ const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
+ for (const s of bm25Normalized) {
330
+ bm25ScoreById.set(s.id, s.score);
331
+ }
332
+
333
+ const wVector: number = hybridWeight?.vector ?? 0.5;
334
+ const wText: number = hybridWeight?.text ?? 0.5;
335
+
336
+ const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
+ const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
+ return {
339
+ id: c.id,
340
+ score: wVector * c.score + wText * bm25Score,
341
+ };
342
+ });
343
+ merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
+ searchResults = merged.slice(0, limit);
345
+ } else {
346
+ searchResults = this.vectorPool.search(
347
+ queryVector,
348
+ limit,
349
+ mode,
350
+ );
351
+ }
352
+
353
+ if (this.debugLog) {
354
+ DyFM_Log.log(
355
+ `Local vector search: found ${searchResults.length} results`
356
+ );
357
+ }
358
+
359
+ // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
+ const results: T[] = [];
361
+ for (const searchResult of searchResults) {
362
+ const dataItem: T | undefined = dataMap.get(searchResult.id);
363
+ if (dataItem) {
364
+ results.push(dataItem);
365
+ }
366
+ }
367
+
368
+ // Töröljük a pool-t a memória felszabadítása érdekében
369
+ this.vectorPool.clearPool();
370
+
371
+ return results;
372
+ } catch (error) {
373
+ // Töröljük a pool-t hiba esetén is
374
+ this.vectorPool.clearPool();
375
+
376
+ throw new DyFM_Error({
377
+ ...this.getDefaultErrorSettings('vectorSearch', error),
378
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
+ });
380
+ }
381
+ }
382
+ }
383
+
384
+ /**
385
+ * @example
386
+ * ```typescript
387
+ * // Create a local vector data service
388
+ * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
+ * new CodeChunk(),
390
+ * codeChunk_dataParams,
391
+ * openAISettings,
392
+ * 'example-issuer'
393
+ * );
394
+ *
395
+ * // Perform vector search with filterBy
396
+ * const results = await localVectorService.vectorSearch({
397
+ * input: 'How to implement authentication?',
398
+ * searchInKey: 'chunkParentedContentVectorized',
399
+ * limit: 5,
400
+ * filterBy: {
401
+ * // Filter by category before vector search
402
+ * category: 'documentation'
403
+ * },
404
+ * searchMode: LVS_Search_Mode.cosineSimilarity
405
+ * });
406
+ *
407
+ * // Results are sorted by similarity score (highest first for cosine)
408
+ * console.log(`Found ${results.length} similar chunks`);
409
+ *
410
+ * // Access the results
411
+ * for (const result of results) {
412
+ * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
+ * }
414
+ * ```
415
+ */
416
+