@futdevpro/nts-dynamo 1.15.86 → 1.15.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (382) hide show
  1. package/.c8rc.json +26 -26
  2. package/.copilot/patterns.json +7 -7
  3. package/.cursor/rules/__assistant_guide.mdc +30 -30
  4. package/.cursor/rules/_ag_backend-structure.mdc +85 -85
  5. package/.cursor/rules/_ag_backend.mdc +16 -16
  6. package/.cursor/rules/_ag_frontend-structure.mdc +86 -86
  7. package/.cursor/rules/_ag_frontend.mdc +39 -39
  8. package/.cursor/rules/_ag_import-rules.mdc +44 -44
  9. package/.cursor/rules/_ag_naming.mdc +115 -115
  10. package/.cursor/rules/_ag_should-be.mdc +6 -6
  11. package/.cursor/rules/ai_development_guide.md +60 -60
  12. package/.cursor/rules/cursor-rules.md +160 -160
  13. package/.cursor/rules/default-command.mdc +464 -464
  14. package/.cursor/rules/error_code_pattern.md +39 -39
  15. package/.cursor/rules/saved rule mcp server use.md +15 -15
  16. package/.dynamo/pipeline.cicd.config.json +4 -0
  17. package/.vscode/settings.json +10 -10
  18. package/HOWTO.md +15 -15
  19. package/LICENSE +21 -21
  20. package/__documentations/nts-integration-tests-2026-03-17.md +26 -26
  21. package/__documentations/plans/BEDROCK-HYPERPLAN.md +95 -95
  22. package/_specifications/BACKLOG.md +92 -92
  23. package/_specifications/TODO.md +15 -15
  24. package/_specifications/agent.md +138 -138
  25. package/build/_collections/global-settings.const.d.ts.map +1 -1
  26. package/build/_collections/global-settings.const.js +7 -0
  27. package/build/_collections/global-settings.const.js.map +1 -1
  28. package/build/_collections/security-headers.util.d.ts +17 -0
  29. package/build/_collections/security-headers.util.d.ts.map +1 -0
  30. package/build/_collections/security-headers.util.js +78 -0
  31. package/build/_collections/security-headers.util.js.map +1 -0
  32. package/build/_models/interfaces/global-settings.interface.d.ts +8 -0
  33. package/build/_models/interfaces/global-settings.interface.d.ts.map +1 -1
  34. package/build/_models/interfaces/security-headers-settings.interface.d.ts +42 -0
  35. package/build/_models/interfaces/security-headers-settings.interface.d.ts.map +1 -0
  36. package/build/_models/interfaces/security-headers-settings.interface.js +3 -0
  37. package/build/_models/interfaces/security-headers-settings.interface.js.map +1 -0
  38. package/build/_services/server/app.server.d.ts +7 -0
  39. package/build/_services/server/app.server.d.ts.map +1 -1
  40. package/build/_services/server/app.server.js +22 -0
  41. package/build/_services/server/app.server.js.map +1 -1
  42. package/build/index.d.ts +2 -0
  43. package/build/index.d.ts.map +1 -1
  44. package/build/index.js +2 -0
  45. package/build/index.js.map +1 -1
  46. package/eslint.config.js +3 -3
  47. package/nodemon.json +24 -24
  48. package/package.json +5 -5
  49. package/pnpm-workspace.yaml +5 -5
  50. package/scripts/run-coverage-tests.js +28 -28
  51. package/spec/support/helpers/spec-reporter-loader.js +359 -359
  52. package/spec/support/helpers/ts-node-helper.js +93 -93
  53. package/spec/support/jasmine.coverage.json +24 -24
  54. package/spec/support/jasmine.json +24 -24
  55. package/src/_collections/archive.util.spec.ts +57 -57
  56. package/src/_collections/archive.util.ts +18 -18
  57. package/src/_collections/atlas-default-db-options.const.ts +9 -9
  58. package/src/_collections/default-fallback-cache-max-age.const.spec.ts +11 -11
  59. package/src/_collections/default-fallback-cache-max-age.const.ts +2 -2
  60. package/src/_collections/default-not-found-page.const.spec.ts +19 -19
  61. package/src/_collections/default-not-found-page.const.ts +22 -22
  62. package/src/_collections/default-socket-path.const.spec.ts +12 -12
  63. package/src/_collections/default-socket-path.const.ts +2 -2
  64. package/src/_collections/get-environment-settings.util.spec.ts +210 -210
  65. package/src/_collections/get-environment-settings.util.ts +48 -48
  66. package/src/_collections/global-settings.const.ts +114 -106
  67. package/src/_collections/sample.env +21 -21
  68. package/src/_collections/security-headers.util.spec.ts +98 -0
  69. package/src/_collections/security-headers.util.ts +96 -0
  70. package/src/_collections/star.controller.spec.ts +224 -224
  71. package/src/_collections/star.controller.ts +129 -129
  72. package/src/_enums/data-model-type.enum.ts +14 -14
  73. package/src/_enums/data-service-function.enum.ts +24 -24
  74. package/src/_enums/predefined-data-types.enum.ts +16 -16
  75. package/src/_enums/route-security.enum.ts +12 -12
  76. package/src/_models/control-models/api-call-params.control-model.spec.ts +152 -152
  77. package/src/_models/control-models/api-call-params.control-model.ts +142 -142
  78. package/src/_models/control-models/app-ext-system-controls.control-model.spec.ts +52 -52
  79. package/src/_models/control-models/app-ext-system-controls.control-model.ts +9 -9
  80. package/src/_models/control-models/app-params.control-model.spec.ts +225 -225
  81. package/src/_models/control-models/app-params.control-model.ts +136 -136
  82. package/src/_models/control-models/app-system-controls.control-model.spec.ts +31 -31
  83. package/src/_models/control-models/app-system-controls.control-model.ts +9 -9
  84. package/src/_models/control-models/endpoint-params.control-model.spec.ts +627 -627
  85. package/src/_models/control-models/endpoint-params.control-model.ts +627 -627
  86. package/src/_models/control-models/http-settings.control-model.spec.ts +77 -77
  87. package/src/_models/control-models/http-settings.control-model.ts +37 -37
  88. package/src/_models/control-models/system-control.control-model.spec.ts +27 -27
  89. package/src/_models/control-models/system-control.control-model.ts +12 -12
  90. package/src/_models/interfaces/certification-settings.interface.ts +7 -7
  91. package/src/_models/interfaces/environment-settings.interface.ts +59 -59
  92. package/src/_models/interfaces/global-log-settings.interface.ts +171 -171
  93. package/src/_models/interfaces/global-service-settings.interface.ts +47 -47
  94. package/src/_models/interfaces/global-settings.interface.ts +244 -235
  95. package/src/_models/interfaces/routing-module-settings.interface.ts +21 -21
  96. package/src/_models/interfaces/security-headers-settings.interface.ts +50 -0
  97. package/src/_models/interfaces/static-client-settings.interface.spec.ts +29 -29
  98. package/src/_models/interfaces/static-client-settings.interface.ts +28 -28
  99. package/src/_models/types/db-update.type.ts +100 -100
  100. package/src/_modules/ai/_models/ai-input-interfaces.ts +117 -117
  101. package/src/_modules/ai/_models/ai-test-generation-result.interface.ts +16 -16
  102. package/src/_modules/ai/_modules/anthropic/_services/aai-user-key.control-service.ts +138 -138
  103. package/src/_modules/ai/_modules/anthropic/index.ts +5 -5
  104. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.spec.ts +242 -242
  105. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.ts +639 -639
  106. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.spec.ts +295 -295
  107. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.ts +518 -518
  108. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.spec.ts +209 -209
  109. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.ts +85 -85
  110. package/src/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.ts +7 -7
  111. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.ts +146 -146
  112. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model.ts +162 -162
  113. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-document.data-model.ts +99 -99
  114. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-code-chunk.interface.ts +68 -68
  115. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface.ts +18 -18
  116. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-page-compare-result.interface.ts +19 -19
  117. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-document-compare-result.interface.ts +25 -25
  118. package/src/_modules/ai/_modules/document-ai/index.ts +30 -30
  119. package/src/_modules/ai/_modules/fdp-ai/_services/fdpai-user-key.control-service.ts +189 -189
  120. package/src/_modules/ai/_modules/fdp-ai/index.ts +5 -5
  121. package/src/_modules/ai/_modules/open-ai/_collections/oai-global-settings.const.ts +9 -9
  122. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests-hu.conts.ts +82 -82
  123. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests.conts.ts +75 -75
  124. package/src/_modules/ai/_modules/open-ai/_enums/oai-gpt-message-role.enum.ts +45 -45
  125. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-global-settings.interface.ts +7 -7
  126. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-gpt-message.interface.ts +7 -7
  127. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-llm-predefined-requests.interface.ts +57 -57
  128. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.ts +292 -292
  129. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.spec.ts +342 -342
  130. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.spec.ts +550 -550
  131. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.ts +630 -630
  132. package/src/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.spec.ts +332 -332
  133. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.spec.ts +462 -462
  134. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.ts +634 -634
  135. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.spec.ts +489 -489
  136. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.tools.spec.ts +173 -173
  137. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.ts +1033 -1033
  138. package/src/_modules/ai/_modules/open-ai/_services/oai-user-key.control-service.ts +157 -157
  139. package/src/_modules/ai/_services/ai-embedding-mock.service.spec.ts +115 -115
  140. package/src/_modules/ai/_services/ai-embedding-mock.service.ts +212 -212
  141. package/src/_modules/ai/_services/ai-embedding-provider.registry.spec.ts +110 -110
  142. package/src/_modules/ai/_services/ai-embedding-provider.registry.ts +110 -110
  143. package/src/_modules/ai/_services/ai-embedding.service-base.spec.ts +98 -98
  144. package/src/_modules/ai/_services/ai-embedding.service-base.ts +48 -48
  145. package/src/_modules/ai/_services/ai-llm-chat.service-base.spec.ts +229 -229
  146. package/src/_modules/ai/_services/ai-llm-chat.service-base.ts +68 -68
  147. package/src/_modules/ai/_services/ai-llm.service-base.spec.ts +250 -250
  148. package/src/_modules/ai/_services/ai-llm.service-base.ts +519 -519
  149. package/src/_modules/ai/_services/ai-provider.service-base.spec.ts +158 -158
  150. package/src/_modules/ai/_services/ai-user-key.service-base.ts +59 -59
  151. package/src/_modules/ai/_services/lmstudio-embedding.control-service.spec.ts +197 -197
  152. package/src/_modules/ai/_services/lmstudio-embedding.control-service.ts +371 -371
  153. package/src/_modules/ai/index.ts +23 -23
  154. package/src/_modules/assistant/_collections/ass-global-settings.const.ts +13 -13
  155. package/src/_modules/assistant/_collections/ass.util.spec.ts +176 -176
  156. package/src/_modules/assistant/_collections/ass.util.ts +50 -50
  157. package/src/_modules/assistant/_models/ass-global-settings.interface.ts +15 -15
  158. package/src/_modules/assistant/_services/ass-io.control-service.spec.ts +140 -140
  159. package/src/_modules/assistant/_services/ass-main.control-service.spec.ts +192 -192
  160. package/src/_modules/assistant/_services/ass-main.control-service.ts +107 -107
  161. package/src/_modules/bot/_collections/bot-default-commands.const.ts +12 -12
  162. package/src/_modules/bot/_collections/bot-global-settings.const.ts +39 -39
  163. package/src/_modules/bot/_models/bot-channel-wrapper.interface.ts +62 -62
  164. package/src/_modules/bot/_models/bot-command.interface.ts +8 -8
  165. package/src/_modules/bot/_models/bot-global-settings.interface.ts +96 -96
  166. package/src/_modules/bot/_models/bot-last-mention-date.interface.ts +6 -6
  167. package/src/_modules/bot/_models/bot-last-message-date.interface.ts +5 -5
  168. package/src/_modules/bot/_models/bot-user-wrapper.interface.ts +41 -41
  169. package/src/_modules/bot/_modules/discord-bot/_models/dib-platform.types.ts +9 -9
  170. package/src/_modules/bot/_modules/discord-bot/_services/dib-messaging-provider.control-service.spec.ts +431 -431
  171. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.spec.ts +160 -160
  172. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.ts +55 -55
  173. package/src/_modules/bot/_modules/dynamo-bot/_models/dyb-platform.types.ts +15 -15
  174. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.spec.ts +374 -374
  175. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.ts +447 -447
  176. package/src/_modules/bot/_modules/dynamo-bot/index.ts +15 -15
  177. package/src/_modules/bot/_modules/slack-bot/_models/slb-platform.types.ts +9 -9
  178. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.spec.ts +344 -344
  179. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.ts +197 -197
  180. package/src/_modules/bot/_modules/teams-bot/_models/teb-platform.types.ts +9 -9
  181. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.spec.ts +345 -345
  182. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.ts +197 -197
  183. package/src/_modules/bot/_services/bot-commands.control-service.spec.ts +116 -116
  184. package/src/_modules/bot/_services/bot-io.control-service.spec.ts +285 -285
  185. package/src/_modules/bot/_services/bot-main.control-service.spec.ts +208 -208
  186. package/src/_modules/bot/_services/bot-messaging-provider.service-base.spec.ts +349 -349
  187. package/src/_modules/bot/_services/bot-routines.control-service.spec.ts +111 -111
  188. package/src/_modules/custom-data/custom-data.controller.spec.ts +49 -49
  189. package/src/_modules/custom-data/custom-data.controller.ts +67 -67
  190. package/src/_modules/custom-data/custom-data.data-service.spec.ts +54 -54
  191. package/src/_modules/custom-data/custom-data.data-service.ts +21 -21
  192. package/src/_modules/custom-data/get-custom-data-routing-module.util.spec.ts +28 -28
  193. package/src/_modules/custom-data/get-custom-data-routing-module.util.ts +24 -24
  194. package/src/_modules/custom-data/index.ts +9 -9
  195. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.spec.ts +161 -161
  196. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.ts +203 -203
  197. package/src/_modules/data-readers/_models/interfaces/dynts-sqlite-reader.interface.ts +33 -33
  198. package/src/_modules/data-readers/index.ts +11 -11
  199. package/src/_modules/defaults/_collections/default-endpoints.util.ts +487 -487
  200. package/src/_modules/defaults/_models/default-user.data-model.ts +72 -72
  201. package/src/_modules/defaults/_services/default-auth.service.spec.ts +269 -269
  202. package/src/_modules/defaults/_services/default-auth.service.ts +177 -177
  203. package/src/_modules/defaults/_services/default-socket-events.service.spec.ts +42 -42
  204. package/src/_modules/defaults/_services/default-socket-events.service.ts +61 -61
  205. package/src/_modules/defaults/_services/default-user.data-service.spec.ts +187 -187
  206. package/src/_modules/defaults/_services/default-user.data-service.ts +98 -98
  207. package/src/_modules/defaults/index.ts +17 -17
  208. package/src/_modules/discord-assistant/_collections/dias-global-settings.const.ts +19 -19
  209. package/src/_modules/discord-assistant/_collections/dias.util.spec.ts +366 -366
  210. package/src/_modules/discord-assistant/_collections/dias.util.ts +132 -132
  211. package/src/_modules/discord-assistant/_models/dias-global-settings.interface.ts +19 -19
  212. package/src/_modules/discord-assistant/_models/dias-knowledge.data-model.ts +52 -52
  213. package/src/_modules/discord-assistant/_services/dias-chunk.data-service.ts +177 -177
  214. package/src/_modules/discord-assistant/_services/dias-io.control-service.spec.ts +108 -108
  215. package/src/_modules/discord-assistant/_services/dias-io.control-service.ts +69 -69
  216. package/src/_modules/discord-assistant/_services/dias-main.control-service.spec.ts +22 -22
  217. package/src/_modules/discord-assistant/_services/dias-main.control-service.ts +27 -27
  218. package/src/_modules/discord-assistant/_services/dias.service-base.spec.ts +195 -195
  219. package/src/_modules/discord-assistant/_services/dias.service-base.ts +76 -76
  220. package/src/_modules/discord-assistant/index.ts +38 -38
  221. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.spec.ts +34 -34
  222. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.ts +11 -11
  223. package/src/_modules/discord-assistant-voiced/index.ts +36 -36
  224. package/src/_modules/discord-bot/_collections/dibo-default-commands.const.ts +16 -16
  225. package/src/_modules/discord-bot/_collections/dibo-global-settings.conts.ts +55 -55
  226. package/src/_modules/discord-bot/_collections/dibo-operations.util.spec.ts +214 -214
  227. package/src/_modules/discord-bot/_collections/dibo-operations.util.ts +387 -387
  228. package/src/_modules/discord-bot/_models/dibo-command.interface.ts +12 -12
  229. package/src/_modules/discord-bot/_models/dibo-global-settings.interface.ts +98 -98
  230. package/src/_modules/discord-bot/_models/dibo-last-mention-date.inteface.ts +7 -7
  231. package/src/_modules/discord-bot/_models/dibo-last-message-date.interface.ts +6 -6
  232. package/src/_modules/discord-bot/_services/dibo-commands.control-service.spec.ts +154 -154
  233. package/src/_modules/discord-bot/_services/dibo-commands.control-service.ts +153 -153
  234. package/src/_modules/discord-bot/_services/dibo-io.control-service.spec.ts +264 -264
  235. package/src/_modules/discord-bot/_services/dibo-io.control-service.ts +306 -306
  236. package/src/_modules/discord-bot/_services/dibo-main.control-service.spec.ts +408 -408
  237. package/src/_modules/discord-bot/_services/dibo-main.control-service.ts +487 -487
  238. package/src/_modules/discord-bot/_services/dibo-routines.control-service.spec.ts +105 -105
  239. package/src/_modules/discord-bot/index.ts +36 -36
  240. package/src/_modules/local-vector-search/_enums/lvs-search-mode.enum.ts +35 -35
  241. package/src/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.ts +59 -59
  242. package/src/_modules/local-vector-search/_models/lvs-search-result.interface.ts +17 -17
  243. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.spec.ts +418 -418
  244. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.ts +276 -276
  245. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.spec.ts +480 -480
  246. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.ts +416 -416
  247. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.spec.ts +198 -198
  248. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.ts +146 -146
  249. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.spec.ts +167 -167
  250. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.ts +106 -106
  251. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.spec.ts +507 -507
  252. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.ts +272 -272
  253. package/src/_modules/local-vector-search/index.ts +16 -16
  254. package/src/_modules/logs/index.ts +11 -11
  255. package/src/_modules/mcp/_models/interfaces/dynts-mcp.interface.ts +111 -111
  256. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.spec.ts +142 -142
  257. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.ts +120 -120
  258. package/src/_modules/mcp/_services/dynts-mcp.adapter.ts +168 -168
  259. package/src/_modules/mcp/index.ts +13 -13
  260. package/src/_modules/messaging/README.md +354 -354
  261. package/src/_modules/messaging/_collections/get-messaging-routing-module.util.ts +26 -26
  262. package/src/_modules/messaging/_collections/msg-global-settings.const.ts +22 -22
  263. package/src/_modules/messaging/_collections/msg.util.spec.ts +226 -226
  264. package/src/_modules/messaging/_models/msg-global-settings.interface.ts +37 -37
  265. package/src/_modules/messaging/_services/msg-conversation.data-service.ts +146 -146
  266. package/src/_modules/messaging/_services/msg-events.service.spec.ts +219 -219
  267. package/src/_modules/messaging/_services/msg-events.service.ts +267 -267
  268. package/src/_modules/messaging/_services/msg-integration.control-service.ts +179 -179
  269. package/src/_modules/messaging/_services/msg-main.control-service.spec.ts +147 -147
  270. package/src/_modules/messaging/_services/msg-main.control-service.ts +571 -571
  271. package/src/_modules/messaging/_services/msg-message.data-service.ts +129 -129
  272. package/src/_modules/messaging/_services/msg.controller.spec.ts +201 -201
  273. package/src/_modules/messaging/index.ts +30 -30
  274. package/src/_modules/mock/app-extended-server.mock.ts +201 -201
  275. package/src/_modules/mock/app-integration-test.mock.ts +51 -51
  276. package/src/_modules/mock/app-params.mock.spec.ts +21 -21
  277. package/src/_modules/mock/app-params.mock.ts +9 -9
  278. package/src/_modules/mock/app-server.mock.ts +188 -188
  279. package/src/_modules/mock/auth-service.mock.spec.ts +47 -47
  280. package/src/_modules/mock/auth-service.mock.ts +28 -28
  281. package/src/_modules/mock/controller.mock.spec.ts +26 -26
  282. package/src/_modules/mock/controller.mock.ts +16 -16
  283. package/src/_modules/mock/data-model.mock.spec.ts +111 -111
  284. package/src/_modules/mock/data-model.mock.ts +82 -82
  285. package/src/_modules/mock/email-service-collection.mock.spec.ts +24 -24
  286. package/src/_modules/mock/email-service-collection.mock.ts +15 -15
  287. package/src/_modules/mock/email-service.mock.spec.ts +17 -17
  288. package/src/_modules/mock/email-service.mock.ts +20 -20
  289. package/src/_modules/mock/email-template.mock.html +14 -14
  290. package/src/_modules/mock/endpoint.mock.ts +91 -91
  291. package/src/_modules/mock/socket-client.mock.spec.ts +40 -40
  292. package/src/_modules/mock/socket-client.mock.ts +45 -45
  293. package/src/_modules/mock/socket-server.mock.spec.ts +44 -44
  294. package/src/_modules/mock/socket-server.mock.ts +46 -46
  295. package/src/_modules/oauth2/_routes/oauth2.controller.spec.ts +107 -107
  296. package/src/_modules/oauth2/_routes/oauth2.controller.ts +98 -98
  297. package/src/_modules/oauth2/_services/oauth2.auth-service.spec.ts +254 -254
  298. package/src/_modules/oauth2/_services/oauth2.auth-service.ts +232 -232
  299. package/src/_modules/oauth2/_services/oauth2.control-service.spec.ts +585 -585
  300. package/src/_modules/oauth2/_services/oauth2.control-service.ts +653 -653
  301. package/src/_modules/oauth2/index.ts +17 -17
  302. package/src/_modules/scoped-config/_enums/dynts-scoped-config-level.enum.ts +22 -22
  303. package/src/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.ts +81 -81
  304. package/src/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.ts +107 -107
  305. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.spec.ts +306 -306
  306. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.ts +295 -295
  307. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.spec.ts +118 -118
  308. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.ts +105 -105
  309. package/src/_modules/scoped-config/index.ts +17 -17
  310. package/src/_modules/server/errors/errors.control-service.spec.ts +238 -238
  311. package/src/_modules/server/errors/errors.control-service.ts +100 -100
  312. package/src/_modules/server/errors/errors.controller.spec.ts +268 -268
  313. package/src/_modules/server/errors/errors.controller.ts +515 -515
  314. package/src/_modules/server/errors/errors.data-service.spec.ts +480 -480
  315. package/src/_modules/server/index.ts +30 -30
  316. package/src/_modules/server/server-status/server-status-snapshot.control-service.spec.ts +70 -70
  317. package/src/_modules/server/server-status/server-status-snapshot.control-service.ts +17 -17
  318. package/src/_modules/server/server-status/server-status-snapshot.data-service.spec.ts +77 -77
  319. package/src/_modules/server/server-status/server-status-snapshot.data-service.ts +37 -37
  320. package/src/_modules/server/server-status/server-status.control-service.spec.ts +576 -576
  321. package/src/_modules/server/server-status/server-status.control-service.ts +396 -396
  322. package/src/_modules/server/server-status/server-status.controller.spec.ts +255 -255
  323. package/src/_modules/server/server-status/server-status.controller.ts +272 -272
  324. package/src/_modules/socket/_enums/socket-security.enum.ts +11 -11
  325. package/src/_modules/socket/_models/socket-client-service-params.control-model.spec.ts +32 -32
  326. package/src/_modules/socket/_models/socket-client-service-params.control-model.ts +22 -22
  327. package/src/_modules/socket/_models/socket-presence.control-model.spec.ts +164 -164
  328. package/src/_modules/socket/_models/socket-presence.control-model.ts +210 -210
  329. package/src/_modules/socket/_models/socket-server-service-params.control-model.spec.ts +46 -46
  330. package/src/_modules/socket/_models/socket-server-service-params.control-model.ts +22 -22
  331. package/src/_modules/socket/_services/socket-client.service.spec.ts +15 -15
  332. package/src/_modules/socket/_services/socket-client.service.ts +260 -260
  333. package/src/_modules/socket/_services/socket-server.service.spec.ts +11 -11
  334. package/src/_modules/socket/app-extended.integration.spec.ts +85 -85
  335. package/src/_modules/socket/app-extended.server.ts +630 -630
  336. package/src/_modules/socket/index.ts +42 -42
  337. package/src/_modules/test/get-test-routing-module.util.spec.ts +28 -28
  338. package/src/_modules/test/get-test-routing-module.util.ts +23 -23
  339. package/src/_modules/test/index.ts +11 -11
  340. package/src/_modules/test/test.controller.spec.ts +72 -72
  341. package/src/_modules/test/test.controller.ts +115 -115
  342. package/src/_modules/usage/get-usage-routing-module.util.ts +22 -22
  343. package/src/_modules/usage/index.ts +15 -15
  344. package/src/_modules/usage/usage.controller.spec.ts +81 -81
  345. package/src/_modules/usage/usage.controller.ts +126 -126
  346. package/src/_modules/usage/usage.data-service.spec.ts +332 -332
  347. package/src/_modules/usage/usage.data-service.ts +185 -185
  348. package/src/_services/base/api.service-base.spec.ts +125 -125
  349. package/src/_services/base/api.service-base.ts +74 -74
  350. package/src/_services/base/archive-data.service.spec.ts +209 -209
  351. package/src/_services/base/archive-data.service.ts +224 -224
  352. package/src/_services/base/data.service.spec.ts +729 -729
  353. package/src/_services/base/data.service.ts +2740 -2740
  354. package/src/_services/base/db.service.spec.ts +73 -73
  355. package/src/_services/base/db.service.ts +1575 -1575
  356. package/src/_services/base/singleton.service-base.spec.ts +28 -28
  357. package/src/_services/base/singleton.service-base.ts +24 -24
  358. package/src/_services/base/singleton.service.spec.ts +114 -114
  359. package/src/_services/base/singleton.service.ts +38 -38
  360. package/src/_services/core/api.service.spec.ts +140 -140
  361. package/src/_services/core/auth.service.spec.ts +159 -159
  362. package/src/_services/core/auth.service.ts +174 -174
  363. package/src/_services/core/email.service.spec.ts +85 -85
  364. package/src/_services/core/email.service.ts +742 -742
  365. package/src/_services/core/global.service.spec.ts +292 -292
  366. package/src/_services/core/global.service.ts +487 -487
  367. package/src/_services/core/memory-guard.service.spec.ts +245 -245
  368. package/src/_services/core/memory-guard.service.ts +481 -481
  369. package/src/_services/core/service-collection.service.spec.ts +46 -46
  370. package/src/_services/core/service-collection.service.ts +6 -6
  371. package/src/_services/route/controller.service.spec.ts +53 -53
  372. package/src/_services/route/controller.service.ts +148 -148
  373. package/src/_services/route/routing-module.service.spec.ts +98 -98
  374. package/src/_services/route/routing-module.service.ts +330 -330
  375. package/src/_services/server/app.server.ts +1941 -1912
  376. package/src/_services/shared.static-service.spec.ts +99 -99
  377. package/src/_services/shared.static-service.ts +78 -78
  378. package/src/index.ts +98 -96
  379. package/tsconfig.app.json +12 -12
  380. package/tsconfig.json +42 -42
  381. package/.dynamo/logs/cicd-pipeline/output.log +0 -2818
  382. package/.dynamo/logs/cicd-pipeline/status.json +0 -94
@@ -1,416 +1,416 @@
1
- import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
- import {
3
- DyFM_DataModel_Params,
4
- DyFM_DataProperty_Params,
5
- DyFM_DBFilter,
6
- DyFM_Metadata,
7
- } from '@futdevpro/fsm-dynamo';
8
- import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
-
10
- import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
- import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
- import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
- import {
14
- DyNTS_LVS_BM25_Corpus,
15
- DyNTS_LVS_BM25_DocScore,
16
- dyNTS_LVS_BM25_minMaxNormalize,
17
- } from './lvs-bm25.util';
18
- import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
- import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
-
21
- /**
22
- * Local Vector Data Service
23
- * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
- * instead of MongoDB vector search
25
- *
26
- * This service loads data from the database, builds an in-memory vector pool,
27
- * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
- *
29
- * @example
30
- * ```typescript
31
- * const service = new DyNTS_OAI_LocalVectorDataService(
32
- * data,
33
- * dataParams,
34
- * openAISettings,
35
- * issuer
36
- * );
37
- *
38
- * const results = await service.vectorSearch({
39
- * input: 'search query',
40
- * searchInKey: 'contentVectorized',
41
- * limit: 10,
42
- * filterBy: { category: 'docs' }
43
- * });
44
- * ```
45
- */
46
- export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
-
48
- /**
49
- * Default search mode
50
- * Cosine similarity is typically preferred for text embeddings
51
- */
52
- defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
-
54
- /**
55
- * Whether to use L2 normalization when storing vectors
56
- * Normalized vectors are more efficient for cosine similarity calculations
57
- */
58
- useL2Normalization: boolean = true;
59
-
60
- /**
61
- * Instance-based vector pool utility
62
- * Minden service instance saját vektor pool-t tart fenn
63
- */
64
- private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
-
66
- constructor(
67
- /**
68
- * Initial data, this will be used by functions on default
69
- */
70
- data: T,
71
- /**
72
- * DB data params will be used to connect to usable dbService on GlobalService
73
- */
74
- dataParams: DyFM_DataModel_Params<T>,
75
- /**
76
- * OpenAI settings
77
- */
78
- openAISettings: DyFM_OAI_Settings,
79
- /**
80
- * Initial set for issuer to be able to follow the issuer's activity
81
- */
82
- issuer: string
83
- ) {
84
- super(data, dataParams, openAISettings, issuer);
85
- }
86
-
87
- /**
88
- * Overrides the parent vectorSearch method to perform local in-memory vector search
89
- *
90
- * Process:
91
- * 1. Load data from DB using filterBy (if provided) or getAll()
92
- * 2. Extract vectorized properties from loaded data
93
- * 3. Build in-memory vector pool
94
- * 4. Vectorize input query
95
- * 5. Perform local vector search
96
- * 6. Map search results back to full data objects
97
- *
98
- * @param set Search parameters
99
- * @returns Array of data objects sorted by similarity score
100
- */
101
- override async vectorSearch(
102
- set: {
103
- input: string;
104
- /** this should be the vectorized property key */
105
- searchInKey: string;
106
- limit?: number;
107
- /**
108
- * Number of candidates that are used to find the best match
109
- * (Not used in local search, but kept for compatibility)
110
- */
111
- numberOfCandidates?: number;
112
- /**
113
- * Filter to narrow down candidates from DB before vector search
114
- */
115
- filterBy?: DyFM_DBFilter<T>;
116
- /**
117
- * Search mode (cosine similarity, L2 distance, or hybrid).
118
- * Defaults to this.defaultSearchMode.
119
- *
120
- * `hybrid` mode combines cosine similarity (vector half) with BM25
121
- * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
- */
123
- searchMode?: LVS_Search_Mode;
124
- /**
125
- * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
- * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
- */
128
- hybridWeight?: { vector: number; text: number };
129
- /**
130
- * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
- * text-search. NEM kell hogy a vectorized property legyen.
132
- */
133
- textSearchKey?: keyof T;
134
- },
135
- ): Promise<T[]> {
136
- try {
137
- if (!set.input) {
138
- throw new DyFM_Error({
139
- ...this.getDefaultErrorSettings(
140
- 'vectorSearch',
141
- new Error('input is required')
142
- ),
143
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
- });
145
- }
146
-
147
- set.limit ??= 3;
148
- const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
- const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
-
151
- // Hybrid mode korai validacio
152
- if (effectiveMode === LVS_Search_Mode.hybrid) {
153
- if (!textSearchKey) {
154
- throw new DyFM_Error({
155
- ...this.getDefaultErrorSettings(
156
- 'vectorSearch',
157
- new Error('textSearchKey is required when searchMode is hybrid'),
158
- ),
159
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
- });
161
- }
162
- if (hybridWeight) {
163
- const w: { vector: number; text: number } = hybridWeight;
164
- if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
- throw new DyFM_Error({
166
- ...this.getDefaultErrorSettings(
167
- 'vectorSearch',
168
- new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
- ),
170
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
- });
172
- }
173
- }
174
- }
175
-
176
- // Validáljuk, hogy a searchInKey létezik-e
177
- const property: DyFM_DataProperty_Params<any, T> =
178
- this.dataParams.properties[searchInKey];
179
-
180
- if (!property) {
181
- throw new DyFM_Error({
182
- ...this.getDefaultErrorSettings(
183
- 'vectorSearch',
184
- new Error(
185
- `Property "${searchInKey}" not found! ` +
186
- `while searching "${this.dataParams.dataName}" ` +
187
- `(The searchable properties are: ` +
188
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
- )
190
- ),
191
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
- __localStack: this.dataParams.stackLocation,
193
- });
194
- }
195
-
196
- // Ellenőrizzük, hogy a property vectorized-e
197
- if (!this.vectorizedProperties.some(
198
- (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
- )) {
200
- throw new DyFM_Error({
201
- ...this.getDefaultErrorSettings(
202
- 'vectorSearch',
203
- new Error(
204
- `Property "${searchInKey}" is not a vectorized property! ` +
205
- `(The vectorized properties are: ` +
206
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
- )
208
- ),
209
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
- __localStack: this.dataParams.stackLocation,
211
- });
212
- }
213
-
214
- // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
- let dataList: T[];
216
- if (filterBy) {
217
- // Ha van filterBy, akkor először szűrünk a DB-ben
218
- if (this.debugLog) {
219
- DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
- }
221
- dataList = await this.findDataList(filterBy, true);
222
- } else {
223
- // Ha nincs filterBy, akkor minden adatot betöltünk
224
- if (this.debugLog) {
225
- DyFM_Log.log(`Local vector search: loading all data...`);
226
- }
227
- dataList = await this.getAll(true);
228
- }
229
-
230
- if (dataList.length === 0) {
231
- if (this.debugLog) {
232
- DyFM_Log.log(`Local vector search: no data found`);
233
- }
234
- return [];
235
- }
236
-
237
- // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
- if (this.debugLog) {
239
- DyFM_Log.log(
240
- `Local vector search: building vector pool from ${dataList.length} items...`
241
- );
242
- }
243
-
244
- // Töröljük a korábbi pool-t
245
- this.vectorPool.clearPool();
246
-
247
- // Hozzáadjuk a vektorokat a pool-hoz
248
- const dataMap: Map<string, T> = new Map<string, T>();
249
- for (const dataItem of dataList) {
250
- if (!dataItem._id) {
251
- // Ha nincs _id, akkor kihagyjuk
252
- if (this.debugLog) {
253
- DyFM_Log.warn(
254
- `Local vector search: skipping item without _id`
255
- );
256
- }
257
- continue;
258
- }
259
-
260
- const vectorizedValue: number[] | undefined =
261
- dataItem[searchInKey] as number[] | undefined;
262
-
263
- if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
- // Ha nincs vectorized érték, akkor kihagyjuk
265
- if (this.debugLog) {
266
- DyFM_Log.warn(
267
- `Local vector search: skipping item "${dataItem._id}" ` +
268
- `without vectorized value for "${searchInKey}"`
269
- );
270
- }
271
- continue;
272
- }
273
-
274
- // Hozzáadjuk a vektort a pool-hoz
275
- this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
- dataMap.set(dataItem._id, dataItem);
277
- }
278
-
279
- if (dataMap.size === 0) {
280
- if (this.debugLog) {
281
- DyFM_Log.log(`Local vector search: no valid vectors found`);
282
- }
283
- return [];
284
- }
285
-
286
- // 3. Vectorizáljuk a query input-ot
287
- if (this.debugLog) {
288
- DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
- }
290
-
291
- const queryVector: number[] = await this.vectorize(
292
- input,
293
- property.embeddingModel
294
- );
295
-
296
- // 4. Végrehajtjuk a local vector search-t
297
- const mode: LVS_Search_Mode = effectiveMode;
298
-
299
- if (this.debugLog) {
300
- DyFM_Log.log(
301
- `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
- );
303
- }
304
-
305
- let searchResults: LVS_SearchResult[];
306
-
307
- if (mode === LVS_Search_Mode.hybrid) {
308
- // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
- const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
- queryVector,
311
- dataMap.size,
312
- LVS_Search_Mode.cosineSimilarity,
313
- );
314
-
315
- // BM25 corpus epitese a `textSearchKey` property-bol
316
- const docs: { id: string; text: string }[] = [];
317
- for (const [docId, dataItem] of dataMap) {
318
- const textValue: unknown = dataItem[textSearchKey as keyof T];
319
- docs.push({
320
- id: docId,
321
- text: typeof textValue === 'string' ? textValue : '',
322
- });
323
- }
324
- const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
- const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
- const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
-
328
- const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
- for (const s of bm25Normalized) {
330
- bm25ScoreById.set(s.id, s.score);
331
- }
332
-
333
- const wVector: number = hybridWeight?.vector ?? 0.5;
334
- const wText: number = hybridWeight?.text ?? 0.5;
335
-
336
- const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
- const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
- return {
339
- id: c.id,
340
- score: wVector * c.score + wText * bm25Score,
341
- };
342
- });
343
- merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
- searchResults = merged.slice(0, limit);
345
- } else {
346
- searchResults = this.vectorPool.search(
347
- queryVector,
348
- limit,
349
- mode,
350
- );
351
- }
352
-
353
- if (this.debugLog) {
354
- DyFM_Log.log(
355
- `Local vector search: found ${searchResults.length} results`
356
- );
357
- }
358
-
359
- // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
- const results: T[] = [];
361
- for (const searchResult of searchResults) {
362
- const dataItem: T | undefined = dataMap.get(searchResult.id);
363
- if (dataItem) {
364
- results.push(dataItem);
365
- }
366
- }
367
-
368
- // Töröljük a pool-t a memória felszabadítása érdekében
369
- this.vectorPool.clearPool();
370
-
371
- return results;
372
- } catch (error) {
373
- // Töröljük a pool-t hiba esetén is
374
- this.vectorPool.clearPool();
375
-
376
- throw new DyFM_Error({
377
- ...this.getDefaultErrorSettings('vectorSearch', error),
378
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
- });
380
- }
381
- }
382
- }
383
-
384
- /**
385
- * @example
386
- * ```typescript
387
- * // Create a local vector data service
388
- * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
- * new CodeChunk(),
390
- * codeChunk_dataParams,
391
- * openAISettings,
392
- * 'example-issuer'
393
- * );
394
- *
395
- * // Perform vector search with filterBy
396
- * const results = await localVectorService.vectorSearch({
397
- * input: 'How to implement authentication?',
398
- * searchInKey: 'chunkParentedContentVectorized',
399
- * limit: 5,
400
- * filterBy: {
401
- * // Filter by category before vector search
402
- * category: 'documentation'
403
- * },
404
- * searchMode: LVS_Search_Mode.cosineSimilarity
405
- * });
406
- *
407
- * // Results are sorted by similarity score (highest first for cosine)
408
- * console.log(`Found ${results.length} similar chunks`);
409
- *
410
- * // Access the results
411
- * for (const result of results) {
412
- * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
- * }
414
- * ```
415
- */
416
-
1
+ import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
+ import {
3
+ DyFM_DataModel_Params,
4
+ DyFM_DataProperty_Params,
5
+ DyFM_DBFilter,
6
+ DyFM_Metadata,
7
+ } from '@futdevpro/fsm-dynamo';
8
+ import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
+
10
+ import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
+ import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
+ import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
+ import {
14
+ DyNTS_LVS_BM25_Corpus,
15
+ DyNTS_LVS_BM25_DocScore,
16
+ dyNTS_LVS_BM25_minMaxNormalize,
17
+ } from './lvs-bm25.util';
18
+ import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
+ import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
+
21
+ /**
22
+ * Local Vector Data Service
23
+ * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
+ * instead of MongoDB vector search
25
+ *
26
+ * This service loads data from the database, builds an in-memory vector pool,
27
+ * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const service = new DyNTS_OAI_LocalVectorDataService(
32
+ * data,
33
+ * dataParams,
34
+ * openAISettings,
35
+ * issuer
36
+ * );
37
+ *
38
+ * const results = await service.vectorSearch({
39
+ * input: 'search query',
40
+ * searchInKey: 'contentVectorized',
41
+ * limit: 10,
42
+ * filterBy: { category: 'docs' }
43
+ * });
44
+ * ```
45
+ */
46
+ export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
+
48
+ /**
49
+ * Default search mode
50
+ * Cosine similarity is typically preferred for text embeddings
51
+ */
52
+ defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
+
54
+ /**
55
+ * Whether to use L2 normalization when storing vectors
56
+ * Normalized vectors are more efficient for cosine similarity calculations
57
+ */
58
+ useL2Normalization: boolean = true;
59
+
60
+ /**
61
+ * Instance-based vector pool utility
62
+ * Minden service instance saját vektor pool-t tart fenn
63
+ */
64
+ private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
+
66
+ constructor(
67
+ /**
68
+ * Initial data, this will be used by functions on default
69
+ */
70
+ data: T,
71
+ /**
72
+ * DB data params will be used to connect to usable dbService on GlobalService
73
+ */
74
+ dataParams: DyFM_DataModel_Params<T>,
75
+ /**
76
+ * OpenAI settings
77
+ */
78
+ openAISettings: DyFM_OAI_Settings,
79
+ /**
80
+ * Initial set for issuer to be able to follow the issuer's activity
81
+ */
82
+ issuer: string
83
+ ) {
84
+ super(data, dataParams, openAISettings, issuer);
85
+ }
86
+
87
+ /**
88
+ * Overrides the parent vectorSearch method to perform local in-memory vector search
89
+ *
90
+ * Process:
91
+ * 1. Load data from DB using filterBy (if provided) or getAll()
92
+ * 2. Extract vectorized properties from loaded data
93
+ * 3. Build in-memory vector pool
94
+ * 4. Vectorize input query
95
+ * 5. Perform local vector search
96
+ * 6. Map search results back to full data objects
97
+ *
98
+ * @param set Search parameters
99
+ * @returns Array of data objects sorted by similarity score
100
+ */
101
+ override async vectorSearch(
102
+ set: {
103
+ input: string;
104
+ /** this should be the vectorized property key */
105
+ searchInKey: string;
106
+ limit?: number;
107
+ /**
108
+ * Number of candidates that are used to find the best match
109
+ * (Not used in local search, but kept for compatibility)
110
+ */
111
+ numberOfCandidates?: number;
112
+ /**
113
+ * Filter to narrow down candidates from DB before vector search
114
+ */
115
+ filterBy?: DyFM_DBFilter<T>;
116
+ /**
117
+ * Search mode (cosine similarity, L2 distance, or hybrid).
118
+ * Defaults to this.defaultSearchMode.
119
+ *
120
+ * `hybrid` mode combines cosine similarity (vector half) with BM25
121
+ * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
+ */
123
+ searchMode?: LVS_Search_Mode;
124
+ /**
125
+ * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
+ * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
+ */
128
+ hybridWeight?: { vector: number; text: number };
129
+ /**
130
+ * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
+ * text-search. NEM kell hogy a vectorized property legyen.
132
+ */
133
+ textSearchKey?: keyof T;
134
+ },
135
+ ): Promise<T[]> {
136
+ try {
137
+ if (!set.input) {
138
+ throw new DyFM_Error({
139
+ ...this.getDefaultErrorSettings(
140
+ 'vectorSearch',
141
+ new Error('input is required')
142
+ ),
143
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
+ });
145
+ }
146
+
147
+ set.limit ??= 3;
148
+ const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
+ const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
+
151
+ // Hybrid mode korai validacio
152
+ if (effectiveMode === LVS_Search_Mode.hybrid) {
153
+ if (!textSearchKey) {
154
+ throw new DyFM_Error({
155
+ ...this.getDefaultErrorSettings(
156
+ 'vectorSearch',
157
+ new Error('textSearchKey is required when searchMode is hybrid'),
158
+ ),
159
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
+ });
161
+ }
162
+ if (hybridWeight) {
163
+ const w: { vector: number; text: number } = hybridWeight;
164
+ if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
+ throw new DyFM_Error({
166
+ ...this.getDefaultErrorSettings(
167
+ 'vectorSearch',
168
+ new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
+ ),
170
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
+ });
172
+ }
173
+ }
174
+ }
175
+
176
+ // Validáljuk, hogy a searchInKey létezik-e
177
+ const property: DyFM_DataProperty_Params<any, T> =
178
+ this.dataParams.properties[searchInKey];
179
+
180
+ if (!property) {
181
+ throw new DyFM_Error({
182
+ ...this.getDefaultErrorSettings(
183
+ 'vectorSearch',
184
+ new Error(
185
+ `Property "${searchInKey}" not found! ` +
186
+ `while searching "${this.dataParams.dataName}" ` +
187
+ `(The searchable properties are: ` +
188
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
+ )
190
+ ),
191
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
+ __localStack: this.dataParams.stackLocation,
193
+ });
194
+ }
195
+
196
+ // Ellenőrizzük, hogy a property vectorized-e
197
+ if (!this.vectorizedProperties.some(
198
+ (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
+ )) {
200
+ throw new DyFM_Error({
201
+ ...this.getDefaultErrorSettings(
202
+ 'vectorSearch',
203
+ new Error(
204
+ `Property "${searchInKey}" is not a vectorized property! ` +
205
+ `(The vectorized properties are: ` +
206
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
+ )
208
+ ),
209
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
+ __localStack: this.dataParams.stackLocation,
211
+ });
212
+ }
213
+
214
+ // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
+ let dataList: T[];
216
+ if (filterBy) {
217
+ // Ha van filterBy, akkor először szűrünk a DB-ben
218
+ if (this.debugLog) {
219
+ DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
+ }
221
+ dataList = await this.findDataList(filterBy, true);
222
+ } else {
223
+ // Ha nincs filterBy, akkor minden adatot betöltünk
224
+ if (this.debugLog) {
225
+ DyFM_Log.log(`Local vector search: loading all data...`);
226
+ }
227
+ dataList = await this.getAll(true);
228
+ }
229
+
230
+ if (dataList.length === 0) {
231
+ if (this.debugLog) {
232
+ DyFM_Log.log(`Local vector search: no data found`);
233
+ }
234
+ return [];
235
+ }
236
+
237
+ // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
+ if (this.debugLog) {
239
+ DyFM_Log.log(
240
+ `Local vector search: building vector pool from ${dataList.length} items...`
241
+ );
242
+ }
243
+
244
+ // Töröljük a korábbi pool-t
245
+ this.vectorPool.clearPool();
246
+
247
+ // Hozzáadjuk a vektorokat a pool-hoz
248
+ const dataMap: Map<string, T> = new Map<string, T>();
249
+ for (const dataItem of dataList) {
250
+ if (!dataItem._id) {
251
+ // Ha nincs _id, akkor kihagyjuk
252
+ if (this.debugLog) {
253
+ DyFM_Log.warn(
254
+ `Local vector search: skipping item without _id`
255
+ );
256
+ }
257
+ continue;
258
+ }
259
+
260
+ const vectorizedValue: number[] | undefined =
261
+ dataItem[searchInKey] as number[] | undefined;
262
+
263
+ if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
+ // Ha nincs vectorized érték, akkor kihagyjuk
265
+ if (this.debugLog) {
266
+ DyFM_Log.warn(
267
+ `Local vector search: skipping item "${dataItem._id}" ` +
268
+ `without vectorized value for "${searchInKey}"`
269
+ );
270
+ }
271
+ continue;
272
+ }
273
+
274
+ // Hozzáadjuk a vektort a pool-hoz
275
+ this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
+ dataMap.set(dataItem._id, dataItem);
277
+ }
278
+
279
+ if (dataMap.size === 0) {
280
+ if (this.debugLog) {
281
+ DyFM_Log.log(`Local vector search: no valid vectors found`);
282
+ }
283
+ return [];
284
+ }
285
+
286
+ // 3. Vectorizáljuk a query input-ot
287
+ if (this.debugLog) {
288
+ DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
+ }
290
+
291
+ const queryVector: number[] = await this.vectorize(
292
+ input,
293
+ property.embeddingModel
294
+ );
295
+
296
+ // 4. Végrehajtjuk a local vector search-t
297
+ const mode: LVS_Search_Mode = effectiveMode;
298
+
299
+ if (this.debugLog) {
300
+ DyFM_Log.log(
301
+ `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
+ );
303
+ }
304
+
305
+ let searchResults: LVS_SearchResult[];
306
+
307
+ if (mode === LVS_Search_Mode.hybrid) {
308
+ // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
+ const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
+ queryVector,
311
+ dataMap.size,
312
+ LVS_Search_Mode.cosineSimilarity,
313
+ );
314
+
315
+ // BM25 corpus epitese a `textSearchKey` property-bol
316
+ const docs: { id: string; text: string }[] = [];
317
+ for (const [docId, dataItem] of dataMap) {
318
+ const textValue: unknown = dataItem[textSearchKey as keyof T];
319
+ docs.push({
320
+ id: docId,
321
+ text: typeof textValue === 'string' ? textValue : '',
322
+ });
323
+ }
324
+ const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
+ const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
+ const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
+
328
+ const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
+ for (const s of bm25Normalized) {
330
+ bm25ScoreById.set(s.id, s.score);
331
+ }
332
+
333
+ const wVector: number = hybridWeight?.vector ?? 0.5;
334
+ const wText: number = hybridWeight?.text ?? 0.5;
335
+
336
+ const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
+ const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
+ return {
339
+ id: c.id,
340
+ score: wVector * c.score + wText * bm25Score,
341
+ };
342
+ });
343
+ merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
+ searchResults = merged.slice(0, limit);
345
+ } else {
346
+ searchResults = this.vectorPool.search(
347
+ queryVector,
348
+ limit,
349
+ mode,
350
+ );
351
+ }
352
+
353
+ if (this.debugLog) {
354
+ DyFM_Log.log(
355
+ `Local vector search: found ${searchResults.length} results`
356
+ );
357
+ }
358
+
359
+ // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
+ const results: T[] = [];
361
+ for (const searchResult of searchResults) {
362
+ const dataItem: T | undefined = dataMap.get(searchResult.id);
363
+ if (dataItem) {
364
+ results.push(dataItem);
365
+ }
366
+ }
367
+
368
+ // Töröljük a pool-t a memória felszabadítása érdekében
369
+ this.vectorPool.clearPool();
370
+
371
+ return results;
372
+ } catch (error) {
373
+ // Töröljük a pool-t hiba esetén is
374
+ this.vectorPool.clearPool();
375
+
376
+ throw new DyFM_Error({
377
+ ...this.getDefaultErrorSettings('vectorSearch', error),
378
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
+ });
380
+ }
381
+ }
382
+ }
383
+
384
+ /**
385
+ * @example
386
+ * ```typescript
387
+ * // Create a local vector data service
388
+ * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
+ * new CodeChunk(),
390
+ * codeChunk_dataParams,
391
+ * openAISettings,
392
+ * 'example-issuer'
393
+ * );
394
+ *
395
+ * // Perform vector search with filterBy
396
+ * const results = await localVectorService.vectorSearch({
397
+ * input: 'How to implement authentication?',
398
+ * searchInKey: 'chunkParentedContentVectorized',
399
+ * limit: 5,
400
+ * filterBy: {
401
+ * // Filter by category before vector search
402
+ * category: 'documentation'
403
+ * },
404
+ * searchMode: LVS_Search_Mode.cosineSimilarity
405
+ * });
406
+ *
407
+ * // Results are sorted by similarity score (highest first for cosine)
408
+ * console.log(`Found ${results.length} similar chunks`);
409
+ *
410
+ * // Access the results
411
+ * for (const result of results) {
412
+ * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
+ * }
414
+ * ```
415
+ */
416
+