@futdevpro/nts-dynamo 1.15.91 → 1.15.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (381) hide show
  1. package/.c8rc.json +26 -26
  2. package/.copilot/patterns.json +7 -7
  3. package/.cursor/rules/__assistant_guide.mdc +30 -30
  4. package/.cursor/rules/_ag_backend-structure.mdc +85 -85
  5. package/.cursor/rules/_ag_backend.mdc +16 -16
  6. package/.cursor/rules/_ag_frontend-structure.mdc +86 -86
  7. package/.cursor/rules/_ag_frontend.mdc +39 -39
  8. package/.cursor/rules/_ag_import-rules.mdc +44 -44
  9. package/.cursor/rules/_ag_naming.mdc +115 -115
  10. package/.cursor/rules/_ag_should-be.mdc +6 -6
  11. package/.cursor/rules/ai_development_guide.md +60 -60
  12. package/.cursor/rules/cursor-rules.md +160 -160
  13. package/.cursor/rules/default-command.mdc +464 -464
  14. package/.cursor/rules/error_code_pattern.md +39 -39
  15. package/.cursor/rules/saved rule mcp server use.md +15 -15
  16. package/.vscode/settings.json +10 -10
  17. package/HOWTO.md +15 -15
  18. package/LICENSE +21 -21
  19. package/__documentations/nts-integration-tests-2026-03-17.md +26 -26
  20. package/__documentations/plans/BEDROCK-HYPERPLAN.md +95 -95
  21. package/_specifications/BACKLOG.md +92 -92
  22. package/_specifications/TODO.md +15 -15
  23. package/_specifications/agent.md +138 -138
  24. package/build/_models/interfaces/db-query-options.interface.d.ts +33 -0
  25. package/build/_models/interfaces/db-query-options.interface.d.ts.map +1 -0
  26. package/build/_models/interfaces/db-query-options.interface.js +3 -0
  27. package/build/_models/interfaces/db-query-options.interface.js.map +1 -0
  28. package/build/_services/base/data.service.d.ts +3 -1
  29. package/build/_services/base/data.service.d.ts.map +1 -1
  30. package/build/_services/base/data.service.js +5 -2
  31. package/build/_services/base/data.service.js.map +1 -1
  32. package/build/_services/base/db.service.d.ts +3 -1
  33. package/build/_services/base/db.service.d.ts.map +1 -1
  34. package/build/_services/base/db.service.js +23 -4
  35. package/build/_services/base/db.service.js.map +1 -1
  36. package/build/index.d.ts +1 -0
  37. package/build/index.d.ts.map +1 -1
  38. package/build/index.js +1 -0
  39. package/build/index.js.map +1 -1
  40. package/eslint.config.js +3 -3
  41. package/nodemon.json +24 -24
  42. package/package.json +2 -2
  43. package/pnpm-workspace.yaml +5 -5
  44. package/scripts/run-coverage-tests.js +28 -28
  45. package/spec/support/helpers/spec-reporter-loader.js +359 -359
  46. package/spec/support/helpers/ts-node-helper.js +93 -93
  47. package/spec/support/jasmine.coverage.json +24 -24
  48. package/spec/support/jasmine.json +24 -24
  49. package/src/_collections/archive.util.spec.ts +57 -57
  50. package/src/_collections/archive.util.ts +18 -18
  51. package/src/_collections/atlas-default-db-options.const.ts +9 -9
  52. package/src/_collections/default-fallback-cache-max-age.const.spec.ts +11 -11
  53. package/src/_collections/default-fallback-cache-max-age.const.ts +2 -2
  54. package/src/_collections/default-not-found-page.const.spec.ts +19 -19
  55. package/src/_collections/default-not-found-page.const.ts +22 -22
  56. package/src/_collections/default-socket-path.const.spec.ts +12 -12
  57. package/src/_collections/default-socket-path.const.ts +2 -2
  58. package/src/_collections/get-environment-settings.util.spec.ts +210 -210
  59. package/src/_collections/get-environment-settings.util.ts +48 -48
  60. package/src/_collections/global-settings.const.ts +109 -109
  61. package/src/_collections/sample.env +21 -21
  62. package/src/_collections/star.controller.spec.ts +224 -224
  63. package/src/_collections/star.controller.ts +129 -129
  64. package/src/_enums/data-model-type.enum.ts +14 -14
  65. package/src/_enums/data-service-function.enum.ts +24 -24
  66. package/src/_enums/predefined-data-types.enum.ts +16 -16
  67. package/src/_enums/route-security.enum.ts +12 -12
  68. package/src/_models/control-models/api-call-params.control-model.spec.ts +152 -152
  69. package/src/_models/control-models/api-call-params.control-model.ts +142 -142
  70. package/src/_models/control-models/app-ext-system-controls.control-model.spec.ts +52 -52
  71. package/src/_models/control-models/app-ext-system-controls.control-model.ts +9 -9
  72. package/src/_models/control-models/app-params.control-model.spec.ts +225 -225
  73. package/src/_models/control-models/app-params.control-model.ts +136 -136
  74. package/src/_models/control-models/app-system-controls.control-model.spec.ts +31 -31
  75. package/src/_models/control-models/app-system-controls.control-model.ts +9 -9
  76. package/src/_models/control-models/endpoint-params.control-model.spec.ts +627 -627
  77. package/src/_models/control-models/endpoint-params.control-model.ts +627 -627
  78. package/src/_models/control-models/http-settings.control-model.spec.ts +77 -77
  79. package/src/_models/control-models/http-settings.control-model.ts +37 -37
  80. package/src/_models/control-models/system-control.control-model.spec.ts +27 -27
  81. package/src/_models/control-models/system-control.control-model.ts +12 -12
  82. package/src/_models/interfaces/certification-settings.interface.ts +7 -7
  83. package/src/_models/interfaces/db-query-options.interface.ts +33 -0
  84. package/src/_models/interfaces/environment-settings.interface.ts +59 -59
  85. package/src/_models/interfaces/global-log-settings.interface.ts +171 -171
  86. package/src/_models/interfaces/global-service-settings.interface.ts +47 -47
  87. package/src/_models/interfaces/global-settings.interface.ts +244 -244
  88. package/src/_models/interfaces/routing-module-settings.interface.ts +21 -21
  89. package/src/_models/interfaces/static-client-settings.interface.spec.ts +29 -29
  90. package/src/_models/interfaces/static-client-settings.interface.ts +28 -28
  91. package/src/_models/types/db-update.type.ts +100 -100
  92. package/src/_modules/ai/_models/ai-input-interfaces.ts +117 -117
  93. package/src/_modules/ai/_models/ai-test-generation-result.interface.ts +16 -16
  94. package/src/_modules/ai/_modules/anthropic/_services/aai-user-key.control-service.ts +138 -138
  95. package/src/_modules/ai/_modules/anthropic/index.ts +5 -5
  96. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.spec.ts +242 -242
  97. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.ts +639 -639
  98. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.spec.ts +295 -295
  99. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.ts +518 -518
  100. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.spec.ts +209 -209
  101. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.ts +85 -85
  102. package/src/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.ts +7 -7
  103. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.ts +146 -146
  104. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model.ts +162 -162
  105. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-document.data-model.ts +99 -99
  106. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-code-chunk.interface.ts +68 -68
  107. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface.ts +18 -18
  108. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-page-compare-result.interface.ts +19 -19
  109. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-document-compare-result.interface.ts +25 -25
  110. package/src/_modules/ai/_modules/document-ai/index.ts +30 -30
  111. package/src/_modules/ai/_modules/fdp-ai/_services/fdpai-user-key.control-service.ts +189 -189
  112. package/src/_modules/ai/_modules/fdp-ai/index.ts +5 -5
  113. package/src/_modules/ai/_modules/open-ai/_collections/oai-global-settings.const.ts +9 -9
  114. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests-hu.conts.ts +82 -82
  115. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests.conts.ts +75 -75
  116. package/src/_modules/ai/_modules/open-ai/_enums/oai-gpt-message-role.enum.ts +45 -45
  117. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-global-settings.interface.ts +7 -7
  118. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-gpt-message.interface.ts +7 -7
  119. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-llm-predefined-requests.interface.ts +57 -57
  120. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.ts +292 -292
  121. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.spec.ts +342 -342
  122. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.spec.ts +550 -550
  123. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.ts +630 -630
  124. package/src/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.spec.ts +332 -332
  125. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.spec.ts +462 -462
  126. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.ts +634 -634
  127. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.spec.ts +489 -489
  128. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.tools.spec.ts +173 -173
  129. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.ts +1033 -1033
  130. package/src/_modules/ai/_modules/open-ai/_services/oai-user-key.control-service.ts +157 -157
  131. package/src/_modules/ai/_services/ai-embedding-mock.service.spec.ts +115 -115
  132. package/src/_modules/ai/_services/ai-embedding-mock.service.ts +212 -212
  133. package/src/_modules/ai/_services/ai-embedding-provider.registry.spec.ts +110 -110
  134. package/src/_modules/ai/_services/ai-embedding-provider.registry.ts +110 -110
  135. package/src/_modules/ai/_services/ai-embedding.service-base.spec.ts +98 -98
  136. package/src/_modules/ai/_services/ai-embedding.service-base.ts +48 -48
  137. package/src/_modules/ai/_services/ai-llm-chat.service-base.spec.ts +229 -229
  138. package/src/_modules/ai/_services/ai-llm-chat.service-base.ts +68 -68
  139. package/src/_modules/ai/_services/ai-llm.service-base.spec.ts +250 -250
  140. package/src/_modules/ai/_services/ai-llm.service-base.ts +519 -519
  141. package/src/_modules/ai/_services/ai-provider.service-base.spec.ts +158 -158
  142. package/src/_modules/ai/_services/ai-user-key.service-base.ts +59 -59
  143. package/src/_modules/ai/_services/lmstudio-embedding.control-service.spec.ts +197 -197
  144. package/src/_modules/ai/_services/lmstudio-embedding.control-service.ts +371 -371
  145. package/src/_modules/ai/index.ts +23 -23
  146. package/src/_modules/assistant/_collections/ass-global-settings.const.ts +13 -13
  147. package/src/_modules/assistant/_collections/ass.util.spec.ts +176 -176
  148. package/src/_modules/assistant/_collections/ass.util.ts +50 -50
  149. package/src/_modules/assistant/_models/ass-global-settings.interface.ts +15 -15
  150. package/src/_modules/assistant/_services/ass-io.control-service.spec.ts +140 -140
  151. package/src/_modules/assistant/_services/ass-main.control-service.spec.ts +192 -192
  152. package/src/_modules/assistant/_services/ass-main.control-service.ts +107 -107
  153. package/src/_modules/bot/_collections/bot-default-commands.const.ts +12 -12
  154. package/src/_modules/bot/_collections/bot-global-settings.const.ts +39 -39
  155. package/src/_modules/bot/_models/bot-channel-wrapper.interface.ts +62 -62
  156. package/src/_modules/bot/_models/bot-command.interface.ts +8 -8
  157. package/src/_modules/bot/_models/bot-global-settings.interface.ts +96 -96
  158. package/src/_modules/bot/_models/bot-last-mention-date.interface.ts +6 -6
  159. package/src/_modules/bot/_models/bot-last-message-date.interface.ts +5 -5
  160. package/src/_modules/bot/_models/bot-user-wrapper.interface.ts +41 -41
  161. package/src/_modules/bot/_modules/discord-bot/_models/dib-platform.types.ts +9 -9
  162. package/src/_modules/bot/_modules/discord-bot/_services/dib-messaging-provider.control-service.spec.ts +431 -431
  163. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.spec.ts +160 -160
  164. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.ts +55 -55
  165. package/src/_modules/bot/_modules/dynamo-bot/_models/dyb-platform.types.ts +15 -15
  166. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.spec.ts +374 -374
  167. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.ts +447 -447
  168. package/src/_modules/bot/_modules/dynamo-bot/index.ts +15 -15
  169. package/src/_modules/bot/_modules/slack-bot/_models/slb-platform.types.ts +9 -9
  170. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.spec.ts +344 -344
  171. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.ts +197 -197
  172. package/src/_modules/bot/_modules/teams-bot/_models/teb-platform.types.ts +9 -9
  173. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.spec.ts +345 -345
  174. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.ts +197 -197
  175. package/src/_modules/bot/_services/bot-commands.control-service.spec.ts +116 -116
  176. package/src/_modules/bot/_services/bot-io.control-service.spec.ts +285 -285
  177. package/src/_modules/bot/_services/bot-main.control-service.spec.ts +208 -208
  178. package/src/_modules/bot/_services/bot-messaging-provider.service-base.spec.ts +349 -349
  179. package/src/_modules/bot/_services/bot-routines.control-service.spec.ts +111 -111
  180. package/src/_modules/custom-data/custom-data.controller.spec.ts +49 -49
  181. package/src/_modules/custom-data/custom-data.controller.ts +67 -67
  182. package/src/_modules/custom-data/custom-data.data-service.spec.ts +54 -54
  183. package/src/_modules/custom-data/custom-data.data-service.ts +21 -21
  184. package/src/_modules/custom-data/get-custom-data-routing-module.util.spec.ts +28 -28
  185. package/src/_modules/custom-data/get-custom-data-routing-module.util.ts +24 -24
  186. package/src/_modules/custom-data/index.ts +9 -9
  187. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.spec.ts +161 -161
  188. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.ts +203 -203
  189. package/src/_modules/data-readers/_models/interfaces/dynts-sqlite-reader.interface.ts +33 -33
  190. package/src/_modules/data-readers/index.ts +11 -11
  191. package/src/_modules/defaults/_collections/default-endpoints.util.ts +487 -487
  192. package/src/_modules/defaults/_models/default-user.data-model.ts +72 -72
  193. package/src/_modules/defaults/_services/default-auth.service.spec.ts +269 -269
  194. package/src/_modules/defaults/_services/default-auth.service.ts +177 -177
  195. package/src/_modules/defaults/_services/default-socket-events.service.spec.ts +42 -42
  196. package/src/_modules/defaults/_services/default-socket-events.service.ts +61 -61
  197. package/src/_modules/defaults/_services/default-user.data-service.spec.ts +187 -187
  198. package/src/_modules/defaults/_services/default-user.data-service.ts +98 -98
  199. package/src/_modules/defaults/index.ts +17 -17
  200. package/src/_modules/discord-assistant/_collections/dias-global-settings.const.ts +19 -19
  201. package/src/_modules/discord-assistant/_collections/dias.util.spec.ts +366 -366
  202. package/src/_modules/discord-assistant/_collections/dias.util.ts +132 -132
  203. package/src/_modules/discord-assistant/_models/dias-global-settings.interface.ts +19 -19
  204. package/src/_modules/discord-assistant/_models/dias-knowledge.data-model.ts +52 -52
  205. package/src/_modules/discord-assistant/_services/dias-chunk.data-service.ts +177 -177
  206. package/src/_modules/discord-assistant/_services/dias-io.control-service.spec.ts +108 -108
  207. package/src/_modules/discord-assistant/_services/dias-io.control-service.ts +69 -69
  208. package/src/_modules/discord-assistant/_services/dias-main.control-service.spec.ts +22 -22
  209. package/src/_modules/discord-assistant/_services/dias-main.control-service.ts +27 -27
  210. package/src/_modules/discord-assistant/_services/dias.service-base.spec.ts +195 -195
  211. package/src/_modules/discord-assistant/_services/dias.service-base.ts +76 -76
  212. package/src/_modules/discord-assistant/index.ts +38 -38
  213. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.spec.ts +34 -34
  214. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.ts +11 -11
  215. package/src/_modules/discord-assistant-voiced/index.ts +36 -36
  216. package/src/_modules/discord-bot/_collections/dibo-default-commands.const.ts +16 -16
  217. package/src/_modules/discord-bot/_collections/dibo-global-settings.conts.ts +55 -55
  218. package/src/_modules/discord-bot/_collections/dibo-operations.util.spec.ts +214 -214
  219. package/src/_modules/discord-bot/_collections/dibo-operations.util.ts +387 -387
  220. package/src/_modules/discord-bot/_models/dibo-command.interface.ts +12 -12
  221. package/src/_modules/discord-bot/_models/dibo-global-settings.interface.ts +98 -98
  222. package/src/_modules/discord-bot/_models/dibo-last-mention-date.inteface.ts +7 -7
  223. package/src/_modules/discord-bot/_models/dibo-last-message-date.interface.ts +6 -6
  224. package/src/_modules/discord-bot/_services/dibo-commands.control-service.spec.ts +154 -154
  225. package/src/_modules/discord-bot/_services/dibo-commands.control-service.ts +153 -153
  226. package/src/_modules/discord-bot/_services/dibo-io.control-service.spec.ts +264 -264
  227. package/src/_modules/discord-bot/_services/dibo-io.control-service.ts +306 -306
  228. package/src/_modules/discord-bot/_services/dibo-main.control-service.spec.ts +408 -408
  229. package/src/_modules/discord-bot/_services/dibo-main.control-service.ts +487 -487
  230. package/src/_modules/discord-bot/_services/dibo-routines.control-service.spec.ts +105 -105
  231. package/src/_modules/discord-bot/index.ts +36 -36
  232. package/src/_modules/local-vector-search/_enums/lvs-search-mode.enum.ts +35 -35
  233. package/src/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.ts +59 -59
  234. package/src/_modules/local-vector-search/_models/lvs-search-result.interface.ts +17 -17
  235. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.spec.ts +418 -418
  236. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.ts +276 -276
  237. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.spec.ts +480 -480
  238. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.ts +416 -416
  239. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.spec.ts +198 -198
  240. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.ts +146 -146
  241. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.spec.ts +167 -167
  242. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.ts +106 -106
  243. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.spec.ts +507 -507
  244. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.ts +272 -272
  245. package/src/_modules/local-vector-search/index.ts +16 -16
  246. package/src/_modules/logs/index.ts +11 -11
  247. package/src/_modules/mcp/_models/interfaces/dynts-mcp.interface.ts +111 -111
  248. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.spec.ts +142 -142
  249. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.ts +120 -120
  250. package/src/_modules/mcp/_services/dynts-mcp.adapter.ts +168 -168
  251. package/src/_modules/mcp/index.ts +13 -13
  252. package/src/_modules/messaging/README.md +354 -354
  253. package/src/_modules/messaging/_collections/get-messaging-routing-module.util.ts +26 -26
  254. package/src/_modules/messaging/_collections/msg-global-settings.const.ts +22 -22
  255. package/src/_modules/messaging/_collections/msg.util.spec.ts +226 -226
  256. package/src/_modules/messaging/_models/msg-global-settings.interface.ts +37 -37
  257. package/src/_modules/messaging/_services/msg-conversation.data-service.ts +146 -146
  258. package/src/_modules/messaging/_services/msg-events.service.spec.ts +219 -219
  259. package/src/_modules/messaging/_services/msg-events.service.ts +267 -267
  260. package/src/_modules/messaging/_services/msg-integration.control-service.ts +179 -179
  261. package/src/_modules/messaging/_services/msg-main.control-service.spec.ts +147 -147
  262. package/src/_modules/messaging/_services/msg-main.control-service.ts +571 -571
  263. package/src/_modules/messaging/_services/msg-message.data-service.ts +129 -129
  264. package/src/_modules/messaging/_services/msg.controller.spec.ts +201 -201
  265. package/src/_modules/messaging/index.ts +30 -30
  266. package/src/_modules/mock/app-extended-server.mock.ts +201 -201
  267. package/src/_modules/mock/app-integration-test.mock.ts +51 -51
  268. package/src/_modules/mock/app-params.mock.spec.ts +21 -21
  269. package/src/_modules/mock/app-params.mock.ts +9 -9
  270. package/src/_modules/mock/app-server.mock.ts +188 -188
  271. package/src/_modules/mock/auth-service.mock.spec.ts +47 -47
  272. package/src/_modules/mock/auth-service.mock.ts +28 -28
  273. package/src/_modules/mock/controller.mock.spec.ts +26 -26
  274. package/src/_modules/mock/controller.mock.ts +16 -16
  275. package/src/_modules/mock/data-model.mock.spec.ts +111 -111
  276. package/src/_modules/mock/data-model.mock.ts +82 -82
  277. package/src/_modules/mock/email-service-collection.mock.spec.ts +24 -24
  278. package/src/_modules/mock/email-service-collection.mock.ts +15 -15
  279. package/src/_modules/mock/email-service.mock.spec.ts +17 -17
  280. package/src/_modules/mock/email-service.mock.ts +20 -20
  281. package/src/_modules/mock/email-template.mock.html +14 -14
  282. package/src/_modules/mock/endpoint.mock.ts +91 -91
  283. package/src/_modules/mock/socket-client.mock.spec.ts +40 -40
  284. package/src/_modules/mock/socket-client.mock.ts +45 -45
  285. package/src/_modules/mock/socket-server.mock.spec.ts +44 -44
  286. package/src/_modules/mock/socket-server.mock.ts +46 -46
  287. package/src/_modules/oauth2/_routes/oauth2.controller.spec.ts +107 -107
  288. package/src/_modules/oauth2/_routes/oauth2.controller.ts +98 -98
  289. package/src/_modules/oauth2/_services/oauth2.auth-service.spec.ts +254 -254
  290. package/src/_modules/oauth2/_services/oauth2.auth-service.ts +232 -232
  291. package/src/_modules/oauth2/_services/oauth2.control-service.spec.ts +585 -585
  292. package/src/_modules/oauth2/_services/oauth2.control-service.ts +653 -653
  293. package/src/_modules/oauth2/index.ts +17 -17
  294. package/src/_modules/scheduler/_models/scheduler-job.interface.ts +35 -35
  295. package/src/_modules/scheduler/get-scheduler-routing-module.util.ts +33 -33
  296. package/src/_modules/scheduler/index.ts +8 -8
  297. package/src/_modules/scheduler/scheduler.controller.spec.ts +42 -42
  298. package/src/_modules/scheduler/scheduler.controller.ts +69 -69
  299. package/src/_modules/scheduler/scheduler.service.spec.ts +141 -141
  300. package/src/_modules/scheduler/scheduler.service.ts +176 -176
  301. package/src/_modules/scoped-config/_enums/dynts-scoped-config-level.enum.ts +22 -22
  302. package/src/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.ts +81 -81
  303. package/src/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.ts +107 -107
  304. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.spec.ts +306 -306
  305. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.ts +295 -295
  306. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.spec.ts +118 -118
  307. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.ts +105 -105
  308. package/src/_modules/scoped-config/index.ts +17 -17
  309. package/src/_modules/server/errors/errors.control-service.spec.ts +238 -238
  310. package/src/_modules/server/errors/errors.control-service.ts +100 -100
  311. package/src/_modules/server/errors/errors.controller.spec.ts +268 -268
  312. package/src/_modules/server/errors/errors.controller.ts +515 -515
  313. package/src/_modules/server/errors/errors.data-service.spec.ts +480 -480
  314. package/src/_modules/server/index.ts +30 -30
  315. package/src/_modules/server/server-status/server-status-snapshot.control-service.spec.ts +70 -70
  316. package/src/_modules/server/server-status/server-status-snapshot.control-service.ts +17 -17
  317. package/src/_modules/server/server-status/server-status-snapshot.data-service.spec.ts +77 -77
  318. package/src/_modules/server/server-status/server-status-snapshot.data-service.ts +37 -37
  319. package/src/_modules/server/server-status/server-status.control-service.spec.ts +576 -576
  320. package/src/_modules/server/server-status/server-status.control-service.ts +396 -396
  321. package/src/_modules/server/server-status/server-status.controller.spec.ts +255 -255
  322. package/src/_modules/server/server-status/server-status.controller.ts +272 -272
  323. package/src/_modules/socket/_enums/socket-security.enum.ts +11 -11
  324. package/src/_modules/socket/_models/socket-client-service-params.control-model.spec.ts +32 -32
  325. package/src/_modules/socket/_models/socket-client-service-params.control-model.ts +22 -22
  326. package/src/_modules/socket/_models/socket-presence.control-model.spec.ts +164 -164
  327. package/src/_modules/socket/_models/socket-presence.control-model.ts +210 -210
  328. package/src/_modules/socket/_models/socket-server-service-params.control-model.spec.ts +46 -46
  329. package/src/_modules/socket/_models/socket-server-service-params.control-model.ts +22 -22
  330. package/src/_modules/socket/_services/socket-client.service.spec.ts +15 -15
  331. package/src/_modules/socket/_services/socket-client.service.ts +260 -260
  332. package/src/_modules/socket/_services/socket-server.service.spec.ts +11 -11
  333. package/src/_modules/socket/app-extended.integration.spec.ts +85 -85
  334. package/src/_modules/socket/app-extended.server.ts +630 -630
  335. package/src/_modules/socket/index.ts +42 -42
  336. package/src/_modules/test/get-test-routing-module.util.spec.ts +28 -28
  337. package/src/_modules/test/get-test-routing-module.util.ts +23 -23
  338. package/src/_modules/test/index.ts +11 -11
  339. package/src/_modules/test/test.controller.spec.ts +72 -72
  340. package/src/_modules/test/test.controller.ts +115 -115
  341. package/src/_modules/usage/get-usage-routing-module.util.ts +22 -22
  342. package/src/_modules/usage/index.ts +15 -15
  343. package/src/_modules/usage/usage.controller.spec.ts +81 -81
  344. package/src/_modules/usage/usage.controller.ts +126 -126
  345. package/src/_modules/usage/usage.data-service.spec.ts +332 -332
  346. package/src/_modules/usage/usage.data-service.ts +185 -185
  347. package/src/_services/base/api.service-base.spec.ts +125 -125
  348. package/src/_services/base/api.service-base.ts +74 -74
  349. package/src/_services/base/archive-data.service.spec.ts +209 -209
  350. package/src/_services/base/archive-data.service.ts +224 -224
  351. package/src/_services/base/data.service.spec.ts +729 -729
  352. package/src/_services/base/data.service.ts +2744 -2740
  353. package/src/_services/base/db.service.spec.ts +160 -73
  354. package/src/_services/base/db.service.ts +1592 -1575
  355. package/src/_services/base/singleton.service-base.spec.ts +28 -28
  356. package/src/_services/base/singleton.service-base.ts +24 -24
  357. package/src/_services/base/singleton.service.spec.ts +114 -114
  358. package/src/_services/base/singleton.service.ts +38 -38
  359. package/src/_services/core/api.service.spec.ts +140 -140
  360. package/src/_services/core/auth.service.spec.ts +159 -159
  361. package/src/_services/core/auth.service.ts +174 -174
  362. package/src/_services/core/email.service.spec.ts +85 -85
  363. package/src/_services/core/email.service.ts +742 -742
  364. package/src/_services/core/global.service.spec.ts +292 -292
  365. package/src/_services/core/global.service.ts +487 -487
  366. package/src/_services/core/memory-guard.service.spec.ts +245 -245
  367. package/src/_services/core/memory-guard.service.ts +481 -481
  368. package/src/_services/core/service-collection.service.spec.ts +46 -46
  369. package/src/_services/core/service-collection.service.ts +6 -6
  370. package/src/_services/route/controller.service.spec.ts +53 -53
  371. package/src/_services/route/controller.service.ts +148 -148
  372. package/src/_services/route/routing-module.service.spec.ts +98 -98
  373. package/src/_services/route/routing-module.service.ts +330 -330
  374. package/src/_services/server/app.server.ts +1941 -1941
  375. package/src/_services/shared.static-service.spec.ts +99 -99
  376. package/src/_services/shared.static-service.ts +78 -78
  377. package/src/index.ts +100 -99
  378. package/tsconfig.app.json +12 -12
  379. package/tsconfig.json +42 -42
  380. package/.dynamo/logs/cicd-pipeline/output.log +0 -2856
  381. package/.dynamo/logs/cicd-pipeline/status.json +0 -74
@@ -1,416 +1,416 @@
1
- import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
- import {
3
- DyFM_DataModel_Params,
4
- DyFM_DataProperty_Params,
5
- DyFM_DBFilter,
6
- DyFM_Metadata,
7
- } from '@futdevpro/fsm-dynamo';
8
- import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
-
10
- import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
- import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
- import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
- import {
14
- DyNTS_LVS_BM25_Corpus,
15
- DyNTS_LVS_BM25_DocScore,
16
- dyNTS_LVS_BM25_minMaxNormalize,
17
- } from './lvs-bm25.util';
18
- import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
- import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
-
21
- /**
22
- * Local Vector Data Service
23
- * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
- * instead of MongoDB vector search
25
- *
26
- * This service loads data from the database, builds an in-memory vector pool,
27
- * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
- *
29
- * @example
30
- * ```typescript
31
- * const service = new DyNTS_OAI_LocalVectorDataService(
32
- * data,
33
- * dataParams,
34
- * openAISettings,
35
- * issuer
36
- * );
37
- *
38
- * const results = await service.vectorSearch({
39
- * input: 'search query',
40
- * searchInKey: 'contentVectorized',
41
- * limit: 10,
42
- * filterBy: { category: 'docs' }
43
- * });
44
- * ```
45
- */
46
- export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
-
48
- /**
49
- * Default search mode
50
- * Cosine similarity is typically preferred for text embeddings
51
- */
52
- defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
-
54
- /**
55
- * Whether to use L2 normalization when storing vectors
56
- * Normalized vectors are more efficient for cosine similarity calculations
57
- */
58
- useL2Normalization: boolean = true;
59
-
60
- /**
61
- * Instance-based vector pool utility
62
- * Minden service instance saját vektor pool-t tart fenn
63
- */
64
- private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
-
66
- constructor(
67
- /**
68
- * Initial data, this will be used by functions on default
69
- */
70
- data: T,
71
- /**
72
- * DB data params will be used to connect to usable dbService on GlobalService
73
- */
74
- dataParams: DyFM_DataModel_Params<T>,
75
- /**
76
- * OpenAI settings
77
- */
78
- openAISettings: DyFM_OAI_Settings,
79
- /**
80
- * Initial set for issuer to be able to follow the issuer's activity
81
- */
82
- issuer: string
83
- ) {
84
- super(data, dataParams, openAISettings, issuer);
85
- }
86
-
87
- /**
88
- * Overrides the parent vectorSearch method to perform local in-memory vector search
89
- *
90
- * Process:
91
- * 1. Load data from DB using filterBy (if provided) or getAll()
92
- * 2. Extract vectorized properties from loaded data
93
- * 3. Build in-memory vector pool
94
- * 4. Vectorize input query
95
- * 5. Perform local vector search
96
- * 6. Map search results back to full data objects
97
- *
98
- * @param set Search parameters
99
- * @returns Array of data objects sorted by similarity score
100
- */
101
- override async vectorSearch(
102
- set: {
103
- input: string;
104
- /** this should be the vectorized property key */
105
- searchInKey: string;
106
- limit?: number;
107
- /**
108
- * Number of candidates that are used to find the best match
109
- * (Not used in local search, but kept for compatibility)
110
- */
111
- numberOfCandidates?: number;
112
- /**
113
- * Filter to narrow down candidates from DB before vector search
114
- */
115
- filterBy?: DyFM_DBFilter<T>;
116
- /**
117
- * Search mode (cosine similarity, L2 distance, or hybrid).
118
- * Defaults to this.defaultSearchMode.
119
- *
120
- * `hybrid` mode combines cosine similarity (vector half) with BM25
121
- * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
- */
123
- searchMode?: LVS_Search_Mode;
124
- /**
125
- * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
- * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
- */
128
- hybridWeight?: { vector: number; text: number };
129
- /**
130
- * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
- * text-search. NEM kell hogy a vectorized property legyen.
132
- */
133
- textSearchKey?: keyof T;
134
- },
135
- ): Promise<T[]> {
136
- try {
137
- if (!set.input) {
138
- throw new DyFM_Error({
139
- ...this.getDefaultErrorSettings(
140
- 'vectorSearch',
141
- new Error('input is required')
142
- ),
143
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
- });
145
- }
146
-
147
- set.limit ??= 3;
148
- const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
- const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
-
151
- // Hybrid mode korai validacio
152
- if (effectiveMode === LVS_Search_Mode.hybrid) {
153
- if (!textSearchKey) {
154
- throw new DyFM_Error({
155
- ...this.getDefaultErrorSettings(
156
- 'vectorSearch',
157
- new Error('textSearchKey is required when searchMode is hybrid'),
158
- ),
159
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
- });
161
- }
162
- if (hybridWeight) {
163
- const w: { vector: number; text: number } = hybridWeight;
164
- if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
- throw new DyFM_Error({
166
- ...this.getDefaultErrorSettings(
167
- 'vectorSearch',
168
- new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
- ),
170
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
- });
172
- }
173
- }
174
- }
175
-
176
- // Validáljuk, hogy a searchInKey létezik-e
177
- const property: DyFM_DataProperty_Params<any, T> =
178
- this.dataParams.properties[searchInKey];
179
-
180
- if (!property) {
181
- throw new DyFM_Error({
182
- ...this.getDefaultErrorSettings(
183
- 'vectorSearch',
184
- new Error(
185
- `Property "${searchInKey}" not found! ` +
186
- `while searching "${this.dataParams.dataName}" ` +
187
- `(The searchable properties are: ` +
188
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
- )
190
- ),
191
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
- __localStack: this.dataParams.stackLocation,
193
- });
194
- }
195
-
196
- // Ellenőrizzük, hogy a property vectorized-e
197
- if (!this.vectorizedProperties.some(
198
- (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
- )) {
200
- throw new DyFM_Error({
201
- ...this.getDefaultErrorSettings(
202
- 'vectorSearch',
203
- new Error(
204
- `Property "${searchInKey}" is not a vectorized property! ` +
205
- `(The vectorized properties are: ` +
206
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
- )
208
- ),
209
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
- __localStack: this.dataParams.stackLocation,
211
- });
212
- }
213
-
214
- // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
- let dataList: T[];
216
- if (filterBy) {
217
- // Ha van filterBy, akkor először szűrünk a DB-ben
218
- if (this.debugLog) {
219
- DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
- }
221
- dataList = await this.findDataList(filterBy, true);
222
- } else {
223
- // Ha nincs filterBy, akkor minden adatot betöltünk
224
- if (this.debugLog) {
225
- DyFM_Log.log(`Local vector search: loading all data...`);
226
- }
227
- dataList = await this.getAll(true);
228
- }
229
-
230
- if (dataList.length === 0) {
231
- if (this.debugLog) {
232
- DyFM_Log.log(`Local vector search: no data found`);
233
- }
234
- return [];
235
- }
236
-
237
- // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
- if (this.debugLog) {
239
- DyFM_Log.log(
240
- `Local vector search: building vector pool from ${dataList.length} items...`
241
- );
242
- }
243
-
244
- // Töröljük a korábbi pool-t
245
- this.vectorPool.clearPool();
246
-
247
- // Hozzáadjuk a vektorokat a pool-hoz
248
- const dataMap: Map<string, T> = new Map<string, T>();
249
- for (const dataItem of dataList) {
250
- if (!dataItem._id) {
251
- // Ha nincs _id, akkor kihagyjuk
252
- if (this.debugLog) {
253
- DyFM_Log.warn(
254
- `Local vector search: skipping item without _id`
255
- );
256
- }
257
- continue;
258
- }
259
-
260
- const vectorizedValue: number[] | undefined =
261
- dataItem[searchInKey] as number[] | undefined;
262
-
263
- if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
- // Ha nincs vectorized érték, akkor kihagyjuk
265
- if (this.debugLog) {
266
- DyFM_Log.warn(
267
- `Local vector search: skipping item "${dataItem._id}" ` +
268
- `without vectorized value for "${searchInKey}"`
269
- );
270
- }
271
- continue;
272
- }
273
-
274
- // Hozzáadjuk a vektort a pool-hoz
275
- this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
- dataMap.set(dataItem._id, dataItem);
277
- }
278
-
279
- if (dataMap.size === 0) {
280
- if (this.debugLog) {
281
- DyFM_Log.log(`Local vector search: no valid vectors found`);
282
- }
283
- return [];
284
- }
285
-
286
- // 3. Vectorizáljuk a query input-ot
287
- if (this.debugLog) {
288
- DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
- }
290
-
291
- const queryVector: number[] = await this.vectorize(
292
- input,
293
- property.embeddingModel
294
- );
295
-
296
- // 4. Végrehajtjuk a local vector search-t
297
- const mode: LVS_Search_Mode = effectiveMode;
298
-
299
- if (this.debugLog) {
300
- DyFM_Log.log(
301
- `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
- );
303
- }
304
-
305
- let searchResults: LVS_SearchResult[];
306
-
307
- if (mode === LVS_Search_Mode.hybrid) {
308
- // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
- const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
- queryVector,
311
- dataMap.size,
312
- LVS_Search_Mode.cosineSimilarity,
313
- );
314
-
315
- // BM25 corpus epitese a `textSearchKey` property-bol
316
- const docs: { id: string; text: string }[] = [];
317
- for (const [docId, dataItem] of dataMap) {
318
- const textValue: unknown = dataItem[textSearchKey as keyof T];
319
- docs.push({
320
- id: docId,
321
- text: typeof textValue === 'string' ? textValue : '',
322
- });
323
- }
324
- const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
- const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
- const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
-
328
- const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
- for (const s of bm25Normalized) {
330
- bm25ScoreById.set(s.id, s.score);
331
- }
332
-
333
- const wVector: number = hybridWeight?.vector ?? 0.5;
334
- const wText: number = hybridWeight?.text ?? 0.5;
335
-
336
- const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
- const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
- return {
339
- id: c.id,
340
- score: wVector * c.score + wText * bm25Score,
341
- };
342
- });
343
- merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
- searchResults = merged.slice(0, limit);
345
- } else {
346
- searchResults = this.vectorPool.search(
347
- queryVector,
348
- limit,
349
- mode,
350
- );
351
- }
352
-
353
- if (this.debugLog) {
354
- DyFM_Log.log(
355
- `Local vector search: found ${searchResults.length} results`
356
- );
357
- }
358
-
359
- // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
- const results: T[] = [];
361
- for (const searchResult of searchResults) {
362
- const dataItem: T | undefined = dataMap.get(searchResult.id);
363
- if (dataItem) {
364
- results.push(dataItem);
365
- }
366
- }
367
-
368
- // Töröljük a pool-t a memória felszabadítása érdekében
369
- this.vectorPool.clearPool();
370
-
371
- return results;
372
- } catch (error) {
373
- // Töröljük a pool-t hiba esetén is
374
- this.vectorPool.clearPool();
375
-
376
- throw new DyFM_Error({
377
- ...this.getDefaultErrorSettings('vectorSearch', error),
378
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
- });
380
- }
381
- }
382
- }
383
-
384
- /**
385
- * @example
386
- * ```typescript
387
- * // Create a local vector data service
388
- * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
- * new CodeChunk(),
390
- * codeChunk_dataParams,
391
- * openAISettings,
392
- * 'example-issuer'
393
- * );
394
- *
395
- * // Perform vector search with filterBy
396
- * const results = await localVectorService.vectorSearch({
397
- * input: 'How to implement authentication?',
398
- * searchInKey: 'chunkParentedContentVectorized',
399
- * limit: 5,
400
- * filterBy: {
401
- * // Filter by category before vector search
402
- * category: 'documentation'
403
- * },
404
- * searchMode: LVS_Search_Mode.cosineSimilarity
405
- * });
406
- *
407
- * // Results are sorted by similarity score (highest first for cosine)
408
- * console.log(`Found ${results.length} similar chunks`);
409
- *
410
- * // Access the results
411
- * for (const result of results) {
412
- * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
- * }
414
- * ```
415
- */
416
-
1
+ import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
+ import {
3
+ DyFM_DataModel_Params,
4
+ DyFM_DataProperty_Params,
5
+ DyFM_DBFilter,
6
+ DyFM_Metadata,
7
+ } from '@futdevpro/fsm-dynamo';
8
+ import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
+
10
+ import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
+ import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
+ import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
+ import {
14
+ DyNTS_LVS_BM25_Corpus,
15
+ DyNTS_LVS_BM25_DocScore,
16
+ dyNTS_LVS_BM25_minMaxNormalize,
17
+ } from './lvs-bm25.util';
18
+ import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
+ import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
+
21
+ /**
22
+ * Local Vector Data Service
23
+ * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
+ * instead of MongoDB vector search
25
+ *
26
+ * This service loads data from the database, builds an in-memory vector pool,
27
+ * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const service = new DyNTS_OAI_LocalVectorDataService(
32
+ * data,
33
+ * dataParams,
34
+ * openAISettings,
35
+ * issuer
36
+ * );
37
+ *
38
+ * const results = await service.vectorSearch({
39
+ * input: 'search query',
40
+ * searchInKey: 'contentVectorized',
41
+ * limit: 10,
42
+ * filterBy: { category: 'docs' }
43
+ * });
44
+ * ```
45
+ */
46
+ export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
+
48
+ /**
49
+ * Default search mode
50
+ * Cosine similarity is typically preferred for text embeddings
51
+ */
52
+ defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
+
54
+ /**
55
+ * Whether to use L2 normalization when storing vectors
56
+ * Normalized vectors are more efficient for cosine similarity calculations
57
+ */
58
+ useL2Normalization: boolean = true;
59
+
60
+ /**
61
+ * Instance-based vector pool utility
62
+ * Minden service instance saját vektor pool-t tart fenn
63
+ */
64
+ private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
+
66
+ constructor(
67
+ /**
68
+ * Initial data, this will be used by functions on default
69
+ */
70
+ data: T,
71
+ /**
72
+ * DB data params will be used to connect to usable dbService on GlobalService
73
+ */
74
+ dataParams: DyFM_DataModel_Params<T>,
75
+ /**
76
+ * OpenAI settings
77
+ */
78
+ openAISettings: DyFM_OAI_Settings,
79
+ /**
80
+ * Initial set for issuer to be able to follow the issuer's activity
81
+ */
82
+ issuer: string
83
+ ) {
84
+ super(data, dataParams, openAISettings, issuer);
85
+ }
86
+
87
+ /**
88
+ * Overrides the parent vectorSearch method to perform local in-memory vector search
89
+ *
90
+ * Process:
91
+ * 1. Load data from DB using filterBy (if provided) or getAll()
92
+ * 2. Extract vectorized properties from loaded data
93
+ * 3. Build in-memory vector pool
94
+ * 4. Vectorize input query
95
+ * 5. Perform local vector search
96
+ * 6. Map search results back to full data objects
97
+ *
98
+ * @param set Search parameters
99
+ * @returns Array of data objects sorted by similarity score
100
+ */
101
+ override async vectorSearch(
102
+ set: {
103
+ input: string;
104
+ /** this should be the vectorized property key */
105
+ searchInKey: string;
106
+ limit?: number;
107
+ /**
108
+ * Number of candidates that are used to find the best match
109
+ * (Not used in local search, but kept for compatibility)
110
+ */
111
+ numberOfCandidates?: number;
112
+ /**
113
+ * Filter to narrow down candidates from DB before vector search
114
+ */
115
+ filterBy?: DyFM_DBFilter<T>;
116
+ /**
117
+ * Search mode (cosine similarity, L2 distance, or hybrid).
118
+ * Defaults to this.defaultSearchMode.
119
+ *
120
+ * `hybrid` mode combines cosine similarity (vector half) with BM25
121
+ * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
+ */
123
+ searchMode?: LVS_Search_Mode;
124
+ /**
125
+ * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
+ * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
+ */
128
+ hybridWeight?: { vector: number; text: number };
129
+ /**
130
+ * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
+ * text-search. NEM kell hogy a vectorized property legyen.
132
+ */
133
+ textSearchKey?: keyof T;
134
+ },
135
+ ): Promise<T[]> {
136
+ try {
137
+ if (!set.input) {
138
+ throw new DyFM_Error({
139
+ ...this.getDefaultErrorSettings(
140
+ 'vectorSearch',
141
+ new Error('input is required')
142
+ ),
143
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
+ });
145
+ }
146
+
147
+ set.limit ??= 3;
148
+ const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
+ const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
+
151
+ // Hybrid mode korai validacio
152
+ if (effectiveMode === LVS_Search_Mode.hybrid) {
153
+ if (!textSearchKey) {
154
+ throw new DyFM_Error({
155
+ ...this.getDefaultErrorSettings(
156
+ 'vectorSearch',
157
+ new Error('textSearchKey is required when searchMode is hybrid'),
158
+ ),
159
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
+ });
161
+ }
162
+ if (hybridWeight) {
163
+ const w: { vector: number; text: number } = hybridWeight;
164
+ if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
+ throw new DyFM_Error({
166
+ ...this.getDefaultErrorSettings(
167
+ 'vectorSearch',
168
+ new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
+ ),
170
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
+ });
172
+ }
173
+ }
174
+ }
175
+
176
+ // Validáljuk, hogy a searchInKey létezik-e
177
+ const property: DyFM_DataProperty_Params<any, T> =
178
+ this.dataParams.properties[searchInKey];
179
+
180
+ if (!property) {
181
+ throw new DyFM_Error({
182
+ ...this.getDefaultErrorSettings(
183
+ 'vectorSearch',
184
+ new Error(
185
+ `Property "${searchInKey}" not found! ` +
186
+ `while searching "${this.dataParams.dataName}" ` +
187
+ `(The searchable properties are: ` +
188
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
+ )
190
+ ),
191
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
+ __localStack: this.dataParams.stackLocation,
193
+ });
194
+ }
195
+
196
+ // Ellenőrizzük, hogy a property vectorized-e
197
+ if (!this.vectorizedProperties.some(
198
+ (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
+ )) {
200
+ throw new DyFM_Error({
201
+ ...this.getDefaultErrorSettings(
202
+ 'vectorSearch',
203
+ new Error(
204
+ `Property "${searchInKey}" is not a vectorized property! ` +
205
+ `(The vectorized properties are: ` +
206
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
+ )
208
+ ),
209
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
+ __localStack: this.dataParams.stackLocation,
211
+ });
212
+ }
213
+
214
+ // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
+ let dataList: T[];
216
+ if (filterBy) {
217
+ // Ha van filterBy, akkor először szűrünk a DB-ben
218
+ if (this.debugLog) {
219
+ DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
+ }
221
+ dataList = await this.findDataList(filterBy, true);
222
+ } else {
223
+ // Ha nincs filterBy, akkor minden adatot betöltünk
224
+ if (this.debugLog) {
225
+ DyFM_Log.log(`Local vector search: loading all data...`);
226
+ }
227
+ dataList = await this.getAll(true);
228
+ }
229
+
230
+ if (dataList.length === 0) {
231
+ if (this.debugLog) {
232
+ DyFM_Log.log(`Local vector search: no data found`);
233
+ }
234
+ return [];
235
+ }
236
+
237
+ // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
+ if (this.debugLog) {
239
+ DyFM_Log.log(
240
+ `Local vector search: building vector pool from ${dataList.length} items...`
241
+ );
242
+ }
243
+
244
+ // Töröljük a korábbi pool-t
245
+ this.vectorPool.clearPool();
246
+
247
+ // Hozzáadjuk a vektorokat a pool-hoz
248
+ const dataMap: Map<string, T> = new Map<string, T>();
249
+ for (const dataItem of dataList) {
250
+ if (!dataItem._id) {
251
+ // Ha nincs _id, akkor kihagyjuk
252
+ if (this.debugLog) {
253
+ DyFM_Log.warn(
254
+ `Local vector search: skipping item without _id`
255
+ );
256
+ }
257
+ continue;
258
+ }
259
+
260
+ const vectorizedValue: number[] | undefined =
261
+ dataItem[searchInKey] as number[] | undefined;
262
+
263
+ if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
+ // Ha nincs vectorized érték, akkor kihagyjuk
265
+ if (this.debugLog) {
266
+ DyFM_Log.warn(
267
+ `Local vector search: skipping item "${dataItem._id}" ` +
268
+ `without vectorized value for "${searchInKey}"`
269
+ );
270
+ }
271
+ continue;
272
+ }
273
+
274
+ // Hozzáadjuk a vektort a pool-hoz
275
+ this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
+ dataMap.set(dataItem._id, dataItem);
277
+ }
278
+
279
+ if (dataMap.size === 0) {
280
+ if (this.debugLog) {
281
+ DyFM_Log.log(`Local vector search: no valid vectors found`);
282
+ }
283
+ return [];
284
+ }
285
+
286
+ // 3. Vectorizáljuk a query input-ot
287
+ if (this.debugLog) {
288
+ DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
+ }
290
+
291
+ const queryVector: number[] = await this.vectorize(
292
+ input,
293
+ property.embeddingModel
294
+ );
295
+
296
+ // 4. Végrehajtjuk a local vector search-t
297
+ const mode: LVS_Search_Mode = effectiveMode;
298
+
299
+ if (this.debugLog) {
300
+ DyFM_Log.log(
301
+ `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
+ );
303
+ }
304
+
305
+ let searchResults: LVS_SearchResult[];
306
+
307
+ if (mode === LVS_Search_Mode.hybrid) {
308
+ // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
+ const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
+ queryVector,
311
+ dataMap.size,
312
+ LVS_Search_Mode.cosineSimilarity,
313
+ );
314
+
315
+ // BM25 corpus epitese a `textSearchKey` property-bol
316
+ const docs: { id: string; text: string }[] = [];
317
+ for (const [docId, dataItem] of dataMap) {
318
+ const textValue: unknown = dataItem[textSearchKey as keyof T];
319
+ docs.push({
320
+ id: docId,
321
+ text: typeof textValue === 'string' ? textValue : '',
322
+ });
323
+ }
324
+ const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
+ const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
+ const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
+
328
+ const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
+ for (const s of bm25Normalized) {
330
+ bm25ScoreById.set(s.id, s.score);
331
+ }
332
+
333
+ const wVector: number = hybridWeight?.vector ?? 0.5;
334
+ const wText: number = hybridWeight?.text ?? 0.5;
335
+
336
+ const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
+ const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
+ return {
339
+ id: c.id,
340
+ score: wVector * c.score + wText * bm25Score,
341
+ };
342
+ });
343
+ merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
+ searchResults = merged.slice(0, limit);
345
+ } else {
346
+ searchResults = this.vectorPool.search(
347
+ queryVector,
348
+ limit,
349
+ mode,
350
+ );
351
+ }
352
+
353
+ if (this.debugLog) {
354
+ DyFM_Log.log(
355
+ `Local vector search: found ${searchResults.length} results`
356
+ );
357
+ }
358
+
359
+ // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
+ const results: T[] = [];
361
+ for (const searchResult of searchResults) {
362
+ const dataItem: T | undefined = dataMap.get(searchResult.id);
363
+ if (dataItem) {
364
+ results.push(dataItem);
365
+ }
366
+ }
367
+
368
+ // Töröljük a pool-t a memória felszabadítása érdekében
369
+ this.vectorPool.clearPool();
370
+
371
+ return results;
372
+ } catch (error) {
373
+ // Töröljük a pool-t hiba esetén is
374
+ this.vectorPool.clearPool();
375
+
376
+ throw new DyFM_Error({
377
+ ...this.getDefaultErrorSettings('vectorSearch', error),
378
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
+ });
380
+ }
381
+ }
382
+ }
383
+
384
+ /**
385
+ * @example
386
+ * ```typescript
387
+ * // Create a local vector data service
388
+ * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
+ * new CodeChunk(),
390
+ * codeChunk_dataParams,
391
+ * openAISettings,
392
+ * 'example-issuer'
393
+ * );
394
+ *
395
+ * // Perform vector search with filterBy
396
+ * const results = await localVectorService.vectorSearch({
397
+ * input: 'How to implement authentication?',
398
+ * searchInKey: 'chunkParentedContentVectorized',
399
+ * limit: 5,
400
+ * filterBy: {
401
+ * // Filter by category before vector search
402
+ * category: 'documentation'
403
+ * },
404
+ * searchMode: LVS_Search_Mode.cosineSimilarity
405
+ * });
406
+ *
407
+ * // Results are sorted by similarity score (highest first for cosine)
408
+ * console.log(`Found ${results.length} similar chunks`);
409
+ *
410
+ * // Access the results
411
+ * for (const result of results) {
412
+ * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
+ * }
414
+ * ```
415
+ */
416
+