@futdevpro/nts-dynamo 1.15.56 → 1.15.58

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (450) hide show
  1. package/.c8rc.json +26 -26
  2. package/.copilot/patterns.json +7 -7
  3. package/.cursor/rules/__assistant_guide.mdc +30 -30
  4. package/.cursor/rules/_ag_backend-structure.mdc +85 -85
  5. package/.cursor/rules/_ag_backend.mdc +16 -16
  6. package/.cursor/rules/_ag_frontend-structure.mdc +86 -86
  7. package/.cursor/rules/_ag_frontend.mdc +39 -39
  8. package/.cursor/rules/_ag_import-rules.mdc +44 -44
  9. package/.cursor/rules/_ag_naming.mdc +115 -115
  10. package/.cursor/rules/_ag_should-be.mdc +6 -6
  11. package/.cursor/rules/ai_development_guide.md +60 -60
  12. package/.cursor/rules/cursor-rules.md +160 -160
  13. package/.cursor/rules/default-command.mdc +464 -464
  14. package/.cursor/rules/error_code_pattern.md +39 -39
  15. package/.cursor/rules/saved rule mcp server use.md +15 -15
  16. package/.dynamo/logs/cicd-pipeline/output.log +2844 -0
  17. package/.dynamo/logs/cicd-pipeline/status.json +94 -0
  18. package/.github/workflows/main.yml +432 -432
  19. package/.vscode/settings.json +10 -10
  20. package/HOWTO.md +15 -15
  21. package/LICENSE +21 -21
  22. package/__documentations/nts-integration-tests-2026-03-17.md +26 -26
  23. package/_specifications/BACKLOG.md +92 -92
  24. package/_specifications/TODO.md +15 -15
  25. package/_specifications/agent.md +138 -138
  26. package/build/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.d.ts +110 -0
  27. package/build/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.d.ts.map +1 -0
  28. package/build/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.js +419 -0
  29. package/build/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.js.map +1 -0
  30. package/build/_modules/ai/_modules/document-ai/_models/interfaces/dai-code-chunk.interface.d.ts +50 -0
  31. package/build/_modules/ai/_modules/document-ai/_models/interfaces/dai-code-chunk.interface.d.ts.map +1 -0
  32. package/build/_modules/ai/_modules/document-ai/_models/interfaces/dai-code-chunk.interface.js +3 -0
  33. package/build/_modules/ai/_modules/document-ai/_models/interfaces/dai-code-chunk.interface.js.map +1 -0
  34. package/build/_modules/ai/_modules/document-ai/index.d.ts +2 -0
  35. package/build/_modules/ai/_modules/document-ai/index.d.ts.map +1 -1
  36. package/build/_modules/ai/_modules/document-ai/index.js +2 -0
  37. package/build/_modules/ai/_modules/document-ai/index.js.map +1 -1
  38. package/build/_modules/ai/_services/ai-embedding-mock.service.d.ts +81 -0
  39. package/build/_modules/ai/_services/ai-embedding-mock.service.d.ts.map +1 -0
  40. package/build/_modules/ai/_services/ai-embedding-mock.service.js +167 -0
  41. package/build/_modules/ai/_services/ai-embedding-mock.service.js.map +1 -0
  42. package/build/_modules/ai/_services/ai-embedding-provider.registry.d.ts +52 -0
  43. package/build/_modules/ai/_services/ai-embedding-provider.registry.d.ts.map +1 -0
  44. package/build/_modules/ai/_services/ai-embedding-provider.registry.js +79 -0
  45. package/build/_modules/ai/_services/ai-embedding-provider.registry.js.map +1 -0
  46. package/build/_modules/ai/_services/lmstudio-embedding.control-service.d.ts +111 -0
  47. package/build/_modules/ai/_services/lmstudio-embedding.control-service.d.ts.map +1 -0
  48. package/build/_modules/ai/_services/lmstudio-embedding.control-service.js +298 -0
  49. package/build/_modules/ai/_services/lmstudio-embedding.control-service.js.map +1 -0
  50. package/build/_modules/ai/index.d.ts +3 -0
  51. package/build/_modules/ai/index.d.ts.map +1 -1
  52. package/build/_modules/ai/index.js +4 -0
  53. package/build/_modules/ai/index.js.map +1 -1
  54. package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.d.ts +59 -0
  55. package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.d.ts.map +1 -0
  56. package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.js +169 -0
  57. package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.js.map +1 -0
  58. package/build/_modules/data-readers/_models/interfaces/dynts-sqlite-reader.interface.d.ts +32 -0
  59. package/build/_modules/data-readers/_models/interfaces/dynts-sqlite-reader.interface.d.ts.map +1 -0
  60. package/build/_modules/data-readers/_models/interfaces/dynts-sqlite-reader.interface.js +8 -0
  61. package/build/_modules/data-readers/_models/interfaces/dynts-sqlite-reader.interface.js.map +1 -0
  62. package/build/_modules/data-readers/index.d.ts +3 -0
  63. package/build/_modules/data-readers/index.d.ts.map +1 -0
  64. package/build/_modules/data-readers/index.js +11 -0
  65. package/build/_modules/data-readers/index.js.map +1 -0
  66. package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.d.ts +36 -0
  67. package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.d.ts.map +1 -0
  68. package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.js +54 -0
  69. package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.js.map +1 -0
  70. package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.d.ts +70 -0
  71. package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.d.ts.map +1 -0
  72. package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.js +123 -0
  73. package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.js.map +1 -0
  74. package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.d.ts +43 -0
  75. package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.d.ts.map +1 -0
  76. package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.js +72 -0
  77. package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.js.map +1 -0
  78. package/build/_modules/local-vector-search/index.d.ts +3 -0
  79. package/build/_modules/local-vector-search/index.d.ts.map +1 -1
  80. package/build/_modules/local-vector-search/index.js +4 -0
  81. package/build/_modules/local-vector-search/index.js.map +1 -1
  82. package/build/_modules/mcp/_models/interfaces/dynts-mcp.interface.d.ts +109 -0
  83. package/build/_modules/mcp/_models/interfaces/dynts-mcp.interface.d.ts.map +1 -0
  84. package/build/_modules/mcp/_models/interfaces/dynts-mcp.interface.js +14 -0
  85. package/build/_modules/mcp/_models/interfaces/dynts-mcp.interface.js.map +1 -0
  86. package/build/_modules/mcp/_services/dynts-mcp-server.service-base.d.ts +71 -0
  87. package/build/_modules/mcp/_services/dynts-mcp-server.service-base.d.ts.map +1 -0
  88. package/build/_modules/mcp/_services/dynts-mcp-server.service-base.js +99 -0
  89. package/build/_modules/mcp/_services/dynts-mcp-server.service-base.js.map +1 -0
  90. package/build/_modules/mcp/_services/dynts-mcp.adapter.d.ts +57 -0
  91. package/build/_modules/mcp/_services/dynts-mcp.adapter.d.ts.map +1 -0
  92. package/build/_modules/mcp/_services/dynts-mcp.adapter.js +134 -0
  93. package/build/_modules/mcp/_services/dynts-mcp.adapter.js.map +1 -0
  94. package/build/_modules/mcp/index.d.ts +4 -0
  95. package/build/_modules/mcp/index.d.ts.map +1 -0
  96. package/build/_modules/mcp/index.js +13 -0
  97. package/build/_modules/mcp/index.js.map +1 -0
  98. package/build/_modules/scoped-config/_enums/dynts-scoped-config-level.enum.d.ts +19 -0
  99. package/build/_modules/scoped-config/_enums/dynts-scoped-config-level.enum.d.ts.map +1 -0
  100. package/build/_modules/scoped-config/_enums/dynts-scoped-config-level.enum.js +23 -0
  101. package/build/_modules/scoped-config/_enums/dynts-scoped-config-level.enum.js.map +1 -0
  102. package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.d.ts +44 -0
  103. package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.d.ts.map +1 -0
  104. package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.js +68 -0
  105. package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.js.map +1 -0
  106. package/build/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.d.ts +89 -0
  107. package/build/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.d.ts.map +1 -0
  108. package/build/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.js +3 -0
  109. package/build/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.js.map +1 -0
  110. package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.d.ts +84 -0
  111. package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.d.ts.map +1 -0
  112. package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.js +220 -0
  113. package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.js.map +1 -0
  114. package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.d.ts +54 -0
  115. package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.d.ts.map +1 -0
  116. package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.js +76 -0
  117. package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.js.map +1 -0
  118. package/build/_modules/scoped-config/index.d.ts +6 -0
  119. package/build/_modules/scoped-config/index.d.ts.map +1 -0
  120. package/build/_modules/scoped-config/index.js +15 -0
  121. package/build/_modules/scoped-config/index.js.map +1 -0
  122. package/eslint.config.js +3 -3
  123. package/nodemon.json +24 -24
  124. package/package.json +58 -2
  125. package/pnpm-workspace.yaml +5 -4
  126. package/scripts/run-coverage-tests.js +28 -28
  127. package/spec/support/helpers/spec-reporter-loader.js +359 -359
  128. package/spec/support/helpers/ts-node-helper.js +93 -93
  129. package/spec/support/jasmine.coverage.json +24 -24
  130. package/spec/support/jasmine.json +24 -24
  131. package/src/_collections/archive.util.spec.ts +57 -57
  132. package/src/_collections/archive.util.ts +18 -18
  133. package/src/_collections/atlas-default-db-options.const.ts +9 -9
  134. package/src/_collections/default-fallback-cache-max-age.const.spec.ts +11 -11
  135. package/src/_collections/default-fallback-cache-max-age.const.ts +2 -2
  136. package/src/_collections/default-not-found-page.const.spec.ts +19 -19
  137. package/src/_collections/default-not-found-page.const.ts +22 -22
  138. package/src/_collections/default-socket-path.const.spec.ts +12 -12
  139. package/src/_collections/default-socket-path.const.ts +2 -2
  140. package/src/_collections/get-environment-settings.util.spec.ts +210 -210
  141. package/src/_collections/get-environment-settings.util.ts +48 -48
  142. package/src/_collections/global-settings.const.ts +82 -82
  143. package/src/_collections/sample.env +21 -21
  144. package/src/_collections/star.controller.spec.ts +224 -224
  145. package/src/_collections/star.controller.ts +129 -129
  146. package/src/_enums/data-model-type.enum.ts +14 -14
  147. package/src/_enums/data-service-function.enum.ts +24 -24
  148. package/src/_enums/predefined-data-types.enum.ts +16 -16
  149. package/src/_enums/route-security.enum.ts +12 -12
  150. package/src/_models/control-models/api-call-params.control-model.spec.ts +152 -152
  151. package/src/_models/control-models/api-call-params.control-model.ts +142 -142
  152. package/src/_models/control-models/app-ext-system-controls.control-model.spec.ts +52 -52
  153. package/src/_models/control-models/app-ext-system-controls.control-model.ts +9 -9
  154. package/src/_models/control-models/app-params.control-model.spec.ts +225 -225
  155. package/src/_models/control-models/app-params.control-model.ts +136 -136
  156. package/src/_models/control-models/app-system-controls.control-model.spec.ts +31 -31
  157. package/src/_models/control-models/app-system-controls.control-model.ts +9 -9
  158. package/src/_models/control-models/endpoint-params.control-model.spec.ts +627 -627
  159. package/src/_models/control-models/endpoint-params.control-model.ts +627 -627
  160. package/src/_models/control-models/http-settings.control-model.spec.ts +77 -77
  161. package/src/_models/control-models/http-settings.control-model.ts +37 -37
  162. package/src/_models/control-models/system-control.control-model.spec.ts +27 -27
  163. package/src/_models/control-models/system-control.control-model.ts +12 -12
  164. package/src/_models/interfaces/certification-settings.interface.ts +7 -7
  165. package/src/_models/interfaces/environment-settings.interface.ts +59 -59
  166. package/src/_models/interfaces/global-log-settings.interface.ts +171 -171
  167. package/src/_models/interfaces/global-service-settings.interface.ts +47 -47
  168. package/src/_models/interfaces/routing-module-settings.interface.ts +21 -21
  169. package/src/_models/interfaces/static-client-settings.interface.spec.ts +29 -29
  170. package/src/_models/interfaces/static-client-settings.interface.ts +28 -28
  171. package/src/_models/types/db-update.type.ts +100 -100
  172. package/src/_modules/ai/_models/ai-input-interfaces.ts +117 -117
  173. package/src/_modules/ai/_models/ai-test-generation-result.interface.ts +16 -16
  174. package/src/_modules/ai/_modules/anthropic/_services/aai-user-key.control-service.ts +138 -138
  175. package/src/_modules/ai/_modules/anthropic/index.ts +5 -5
  176. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.spec.ts +242 -242
  177. package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.ts +639 -639
  178. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.spec.ts +295 -0
  179. package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.ts +520 -0
  180. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.spec.ts +209 -209
  181. package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.ts +85 -85
  182. package/src/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.ts +7 -7
  183. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.ts +146 -146
  184. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model.ts +162 -162
  185. package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-document.data-model.ts +99 -99
  186. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-code-chunk.interface.ts +68 -0
  187. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface.ts +18 -18
  188. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-page-compare-result.interface.ts +19 -19
  189. package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-document-compare-result.interface.ts +25 -25
  190. package/src/_modules/ai/_modules/document-ai/index.ts +30 -28
  191. package/src/_modules/ai/_modules/fdp-ai/_services/fdpai-user-key.control-service.ts +189 -189
  192. package/src/_modules/ai/_modules/fdp-ai/index.ts +5 -5
  193. package/src/_modules/ai/_modules/open-ai/_collections/oai-global-settings.const.ts +9 -9
  194. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests-hu.conts.ts +82 -82
  195. package/src/_modules/ai/_modules/open-ai/_collections/oai-llm-predefined-requests.conts.ts +75 -75
  196. package/src/_modules/ai/_modules/open-ai/_enums/oai-gpt-message-role.enum.ts +45 -45
  197. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-global-settings.interface.ts +7 -7
  198. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-gpt-message.interface.ts +7 -7
  199. package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-llm-predefined-requests.interface.ts +57 -57
  200. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.ts +292 -292
  201. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.spec.ts +342 -342
  202. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.spec.ts +550 -550
  203. package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.ts +630 -630
  204. package/src/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.spec.ts +332 -332
  205. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.spec.ts +462 -462
  206. package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.ts +634 -634
  207. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.spec.ts +489 -489
  208. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.tools.spec.ts +173 -173
  209. package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.ts +1033 -1033
  210. package/src/_modules/ai/_modules/open-ai/_services/oai-user-key.control-service.ts +157 -157
  211. package/src/_modules/ai/_services/ai-embedding-mock.service.spec.ts +115 -0
  212. package/src/_modules/ai/_services/ai-embedding-mock.service.ts +219 -0
  213. package/src/_modules/ai/_services/ai-embedding-provider.registry.spec.ts +110 -0
  214. package/src/_modules/ai/_services/ai-embedding-provider.registry.ts +110 -0
  215. package/src/_modules/ai/_services/ai-embedding.service-base.spec.ts +98 -98
  216. package/src/_modules/ai/_services/ai-embedding.service-base.ts +48 -48
  217. package/src/_modules/ai/_services/ai-llm-chat.service-base.spec.ts +229 -229
  218. package/src/_modules/ai/_services/ai-llm-chat.service-base.ts +68 -68
  219. package/src/_modules/ai/_services/ai-llm.service-base.spec.ts +250 -250
  220. package/src/_modules/ai/_services/ai-llm.service-base.ts +519 -519
  221. package/src/_modules/ai/_services/ai-provider.service-base.spec.ts +158 -158
  222. package/src/_modules/ai/_services/ai-user-key.service-base.ts +59 -59
  223. package/src/_modules/ai/_services/lmstudio-embedding.control-service.spec.ts +197 -0
  224. package/src/_modules/ai/_services/lmstudio-embedding.control-service.ts +378 -0
  225. package/src/_modules/ai/index.ts +18 -13
  226. package/src/_modules/assistant/_collections/ass-global-settings.const.ts +13 -13
  227. package/src/_modules/assistant/_collections/ass.util.spec.ts +176 -176
  228. package/src/_modules/assistant/_collections/ass.util.ts +50 -50
  229. package/src/_modules/assistant/_models/ass-global-settings.interface.ts +15 -15
  230. package/src/_modules/assistant/_services/ass-io.control-service.spec.ts +140 -140
  231. package/src/_modules/assistant/_services/ass-main.control-service.spec.ts +192 -192
  232. package/src/_modules/assistant/_services/ass-main.control-service.ts +107 -107
  233. package/src/_modules/bot/_collections/bot-default-commands.const.ts +12 -12
  234. package/src/_modules/bot/_collections/bot-global-settings.const.ts +39 -39
  235. package/src/_modules/bot/_models/bot-channel-wrapper.interface.ts +62 -62
  236. package/src/_modules/bot/_models/bot-command.interface.ts +8 -8
  237. package/src/_modules/bot/_models/bot-global-settings.interface.ts +96 -96
  238. package/src/_modules/bot/_models/bot-last-mention-date.interface.ts +6 -6
  239. package/src/_modules/bot/_models/bot-last-message-date.interface.ts +5 -5
  240. package/src/_modules/bot/_models/bot-user-wrapper.interface.ts +41 -41
  241. package/src/_modules/bot/_modules/discord-bot/_models/dib-platform.types.ts +9 -9
  242. package/src/_modules/bot/_modules/discord-bot/_services/dib-messaging-provider.control-service.spec.ts +431 -431
  243. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.spec.ts +160 -160
  244. package/src/_modules/bot/_modules/dynamo-bot/_collections/dyb-operations.util.ts +55 -55
  245. package/src/_modules/bot/_modules/dynamo-bot/_models/dyb-platform.types.ts +15 -15
  246. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.spec.ts +374 -374
  247. package/src/_modules/bot/_modules/dynamo-bot/_services/dyb-messaging-provider.control-service.ts +447 -447
  248. package/src/_modules/bot/_modules/dynamo-bot/index.ts +15 -15
  249. package/src/_modules/bot/_modules/slack-bot/_models/slb-platform.types.ts +9 -9
  250. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.spec.ts +344 -344
  251. package/src/_modules/bot/_modules/slack-bot/_services/slb-messaging-provider.control-service.ts +197 -197
  252. package/src/_modules/bot/_modules/teams-bot/_models/teb-platform.types.ts +9 -9
  253. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.spec.ts +345 -345
  254. package/src/_modules/bot/_modules/teams-bot/_services/teb-messaging-provider.control-service.ts +197 -197
  255. package/src/_modules/bot/_services/bot-commands.control-service.spec.ts +116 -116
  256. package/src/_modules/bot/_services/bot-io.control-service.spec.ts +285 -285
  257. package/src/_modules/bot/_services/bot-main.control-service.spec.ts +208 -208
  258. package/src/_modules/bot/_services/bot-messaging-provider.service-base.spec.ts +349 -349
  259. package/src/_modules/bot/_services/bot-routines.control-service.spec.ts +111 -111
  260. package/src/_modules/custom-data/custom-data.controller.spec.ts +49 -49
  261. package/src/_modules/custom-data/custom-data.controller.ts +67 -67
  262. package/src/_modules/custom-data/custom-data.data-service.spec.ts +54 -54
  263. package/src/_modules/custom-data/custom-data.data-service.ts +21 -21
  264. package/src/_modules/custom-data/get-custom-data-routing-module.util.spec.ts +28 -28
  265. package/src/_modules/custom-data/get-custom-data-routing-module.util.ts +24 -24
  266. package/src/_modules/custom-data/index.ts +9 -9
  267. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.spec.ts +161 -0
  268. package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.ts +192 -0
  269. package/src/_modules/data-readers/_models/interfaces/dynts-sqlite-reader.interface.ts +33 -0
  270. package/src/_modules/data-readers/index.ts +11 -0
  271. package/src/_modules/defaults/_collections/default-endpoints.util.ts +487 -487
  272. package/src/_modules/defaults/_models/default-user.data-model.ts +72 -72
  273. package/src/_modules/defaults/_services/default-auth.service.spec.ts +269 -269
  274. package/src/_modules/defaults/_services/default-auth.service.ts +177 -177
  275. package/src/_modules/defaults/_services/default-socket-events.service.spec.ts +42 -42
  276. package/src/_modules/defaults/_services/default-socket-events.service.ts +61 -61
  277. package/src/_modules/defaults/_services/default-user.data-service.spec.ts +187 -187
  278. package/src/_modules/defaults/_services/default-user.data-service.ts +98 -98
  279. package/src/_modules/defaults/index.ts +17 -17
  280. package/src/_modules/discord-assistant/_collections/dias-global-settings.const.ts +19 -19
  281. package/src/_modules/discord-assistant/_collections/dias.util.spec.ts +366 -366
  282. package/src/_modules/discord-assistant/_collections/dias.util.ts +132 -132
  283. package/src/_modules/discord-assistant/_models/dias-global-settings.interface.ts +19 -19
  284. package/src/_modules/discord-assistant/_models/dias-knowledge.data-model.ts +52 -52
  285. package/src/_modules/discord-assistant/_services/dias-chunk.data-service.ts +177 -177
  286. package/src/_modules/discord-assistant/_services/dias-io.control-service.spec.ts +108 -108
  287. package/src/_modules/discord-assistant/_services/dias-io.control-service.ts +69 -69
  288. package/src/_modules/discord-assistant/_services/dias-main.control-service.spec.ts +22 -22
  289. package/src/_modules/discord-assistant/_services/dias-main.control-service.ts +27 -27
  290. package/src/_modules/discord-assistant/_services/dias.service-base.spec.ts +195 -195
  291. package/src/_modules/discord-assistant/_services/dias.service-base.ts +76 -76
  292. package/src/_modules/discord-assistant/index.ts +38 -38
  293. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.spec.ts +34 -34
  294. package/src/_modules/discord-assistant-voiced/_services/dias-discord-bot.control-service.ts +11 -11
  295. package/src/_modules/discord-assistant-voiced/index.ts +36 -36
  296. package/src/_modules/discord-bot/_collections/dibo-default-commands.const.ts +16 -16
  297. package/src/_modules/discord-bot/_collections/dibo-global-settings.conts.ts +55 -55
  298. package/src/_modules/discord-bot/_collections/dibo-operations.util.spec.ts +214 -214
  299. package/src/_modules/discord-bot/_collections/dibo-operations.util.ts +387 -387
  300. package/src/_modules/discord-bot/_models/dibo-command.interface.ts +12 -12
  301. package/src/_modules/discord-bot/_models/dibo-global-settings.interface.ts +98 -98
  302. package/src/_modules/discord-bot/_models/dibo-last-mention-date.inteface.ts +7 -7
  303. package/src/_modules/discord-bot/_models/dibo-last-message-date.interface.ts +6 -6
  304. package/src/_modules/discord-bot/_services/dibo-commands.control-service.spec.ts +154 -154
  305. package/src/_modules/discord-bot/_services/dibo-commands.control-service.ts +153 -153
  306. package/src/_modules/discord-bot/_services/dibo-io.control-service.spec.ts +264 -264
  307. package/src/_modules/discord-bot/_services/dibo-io.control-service.ts +306 -306
  308. package/src/_modules/discord-bot/_services/dibo-main.control-service.spec.ts +408 -408
  309. package/src/_modules/discord-bot/_services/dibo-main.control-service.ts +487 -487
  310. package/src/_modules/discord-bot/_services/dibo-routines.control-service.spec.ts +105 -105
  311. package/src/_modules/discord-bot/index.ts +36 -36
  312. package/src/_modules/local-vector-search/_enums/lvs-search-mode.enum.ts +35 -35
  313. package/src/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.ts +59 -0
  314. package/src/_modules/local-vector-search/_models/lvs-search-result.interface.ts +17 -17
  315. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.spec.ts +418 -418
  316. package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.ts +276 -276
  317. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.spec.ts +480 -480
  318. package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.ts +416 -416
  319. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.spec.ts +198 -0
  320. package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.ts +146 -0
  321. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.spec.ts +167 -0
  322. package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.ts +106 -0
  323. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.spec.ts +393 -393
  324. package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.ts +220 -220
  325. package/src/_modules/local-vector-search/index.ts +17 -12
  326. package/src/_modules/logs/index.ts +11 -11
  327. package/src/_modules/mcp/_models/interfaces/dynts-mcp.interface.ts +111 -0
  328. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.spec.ts +142 -0
  329. package/src/_modules/mcp/_services/dynts-mcp-server.service-base.ts +120 -0
  330. package/src/_modules/mcp/_services/dynts-mcp.adapter.ts +157 -0
  331. package/src/_modules/mcp/index.ts +13 -0
  332. package/src/_modules/messaging/README.md +354 -354
  333. package/src/_modules/messaging/_collections/get-messaging-routing-module.util.ts +26 -26
  334. package/src/_modules/messaging/_collections/msg-global-settings.const.ts +22 -22
  335. package/src/_modules/messaging/_collections/msg.util.spec.ts +226 -226
  336. package/src/_modules/messaging/_models/msg-global-settings.interface.ts +37 -37
  337. package/src/_modules/messaging/_services/msg-conversation.data-service.ts +146 -146
  338. package/src/_modules/messaging/_services/msg-events.service.spec.ts +219 -219
  339. package/src/_modules/messaging/_services/msg-events.service.ts +267 -267
  340. package/src/_modules/messaging/_services/msg-integration.control-service.ts +179 -179
  341. package/src/_modules/messaging/_services/msg-main.control-service.spec.ts +147 -147
  342. package/src/_modules/messaging/_services/msg-main.control-service.ts +571 -571
  343. package/src/_modules/messaging/_services/msg-message.data-service.ts +129 -129
  344. package/src/_modules/messaging/_services/msg.controller.spec.ts +201 -201
  345. package/src/_modules/messaging/index.ts +30 -30
  346. package/src/_modules/mock/app-extended-server.mock.ts +201 -201
  347. package/src/_modules/mock/app-integration-test.mock.ts +51 -51
  348. package/src/_modules/mock/app-params.mock.spec.ts +21 -21
  349. package/src/_modules/mock/app-params.mock.ts +9 -9
  350. package/src/_modules/mock/app-server.mock.ts +188 -188
  351. package/src/_modules/mock/auth-service.mock.spec.ts +47 -47
  352. package/src/_modules/mock/auth-service.mock.ts +28 -28
  353. package/src/_modules/mock/controller.mock.spec.ts +26 -26
  354. package/src/_modules/mock/controller.mock.ts +16 -16
  355. package/src/_modules/mock/data-model.mock.spec.ts +111 -111
  356. package/src/_modules/mock/data-model.mock.ts +82 -82
  357. package/src/_modules/mock/email-service-collection.mock.spec.ts +24 -24
  358. package/src/_modules/mock/email-service-collection.mock.ts +15 -15
  359. package/src/_modules/mock/email-service.mock.spec.ts +17 -17
  360. package/src/_modules/mock/email-service.mock.ts +20 -20
  361. package/src/_modules/mock/email-template.mock.html +14 -14
  362. package/src/_modules/mock/endpoint.mock.ts +91 -91
  363. package/src/_modules/mock/socket-client.mock.spec.ts +40 -40
  364. package/src/_modules/mock/socket-client.mock.ts +45 -45
  365. package/src/_modules/mock/socket-server.mock.spec.ts +44 -44
  366. package/src/_modules/mock/socket-server.mock.ts +46 -46
  367. package/src/_modules/oauth2/_routes/oauth2.controller.spec.ts +107 -107
  368. package/src/_modules/oauth2/_routes/oauth2.controller.ts +98 -98
  369. package/src/_modules/oauth2/_services/oauth2.auth-service.spec.ts +254 -254
  370. package/src/_modules/oauth2/_services/oauth2.auth-service.ts +232 -232
  371. package/src/_modules/oauth2/_services/oauth2.control-service.spec.ts +585 -585
  372. package/src/_modules/oauth2/_services/oauth2.control-service.ts +653 -653
  373. package/src/_modules/oauth2/index.ts +17 -17
  374. package/src/_modules/scoped-config/_enums/dynts-scoped-config-level.enum.ts +22 -0
  375. package/src/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.ts +81 -0
  376. package/src/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.ts +109 -0
  377. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.spec.ts +306 -0
  378. package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.ts +295 -0
  379. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.spec.ts +118 -0
  380. package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.ts +105 -0
  381. package/src/_modules/scoped-config/index.ts +17 -0
  382. package/src/_modules/server/errors/errors.control-service.spec.ts +238 -238
  383. package/src/_modules/server/errors/errors.control-service.ts +85 -85
  384. package/src/_modules/server/errors/errors.controller.spec.ts +241 -241
  385. package/src/_modules/server/errors/errors.controller.ts +431 -431
  386. package/src/_modules/server/errors/errors.data-service.spec.ts +361 -361
  387. package/src/_modules/server/index.ts +30 -30
  388. package/src/_modules/server/server-status/server-status-snapshot.control-service.spec.ts +70 -70
  389. package/src/_modules/server/server-status/server-status-snapshot.control-service.ts +17 -17
  390. package/src/_modules/server/server-status/server-status-snapshot.data-service.spec.ts +77 -77
  391. package/src/_modules/server/server-status/server-status-snapshot.data-service.ts +37 -37
  392. package/src/_modules/server/server-status/server-status.control-service.spec.ts +576 -576
  393. package/src/_modules/server/server-status/server-status.control-service.ts +396 -396
  394. package/src/_modules/server/server-status/server-status.controller.spec.ts +240 -240
  395. package/src/_modules/server/server-status/server-status.controller.ts +253 -253
  396. package/src/_modules/socket/_enums/socket-security.enum.ts +11 -11
  397. package/src/_modules/socket/_models/socket-client-service-params.control-model.spec.ts +32 -32
  398. package/src/_modules/socket/_models/socket-client-service-params.control-model.ts +22 -22
  399. package/src/_modules/socket/_models/socket-presence.control-model.spec.ts +164 -164
  400. package/src/_modules/socket/_models/socket-presence.control-model.ts +210 -210
  401. package/src/_modules/socket/_models/socket-server-service-params.control-model.spec.ts +46 -46
  402. package/src/_modules/socket/_models/socket-server-service-params.control-model.ts +22 -22
  403. package/src/_modules/socket/_services/socket-client.service.spec.ts +15 -15
  404. package/src/_modules/socket/_services/socket-client.service.ts +260 -260
  405. package/src/_modules/socket/_services/socket-server.service.spec.ts +11 -11
  406. package/src/_modules/socket/app-extended.integration.spec.ts +85 -85
  407. package/src/_modules/socket/app-extended.server.ts +630 -630
  408. package/src/_modules/socket/index.ts +42 -42
  409. package/src/_modules/test/get-test-routing-module.util.spec.ts +28 -28
  410. package/src/_modules/test/get-test-routing-module.util.ts +23 -23
  411. package/src/_modules/test/index.ts +11 -11
  412. package/src/_modules/test/test.controller.spec.ts +72 -72
  413. package/src/_modules/test/test.controller.ts +115 -115
  414. package/src/_modules/usage/get-usage-routing-module.util.ts +22 -22
  415. package/src/_modules/usage/index.ts +15 -15
  416. package/src/_modules/usage/usage.controller.spec.ts +81 -81
  417. package/src/_modules/usage/usage.controller.ts +126 -126
  418. package/src/_modules/usage/usage.data-service.spec.ts +332 -332
  419. package/src/_modules/usage/usage.data-service.ts +185 -185
  420. package/src/_services/base/api.service-base.spec.ts +125 -125
  421. package/src/_services/base/api.service-base.ts +74 -74
  422. package/src/_services/base/archive-data.service.spec.ts +196 -196
  423. package/src/_services/base/archive-data.service.ts +216 -216
  424. package/src/_services/base/data.service.spec.ts +674 -674
  425. package/src/_services/base/data.service.ts +2719 -2719
  426. package/src/_services/base/db.service.spec.ts +73 -73
  427. package/src/_services/base/db.service.ts +1575 -1575
  428. package/src/_services/base/singleton.service-base.spec.ts +28 -28
  429. package/src/_services/base/singleton.service-base.ts +24 -24
  430. package/src/_services/base/singleton.service.spec.ts +114 -114
  431. package/src/_services/base/singleton.service.ts +38 -38
  432. package/src/_services/core/api.service.spec.ts +140 -140
  433. package/src/_services/core/auth.service.spec.ts +159 -159
  434. package/src/_services/core/auth.service.ts +174 -174
  435. package/src/_services/core/email.service.spec.ts +85 -85
  436. package/src/_services/core/email.service.ts +742 -742
  437. package/src/_services/core/global.service.spec.ts +275 -275
  438. package/src/_services/core/global.service.ts +461 -461
  439. package/src/_services/core/service-collection.service.spec.ts +46 -46
  440. package/src/_services/core/service-collection.service.ts +6 -6
  441. package/src/_services/route/controller.service.spec.ts +53 -53
  442. package/src/_services/route/controller.service.ts +148 -148
  443. package/src/_services/route/routing-module.service.spec.ts +98 -98
  444. package/src/_services/route/routing-module.service.ts +330 -330
  445. package/src/_services/server/app.server.ts +1726 -1726
  446. package/src/_services/shared.static-service.spec.ts +99 -99
  447. package/src/_services/shared.static-service.ts +78 -78
  448. package/src/index.ts +96 -96
  449. package/tsconfig.app.json +12 -12
  450. package/tsconfig.json +42 -42
@@ -1,416 +1,416 @@
1
- import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
- import {
3
- DyFM_DataModel_Params,
4
- DyFM_DataProperty_Params,
5
- DyFM_DBFilter,
6
- DyFM_Metadata,
7
- } from '@futdevpro/fsm-dynamo';
8
- import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
-
10
- import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
- import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
- import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
- import {
14
- DyNTS_LVS_BM25_Corpus,
15
- DyNTS_LVS_BM25_DocScore,
16
- dyNTS_LVS_BM25_minMaxNormalize,
17
- } from './lvs-bm25.util';
18
- import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
- import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
-
21
- /**
22
- * Local Vector Data Service
23
- * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
- * instead of MongoDB vector search
25
- *
26
- * This service loads data from the database, builds an in-memory vector pool,
27
- * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
- *
29
- * @example
30
- * ```typescript
31
- * const service = new DyNTS_OAI_LocalVectorDataService(
32
- * data,
33
- * dataParams,
34
- * openAISettings,
35
- * issuer
36
- * );
37
- *
38
- * const results = await service.vectorSearch({
39
- * input: 'search query',
40
- * searchInKey: 'contentVectorized',
41
- * limit: 10,
42
- * filterBy: { category: 'docs' }
43
- * });
44
- * ```
45
- */
46
- export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
-
48
- /**
49
- * Default search mode
50
- * Cosine similarity is typically preferred for text embeddings
51
- */
52
- defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
-
54
- /**
55
- * Whether to use L2 normalization when storing vectors
56
- * Normalized vectors are more efficient for cosine similarity calculations
57
- */
58
- useL2Normalization: boolean = true;
59
-
60
- /**
61
- * Instance-based vector pool utility
62
- * Minden service instance saját vektor pool-t tart fenn
63
- */
64
- private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
-
66
- constructor(
67
- /**
68
- * Initial data, this will be used by functions on default
69
- */
70
- data: T,
71
- /**
72
- * DB data params will be used to connect to usable dbService on GlobalService
73
- */
74
- dataParams: DyFM_DataModel_Params<T>,
75
- /**
76
- * OpenAI settings
77
- */
78
- openAISettings: DyFM_OAI_Settings,
79
- /**
80
- * Initial set for issuer to be able to follow the issuer's activity
81
- */
82
- issuer: string
83
- ) {
84
- super(data, dataParams, openAISettings, issuer);
85
- }
86
-
87
- /**
88
- * Overrides the parent vectorSearch method to perform local in-memory vector search
89
- *
90
- * Process:
91
- * 1. Load data from DB using filterBy (if provided) or getAll()
92
- * 2. Extract vectorized properties from loaded data
93
- * 3. Build in-memory vector pool
94
- * 4. Vectorize input query
95
- * 5. Perform local vector search
96
- * 6. Map search results back to full data objects
97
- *
98
- * @param set Search parameters
99
- * @returns Array of data objects sorted by similarity score
100
- */
101
- override async vectorSearch(
102
- set: {
103
- input: string;
104
- /** this should be the vectorized property key */
105
- searchInKey: string;
106
- limit?: number;
107
- /**
108
- * Number of candidates that are used to find the best match
109
- * (Not used in local search, but kept for compatibility)
110
- */
111
- numberOfCandidates?: number;
112
- /**
113
- * Filter to narrow down candidates from DB before vector search
114
- */
115
- filterBy?: DyFM_DBFilter<T>;
116
- /**
117
- * Search mode (cosine similarity, L2 distance, or hybrid).
118
- * Defaults to this.defaultSearchMode.
119
- *
120
- * `hybrid` mode combines cosine similarity (vector half) with BM25
121
- * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
- */
123
- searchMode?: LVS_Search_Mode;
124
- /**
125
- * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
- * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
- */
128
- hybridWeight?: { vector: number; text: number };
129
- /**
130
- * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
- * text-search. NEM kell hogy a vectorized property legyen.
132
- */
133
- textSearchKey?: keyof T;
134
- },
135
- ): Promise<T[]> {
136
- try {
137
- if (!set.input) {
138
- throw new DyFM_Error({
139
- ...this.getDefaultErrorSettings(
140
- 'vectorSearch',
141
- new Error('input is required')
142
- ),
143
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
- });
145
- }
146
-
147
- set.limit ??= 3;
148
- const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
- const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
-
151
- // Hybrid mode korai validacio
152
- if (effectiveMode === LVS_Search_Mode.hybrid) {
153
- if (!textSearchKey) {
154
- throw new DyFM_Error({
155
- ...this.getDefaultErrorSettings(
156
- 'vectorSearch',
157
- new Error('textSearchKey is required when searchMode is hybrid'),
158
- ),
159
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
- });
161
- }
162
- if (hybridWeight) {
163
- const w: { vector: number; text: number } = hybridWeight;
164
- if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
- throw new DyFM_Error({
166
- ...this.getDefaultErrorSettings(
167
- 'vectorSearch',
168
- new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
- ),
170
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
- });
172
- }
173
- }
174
- }
175
-
176
- // Validáljuk, hogy a searchInKey létezik-e
177
- const property: DyFM_DataProperty_Params<any, T> =
178
- this.dataParams.properties[searchInKey];
179
-
180
- if (!property) {
181
- throw new DyFM_Error({
182
- ...this.getDefaultErrorSettings(
183
- 'vectorSearch',
184
- new Error(
185
- `Property "${searchInKey}" not found! ` +
186
- `while searching "${this.dataParams.dataName}" ` +
187
- `(The searchable properties are: ` +
188
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
- )
190
- ),
191
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
- __localStack: this.dataParams.stackLocation,
193
- });
194
- }
195
-
196
- // Ellenőrizzük, hogy a property vectorized-e
197
- if (!this.vectorizedProperties.some(
198
- (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
- )) {
200
- throw new DyFM_Error({
201
- ...this.getDefaultErrorSettings(
202
- 'vectorSearch',
203
- new Error(
204
- `Property "${searchInKey}" is not a vectorized property! ` +
205
- `(The vectorized properties are: ` +
206
- `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
- )
208
- ),
209
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
- __localStack: this.dataParams.stackLocation,
211
- });
212
- }
213
-
214
- // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
- let dataList: T[];
216
- if (filterBy) {
217
- // Ha van filterBy, akkor először szűrünk a DB-ben
218
- if (this.debugLog) {
219
- DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
- }
221
- dataList = await this.findDataList(filterBy, true);
222
- } else {
223
- // Ha nincs filterBy, akkor minden adatot betöltünk
224
- if (this.debugLog) {
225
- DyFM_Log.log(`Local vector search: loading all data...`);
226
- }
227
- dataList = await this.getAll(true);
228
- }
229
-
230
- if (dataList.length === 0) {
231
- if (this.debugLog) {
232
- DyFM_Log.log(`Local vector search: no data found`);
233
- }
234
- return [];
235
- }
236
-
237
- // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
- if (this.debugLog) {
239
- DyFM_Log.log(
240
- `Local vector search: building vector pool from ${dataList.length} items...`
241
- );
242
- }
243
-
244
- // Töröljük a korábbi pool-t
245
- this.vectorPool.clearPool();
246
-
247
- // Hozzáadjuk a vektorokat a pool-hoz
248
- const dataMap: Map<string, T> = new Map<string, T>();
249
- for (const dataItem of dataList) {
250
- if (!dataItem._id) {
251
- // Ha nincs _id, akkor kihagyjuk
252
- if (this.debugLog) {
253
- DyFM_Log.warn(
254
- `Local vector search: skipping item without _id`
255
- );
256
- }
257
- continue;
258
- }
259
-
260
- const vectorizedValue: number[] | undefined =
261
- dataItem[searchInKey] as number[] | undefined;
262
-
263
- if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
- // Ha nincs vectorized érték, akkor kihagyjuk
265
- if (this.debugLog) {
266
- DyFM_Log.warn(
267
- `Local vector search: skipping item "${dataItem._id}" ` +
268
- `without vectorized value for "${searchInKey}"`
269
- );
270
- }
271
- continue;
272
- }
273
-
274
- // Hozzáadjuk a vektort a pool-hoz
275
- this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
- dataMap.set(dataItem._id, dataItem);
277
- }
278
-
279
- if (dataMap.size === 0) {
280
- if (this.debugLog) {
281
- DyFM_Log.log(`Local vector search: no valid vectors found`);
282
- }
283
- return [];
284
- }
285
-
286
- // 3. Vectorizáljuk a query input-ot
287
- if (this.debugLog) {
288
- DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
- }
290
-
291
- const queryVector: number[] = await this.vectorize(
292
- input,
293
- property.embeddingModel
294
- );
295
-
296
- // 4. Végrehajtjuk a local vector search-t
297
- const mode: LVS_Search_Mode = effectiveMode;
298
-
299
- if (this.debugLog) {
300
- DyFM_Log.log(
301
- `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
- );
303
- }
304
-
305
- let searchResults: LVS_SearchResult[];
306
-
307
- if (mode === LVS_Search_Mode.hybrid) {
308
- // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
- const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
- queryVector,
311
- dataMap.size,
312
- LVS_Search_Mode.cosineSimilarity,
313
- );
314
-
315
- // BM25 corpus epitese a `textSearchKey` property-bol
316
- const docs: { id: string; text: string }[] = [];
317
- for (const [docId, dataItem] of dataMap) {
318
- const textValue: unknown = dataItem[textSearchKey as keyof T];
319
- docs.push({
320
- id: docId,
321
- text: typeof textValue === 'string' ? textValue : '',
322
- });
323
- }
324
- const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
- const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
- const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
-
328
- const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
- for (const s of bm25Normalized) {
330
- bm25ScoreById.set(s.id, s.score);
331
- }
332
-
333
- const wVector: number = hybridWeight?.vector ?? 0.5;
334
- const wText: number = hybridWeight?.text ?? 0.5;
335
-
336
- const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
- const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
- return {
339
- id: c.id,
340
- score: wVector * c.score + wText * bm25Score,
341
- };
342
- });
343
- merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
- searchResults = merged.slice(0, limit);
345
- } else {
346
- searchResults = this.vectorPool.search(
347
- queryVector,
348
- limit,
349
- mode,
350
- );
351
- }
352
-
353
- if (this.debugLog) {
354
- DyFM_Log.log(
355
- `Local vector search: found ${searchResults.length} results`
356
- );
357
- }
358
-
359
- // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
- const results: T[] = [];
361
- for (const searchResult of searchResults) {
362
- const dataItem: T | undefined = dataMap.get(searchResult.id);
363
- if (dataItem) {
364
- results.push(dataItem);
365
- }
366
- }
367
-
368
- // Töröljük a pool-t a memória felszabadítása érdekében
369
- this.vectorPool.clearPool();
370
-
371
- return results;
372
- } catch (error) {
373
- // Töröljük a pool-t hiba esetén is
374
- this.vectorPool.clearPool();
375
-
376
- throw new DyFM_Error({
377
- ...this.getDefaultErrorSettings('vectorSearch', error),
378
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
- });
380
- }
381
- }
382
- }
383
-
384
- /**
385
- * @example
386
- * ```typescript
387
- * // Create a local vector data service
388
- * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
- * new CodeChunk(),
390
- * codeChunk_dataParams,
391
- * openAISettings,
392
- * 'example-issuer'
393
- * );
394
- *
395
- * // Perform vector search with filterBy
396
- * const results = await localVectorService.vectorSearch({
397
- * input: 'How to implement authentication?',
398
- * searchInKey: 'chunkParentedContentVectorized',
399
- * limit: 5,
400
- * filterBy: {
401
- * // Filter by category before vector search
402
- * category: 'documentation'
403
- * },
404
- * searchMode: LVS_Search_Mode.cosineSimilarity
405
- * });
406
- *
407
- * // Results are sorted by similarity score (highest first for cosine)
408
- * console.log(`Found ${results.length} similar chunks`);
409
- *
410
- * // Access the results
411
- * for (const result of results) {
412
- * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
- * }
414
- * ```
415
- */
416
-
1
+ import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
2
+ import {
3
+ DyFM_DataModel_Params,
4
+ DyFM_DataProperty_Params,
5
+ DyFM_DBFilter,
6
+ DyFM_Metadata,
7
+ } from '@futdevpro/fsm-dynamo';
8
+ import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
9
+
10
+ import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
11
+ import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
12
+ import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
13
+ import {
14
+ DyNTS_LVS_BM25_Corpus,
15
+ DyNTS_LVS_BM25_DocScore,
16
+ dyNTS_LVS_BM25_minMaxNormalize,
17
+ } from './lvs-bm25.util';
18
+ import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
19
+ import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
20
+
21
+ /**
22
+ * Local Vector Data Service
23
+ * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
24
+ * instead of MongoDB vector search
25
+ *
26
+ * This service loads data from the database, builds an in-memory vector pool,
27
+ * and performs brute-force vector similarity search using cosine similarity or L2 distance
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const service = new DyNTS_OAI_LocalVectorDataService(
32
+ * data,
33
+ * dataParams,
34
+ * openAISettings,
35
+ * issuer
36
+ * );
37
+ *
38
+ * const results = await service.vectorSearch({
39
+ * input: 'search query',
40
+ * searchInKey: 'contentVectorized',
41
+ * limit: 10,
42
+ * filterBy: { category: 'docs' }
43
+ * });
44
+ * ```
45
+ */
46
+ export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
47
+
48
+ /**
49
+ * Default search mode
50
+ * Cosine similarity is typically preferred for text embeddings
51
+ */
52
+ defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
53
+
54
+ /**
55
+ * Whether to use L2 normalization when storing vectors
56
+ * Normalized vectors are more efficient for cosine similarity calculations
57
+ */
58
+ useL2Normalization: boolean = true;
59
+
60
+ /**
61
+ * Instance-based vector pool utility
62
+ * Minden service instance saját vektor pool-t tart fenn
63
+ */
64
+ private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
65
+
66
+ constructor(
67
+ /**
68
+ * Initial data, this will be used by functions on default
69
+ */
70
+ data: T,
71
+ /**
72
+ * DB data params will be used to connect to usable dbService on GlobalService
73
+ */
74
+ dataParams: DyFM_DataModel_Params<T>,
75
+ /**
76
+ * OpenAI settings
77
+ */
78
+ openAISettings: DyFM_OAI_Settings,
79
+ /**
80
+ * Initial set for issuer to be able to follow the issuer's activity
81
+ */
82
+ issuer: string
83
+ ) {
84
+ super(data, dataParams, openAISettings, issuer);
85
+ }
86
+
87
+ /**
88
+ * Overrides the parent vectorSearch method to perform local in-memory vector search
89
+ *
90
+ * Process:
91
+ * 1. Load data from DB using filterBy (if provided) or getAll()
92
+ * 2. Extract vectorized properties from loaded data
93
+ * 3. Build in-memory vector pool
94
+ * 4. Vectorize input query
95
+ * 5. Perform local vector search
96
+ * 6. Map search results back to full data objects
97
+ *
98
+ * @param set Search parameters
99
+ * @returns Array of data objects sorted by similarity score
100
+ */
101
+ override async vectorSearch(
102
+ set: {
103
+ input: string;
104
+ /** this should be the vectorized property key */
105
+ searchInKey: string;
106
+ limit?: number;
107
+ /**
108
+ * Number of candidates that are used to find the best match
109
+ * (Not used in local search, but kept for compatibility)
110
+ */
111
+ numberOfCandidates?: number;
112
+ /**
113
+ * Filter to narrow down candidates from DB before vector search
114
+ */
115
+ filterBy?: DyFM_DBFilter<T>;
116
+ /**
117
+ * Search mode (cosine similarity, L2 distance, or hybrid).
118
+ * Defaults to this.defaultSearchMode.
119
+ *
120
+ * `hybrid` mode combines cosine similarity (vector half) with BM25
121
+ * text scoring (text half) — `textSearchKey` is REQUIRED in hybrid mode.
122
+ */
123
+ searchMode?: LVS_Search_Mode;
124
+ /**
125
+ * Csak `hybrid` modban — weighted score-merge a cosine es a BM25 kozott.
126
+ * Default: { vector: 0.5, text: 0.5 }. Mindkettonek 0..1 tartomany javasolt.
127
+ */
128
+ hybridWeight?: { vector: number; text: number };
129
+ /**
130
+ * Csak `hybrid` modban — KOTELEZO; melyik string property-n fut a BM25
131
+ * text-search. NEM kell hogy a vectorized property legyen.
132
+ */
133
+ textSearchKey?: keyof T;
134
+ },
135
+ ): Promise<T[]> {
136
+ try {
137
+ if (!set.input) {
138
+ throw new DyFM_Error({
139
+ ...this.getDefaultErrorSettings(
140
+ 'vectorSearch',
141
+ new Error('input is required')
142
+ ),
143
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
144
+ });
145
+ }
146
+
147
+ set.limit ??= 3;
148
+ const { input, searchInKey, limit, filterBy, searchMode, hybridWeight, textSearchKey } = set;
149
+ const effectiveMode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
150
+
151
+ // Hybrid mode korai validacio
152
+ if (effectiveMode === LVS_Search_Mode.hybrid) {
153
+ if (!textSearchKey) {
154
+ throw new DyFM_Error({
155
+ ...this.getDefaultErrorSettings(
156
+ 'vectorSearch',
157
+ new Error('textSearchKey is required when searchMode is hybrid'),
158
+ ),
159
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS4`,
160
+ });
161
+ }
162
+ if (hybridWeight) {
163
+ const w: { vector: number; text: number } = hybridWeight;
164
+ if (!Number.isFinite(w.vector) || !Number.isFinite(w.text) || w.vector < 0 || w.text < 0) {
165
+ throw new DyFM_Error({
166
+ ...this.getDefaultErrorSettings(
167
+ 'vectorSearch',
168
+ new Error('hybridWeight.vector and .text must be non-negative finite numbers'),
169
+ ),
170
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS5`,
171
+ });
172
+ }
173
+ }
174
+ }
175
+
176
+ // Validáljuk, hogy a searchInKey létezik-e
177
+ const property: DyFM_DataProperty_Params<any, T> =
178
+ this.dataParams.properties[searchInKey];
179
+
180
+ if (!property) {
181
+ throw new DyFM_Error({
182
+ ...this.getDefaultErrorSettings(
183
+ 'vectorSearch',
184
+ new Error(
185
+ `Property "${searchInKey}" not found! ` +
186
+ `while searching "${this.dataParams.dataName}" ` +
187
+ `(The searchable properties are: ` +
188
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
189
+ )
190
+ ),
191
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
192
+ __localStack: this.dataParams.stackLocation,
193
+ });
194
+ }
195
+
196
+ // Ellenőrizzük, hogy a property vectorized-e
197
+ if (!this.vectorizedProperties.some(
198
+ (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
199
+ )) {
200
+ throw new DyFM_Error({
201
+ ...this.getDefaultErrorSettings(
202
+ 'vectorSearch',
203
+ new Error(
204
+ `Property "${searchInKey}" is not a vectorized property! ` +
205
+ `(The vectorized properties are: ` +
206
+ `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
207
+ )
208
+ ),
209
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
210
+ __localStack: this.dataParams.stackLocation,
211
+ });
212
+ }
213
+
214
+ // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
215
+ let dataList: T[];
216
+ if (filterBy) {
217
+ // Ha van filterBy, akkor először szűrünk a DB-ben
218
+ if (this.debugLog) {
219
+ DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
220
+ }
221
+ dataList = await this.findDataList(filterBy, true);
222
+ } else {
223
+ // Ha nincs filterBy, akkor minden adatot betöltünk
224
+ if (this.debugLog) {
225
+ DyFM_Log.log(`Local vector search: loading all data...`);
226
+ }
227
+ dataList = await this.getAll(true);
228
+ }
229
+
230
+ if (dataList.length === 0) {
231
+ if (this.debugLog) {
232
+ DyFM_Log.log(`Local vector search: no data found`);
233
+ }
234
+ return [];
235
+ }
236
+
237
+ // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
238
+ if (this.debugLog) {
239
+ DyFM_Log.log(
240
+ `Local vector search: building vector pool from ${dataList.length} items...`
241
+ );
242
+ }
243
+
244
+ // Töröljük a korábbi pool-t
245
+ this.vectorPool.clearPool();
246
+
247
+ // Hozzáadjuk a vektorokat a pool-hoz
248
+ const dataMap: Map<string, T> = new Map<string, T>();
249
+ for (const dataItem of dataList) {
250
+ if (!dataItem._id) {
251
+ // Ha nincs _id, akkor kihagyjuk
252
+ if (this.debugLog) {
253
+ DyFM_Log.warn(
254
+ `Local vector search: skipping item without _id`
255
+ );
256
+ }
257
+ continue;
258
+ }
259
+
260
+ const vectorizedValue: number[] | undefined =
261
+ dataItem[searchInKey] as number[] | undefined;
262
+
263
+ if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
264
+ // Ha nincs vectorized érték, akkor kihagyjuk
265
+ if (this.debugLog) {
266
+ DyFM_Log.warn(
267
+ `Local vector search: skipping item "${dataItem._id}" ` +
268
+ `without vectorized value for "${searchInKey}"`
269
+ );
270
+ }
271
+ continue;
272
+ }
273
+
274
+ // Hozzáadjuk a vektort a pool-hoz
275
+ this.vectorPool.addVector(dataItem._id, vectorizedValue);
276
+ dataMap.set(dataItem._id, dataItem);
277
+ }
278
+
279
+ if (dataMap.size === 0) {
280
+ if (this.debugLog) {
281
+ DyFM_Log.log(`Local vector search: no valid vectors found`);
282
+ }
283
+ return [];
284
+ }
285
+
286
+ // 3. Vectorizáljuk a query input-ot
287
+ if (this.debugLog) {
288
+ DyFM_Log.log(`Local vector search: vectorizing query input...`);
289
+ }
290
+
291
+ const queryVector: number[] = await this.vectorize(
292
+ input,
293
+ property.embeddingModel
294
+ );
295
+
296
+ // 4. Végrehajtjuk a local vector search-t
297
+ const mode: LVS_Search_Mode = effectiveMode;
298
+
299
+ if (this.debugLog) {
300
+ DyFM_Log.log(
301
+ `Local vector search: searching with mode "${mode}", limit: ${limit}...`
302
+ );
303
+ }
304
+
305
+ let searchResults: LVS_SearchResult[];
306
+
307
+ if (mode === LVS_Search_Mode.hybrid) {
308
+ // Hybrid: cosine ALL candidate-re + BM25 ALL candidate-re + min-max norm + weighted sum
309
+ const allCandidatesCosine: LVS_SearchResult[] = this.vectorPool.search(
310
+ queryVector,
311
+ dataMap.size,
312
+ LVS_Search_Mode.cosineSimilarity,
313
+ );
314
+
315
+ // BM25 corpus epitese a `textSearchKey` property-bol
316
+ const docs: { id: string; text: string }[] = [];
317
+ for (const [docId, dataItem] of dataMap) {
318
+ const textValue: unknown = dataItem[textSearchKey as keyof T];
319
+ docs.push({
320
+ id: docId,
321
+ text: typeof textValue === 'string' ? textValue : '',
322
+ });
323
+ }
324
+ const bm25Corpus: DyNTS_LVS_BM25_Corpus = new DyNTS_LVS_BM25_Corpus(docs);
325
+ const bm25Raw: DyNTS_LVS_BM25_DocScore[] = bm25Corpus.score(input);
326
+ const bm25Normalized: DyNTS_LVS_BM25_DocScore[] = dyNTS_LVS_BM25_minMaxNormalize(bm25Raw);
327
+
328
+ const bm25ScoreById: Map<string, number> = new Map<string, number>();
329
+ for (const s of bm25Normalized) {
330
+ bm25ScoreById.set(s.id, s.score);
331
+ }
332
+
333
+ const wVector: number = hybridWeight?.vector ?? 0.5;
334
+ const wText: number = hybridWeight?.text ?? 0.5;
335
+
336
+ const merged: LVS_SearchResult[] = allCandidatesCosine.map((c: LVS_SearchResult): LVS_SearchResult => {
337
+ const bm25Score: number = bm25ScoreById.get(c.id) ?? 0;
338
+ return {
339
+ id: c.id,
340
+ score: wVector * c.score + wText * bm25Score,
341
+ };
342
+ });
343
+ merged.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
344
+ searchResults = merged.slice(0, limit);
345
+ } else {
346
+ searchResults = this.vectorPool.search(
347
+ queryVector,
348
+ limit,
349
+ mode,
350
+ );
351
+ }
352
+
353
+ if (this.debugLog) {
354
+ DyFM_Log.log(
355
+ `Local vector search: found ${searchResults.length} results`
356
+ );
357
+ }
358
+
359
+ // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
360
+ const results: T[] = [];
361
+ for (const searchResult of searchResults) {
362
+ const dataItem: T | undefined = dataMap.get(searchResult.id);
363
+ if (dataItem) {
364
+ results.push(dataItem);
365
+ }
366
+ }
367
+
368
+ // Töröljük a pool-t a memória felszabadítása érdekében
369
+ this.vectorPool.clearPool();
370
+
371
+ return results;
372
+ } catch (error) {
373
+ // Töröljük a pool-t hiba esetén is
374
+ this.vectorPool.clearPool();
375
+
376
+ throw new DyFM_Error({
377
+ ...this.getDefaultErrorSettings('vectorSearch', error),
378
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
379
+ });
380
+ }
381
+ }
382
+ }
383
+
384
+ /**
385
+ * @example
386
+ * ```typescript
387
+ * // Create a local vector data service
388
+ * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
389
+ * new CodeChunk(),
390
+ * codeChunk_dataParams,
391
+ * openAISettings,
392
+ * 'example-issuer'
393
+ * );
394
+ *
395
+ * // Perform vector search with filterBy
396
+ * const results = await localVectorService.vectorSearch({
397
+ * input: 'How to implement authentication?',
398
+ * searchInKey: 'chunkParentedContentVectorized',
399
+ * limit: 5,
400
+ * filterBy: {
401
+ * // Filter by category before vector search
402
+ * category: 'documentation'
403
+ * },
404
+ * searchMode: LVS_Search_Mode.cosineSimilarity
405
+ * });
406
+ *
407
+ * // Results are sorted by similarity score (highest first for cosine)
408
+ * console.log(`Found ${results.length} similar chunks`);
409
+ *
410
+ * // Access the results
411
+ * for (const result of results) {
412
+ * console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
413
+ * }
414
+ * ```
415
+ */
416
+