@carlonicora/nestjs-neo4jsonapi 1.105.0 → 1.106.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/dist/agents/graph.creator/services/graph.creator.service.d.ts +1 -1
  2. package/dist/agents/graph.creator/services/graph.creator.service.d.ts.map +1 -1
  3. package/dist/agents/graph.creator/services/graph.creator.service.js +14 -0
  4. package/dist/agents/graph.creator/services/graph.creator.service.js.map +1 -1
  5. package/dist/bootstrap/app.module.factory.d.ts.map +1 -1
  6. package/dist/bootstrap/app.module.factory.js +1 -0
  7. package/dist/bootstrap/app.module.factory.js.map +1 -1
  8. package/dist/bootstrap/bootstrap.options.d.ts +3 -0
  9. package/dist/bootstrap/bootstrap.options.d.ts.map +1 -1
  10. package/dist/common/interfaces/agents/graph.creator.interface.d.ts +4 -0
  11. package/dist/common/interfaces/agents/graph.creator.interface.d.ts.map +1 -1
  12. package/dist/common/repositories/ai-source-query.provider.d.ts +19 -0
  13. package/dist/common/repositories/ai-source-query.provider.d.ts.map +1 -0
  14. package/dist/common/repositories/ai-source-query.provider.js +5 -0
  15. package/dist/common/repositories/ai-source-query.provider.js.map +1 -0
  16. package/dist/common/repositories/default.ai-source-query.provider.d.ts +19 -0
  17. package/dist/common/repositories/default.ai-source-query.provider.d.ts.map +1 -0
  18. package/dist/common/repositories/default.ai-source-query.provider.js +36 -0
  19. package/dist/common/repositories/default.ai-source-query.provider.js.map +1 -0
  20. package/dist/common/repositories/index.d.ts +2 -0
  21. package/dist/common/repositories/index.d.ts.map +1 -1
  22. package/dist/common/repositories/index.js +2 -0
  23. package/dist/common/repositories/index.js.map +1 -1
  24. package/dist/common/types/data.limits.d.ts +2 -2
  25. package/dist/common/types/data.limits.d.ts.map +1 -1
  26. package/dist/config/interfaces/base.config.interface.d.ts +2 -0
  27. package/dist/config/interfaces/base.config.interface.d.ts.map +1 -1
  28. package/dist/config/interfaces/config.embedding.context.interface.d.ts +18 -0
  29. package/dist/config/interfaces/config.embedding.context.interface.d.ts.map +1 -0
  30. package/dist/config/interfaces/config.embedding.context.interface.js +3 -0
  31. package/dist/config/interfaces/config.embedding.context.interface.js.map +1 -0
  32. package/dist/config/interfaces/index.d.ts +1 -0
  33. package/dist/config/interfaces/index.d.ts.map +1 -1
  34. package/dist/config/interfaces/index.js +1 -0
  35. package/dist/config/interfaces/index.js.map +1 -1
  36. package/dist/core/core.module.d.ts +4 -0
  37. package/dist/core/core.module.d.ts.map +1 -1
  38. package/dist/core/core.module.js +7 -2
  39. package/dist/core/core.module.js.map +1 -1
  40. package/dist/foundations/atomicfact/repositories/atomicfact.repository.d.ts +3 -8
  41. package/dist/foundations/atomicfact/repositories/atomicfact.repository.d.ts.map +1 -1
  42. package/dist/foundations/atomicfact/repositories/atomicfact.repository.js +19 -46
  43. package/dist/foundations/atomicfact/repositories/atomicfact.repository.js.map +1 -1
  44. package/dist/foundations/chunk/chunk.module.d.ts +6 -3
  45. package/dist/foundations/chunk/chunk.module.d.ts.map +1 -1
  46. package/dist/foundations/chunk/chunk.module.js +25 -5
  47. package/dist/foundations/chunk/chunk.module.js.map +1 -1
  48. package/dist/foundations/chunk/controllers/chunk.controller.d.ts +19 -1
  49. package/dist/foundations/chunk/controllers/chunk.controller.d.ts.map +1 -1
  50. package/dist/foundations/chunk/controllers/chunk.controller.js +59 -2
  51. package/dist/foundations/chunk/controllers/chunk.controller.js.map +1 -1
  52. package/dist/foundations/chunk/entities/chunk.entity.d.ts +8 -0
  53. package/dist/foundations/chunk/entities/chunk.entity.d.ts.map +1 -1
  54. package/dist/foundations/chunk/entities/chunk.entity.js +20 -0
  55. package/dist/foundations/chunk/entities/chunk.entity.js.map +1 -1
  56. package/dist/foundations/chunk/processors/chunk.embedding.processor.d.ts +21 -0
  57. package/dist/foundations/chunk/processors/chunk.embedding.processor.d.ts.map +1 -0
  58. package/dist/foundations/chunk/processors/chunk.embedding.processor.js +98 -0
  59. package/dist/foundations/chunk/processors/chunk.embedding.processor.js.map +1 -0
  60. package/dist/foundations/chunk/repositories/chunk.repository.d.ts +36 -8
  61. package/dist/foundations/chunk/repositories/chunk.repository.d.ts.map +1 -1
  62. package/dist/foundations/chunk/repositories/chunk.repository.js +184 -50
  63. package/dist/foundations/chunk/repositories/chunk.repository.js.map +1 -1
  64. package/dist/foundations/chunk/services/chunk.service.d.ts +9 -0
  65. package/dist/foundations/chunk/services/chunk.service.d.ts.map +1 -1
  66. package/dist/foundations/chunk/services/chunk.service.js +87 -0
  67. package/dist/foundations/chunk/services/chunk.service.js.map +1 -1
  68. package/dist/foundations/chunk/services/embedding-context.d.ts +19 -0
  69. package/dist/foundations/chunk/services/embedding-context.d.ts.map +1 -0
  70. package/dist/foundations/chunk/services/embedding-context.js +27 -0
  71. package/dist/foundations/chunk/services/embedding-context.js.map +1 -0
  72. package/dist/foundations/chunk/services/reciprocal-rank-fusion.d.ts +2 -0
  73. package/dist/foundations/chunk/services/reciprocal-rank-fusion.d.ts.map +1 -0
  74. package/dist/foundations/chunk/services/reciprocal-rank-fusion.js +13 -0
  75. package/dist/foundations/chunk/services/reciprocal-rank-fusion.js.map +1 -0
  76. package/dist/foundations/keyconcept/controllers/keyconcept.controller.d.ts +15 -0
  77. package/dist/foundations/keyconcept/controllers/keyconcept.controller.d.ts.map +1 -0
  78. package/dist/foundations/keyconcept/controllers/keyconcept.controller.js +56 -0
  79. package/dist/foundations/keyconcept/controllers/keyconcept.controller.js.map +1 -0
  80. package/dist/foundations/keyconcept/index.d.ts +2 -0
  81. package/dist/foundations/keyconcept/index.d.ts.map +1 -1
  82. package/dist/foundations/keyconcept/index.js +5 -1
  83. package/dist/foundations/keyconcept/index.js.map +1 -1
  84. package/dist/foundations/keyconcept/keyconcept.module.d.ts.map +1 -1
  85. package/dist/foundations/keyconcept/keyconcept.module.js +7 -2
  86. package/dist/foundations/keyconcept/keyconcept.module.js.map +1 -1
  87. package/dist/foundations/keyconcept/processors/keyconcept.embedding.processor.d.ts +22 -0
  88. package/dist/foundations/keyconcept/processors/keyconcept.embedding.processor.d.ts.map +1 -0
  89. package/dist/foundations/keyconcept/processors/keyconcept.embedding.processor.js +98 -0
  90. package/dist/foundations/keyconcept/processors/keyconcept.embedding.processor.js.map +1 -0
  91. package/dist/foundations/keyconcept/repositories/keyconcept.repository.d.ts +9 -15
  92. package/dist/foundations/keyconcept/repositories/keyconcept.repository.d.ts.map +1 -1
  93. package/dist/foundations/keyconcept/repositories/keyconcept.repository.js +65 -99
  94. package/dist/foundations/keyconcept/repositories/keyconcept.repository.js.map +1 -1
  95. package/package.json +1 -1
@@ -8,19 +8,23 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
8
8
  var __metadata = (this && this.__metadata) || function (k, v) {
9
9
  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
10
  };
11
+ var __param = (this && this.__param) || function (paramIndex, decorator) {
12
+ return function (target, key) { decorator(target, key, paramIndex); }
13
+ };
11
14
  Object.defineProperty(exports, "__esModule", { value: true });
12
15
  exports.AtomicFactRepository = void 0;
13
16
  const common_1 = require("@nestjs/common");
14
17
  const nestjs_cls_1 = require("nestjs-cls");
15
- const ai_source_query_1 = require("../../../common/repositories/ai.source.query");
18
+ const ai_source_query_provider_1 = require("../../../common/repositories/ai-source-query.provider");
16
19
  const neo4j_service_1 = require("../../../core/neo4j/services/neo4j.service");
17
20
  const security_service_1 = require("../../../core/security/services/security.service");
18
21
  const atomic_fact_model_1 = require("../../atomicfact/entities/atomic.fact.model");
19
22
  let AtomicFactRepository = class AtomicFactRepository {
20
- constructor(neo4j, securityService, clsService) {
23
+ constructor(neo4j, securityService, clsService, aiSourceQuery) {
21
24
  this.neo4j = neo4j;
22
25
  this.securityService = securityService;
23
26
  this.clsService = clsService;
27
+ this.aiSourceQuery = aiSourceQuery;
24
28
  }
25
29
  async onModuleInit() {
26
30
  await this.neo4j.writeOne({
@@ -28,60 +32,28 @@ let AtomicFactRepository = class AtomicFactRepository {
28
32
  });
29
33
  }
30
34
  async findAtomicFactsByKeyConcepts(params) {
31
- // SECURITY: HowTo retrieval intentionally bypasses company filtering.
32
- // Dispatch to the private method so the bypass is visible and quarantined here.
33
- if (params.dataLimits.howToMode || params.dataLimits.limitToHowToId) {
34
- return this.findAtomicFactsByKeyConceptsFromHowTos(params);
35
- }
36
35
  const query = this.neo4j.initQuery({ serialiser: atomic_fact_model_1.AtomicFactModel });
37
- query.queryParams = {
38
- ...query.queryParams,
39
- keyConcepts: params.keyConcepts,
40
- skipChunkIds: params.skipChunkIds,
41
- skipAtomicFactIds: params.skipAtomicFactIds,
42
- };
43
- query.query += `
44
- MATCH (data)-[:BELONGS_TO]->(company)
45
- ${(0, ai_source_query_1.aiSourceQuery)({
36
+ const scope = this.aiSourceQuery.build({
37
+ dataLimits: params.dataLimits,
46
38
  currentUserId: this.clsService.get("userId"),
47
39
  securityService: this.securityService,
48
- dataLimits: params.dataLimits,
49
40
  returnsData: true,
50
- })}
51
- MATCH (keyconcept:KeyConcept)<-[:HAS_KEY_CONCEPT]-(atomicfact:AtomicFact)<-[:HAS_ATOMIC_FACT]-(atomicfact_chunk:Chunk)<-[:HAS_CHUNK]-(data)
52
- WHERE keyconcept.value IN $keyConcepts
53
- ${params.skipChunkIds.length > 0 ? `AND NOT atomicfact_chunk.id IN $skipChunkIds` : ""}
54
- ${params.skipAtomicFactIds.length > 0 ? `AND NOT atomicfact.id IN $skipAtomicFactIds` : ""}
55
- RETURN atomicfact, atomicfact_chunk
56
- `;
57
- return this.neo4j.readMany(query);
58
- }
59
- /**
60
- * Help-content retrieval. Intentionally bypasses company filtering — `(data:HowTo)`
61
- * nodes are global and have no `BELONGS_TO Company` edge. Only callable from inside
62
- * this repository; `findAtomicFactsByKeyConcepts` dispatches here based on
63
- * `dataLimits.howToMode` / `dataLimits.limitToHowToId`.
64
- */
65
- async findAtomicFactsByKeyConceptsFromHowTos(params) {
66
- const query = this.neo4j.initQuery({ serialiser: atomic_fact_model_1.AtomicFactModel });
41
+ });
67
42
  query.queryParams = {
68
43
  ...query.queryParams,
69
44
  keyConcepts: params.keyConcepts,
70
45
  skipChunkIds: params.skipChunkIds,
71
46
  skipAtomicFactIds: params.skipAtomicFactIds,
72
- ...(params.dataLimits.limitToHowToId ? { limitToHowToId: params.dataLimits.limitToHowToId } : {}),
47
+ ...scope.params,
73
48
  };
74
- const limitClause = params.dataLimits.limitToHowToId ? `WHERE data.id = $limitToHowToId` : "";
75
49
  query.query += `
76
- MATCH (data:HowTo)
77
- ${limitClause}
78
- WITH data
79
- MATCH (keyconcept:KeyConcept)<-[:HAS_KEY_CONCEPT]-(atomicfact:AtomicFact)<-[:HAS_ATOMIC_FACT]-(atomicfact_chunk:Chunk)<-[:HAS_CHUNK]-(data)
80
- WHERE keyconcept.value IN $keyConcepts
81
- ${params.skipChunkIds.length > 0 ? `AND NOT atomicfact_chunk.id IN $skipChunkIds` : ""}
82
- ${params.skipAtomicFactIds.length > 0 ? `AND NOT atomicfact.id IN $skipAtomicFactIds` : ""}
83
- RETURN atomicfact, atomicfact_chunk
84
- `;
50
+ ${scope.cypher}
51
+ MATCH (keyconcept:KeyConcept)<-[:HAS_KEY_CONCEPT]-(atomicfact:AtomicFact)<-[:HAS_ATOMIC_FACT]-(atomicfact_chunk:Chunk)<-[:HAS_CHUNK]-(data)
52
+ WHERE keyconcept.value IN $keyConcepts
53
+ ${params.skipChunkIds.length > 0 ? `AND NOT atomicfact_chunk.id IN $skipChunkIds` : ""}
54
+ ${params.skipAtomicFactIds.length > 0 ? `AND NOT atomicfact.id IN $skipAtomicFactIds` : ""}
55
+ RETURN atomicfact, atomicfact_chunk
56
+ `;
85
57
  return this.neo4j.readMany(query);
86
58
  }
87
59
  async findAtomicFactsByChunkId(params) {
@@ -137,8 +109,9 @@ let AtomicFactRepository = class AtomicFactRepository {
137
109
  exports.AtomicFactRepository = AtomicFactRepository;
138
110
  exports.AtomicFactRepository = AtomicFactRepository = __decorate([
139
111
  (0, common_1.Injectable)(),
112
+ __param(3, (0, common_1.Inject)(ai_source_query_provider_1.AI_SOURCE_QUERY)),
140
113
  __metadata("design:paramtypes", [neo4j_service_1.Neo4jService,
141
114
  security_service_1.SecurityService,
142
- nestjs_cls_1.ClsService])
115
+ nestjs_cls_1.ClsService, Object])
143
116
  ], AtomicFactRepository);
144
117
  //# sourceMappingURL=atomicfact.repository.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"atomicfact.repository.js","sourceRoot":"","sources":["../../../../src/foundations/atomicfact/repositories/atomicfact.repository.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA0D;AAC1D,2CAAwC;AACxC,kFAA6E;AAE7E,8EAA0E;AAC1E,uFAAmF;AAEnF,mFAA8E;AAGvE,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;IAC/B,YACmB,KAAmB,EACnB,eAAgC,EAChC,UAAsB;QAFtB,UAAK,GAAL,KAAK,CAAc;QACnB,oBAAe,GAAf,eAAe,CAAiB;QAChC,eAAU,GAAV,UAAU,CAAY;IACtC,CAAC;IAEJ,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;YACxB,KAAK,EAAE,2GAA2G;SACnH,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,4BAA4B,CAAC,MAKlC;QACC,sEAAsE;QACtE,gFAAgF;QAChF,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC,cAAc,EAAE,CAAC;YACpE,OAAO,IAAI,CAAC,sCAAsC,CAAC,MAAM,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,mCAAe,EAAE,CAAC,CAAC;QAEpE,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;SAC5C,CAAC;QAEF,KAAK,CAAC,KAAK,IAAI;;UAET,IAAA,+BAAa,EAAC;YACd,aAAa,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC5C,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,WAAW,EAAE,IAAI;SAClB,CAAC;;;UAGA,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,8CAA8C,CAAC,CAAC,CAAC,EAAE;UACpF,MAAM,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,EAAE;;KAE7F,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,sCAAsC,CAAC,MAKpD;QACC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,mCAAe,EAAE,CAAC,CAAC;QAEpE,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,UAAU,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClG,CAAC;QAEF,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE9F,KAAK,CAAC,KAAK,IAAI;;UAET,WAAW;;;;UAIX,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,8CAA8C,CAAC,CAAC,CAAC,EAAE;UACpF,MAAM,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,EAAE;;KAE7F,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,wBAAwB,CAAC,MAA2B;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,mCAAe,EAAE,CAAC,CAAC;QAEpE,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC;QAEF,KAAK,CAAC,KAAK,IAAI;;;KAGd,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,MAAgC;QACvD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,mCAAe,EAAE,CAAC,CAAC;QAEpE,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,YAAY,EAAE,MAAM,CAAC,YAAY;SAClC,CAAC;QAEF,KAAK,CAAC,KAAK,IAAI;;;KAGd,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,6BAA6B;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAErC,KAAK,CAAC,KAAK,GAAG;;;;KAIb,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,MAAkE;QACvF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAErC,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,iBAAiB,EAAE,MAAM,CAAC,OAAO;SAClC,CAAC;QAEF,KAAK,CAAC,KAAK,IAAI;;;;;KAKd,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;CACF,CAAA;AAxJY,oDAAoB;+BAApB,oBAAoB;IADhC,IAAA,mBAAU,GAAE;qCAGe,4BAAY;QACF,kCAAe;QACpB,uBAAU;GAJ9B,oBAAoB,CAwJhC"}
1
+ {"version":3,"file":"atomicfact.repository.js","sourceRoot":"","sources":["../../../../src/foundations/atomicfact/repositories/atomicfact.repository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAAkE;AAClE,2CAAwC;AACxC,oGAA+G;AAE/G,8EAA0E;AAC1E,uFAAmF;AAEnF,mFAA8E;AAGvE,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;IAC/B,YACmB,KAAmB,EACnB,eAAgC,EAChC,UAAsB,EACG,aAAoC;QAH7D,UAAK,GAAL,KAAK,CAAc;QACnB,oBAAe,GAAf,eAAe,CAAiB;QAChC,eAAU,GAAV,UAAU,CAAY;QACG,kBAAa,GAAb,aAAa,CAAuB;IAC7E,CAAC;IAEJ,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;YACxB,KAAK,EAAE,2GAA2G;SACnH,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,4BAA4B,CAAC,MAKlC;QACC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,mCAAe,EAAE,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC;YACrC,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,aAAa,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC5C,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,GAAG,KAAK,CAAC,MAAM;SAChB,CAAC;QACF,KAAK,CAAC,KAAK,IAAI;QACX,KAAK,CAAC,MAAM;;;QAGZ,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,8CAA8C,CAAC,CAAC,CAAC,EAAE;QACpF,MAAM,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,EAAE;;GAE7F,CAAC;QACA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,wBAAwB,CAAC,MAA2B;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,mCAAe,EAAE,CAAC,CAAC;QAEpE,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC;QAEF,KAAK,CAAC,KAAK,IAAI;;;KAGd,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,MAAgC;QACvD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,mCAAe,EAAE,CAAC,CAAC;QAEpE,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,YAAY,EAAE,MAAM,CAAC,YAAY;SAClC,CAAC;QAEF,KAAK,CAAC,KAAK,IAAI;;;KAGd,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,6BAA6B;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAErC,KAAK,CAAC,KAAK,GAAG;;;;KAIb,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,MAAkE;QACvF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAErC,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,iBAAiB,EAAE,MAAM,CAAC,OAAO;SAClC,CAAC;QAEF,KAAK,CAAC,KAAK,IAAI;;;;;KAKd,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;CACF,CAAA;AA3GY,oDAAoB;+BAApB,oBAAoB;IADhC,IAAA,mBAAU,GAAE;IAMR,WAAA,IAAA,eAAM,EAAC,0CAAe,CAAC,CAAA;qCAHA,4BAAY;QACF,kCAAe;QACpB,uBAAU;GAJ9B,oBAAoB,CA2GhC"}
@@ -2,9 +2,12 @@ import { OnModuleInit } from "@nestjs/common";
2
2
  /**
3
3
  * ChunkModule - Handles document chunking and graph generation.
4
4
  *
5
- * This is a fully static module. Queue registration is handled centrally
6
- * by QueueModule (which is @Global), so ChunkProcessor can inject queues
7
- * via @InjectQueue() without needing to register them here.
5
+ * The content-processing queue (CHUNK) is provided by the @Global QueueModule.
6
+ * The embedding-rebuild queues backing ChunkController's @InjectQueue() and the
7
+ * ChunkEmbeddingProcessor are registered LOCALLY here so the module is
8
+ * self-contained — every consuming app (api/corpus/neural-erp/phlow) loads
9
+ * ChunkModule without having to register these queues in its own config, and
10
+ * without declaring the rebuild routes itself (which previously duplicated).
8
11
  */
9
12
  export declare class ChunkModule implements OnModuleInit {
10
13
  onModuleInit(): void;
@@ -1 +1 @@
1
- {"version":3,"file":"chunk.module.d.ts","sourceRoot":"","sources":["../../../src/foundations/chunk/chunk.module.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAetD;;;;;;GAMG;AACH,qBAMa,WAAY,YAAW,YAAY;IAC9C,YAAY;CAGb"}
1
+ {"version":3,"file":"chunk.module.d.ts","sourceRoot":"","sources":["../../../src/foundations/chunk/chunk.module.ts"],"names":[],"mappings":"AACA,OAAO,EAAU,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAgBtD;;;;;;;;;GASG;AACH,qBAqBa,WAAY,YAAW,YAAY;IAC9C,YAAY;CAGb"}
@@ -7,6 +7,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
7
7
  };
8
8
  Object.defineProperty(exports, "__esModule", { value: true });
9
9
  exports.ChunkModule = void 0;
10
+ const bullmq_1 = require("@nestjs/bullmq");
10
11
  const common_1 = require("@nestjs/common");
11
12
  const graph_creator_module_1 = require("../../agents/graph.creator/graph.creator.module");
12
13
  const conditional_service_decorator_1 = require("../../common/decorators/conditional-service.decorator");
@@ -18,15 +19,19 @@ const s3_module_1 = require("../s3/s3.module");
18
19
  const tokenusage_module_1 = require("../tokenusage/tokenusage.module");
19
20
  const chunk_controller_1 = require("./controllers/chunk.controller");
20
21
  const chunk_entity_1 = require("./entities/chunk.entity");
22
+ const chunk_embedding_processor_1 = require("./processors/chunk.embedding.processor");
21
23
  const chunk_processor_1 = require("./processors/chunk.processor");
22
24
  const chunk_repository_1 = require("./repositories/chunk.repository");
23
25
  const chunk_service_1 = require("./services/chunk.service");
24
26
  /**
25
27
  * ChunkModule - Handles document chunking and graph generation.
26
28
  *
27
- * This is a fully static module. Queue registration is handled centrally
28
- * by QueueModule (which is @Global), so ChunkProcessor can inject queues
29
- * via @InjectQueue() without needing to register them here.
29
+ * The content-processing queue (CHUNK) is provided by the @Global QueueModule.
30
+ * The embedding-rebuild queues backing ChunkController's @InjectQueue() and the
31
+ * ChunkEmbeddingProcessor are registered LOCALLY here so the module is
32
+ * self-contained — every consuming app (api/corpus/neural-erp/phlow) loads
33
+ * ChunkModule without having to register these queues in its own config, and
34
+ * without declaring the rebuild routes itself (which previously duplicated).
30
35
  */
31
36
  let ChunkModule = class ChunkModule {
32
37
  onModuleInit() {
@@ -37,9 +42,24 @@ exports.ChunkModule = ChunkModule;
37
42
  exports.ChunkModule = ChunkModule = __decorate([
38
43
  (0, common_1.Module)({
39
44
  controllers: [chunk_controller_1.ChunkController],
40
- providers: [chunk_entity_1.ChunkDescriptor.model.serialiser, chunk_service_1.ChunkService, chunk_repository_1.ChunkRepository, (0, conditional_service_decorator_1.createWorkerProvider)(chunk_processor_1.ChunkProcessor)],
45
+ providers: [
46
+ chunk_entity_1.ChunkDescriptor.model.serialiser,
47
+ chunk_service_1.ChunkService,
48
+ chunk_repository_1.ChunkRepository,
49
+ (0, conditional_service_decorator_1.createWorkerProvider)(chunk_processor_1.ChunkProcessor),
50
+ (0, conditional_service_decorator_1.createWorkerProvider)(chunk_embedding_processor_1.ChunkEmbeddingProcessor),
51
+ ],
41
52
  exports: [chunk_service_1.ChunkService, chunk_repository_1.ChunkRepository],
42
- imports: [atomicfact_module_1.AtomicFactModule, graph_creator_module_1.GraphCreatorModule, keyconcept_module_1.KeyConceptModule, s3_module_1.S3Module, llm_module_1.LLMModule, tokenusage_module_1.TokenUsageModule],
53
+ imports: [
54
+ atomicfact_module_1.AtomicFactModule,
55
+ graph_creator_module_1.GraphCreatorModule,
56
+ keyconcept_module_1.KeyConceptModule,
57
+ s3_module_1.S3Module,
58
+ llm_module_1.LLMModule,
59
+ tokenusage_module_1.TokenUsageModule,
60
+ bullmq_1.BullModule.registerQueue({ name: "embedding-chunks" }),
61
+ bullmq_1.BullModule.registerQueue({ name: "embedding-keyconcepts" }),
62
+ ],
43
63
  })
44
64
  ], ChunkModule);
45
65
  //# sourceMappingURL=chunk.module.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"chunk.module.js","sourceRoot":"","sources":["../../../src/foundations/chunk/chunk.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAsD;AACtD,0FAAqF;AACrF,yGAA6F;AAC7F,+DAAiE;AACjE,0DAAsD;AACtD,uEAAmE;AACnE,uEAAmE;AACnE,+CAA2C;AAC3C,uEAAmE;AACnE,qEAAiE;AACjE,0DAA0D;AAC1D,kEAA8D;AAC9D,sEAAkE;AAClE,4DAAwD;AAExD;;;;;;GAMG;AAOI,IAAM,WAAW,GAAjB,MAAM,WAAW;IACtB,YAAY;QACV,wBAAa,CAAC,QAAQ,CAAC,8BAAe,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC;CACF,CAAA;AAJY,kCAAW;sBAAX,WAAW;IANvB,IAAA,eAAM,EAAC;QACN,WAAW,EAAE,CAAC,kCAAe,CAAC;QAC9B,SAAS,EAAE,CAAC,8BAAe,CAAC,KAAK,CAAC,UAAU,EAAE,4BAAY,EAAE,kCAAe,EAAE,IAAA,oDAAoB,EAAC,gCAAc,CAAC,CAAC;QAClH,OAAO,EAAE,CAAC,4BAAY,EAAE,kCAAe,CAAC;QACxC,OAAO,EAAE,CAAC,oCAAgB,EAAE,yCAAkB,EAAE,oCAAgB,EAAE,oBAAQ,EAAE,sBAAS,EAAE,oCAAgB,CAAC;KACzG,CAAC;GACW,WAAW,CAIvB"}
1
+ {"version":3,"file":"chunk.module.js","sourceRoot":"","sources":["../../../src/foundations/chunk/chunk.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAA4C;AAC5C,2CAAsD;AACtD,0FAAqF;AACrF,yGAA6F;AAC7F,+DAAiE;AACjE,0DAAsD;AACtD,uEAAmE;AACnE,uEAAmE;AACnE,+CAA2C;AAC3C,uEAAmE;AACnE,qEAAiE;AACjE,0DAA0D;AAC1D,sFAAiF;AACjF,kEAA8D;AAC9D,sEAAkE;AAClE,4DAAwD;AAExD;;;;;;;;;GASG;AAsBI,IAAM,WAAW,GAAjB,MAAM,WAAW;IACtB,YAAY;QACV,wBAAa,CAAC,QAAQ,CAAC,8BAAe,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC;CACF,CAAA;AAJY,kCAAW;sBAAX,WAAW;IArBvB,IAAA,eAAM,EAAC;QACN,WAAW,EAAE,CAAC,kCAAe,CAAC;QAC9B,SAAS,EAAE;YACT,8BAAe,CAAC,KAAK,CAAC,UAAU;YAChC,4BAAY;YACZ,kCAAe;YACf,IAAA,oDAAoB,EAAC,gCAAc,CAAC;YACpC,IAAA,oDAAoB,EAAC,mDAAuB,CAAC;SAC9C;QACD,OAAO,EAAE,CAAC,4BAAY,EAAE,kCAAe,CAAC;QACxC,OAAO,EAAE;YACP,oCAAgB;YAChB,yCAAkB;YAClB,oCAAgB;YAChB,oBAAQ;YACR,sBAAS;YACT,oCAAgB;YAChB,mBAAU,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC;YACtD,mBAAU,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC;SAC5D;KACF,CAAC;GACW,WAAW,CAIvB"}
@@ -1,7 +1,25 @@
1
+ import { ConfigService } from "@nestjs/config";
2
+ import { Queue } from "bullmq";
3
+ import { ClsService } from "nestjs-cls";
4
+ import { BaseConfigInterface } from "../../../config/interfaces/base.config.interface";
1
5
  import { ChunkService } from "../../chunk/services/chunk.service";
2
6
  export declare class ChunkController {
3
7
  private readonly chunkService;
4
- constructor(chunkService: ChunkService);
8
+ private readonly clsService;
9
+ private readonly embeddingQueue;
10
+ private readonly keyConceptQueue;
11
+ private readonly rebuildChunksJobName;
12
+ private readonly rebuildKeyConceptsJobName;
13
+ constructor(chunkService: ChunkService, clsService: ClsService, embeddingQueue: Queue, keyConceptQueue: Queue, configService: ConfigService<BaseConfigInterface>);
5
14
  findById(chunkId: string): Promise<import("../../..").JsonApiDataInterface>;
15
+ rebuildEmbeddings(): Promise<{
16
+ jobId: string;
17
+ message: string;
18
+ }>;
19
+ rebuildAllEmbeddings(): Promise<{
20
+ chunkJobId: string;
21
+ keyConceptJobId: string;
22
+ message: string;
23
+ }>;
6
24
  }
7
25
  //# sourceMappingURL=chunk.controller.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"chunk.controller.d.ts","sourceRoot":"","sources":["../../../../src/foundations/chunk/controllers/chunk.controller.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAElE,qBACa,eAAe;IACd,OAAO,CAAC,QAAQ,CAAC,YAAY;gBAAZ,YAAY,EAAE,YAAY;IAIjD,QAAQ,CAAmB,OAAO,EAAE,MAAM;CAGjD"}
1
+ {"version":3,"file":"chunk.controller.d.ts","sourceRoot":"","sources":["../../../../src/foundations/chunk/controllers/chunk.controller.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kDAAkD,CAAC;AAEvF,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAQlE,qBACa,eAAe;IAKxB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,UAAU;IACU,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC1B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAP5E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAS;gBAGhC,YAAY,EAAE,YAAY,EAC1B,UAAU,EAAE,UAAU,EACe,cAAc,EAAE,KAAK,EAChB,eAAe,EAAE,KAAK,EACjF,aAAa,EAAE,aAAa,CAAC,mBAAmB,CAAC;IAS7C,QAAQ,CAAmB,OAAO,EAAE,MAAM;IAM1C,iBAAiB,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAkBhE,oBAAoB,IAAI,OAAO,CAAC;QACpC,UAAU,EAAE,MAAM,CAAC;QACnB,eAAe,EAAE,MAAM,CAAC;QACxB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CAuBH"}
@@ -13,17 +13,54 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
13
13
  };
14
14
  Object.defineProperty(exports, "__esModule", { value: true });
15
15
  exports.ChunkController = void 0;
16
+ const bullmq_1 = require("@nestjs/bullmq");
16
17
  const common_1 = require("@nestjs/common");
18
+ const config_1 = require("@nestjs/config");
19
+ const bullmq_2 = require("bullmq");
20
+ const nestjs_cls_1 = require("nestjs-cls");
17
21
  const jwt_auth_guard_1 = require("../../../common/guards/jwt.auth.guard");
18
22
  const chunk_meta_1 = require("../../chunk/entities/chunk.meta");
19
23
  const chunk_service_1 = require("../../chunk/services/chunk.service");
24
+ // App embedding-rebuild queues. The package QueueId enum is a base apps extend;
25
+ // apps register these queues via config.chunkQueues.queueIds (or the module below
26
+ // registers them locally). Job names come from config.jobNames.process.
27
+ const EMBEDDING_CHUNKS_QUEUE = "embedding-chunks";
28
+ const EMBEDDING_KEYCONCEPTS_QUEUE = "embedding-keyconcepts";
20
29
  let ChunkController = class ChunkController {
21
- constructor(chunkService) {
30
+ constructor(chunkService, clsService, embeddingQueue, keyConceptQueue, configService) {
22
31
  this.chunkService = chunkService;
32
+ this.clsService = clsService;
33
+ this.embeddingQueue = embeddingQueue;
34
+ this.keyConceptQueue = keyConceptQueue;
35
+ const jobNames = configService.get("jobNames", { infer: true });
36
+ this.rebuildChunksJobName = jobNames?.process?.rebuild_chunks ?? "embedding:rebuild_chunks";
37
+ this.rebuildKeyConceptsJobName = jobNames?.process?.rebuild_keyconcepts ?? "embedding:rebuild_keyconcepts";
23
38
  }
24
39
  async findById(chunkId) {
25
40
  return await this.chunkService.findById({ chunkId: chunkId });
26
41
  }
42
+ async rebuildEmbeddings() {
43
+ const companyId = this.clsService.get("companyId");
44
+ const userId = this.clsService.get("userId");
45
+ const job = await this.embeddingQueue.add(this.rebuildChunksJobName, { companyId, userId }, { removeOnComplete: true, removeOnFail: false });
46
+ return {
47
+ jobId: job.id,
48
+ message: "Chunk embedding rebuild job queued successfully",
49
+ };
50
+ }
51
+ async rebuildAllEmbeddings() {
52
+ const companyId = this.clsService.get("companyId");
53
+ const userId = this.clsService.get("userId");
54
+ const [chunkJob, keyConceptJob] = await Promise.all([
55
+ this.embeddingQueue.add(this.rebuildChunksJobName, { companyId, userId }, { removeOnComplete: true, removeOnFail: false }),
56
+ this.keyConceptQueue.add(this.rebuildKeyConceptsJobName, { companyId, userId }, { removeOnComplete: true, removeOnFail: false }),
57
+ ]);
58
+ return {
59
+ chunkJobId: chunkJob.id,
60
+ keyConceptJobId: keyConceptJob.id,
61
+ message: "All embedding rebuild jobs queued successfully",
62
+ };
63
+ }
27
64
  };
28
65
  exports.ChunkController = ChunkController;
29
66
  __decorate([
@@ -34,8 +71,28 @@ __decorate([
34
71
  __metadata("design:paramtypes", [String]),
35
72
  __metadata("design:returntype", Promise)
36
73
  ], ChunkController.prototype, "findById", null);
74
+ __decorate([
75
+ (0, common_1.UseGuards)(jwt_auth_guard_1.JwtAuthGuard),
76
+ (0, common_1.Post)(`${chunk_meta_1.chunkMeta.endpoint}/rebuild-embeddings`),
77
+ __metadata("design:type", Function),
78
+ __metadata("design:paramtypes", []),
79
+ __metadata("design:returntype", Promise)
80
+ ], ChunkController.prototype, "rebuildEmbeddings", null);
81
+ __decorate([
82
+ (0, common_1.UseGuards)(jwt_auth_guard_1.JwtAuthGuard),
83
+ (0, common_1.Post)(`${chunk_meta_1.chunkMeta.endpoint}/rebuild-all-embeddings`),
84
+ __metadata("design:type", Function),
85
+ __metadata("design:paramtypes", []),
86
+ __metadata("design:returntype", Promise)
87
+ ], ChunkController.prototype, "rebuildAllEmbeddings", null);
37
88
  exports.ChunkController = ChunkController = __decorate([
38
89
  (0, common_1.Controller)(),
39
- __metadata("design:paramtypes", [chunk_service_1.ChunkService])
90
+ __param(2, (0, bullmq_1.InjectQueue)(EMBEDDING_CHUNKS_QUEUE)),
91
+ __param(3, (0, bullmq_1.InjectQueue)(EMBEDDING_KEYCONCEPTS_QUEUE)),
92
+ __metadata("design:paramtypes", [chunk_service_1.ChunkService,
93
+ nestjs_cls_1.ClsService,
94
+ bullmq_2.Queue,
95
+ bullmq_2.Queue,
96
+ config_1.ConfigService])
40
97
  ], ChunkController);
41
98
  //# sourceMappingURL=chunk.controller.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"chunk.controller.js","sourceRoot":"","sources":["../../../../src/foundations/chunk/controllers/chunk.controller.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAAmE;AAEnE,0EAAqE;AACrE,gEAA4D;AAC5D,sEAAkE;AAG3D,IAAM,eAAe,GAArB,MAAM,eAAe;IAC1B,YAA6B,YAA0B;QAA1B,iBAAY,GAAZ,YAAY,CAAc;IAAG,CAAC;IAIrD,AAAN,KAAK,CAAC,QAAQ,CAAmB,OAAe;QAC9C,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC;CACF,CAAA;AARY,0CAAe;AAKpB;IAFL,IAAA,kBAAS,EAAC,6BAAY,CAAC;IACvB,IAAA,YAAG,EAAC,GAAG,sBAAS,CAAC,QAAQ,WAAW,CAAC;IACtB,WAAA,IAAA,cAAK,EAAC,SAAS,CAAC,CAAA;;;;+CAE/B;0BAPU,eAAe;IAD3B,IAAA,mBAAU,GAAE;qCAEgC,4BAAY;GAD5C,eAAe,CAQ3B"}
1
+ {"version":3,"file":"chunk.controller.js","sourceRoot":"","sources":["../../../../src/foundations/chunk/controllers/chunk.controller.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAA6C;AAC7C,2CAAyE;AACzE,2CAA+C;AAC/C,mCAA+B;AAC/B,2CAAwC;AAExC,0EAAqE;AAErE,gEAA4D;AAC5D,sEAAkE;AAElE,gFAAgF;AAChF,kFAAkF;AAClF,wEAAwE;AACxE,MAAM,sBAAsB,GAAG,kBAAkB,CAAC;AAClD,MAAM,2BAA2B,GAAG,uBAAuB,CAAC;AAGrD,IAAM,eAAe,GAArB,MAAM,eAAe;IAI1B,YACmB,YAA0B,EAC1B,UAAsB,EACe,cAAqB,EAChB,eAAsB,EACjF,aAAiD;QAJhC,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACe,mBAAc,GAAd,cAAc,CAAO;QAChB,oBAAe,GAAf,eAAe,CAAO;QAGjF,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,oBAAoB,GAAG,QAAQ,EAAE,OAAO,EAAE,cAAc,IAAI,0BAA0B,CAAC;QAC5F,IAAI,CAAC,yBAAyB,GAAG,QAAQ,EAAE,OAAO,EAAE,mBAAmB,IAAI,+BAA+B,CAAC;IAC7G,CAAC;IAIK,AAAN,KAAK,CAAC,QAAQ,CAAmB,OAAe;QAC9C,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC;IAIK,AAAN,KAAK,CAAC,iBAAiB;QACrB,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE7C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CACvC,IAAI,CAAC,oBAAoB,EACzB,EAAE,SAAS,EAAE,MAAM,EAAE,EACrB,EAAE,gBAAgB,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,CAChD,CAAC;QAEF,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,OAAO,EAAE,iDAAiD;SAC3D,CAAC;IACJ,CAAC;IAIK,AAAN,KAAK,CAAC,oBAAoB;QAKxB,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE7C,MAAM,CAAC,QAAQ,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClD,IAAI,CAAC,cAAc,CAAC,GAAG,CACrB,IAAI,CAAC,oBAAoB,EACzB,EAAE,SAAS,EAAE,MAAM,EAAE,EACrB,EAAE,gBAAgB,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,CAChD;YACD,IAAI,CAAC,eAAe,CAAC,GAAG,CACtB,IAAI,CAAC,yBAAyB,EAC9B,EAAE,SAAS,EAAE,MAAM,EAAE,EACrB,EAAE,gBAAgB,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,CAChD;SACF,CAAC,CAAC;QAEH,OAAO;YACL,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,eAAe,EAAE,aAAa,CAAC,EAAE;YACjC,OAAO,EAAE,gDAAgD;SAC1D,CAAC;IACJ,CAAC;CACF,CAAA;AArEY,0CAAe;AAkBpB;IAFL,IAAA,kBAAS,EAAC,6BAAY,CAAC;IACvB,IAAA,YAAG,EAAC,GAAG,sBAAS,CAAC,QAAQ,WAAW,CAAC;IACtB,WAAA,IAAA,cAAK,EAAC,SAAS,CAAC,CAAA;;;;+CAE/B;AAIK;IAFL,IAAA,kBAAS,EAAC,6BAAY,CAAC;IACvB,IAAA,aAAI,EAAC,GAAG,sBAAS,CAAC,QAAQ,qBAAqB,CAAC;;;;wDAehD;AAIK;IAFL,IAAA,kBAAS,EAAC,6BAAY,CAAC;IACvB,IAAA,aAAI,EAAC,GAAG,sBAAS,CAAC,QAAQ,yBAAyB,CAAC;;;;2DA2BpD;0BApEU,eAAe;IAD3B,IAAA,mBAAU,GAAE;IAQR,WAAA,IAAA,oBAAW,EAAC,sBAAsB,CAAC,CAAA;IACnC,WAAA,IAAA,oBAAW,EAAC,2BAA2B,CAAC,CAAA;qCAHV,4BAAY;QACd,uBAAU;QAC+B,cAAK;QACC,cAAK;QAClE,sBAAa;GATnB,eAAe,CAqE3B"}
@@ -1,5 +1,9 @@
1
1
  import { Entity } from "../../../common";
2
2
  import { PolymorphicDiscriminatorData } from "../../../common/interfaces/entity.schema.interface";
3
+ export interface ChunkDate {
4
+ date: string;
5
+ description: string;
6
+ }
3
7
  export type Chunk = Entity & {
4
8
  content: string;
5
9
  position?: number;
@@ -9,6 +13,10 @@ export type Chunk = Entity & {
9
13
  aiStatus?: string;
10
14
  embedding?: number[];
11
15
  source?: unknown;
16
+ heading?: string;
17
+ /** Stored on the node as a JSON string; exposed here parsed via a computed field. */
18
+ dates?: ChunkDate[];
19
+ propagatedDates?: ChunkDate[];
12
20
  };
13
21
  export declare const ChunkDescriptor: import("../../../common").EntityDescriptor<Chunk, {
14
22
  source: {
@@ -1 +1 @@
1
- {"version":3,"file":"chunk.entity.d.ts","sourceRoot":"","sources":["../../../../src/foundations/chunk/entities/chunk.entity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAgB,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oDAAoD,CAAC;AAKlG,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,eAAe;;;;;;;;;;kCAkCE,4BAA4B;;;EAUxD,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,OAAO,eAAe,CAAC"}
1
+ {"version":3,"file":"chunk.entity.d.ts","sourceRoot":"","sources":["../../../../src/foundations/chunk/entities/chunk.entity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAgB,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,4BAA4B,EAAE,MAAM,oDAAoD,CAAC;AAKlG,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qFAAqF;IACrF,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IACpB,eAAe,CAAC,EAAE,SAAS,EAAE,CAAC;CAC/B,CAAC;AAgBF,eAAO,MAAM,eAAe;;;;;;;;;;kCAwCE,4BAA4B;;;EAUxD,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,OAAO,eAAe,CAAC"}
@@ -5,6 +5,21 @@ const common_1 = require("../../../common");
5
5
  const registry_1 = require("../../../common/registries/registry");
6
6
  const s3_1 = require("../../s3");
7
7
  const chunk_meta_1 = require("./chunk.meta");
8
+ /** Parse a `dates` / `propagatedDates` value that is persisted as a JSON-string blob. */
9
+ const parseChunkDates = (raw) => {
10
+ if (Array.isArray(raw))
11
+ return raw;
12
+ if (typeof raw === "string" && raw.trim()) {
13
+ try {
14
+ const parsed = JSON.parse(raw);
15
+ return Array.isArray(parsed) ? parsed : [];
16
+ }
17
+ catch {
18
+ return [];
19
+ }
20
+ }
21
+ return [];
22
+ };
8
23
  exports.ChunkDescriptor = (0, common_1.defineEntity)()({
9
24
  ...chunk_meta_1.chunkMeta,
10
25
  injectServices: [s3_1.S3Service],
@@ -14,6 +29,7 @@ exports.ChunkDescriptor = (0, common_1.defineEntity)()({
14
29
  nodeId: { type: "string", meta: true },
15
30
  nodeType: { type: "string", meta: true },
16
31
  aiStatus: { type: "string" },
32
+ heading: { type: "string" },
17
33
  embedding: { type: "number[]", excludeFromJsonApi: true },
18
34
  imagePath: {
19
35
  type: "string",
@@ -27,6 +43,10 @@ exports.ChunkDescriptor = (0, common_1.defineEntity)()({
27
43
  },
28
44
  },
29
45
  },
46
+ computed: {
47
+ dates: { compute: (params) => parseChunkDates(params.data?.dates) },
48
+ propagatedDates: { compute: (params) => parseChunkDates(params.data?.propagatedDates) },
49
+ },
30
50
  relationships: {
31
51
  source: {
32
52
  model: chunk_meta_1.chunkMeta, // overridden per-row by polymorphic discriminator
@@ -1 +1 @@
1
- {"version":3,"file":"chunk.entity.js","sourceRoot":"","sources":["../../../../src/foundations/chunk/entities/chunk.entity.ts"],"names":[],"mappings":";;;AAAA,4CAAuD;AAEvD,kEAAoE;AACpE,iCAAqC;AACrC,6CAAyC;AAa5B,QAAA,eAAe,GAAG,IAAA,qBAAY,GAAS,CAAC;IACnD,GAAG,sBAAS;IAEZ,cAAc,EAAE,CAAC,cAAS,CAAC;IAE3B,MAAM,EAAE;QACN,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC3C,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC5B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE;QACtC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE;QACxC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC5B,SAAS,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,kBAAkB,EAAE,IAAI,EAAE;QACzD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;gBAClC,IAAI,CAAC,IAAI,CAAC,SAAS;oBAAE,OAAO,SAAS,CAAC;gBACtC,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,iBAAiB,CAAC;oBAChD,GAAG,EAAE,IAAI,CAAC,SAAS;oBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;iBACtB,CAAC,CAAC;YACL,CAAC;SACF;KACF;IAED,aAAa,EAAE;QACb,MAAM,EAAE;YACN,KAAK,EAAE,sBAAS,EAAE,kDAAkD;YACpE,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,WAAW;YACzB,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE;gBACX,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,CAAC,IAAkC,EAAE,EAAE;oBACpD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;wBAChC,MAAM,KAAK,GAAG,wBAAa,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;wBAClD,IAAI,KAAK;4BAAE,OAAO,KAAK,CAAC;oBAC1B,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACzG,CAAC;aACF;SACF;KACF;CACF,CAAC,CAAC"}
1
+ {"version":3,"file":"chunk.entity.js","sourceRoot":"","sources":["../../../../src/foundations/chunk/entities/chunk.entity.ts"],"names":[],"mappings":";;;AAAA,4CAAuD;AAEvD,kEAAoE;AACpE,iCAAqC;AACrC,6CAAyC;AAsBzC,yFAAyF;AACzF,MAAM,eAAe,GAAG,CAAC,GAAY,EAAe,EAAE;IACpD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,GAAkB,CAAC;IAClD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAE,MAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC;AAEW,QAAA,eAAe,GAAG,IAAA,qBAAY,GAAS,CAAC;IACnD,GAAG,sBAAS;IAEZ,cAAc,EAAE,CAAC,cAAS,CAAC;IAE3B,MAAM,EAAE;QACN,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC3C,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC5B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE;QACtC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE;QACxC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC5B,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC3B,SAAS,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,kBAAkB,EAAE,IAAI,EAAE;QACzD,SAAS,EAAE;YACT,IAAI,EAAE,QAAQ;YACd,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;gBAClC,IAAI,CAAC,IAAI,CAAC,SAAS;oBAAE,OAAO,SAAS,CAAC;gBACtC,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,iBAAiB,CAAC;oBAChD,GAAG,EAAE,IAAI,CAAC,SAAS;oBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;iBACtB,CAAC,CAAC;YACL,CAAC;SACF;KACF;IAED,QAAQ,EAAE;QACR,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE;QACnE,eAAe,EAAE,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE;KACxF;IAED,aAAa,EAAE;QACb,MAAM,EAAE;YACN,KAAK,EAAE,sBAAS,EAAE,kDAAkD;YACpE,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,WAAW;YACzB,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE;gBACX,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,CAAC,IAAkC,EAAE,EAAE;oBACpD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;wBAChC,MAAM,KAAK,GAAG,wBAAa,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;wBAClD,IAAI,KAAK;4BAAE,OAAO,KAAK,CAAC;oBAC1B,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACzG,CAAC;aACF;SACF;KACF;CACF,CAAC,CAAC"}
@@ -0,0 +1,21 @@
1
+ import { WorkerHost } from "@nestjs/bullmq";
2
+ import { ConfigService } from "@nestjs/config";
3
+ import { Job } from "bullmq";
4
+ import { ClsService } from "nestjs-cls";
5
+ import { BaseConfigInterface } from "../../../config/interfaces/base.config.interface";
6
+ import { EmbedderService } from "../../../core";
7
+ import { ChunkRepository } from "../repositories/chunk.repository";
8
+ export declare class ChunkEmbeddingProcessor extends WorkerHost {
9
+ private readonly clsService;
10
+ private readonly chunkRepository;
11
+ private readonly embedderService;
12
+ private readonly rebuildChunksJobName;
13
+ constructor(clsService: ClsService, chunkRepository: ChunkRepository, embedderService: EmbedderService, configService: ConfigService<BaseConfigInterface>);
14
+ onActive(job: Job): void;
15
+ onError(job: Job): void;
16
+ onCompleted(job: Job): void;
17
+ process(job: Job): Promise<{
18
+ processed: number;
19
+ }>;
20
+ }
21
+ //# sourceMappingURL=chunk.embedding.processor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"chunk.embedding.processor.d.ts","sourceRoot":"","sources":["../../../../src/foundations/chunk/processors/chunk.embedding.processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kDAAkD,CAAC;AACvF,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAInE,qBACa,uBAAwB,SAAQ,UAAU;IAInD,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,eAAe;IALlC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;gBAG3B,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,eAAe,EAChC,eAAe,EAAE,eAAe,EACjD,aAAa,EAAE,aAAa,CAAC,mBAAmB,CAAC;IAQnD,QAAQ,CAAC,GAAG,EAAE,GAAG;IAKjB,OAAO,CAAC,GAAG,EAAE,GAAG;IAKhB,WAAW,CAAC,GAAG,EAAE,GAAG;IAId,OAAO,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;CA0CxD"}
@@ -0,0 +1,98 @@
1
+ "use strict";
2
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
7
+ };
8
+ var __metadata = (this && this.__metadata) || function (k, v) {
9
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.ChunkEmbeddingProcessor = void 0;
13
+ const bullmq_1 = require("@nestjs/bullmq");
14
+ const config_1 = require("@nestjs/config");
15
+ const bullmq_2 = require("bullmq");
16
+ const nestjs_cls_1 = require("nestjs-cls");
17
+ const core_1 = require("../../../core");
18
+ const chunk_repository_1 = require("../repositories/chunk.repository");
19
+ const EMBEDDING_CHUNKS_QUEUE = "embedding-chunks";
20
+ let ChunkEmbeddingProcessor = class ChunkEmbeddingProcessor extends bullmq_1.WorkerHost {
21
+ constructor(clsService, chunkRepository, embedderService, configService) {
22
+ super();
23
+ this.clsService = clsService;
24
+ this.chunkRepository = chunkRepository;
25
+ this.embedderService = embedderService;
26
+ this.rebuildChunksJobName =
27
+ configService.get("jobNames", { infer: true })?.process?.rebuild_chunks ?? "embedding:rebuild_chunks";
28
+ }
29
+ onActive(job) {
30
+ console.log(`Processing ${job.name} job`);
31
+ }
32
+ onError(job) {
33
+ console.error(`Error processing ${job.name} job (ID: ${job.id}). Reason: ${job.failedReason}`);
34
+ }
35
+ onCompleted(job) {
36
+ console.log(`Completed ${job.name} job (ID: ${job.id})`);
37
+ }
38
+ async process(job) {
39
+ if (job.name !== this.rebuildChunksJobName) {
40
+ throw new Error(`Job ${job.name} not handled by ChunkEmbeddingProcessor`);
41
+ }
42
+ return await this.clsService.run(async () => {
43
+ this.clsService.set("companyId", job.data.companyId);
44
+ this.clsService.set("userId", job.data.userId);
45
+ this.clsService.set("isAutomatedJob", true);
46
+ // 1. Recreate vector index with new dimensions
47
+ await this.chunkRepository.recreateVectorIndex();
48
+ // 2. Get all chunks
49
+ const allChunks = await this.chunkRepository.findAllChunks();
50
+ const chunksWithContent = allChunks.filter((c) => c.content);
51
+ // 3. Process in batches
52
+ const batchSize = 100;
53
+ let processed = 0;
54
+ for (let i = 0; i < chunksWithContent.length; i += batchSize) {
55
+ const batch = chunksWithContent.slice(i, i + batchSize);
56
+ const texts = batch.map((c) => c.content);
57
+ const embeddings = await this.embedderService.vectoriseTextBatch(texts);
58
+ for (let j = 0; j < batch.length; j++) {
59
+ await this.chunkRepository.updateEmbedding({
60
+ chunkId: batch[j].id,
61
+ embedding: embeddings[j],
62
+ });
63
+ processed++;
64
+ }
65
+ const progress = Math.round((processed / chunksWithContent.length) * 100);
66
+ await job.updateProgress(progress);
67
+ }
68
+ return { processed };
69
+ });
70
+ }
71
+ };
72
+ exports.ChunkEmbeddingProcessor = ChunkEmbeddingProcessor;
73
+ __decorate([
74
+ (0, bullmq_1.OnWorkerEvent)("active"),
75
+ __metadata("design:type", Function),
76
+ __metadata("design:paramtypes", [bullmq_2.Job]),
77
+ __metadata("design:returntype", void 0)
78
+ ], ChunkEmbeddingProcessor.prototype, "onActive", null);
79
+ __decorate([
80
+ (0, bullmq_1.OnWorkerEvent)("failed"),
81
+ __metadata("design:type", Function),
82
+ __metadata("design:paramtypes", [bullmq_2.Job]),
83
+ __metadata("design:returntype", void 0)
84
+ ], ChunkEmbeddingProcessor.prototype, "onError", null);
85
+ __decorate([
86
+ (0, bullmq_1.OnWorkerEvent)("completed"),
87
+ __metadata("design:type", Function),
88
+ __metadata("design:paramtypes", [bullmq_2.Job]),
89
+ __metadata("design:returntype", void 0)
90
+ ], ChunkEmbeddingProcessor.prototype, "onCompleted", null);
91
+ exports.ChunkEmbeddingProcessor = ChunkEmbeddingProcessor = __decorate([
92
+ (0, bullmq_1.Processor)(EMBEDDING_CHUNKS_QUEUE, { concurrency: 1, lockDuration: 1000 * 60 * 30 }),
93
+ __metadata("design:paramtypes", [nestjs_cls_1.ClsService,
94
+ chunk_repository_1.ChunkRepository,
95
+ core_1.EmbedderService,
96
+ config_1.ConfigService])
97
+ ], ChunkEmbeddingProcessor);
98
+ //# sourceMappingURL=chunk.embedding.processor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"chunk.embedding.processor.js","sourceRoot":"","sources":["../../../../src/foundations/chunk/processors/chunk.embedding.processor.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAAsE;AACtE,2CAA+C;AAC/C,mCAA6B;AAC7B,2CAAwC;AAExC,wCAAgD;AAChD,uEAAmE;AAEnE,MAAM,sBAAsB,GAAG,kBAAkB,CAAC;AAG3C,IAAM,uBAAuB,GAA7B,MAAM,uBAAwB,SAAQ,mBAAU;IAGrD,YACmB,UAAsB,EACtB,eAAgC,EAChC,eAAgC,EACjD,aAAiD;QAEjD,KAAK,EAAE,CAAC;QALS,eAAU,GAAV,UAAU,CAAY;QACtB,oBAAe,GAAf,eAAe,CAAiB;QAChC,oBAAe,GAAf,eAAe,CAAiB;QAIjD,IAAI,CAAC,oBAAoB;YACvB,aAAa,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,cAAc,IAAI,0BAA0B,CAAC;IAC1G,CAAC;IAGD,QAAQ,CAAC,GAAQ;QACf,OAAO,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC;IAC5C,CAAC;IAGD,OAAO,CAAC,GAAQ;QACd,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,CAAC,IAAI,aAAa,GAAG,CAAC,EAAE,cAAc,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC;IACjG,CAAC;IAGD,WAAW,CAAC,GAAQ;QAClB,OAAO,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,IAAI,aAAa,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAQ;QACpB,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,IAAI,yCAAyC,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;YAE5C,+CAA+C;YAC/C,MAAM,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,CAAC;YAEjD,oBAAoB;YACpB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;YAC7D,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAE7D,wBAAwB;YACxB,MAAM,SAAS,GAAG,GAAG,CAAC;YACtB,IAAI,SAAS,GAAG,CAAC,CAAC;YAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC7D,MAAM,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBAExD,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBAC1C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBAExE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACtC,MAAM,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC;wBACzC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;wBACpB,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;qBACzB,CAAC,CAAC;oBACH,SAAS,EAAE,CAAC;gBACd,CAAC;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;gBAC1E,MAAM,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACrC,CAAC;YAED,OAAO,EAAE,SAAS,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;CACF,CAAA;AAvEY,0DAAuB;AAelC;IADC,IAAA,sBAAa,EAAC,QAAQ,CAAC;;qCACV,YAAG;;uDAEhB;AAGD;IADC,IAAA,sBAAa,EAAC,QAAQ,CAAC;;qCACX,YAAG;;sDAEf;AAGD;IADC,IAAA,sBAAa,EAAC,WAAW,CAAC;;qCACV,YAAG;;0DAEnB;kCA3BU,uBAAuB;IADnC,IAAA,kBAAS,EAAC,sBAAsB,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;qCAKnD,uBAAU;QACL,kCAAe;QACf,sBAAe;QAClC,sBAAa;GAPnB,uBAAuB,CAuEnC"}