@n8n/backend-module-insights 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/LICENSE.md +88 -0
  2. package/dist/build.tsbuildinfo +1 -0
  3. package/dist/index.d.ts +14 -0
  4. package/dist/index.js +28 -0
  5. package/dist/index.js.map +1 -0
  6. package/dist/insights-by-period-query.helper.d.ts +15 -0
  7. package/dist/insights-by-period-query.helper.js +45 -0
  8. package/dist/insights-by-period-query.helper.js.map +1 -0
  9. package/dist/insights-by-period.entity.d.ts +17 -0
  10. package/dist/insights-by-period.entity.js +73 -0
  11. package/dist/insights-by-period.entity.js.map +1 -0
  12. package/dist/insights-by-period.repository.d.ts +100 -0
  13. package/dist/insights-by-period.repository.js +347 -0
  14. package/dist/insights-by-period.repository.js.map +1 -0
  15. package/dist/insights-collection.service.d.ts +30 -0
  16. package/dist/insights-collection.service.js +280 -0
  17. package/dist/insights-collection.service.js.map +1 -0
  18. package/dist/insights-compaction.service.d.ts +19 -0
  19. package/dist/insights-compaction.service.js +161 -0
  20. package/dist/insights-compaction.service.js.map +1 -0
  21. package/dist/insights-compaction.task.d.ts +14 -0
  22. package/dist/insights-compaction.task.js +35 -0
  23. package/dist/insights-compaction.task.js.map +1 -0
  24. package/dist/insights-metadata.entity.d.ts +8 -0
  25. package/dist/insights-metadata.entity.js +40 -0
  26. package/dist/insights-metadata.entity.js.map +1 -0
  27. package/dist/insights-metadata.repository.d.ts +7 -0
  28. package/dist/insights-metadata.repository.js +36 -0
  29. package/dist/insights-metadata.repository.js.map +1 -0
  30. package/dist/insights-pruning.service.d.ts +11 -0
  31. package/dist/insights-pruning.service.js +49 -0
  32. package/dist/insights-pruning.service.js.map +1 -0
  33. package/dist/insights-pruning.task.d.ts +15 -0
  34. package/dist/insights-pruning.task.js +36 -0
  35. package/dist/insights-pruning.task.js.map +1 -0
  36. package/dist/insights-raw.entity.d.ts +13 -0
  37. package/dist/insights-raw.entity.js +61 -0
  38. package/dist/insights-raw.entity.js.map +1 -0
  39. package/dist/insights-raw.repository.d.ts +12 -0
  40. package/dist/insights-raw.repository.js +36 -0
  41. package/dist/insights-raw.repository.js.map +1 -0
  42. package/dist/insights-shared.d.ts +21 -0
  43. package/dist/insights-shared.js +30 -0
  44. package/dist/insights-shared.js.map +1 -0
  45. package/dist/insights.config.d.ts +13 -0
  46. package/dist/insights.config.js +80 -0
  47. package/dist/insights.config.js.map +1 -0
  48. package/dist/insights.constants.d.ts +4 -0
  49. package/dist/insights.constants.js +24 -0
  50. package/dist/insights.constants.js.map +1 -0
  51. package/dist/insights.controller.d.ts +16 -0
  52. package/dist/insights.controller.js +159 -0
  53. package/dist/insights.controller.js.map +1 -0
  54. package/dist/insights.module.d.ts +17 -0
  55. package/dist/insights.module.js +51 -0
  56. package/dist/insights.module.js.map +1 -0
  57. package/dist/insights.service.d.ts +73 -0
  58. package/dist/insights.service.js +257 -0
  59. package/dist/insights.service.js.map +1 -0
  60. package/dist/insights.settings.d.ts +20 -0
  61. package/dist/insights.settings.js +48 -0
  62. package/dist/insights.settings.js.map +1 -0
  63. package/dist/testing/db-utils.d.ts +22 -0
  64. package/dist/testing/db-utils.js +76 -0
  65. package/dist/testing/db-utils.js.map +1 -0
  66. package/dist/testing.d.ts +1 -0
  67. package/dist/testing.js +9 -0
  68. package/dist/testing.js.map +1 -0
  69. package/package.json +97 -0
@@ -0,0 +1,280 @@
1
+ "use strict";
2
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
7
+ };
8
+ var __metadata = (this && this.__metadata) || function (k, v) {
9
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.InsightsCollectionService = void 0;
13
+ const backend_common_1 = require("@n8n/backend-common");
14
+ const backend_services_1 = require("@n8n/backend-services");
15
+ const db_1 = require("@n8n/db");
16
+ const decorators_1 = require("@n8n/decorators");
17
+ const di_1 = require("@n8n/di");
18
+ const luxon_1 = require("luxon");
19
+ const insights_metadata_entity_1 = require("./insights-metadata.entity");
20
+ const insights_raw_entity_1 = require("./insights-raw.entity");
21
+ const insights_metadata_repository_1 = require("./insights-metadata.repository");
22
+ const insights_raw_repository_1 = require("./insights-raw.repository");
23
+ const insights_config_1 = require("./insights.config");
24
+ const shouldSkipStatus = {
25
+ success: false,
26
+ crashed: false,
27
+ error: false,
28
+ canceled: true,
29
+ new: true,
30
+ running: true,
31
+ unknown: true,
32
+ waiting: true,
33
+ };
34
+ const shouldSkipMode = {
35
+ cli: false,
36
+ error: false,
37
+ retry: false,
38
+ trigger: false,
39
+ webhook: false,
40
+ evaluation: false,
41
+ integrated: true,
42
+ internal: true,
43
+ manual: true,
44
+ chat: true,
45
+ agent: true,
46
+ };
47
+ const MIN_RUNTIME = 0;
48
+ const MAX_RUNTIME = 2 ** 31 - 1;
49
+ function integerValueForInsightsRaw(value) {
50
+ if (!Number.isFinite(value)) {
51
+ return 0;
52
+ }
53
+ return Math.round(value);
54
+ }
55
+ let InsightsCollectionService = class InsightsCollectionService {
56
+ constructor(sharedWorkflowRepository, insightsRawRepository, insightsMetadataRepository, insightsConfig, logger) {
57
+ this.sharedWorkflowRepository = sharedWorkflowRepository;
58
+ this.insightsRawRepository = insightsRawRepository;
59
+ this.insightsMetadataRepository = insightsMetadataRepository;
60
+ this.insightsConfig = insightsConfig;
61
+ this.logger = logger;
62
+ this.cachedMetadata = new Map();
63
+ this.bufferedInsights = new Set();
64
+ this.isAsynchronouslySavingInsights = true;
65
+ this.flushesInProgress = new Set();
66
+ this.isInitialized = false;
67
+ this.logger = this.logger.scoped('insights');
68
+ }
69
+ init() {
70
+ this.isInitialized = true;
71
+ this.isAsynchronouslySavingInsights = true;
72
+ this.scheduleFlushing();
73
+ this.logger.debug('Started flushing timer');
74
+ }
75
+ scheduleFlushing() {
76
+ if (!this.isInitialized)
77
+ return;
78
+ this.cancelScheduledFlushing();
79
+ this.flushInsightsRawBufferTimer = setTimeout(async () => await this.flushEvents(), this.insightsConfig.flushIntervalSeconds * 1000);
80
+ }
81
+ cancelScheduledFlushing() {
82
+ if (this.flushInsightsRawBufferTimer !== undefined) {
83
+ clearTimeout(this.flushInsightsRawBufferTimer);
84
+ this.flushInsightsRawBufferTimer = undefined;
85
+ }
86
+ }
87
+ stopFlushingTimer() {
88
+ this.cancelScheduledFlushing();
89
+ this.logger.debug('Stopped flushing timer');
90
+ }
91
+ async shutdown() {
92
+ this.stopFlushingTimer();
93
+ this.isAsynchronouslySavingInsights = false;
94
+ this.logger.debug('Flushing remaining insights before shutdown');
95
+ await Promise.all([...this.flushesInProgress, this.flushEvents()]);
96
+ this.isInitialized = false;
97
+ }
98
+ async handleWorkflowExecuteAfter(ctx) {
99
+ if (!this.isInitialized) {
100
+ return;
101
+ }
102
+ if (shouldSkipStatus[ctx.runData.status] || shouldSkipMode[ctx.runData.mode]) {
103
+ return;
104
+ }
105
+ if (ctx.source === 'instance_ai') {
106
+ return;
107
+ }
108
+ const status = ctx.runData.status === 'success' ? 'success' : 'failure';
109
+ const commonWorkflowData = {
110
+ workflowId: ctx.workflow.id,
111
+ workflowName: ctx.workflow.name,
112
+ timestamp: luxon_1.DateTime.utc().toJSDate(),
113
+ };
114
+ this.bufferedInsights.add({
115
+ ...commonWorkflowData,
116
+ type: status,
117
+ value: 1,
118
+ });
119
+ if ((0, backend_services_1.isBillableExecution)(ctx.runData, ctx.source)) {
120
+ this.bufferedInsights.add({
121
+ ...commonWorkflowData,
122
+ type: 'billable',
123
+ value: 1,
124
+ });
125
+ }
126
+ if (ctx.runData.stoppedAt) {
127
+ const runtimeMs = ctx.runData.stoppedAt.getTime() - ctx.runData.startedAt.getTime();
128
+ if (runtimeMs < MIN_RUNTIME || runtimeMs > MAX_RUNTIME) {
129
+ this.logger.warn(`Invalid runtime detected: ${runtimeMs}ms, clamping to safe range`);
130
+ }
131
+ const value = Math.min(Math.max(runtimeMs, MIN_RUNTIME), MAX_RUNTIME);
132
+ this.bufferedInsights.add({
133
+ ...commonWorkflowData,
134
+ type: 'runtime_ms',
135
+ value,
136
+ });
137
+ }
138
+ if (status === 'success' && ctx.runData.mode !== 'error') {
139
+ const finalTimeSaved = this.calculateTimeSaved(ctx);
140
+ if (finalTimeSaved !== undefined) {
141
+ this.bufferedInsights.add({
142
+ ...commonWorkflowData,
143
+ type: 'time_saved_min',
144
+ value: finalTimeSaved,
145
+ });
146
+ }
147
+ }
148
+ if (!this.isAsynchronouslySavingInsights) {
149
+ this.logger.debug('Flushing insights synchronously (shutdown in progress)');
150
+ await this.flushEvents();
151
+ }
152
+ if (this.bufferedInsights.size >= this.insightsConfig.flushBatchSize) {
153
+ this.logger.debug(`Buffer is full (${this.bufferedInsights.size} insights), flushing events`);
154
+ void this.flushEvents();
155
+ }
156
+ }
157
+ async saveInsightsMetadataAndRaw(insightsRawToInsertBuffer) {
158
+ this.logger.debug(`Flushing ${insightsRawToInsertBuffer.size} insights`);
159
+ const workflowIdNames = new Map();
160
+ for (const event of insightsRawToInsertBuffer) {
161
+ workflowIdNames.set(event.workflowId, event.workflowName);
162
+ }
163
+ const ownerProjects = await this.sharedWorkflowRepository.findOwnerProjectsByWorkflowIds([
164
+ ...workflowIdNames.keys(),
165
+ ]);
166
+ const metadataToUpsert = [];
167
+ for (const [workflowId, workflowName] of workflowIdNames) {
168
+ const project = ownerProjects.get(workflowId);
169
+ if (!project)
170
+ continue;
171
+ const cachedMetadata = this.cachedMetadata.get(workflowId);
172
+ if (!cachedMetadata ||
173
+ cachedMetadata.projectId !== project.id ||
174
+ cachedMetadata.projectName !== project.name ||
175
+ cachedMetadata.workflowName !== workflowName) {
176
+ const metadata = new insights_metadata_entity_1.InsightsMetadata();
177
+ metadata.projectId = project.id;
178
+ metadata.projectName = project.name;
179
+ metadata.workflowId = workflowId;
180
+ metadata.workflowName = workflowName;
181
+ metadataToUpsert.push(metadata);
182
+ }
183
+ }
184
+ if (metadataToUpsert.length > 0) {
185
+ this.logger.debug(`Saving ${metadataToUpsert.length} insights metadata for workflows`);
186
+ await this.insightsMetadataRepository.upsertWorkflowMetadata(metadataToUpsert);
187
+ const upsertMetadata = await this.insightsMetadataRepository.findByWorkflowIds(metadataToUpsert.map((metadata) => metadata.workflowId));
188
+ for (const metadata of upsertMetadata) {
189
+ this.cachedMetadata.set(metadata.workflowId, metadata);
190
+ }
191
+ }
192
+ const events = [];
193
+ const workflowIdsWithoutMetadata = new Set();
194
+ for (const event of insightsRawToInsertBuffer) {
195
+ const insight = new insights_raw_entity_1.InsightsRaw();
196
+ const metadata = this.cachedMetadata.get(event.workflowId);
197
+ if (!metadata) {
198
+ workflowIdsWithoutMetadata.add(event.workflowId);
199
+ continue;
200
+ }
201
+ insight.metaId = metadata.metaId;
202
+ insight.type = event.type;
203
+ insight.value = integerValueForInsightsRaw(event.value);
204
+ insight.timestamp = event.timestamp;
205
+ events.push(insight);
206
+ }
207
+ if (workflowIdsWithoutMetadata.size > 0) {
208
+ this.logger.warn('Dropped insights for workflows with no shared workflow', {
209
+ workflowIds: [...workflowIdsWithoutMetadata],
210
+ });
211
+ }
212
+ if (events.length === 0)
213
+ return;
214
+ this.logger.debug(`Inserting ${events.length} insights raw`);
215
+ await this.insightsRawRepository.insert(events);
216
+ }
217
+ async flushEvents() {
218
+ if (!this.isInitialized) {
219
+ return;
220
+ }
221
+ if (this.bufferedInsights.size === 0) {
222
+ this.scheduleFlushing();
223
+ return;
224
+ }
225
+ this.cancelScheduledFlushing();
226
+ const bufferedInsightsToFlush = new Set(this.bufferedInsights);
227
+ this.bufferedInsights.clear();
228
+ let flushPromise = undefined;
229
+ flushPromise = (async () => {
230
+ try {
231
+ await this.saveInsightsMetadataAndRaw(bufferedInsightsToFlush);
232
+ }
233
+ catch (e) {
234
+ this.logger.error('Error while saving insights metadata and raw data', { error: e });
235
+ for (const event of bufferedInsightsToFlush) {
236
+ this.bufferedInsights.add(event);
237
+ }
238
+ }
239
+ finally {
240
+ this.scheduleFlushing();
241
+ this.flushesInProgress.delete(flushPromise);
242
+ }
243
+ })();
244
+ this.flushesInProgress.add(flushPromise);
245
+ await flushPromise;
246
+ }
247
+ calculateTimeSaved(ctx) {
248
+ const workflowTimeSaved = ctx.workflow.settings?.timeSavedPerExecution;
249
+ if (ctx.workflow.settings?.timeSavedMode !== 'dynamic') {
250
+ return workflowTimeSaved ?? 0;
251
+ }
252
+ const nodeTimeSaved = this.extractTimeSavedFromNodes(ctx.runData);
253
+ return nodeTimeSaved;
254
+ }
255
+ extractTimeSavedFromNodes(runData) {
256
+ let totalMinutes = 0;
257
+ const resultData = runData.data.resultData?.runData ?? {};
258
+ for (const nodeName in resultData) {
259
+ const taskData = resultData[nodeName];
260
+ for (const taskDataEntry of taskData) {
261
+ if (taskDataEntry?.metadata?.timeSaved) {
262
+ totalMinutes += taskDataEntry?.metadata?.timeSaved.minutes;
263
+ }
264
+ }
265
+ }
266
+ return totalMinutes;
267
+ }
268
+ };
269
+ exports.InsightsCollectionService = InsightsCollectionService;
270
+ __decorate([
271
+ (0, decorators_1.OnLifecycleEvent)('workflowExecuteAfter'),
272
+ __metadata("design:type", Function),
273
+ __metadata("design:paramtypes", [Object]),
274
+ __metadata("design:returntype", Promise)
275
+ ], InsightsCollectionService.prototype, "handleWorkflowExecuteAfter", null);
276
+ exports.InsightsCollectionService = InsightsCollectionService = __decorate([
277
+ (0, di_1.Service)(),
278
+ __metadata("design:paramtypes", [db_1.SharedWorkflowRepository, insights_raw_repository_1.InsightsRawRepository, insights_metadata_repository_1.InsightsMetadataRepository, insights_config_1.InsightsConfig, backend_common_1.Logger])
279
+ ], InsightsCollectionService);
280
+ //# sourceMappingURL=insights-collection.service.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"insights-collection.service.js","sourceRoot":"","sources":["../src/insights-collection.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wDAA6C;AAC7C,4DAA4D;AAC5D,gCAAmD;AACnD,gDAAqF;AACrF,gCAAkC;AAClC,iCAAiC;AAGjC,yEAA8D;AAC9D,+DAAoD;AACpD,iFAA4E;AAC5E,uEAAkE;AAClE,uDAAmD;AAEnD,MAAM,gBAAgB,GAAqC;IAC1D,OAAO,EAAE,KAAK;IACd,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,KAAK;IAEZ,QAAQ,EAAE,IAAI;IACd,GAAG,EAAE,IAAI;IACT,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,IAAI;CACb,CAAC;AAEF,MAAM,cAAc,GAAyC;IAC5D,GAAG,EAAE,KAAK;IACV,KAAK,EAAE,KAAK;IACZ,KAAK,EAAE,KAAK;IACZ,OAAO,EAAE,KAAK;IACd,OAAO,EAAE,KAAK;IACd,UAAU,EAAE,KAAK;IAGjB,UAAU,EAAE,IAAI;IAGhB,QAAQ,EAAE,IAAI;IAEd,MAAM,EAAE,IAAI;IAGZ,IAAI,EAAE,IAAI;IAGV,KAAK,EAAE,IAAI;CACX,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,CAAC;AAGtB,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAMhC,SAAS,0BAA0B,CAAC,KAAa;IAChD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,CAAC;IACV,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAYM,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAarC,YACkB,wBAAkD,EAClD,qBAA4C,EAC5C,0BAAsD,EACtD,cAA8B,EAC9B,MAAc;QAJd,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,0BAAqB,GAArB,qBAAqB,CAAuB;QAC5C,+BAA0B,GAA1B,0BAA0B,CAA4B;QACtD,mBAAc,GAAd,cAAc,CAAgB;QAC9B,WAAM,GAAN,MAAM,CAAQ;QAjBf,mBAAc,GAAkC,IAAI,GAAG,EAAE,CAAC;QAEnE,qBAAgB,GAAyB,IAAI,GAAG,EAAE,CAAC;QAInD,mCAA8B,GAAG,IAAI,CAAC;QAEtC,sBAAiB,GAAuB,IAAI,GAAG,EAAE,CAAC;QAElD,kBAAa,GAAG,KAAK,CAAC;QAS7B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI;QACH,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,8BAA8B,GAAG,IAAI,CAAC;QAE3C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC7C,CAAC;IAED,gBAAgB;QAEf,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,OAAO;QAEhC,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAC/B,IAAI,CAAC,2BAA2B,GAAG,UAAU,CAC5C,KAAK,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,EACpC,IAAI,CAAC,cAAc,CAAC,oBAAoB,GAAG,IAAI,CAC/C,CAAC;IACH,CAAC;IAED,uBAAuB;QACtB,IAAI,IAAI,CAAC,2BAA2B,KAAK,SAAS,EAAE,CAAC;YACpD,YAAY,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC/C,IAAI,CAAC,2BAA2B,GAAG,SAAS,CAAC;QAC9C,CAAC;IACF,CAAC;IAED,iBAAiB;QAChB,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAC/B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,QAAQ;QACb,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAIzB,IAAI,CAAC,8BAA8B,GAAG,KAAK,CAAC;QAI5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAEnE,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;IAC5B,CAAC;IAGK,AAAN,KAAK,CAAC,0BAA0B,CAAC,GAAgC;QAEhE,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACzB,OAAO;QACR,CAAC;QAED,IAAI,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9E,OAAO;QACR,CAAC;QAKD,IAAI,GAAG,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;YAClC,OAAO;QACR,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAExE,MAAM,kBAAkB,GAAG;YAC1B,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC3B,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,IAAI;YAC/B,SAAS,EAAE,gBAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SACpC,CAAC;QAGF,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC;YACzB,GAAG,kBAAkB;YACrB,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC;SACR,CAAC,CAAC;QAEH,IAAI,IAAA,sCAAmB,EAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC;gBACzB,GAAG,kBAAkB;gBACrB,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,CAAC;aACR,CAAC,CAAC;QACJ,CAAC;QAGD,IAAI,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACpF,IAAI,SAAS,GAAG,WAAW,IAAI,SAAS,GAAG,WAAW,EAAE,CAAC;gBACxD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,SAAS,4BAA4B,CAAC,CAAC;YACtF,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;YACtE,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC;gBACzB,GAAG,kBAAkB;gBACrB,IAAI,EAAE,YAAY;gBAClB,KAAK;aACL,CAAC,CAAC;QACJ,CAAC;QAGD,IAAI,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1D,MAAM,cAAc,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACpD,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC;oBACzB,GAAG,kBAAkB;oBACrB,IAAI,EAAE,gBAAgB;oBACtB,KAAK,EAAE,cAAc;iBACrB,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,8BAA8B,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;YAE5E,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC1B,CAAC;QAGD,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;YACtE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,gBAAgB,CAAC,IAAI,6BAA6B,CAAC,CAAC;YAE9F,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,0BAA0B,CAAC,yBAA+C;QACvF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,yBAAyB,CAAC,IAAI,WAAW,CAAC,CAAC;QACzE,MAAM,eAAe,GAAwB,IAAI,GAAG,EAAE,CAAC;QAEvD,KAAK,MAAM,KAAK,IAAI,yBAAyB,EAAE,CAAC;YAC/C,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,8BAA8B,CAAC;YACxF,GAAG,eAAe,CAAC,IAAI,EAAE;SACzB,CAAC,CAAC;QAIH,MAAM,gBAAgB,GAAuB,EAAE,CAAC;QAChD,KAAK,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,IAAI,eAAe,EAAE,CAAC;YAC1D,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC9C,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC3D,IACC,CAAC,cAAc;gBACf,cAAc,CAAC,SAAS,KAAK,OAAO,CAAC,EAAE;gBACvC,cAAc,CAAC,WAAW,KAAK,OAAO,CAAC,IAAI;gBAC3C,cAAc,CAAC,YAAY,KAAK,YAAY,EAC3C,CAAC;gBACF,MAAM,QAAQ,GAAG,IAAI,2CAAgB,EAAE,CAAC;gBACxC,QAAQ,CAAC,SAAS,GAAG,OAAO,CAAC,EAAE,CAAC;gBAChC,QAAQ,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;gBACpC,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;gBACjC,QAAQ,CAAC,YAAY,GAAG,YAAY,CAAC;gBAErC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;QACF,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,gBAAgB,CAAC,MAAM,kCAAkC,CAAC,CAAC;YACvF,MAAM,IAAI,CAAC,0BAA0B,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;YAE/E,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,iBAAiB,CAC7E,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CACvD,CAAC;YACF,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;gBACvC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YACxD,CAAC;QACF,CAAC;QAED,MAAM,MAAM,GAAkB,EAAE,CAAC;QACjC,MAAM,0BAA0B,GAAG,IAAI,GAAG,EAAU,CAAC;QACrD,KAAK,MAAM,KAAK,IAAI,yBAAyB,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,iCAAW,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAC3D,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAKf,0BAA0B,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACjD,SAAS;YACV,CAAC;YACD,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;YACjC,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;YAC1B,OAAO,CAAC,KAAK,GAAG,0BAA0B,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACxD,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;YAEpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,IAAI,0BAA0B,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wDAAwD,EAAE;gBAC1E,WAAW,EAAE,CAAC,GAAG,0BAA0B,CAAC;aAC5C,CAAC,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,MAAM,eAAe,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,WAAW;QAEhB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACzB,OAAO;QACR,CAAC;QAGD,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAEtC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,OAAO;QACR,CAAC;QAGD,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAI/B,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/D,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAE9B,IAAI,YAAY,GAA8B,SAAS,CAAC;QACxD,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1B,IAAI,CAAC;gBACJ,MAAM,IAAI,CAAC,0BAA0B,CAAC,uBAAuB,CAAC,CAAC;YAChE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mDAAmD,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBACrF,KAAK,MAAM,KAAK,IAAI,uBAAuB,EAAE,CAAC;oBAC7C,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAClC,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,YAAa,CAAC,CAAC;YAC9C,CAAC;QACF,CAAC,CAAC,EAAE,CAAC;QAGL,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACzC,MAAM,YAAY,CAAC;IACpB,CAAC;IAMO,kBAAkB,CAAC,GAAgC;QAC1D,MAAM,iBAAiB,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,qBAAqB,CAAC;QAGvE,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,aAAa,KAAK,SAAS,EAAE,CAAC;YACxD,OAAO,iBAAiB,IAAI,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAElE,OAAO,aAAa,CAAC;IACtB,CAAC;IAMO,yBAAyB,CAAC,OAAa;QAC9C,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,IAAI,EAAE,CAAC;QAG1D,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YAGtC,KAAK,MAAM,aAAa,IAAI,QAAQ,EAAE,CAAC;gBACtC,IAAI,aAAa,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;oBACxC,YAAY,IAAI,aAAa,EAAE,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC;gBAC5D,CAAC;YACF,CAAC;QACF,CAAC;QAED,OAAO,YAAY,CAAC;IACrB,CAAC;CACD,CAAA;;AAlPM;IADL,IAAA,6BAAgB,EAAC,sBAAsB,CAAC;;;;2EA+ExC;oCApJW,yBAAyB;IADrC,IAAA,YAAO,GAAE;qCAemC,6BAAwB,EAC3B,+CAAqB,EAChB,yDAA0B,EACtC,gCAAc,EACtB,uBAAM;GAlBpB,yBAAyB,CAwTrC"}
@@ -0,0 +1,19 @@
1
+ import { Logger } from '@n8n/backend-common';
2
+ import { InsightsByPeriodRepository } from './insights-by-period.repository';
3
+ import { InsightsRawRepository } from './insights-raw.repository';
4
+ import { InsightsConfig } from './insights.config';
5
+ export declare class InsightsCompactionService {
6
+ private readonly insightsByPeriodRepository;
7
+ private readonly insightsRawRepository;
8
+ private readonly insightsConfig;
9
+ private readonly logger;
10
+ constructor(insightsByPeriodRepository: InsightsByPeriodRepository, insightsRawRepository: InsightsRawRepository, insightsConfig: InsightsConfig, logger: Logger);
11
+ compactInsights(signal: AbortSignal): Promise<void>;
12
+ private compactStage;
13
+ private getCompactionRunStopReason;
14
+ private logCompactionRunLimitReached;
15
+ private waitBeforeNextBatchIfFull;
16
+ compactRawToHour(): Promise<number>;
17
+ compactHourToDay(): Promise<number>;
18
+ compactDayToWeek(): Promise<number>;
19
+ }
@@ -0,0 +1,161 @@
1
+ "use strict";
2
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
7
+ };
8
+ var __metadata = (this && this.__metadata) || function (k, v) {
9
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.InsightsCompactionService = void 0;
13
+ const backend_common_1 = require("@n8n/backend-common");
14
+ const constants_1 = require("@n8n/constants");
15
+ const di_1 = require("@n8n/di");
16
+ const sleep_1 = require("@n8n/utils/sleep");
17
+ const insights_by_period_repository_1 = require("./insights-by-period.repository");
18
+ const insights_raw_repository_1 = require("./insights-raw.repository");
19
+ const insights_config_1 = require("./insights.config");
20
+ let InsightsCompactionService = class InsightsCompactionService {
21
+ constructor(insightsByPeriodRepository, insightsRawRepository, insightsConfig, logger) {
22
+ this.insightsByPeriodRepository = insightsByPeriodRepository;
23
+ this.insightsRawRepository = insightsRawRepository;
24
+ this.insightsConfig = insightsConfig;
25
+ this.logger = logger;
26
+ this.logger = this.logger.scoped('insights');
27
+ }
28
+ async compactInsights(signal) {
29
+ const runState = {
30
+ startedAt: Date.now(),
31
+ batchesProcessed: 0,
32
+ rowsCompacted: 0,
33
+ signal,
34
+ };
35
+ const stoppedAfterRawToHour = await this.compactStage({
36
+ stageName: 'raw-to-hour',
37
+ beforeBatchMessage: 'Compacting raw data to hourly aggregates',
38
+ afterBatchMessage: (rowsCompacted) => `Compacted ${rowsCompacted} raw data to hourly aggregates`,
39
+ compactBatch: this.compactRawToHour.bind(this),
40
+ runState,
41
+ });
42
+ if (stoppedAfterRawToHour)
43
+ return;
44
+ const stoppedAfterHourToDay = await this.compactStage({
45
+ stageName: 'hour-to-day',
46
+ beforeBatchMessage: 'Compacting hourly data to daily aggregates',
47
+ afterBatchMessage: (rowsCompacted) => `Compacted ${rowsCompacted} hourly data to daily aggregates`,
48
+ compactBatch: this.compactHourToDay.bind(this),
49
+ runState,
50
+ });
51
+ if (stoppedAfterHourToDay)
52
+ return;
53
+ await this.compactStage({
54
+ stageName: 'day-to-week',
55
+ beforeBatchMessage: 'Compacting daily data to weekly aggregates',
56
+ afterBatchMessage: (rowsCompacted) => `Compacted ${rowsCompacted} daily data to weekly aggregates`,
57
+ compactBatch: this.compactDayToWeek.bind(this),
58
+ runState,
59
+ });
60
+ }
61
+ async compactStage({ stageName, beforeBatchMessage, afterBatchMessage, compactBatch, runState, }) {
62
+ let numberOfCompactedData;
63
+ do {
64
+ const stopReason = this.getCompactionRunStopReason(runState);
65
+ if (stopReason !== undefined) {
66
+ this.logCompactionRunLimitReached(stopReason, stageName, runState);
67
+ return true;
68
+ }
69
+ this.logger.debug(beforeBatchMessage);
70
+ numberOfCompactedData = await compactBatch();
71
+ this.logger.debug(afterBatchMessage(numberOfCompactedData));
72
+ runState.batchesProcessed++;
73
+ runState.rowsCompacted += numberOfCompactedData;
74
+ const stopReasonAfterBatch = this.getCompactionRunStopReason(runState);
75
+ if (stopReasonAfterBatch !== undefined) {
76
+ this.logCompactionRunLimitReached(stopReasonAfterBatch, stageName, runState);
77
+ return true;
78
+ }
79
+ await this.waitBeforeNextBatchIfFull(numberOfCompactedData, runState);
80
+ } while (numberOfCompactedData === this.insightsConfig.compactionBatchSize);
81
+ return false;
82
+ }
83
+ getCompactionRunStopReason(runState) {
84
+ if (runState.signal.aborted) {
85
+ return 'aborted';
86
+ }
87
+ if (this.insightsConfig.compactionMaxBatchesPerRun > 0 &&
88
+ runState.batchesProcessed >= this.insightsConfig.compactionMaxBatchesPerRun) {
89
+ return 'max-batches';
90
+ }
91
+ if (this.insightsConfig.compactionMaxRuntimeSeconds > 0 &&
92
+ Date.now() - runState.startedAt >=
93
+ this.insightsConfig.compactionMaxRuntimeSeconds * constants_1.Time.seconds.toMilliseconds) {
94
+ return 'max-runtime';
95
+ }
96
+ return undefined;
97
+ }
98
+ logCompactionRunLimitReached(reason, stageName, runState) {
99
+ const details = {
100
+ reason,
101
+ stageName,
102
+ batchesProcessed: runState.batchesProcessed,
103
+ rowsCompacted: runState.rowsCompacted,
104
+ compactionMaxBatchesPerRun: this.insightsConfig.compactionMaxBatchesPerRun,
105
+ compactionMaxRuntimeSeconds: this.insightsConfig.compactionMaxRuntimeSeconds,
106
+ };
107
+ if (reason === 'aborted') {
108
+ this.logger.debug('Stopping insights compaction because the run was aborted', details);
109
+ }
110
+ else {
111
+ this.logger.warn('Stopping insights compaction because a per-run limit was reached', details);
112
+ }
113
+ }
114
+ async waitBeforeNextBatchIfFull(numberOfCompactedData, runState) {
115
+ if (numberOfCompactedData !== this.insightsConfig.compactionBatchSize ||
116
+ this.insightsConfig.compactionBatchDelayMilliseconds <= 0) {
117
+ return;
118
+ }
119
+ try {
120
+ await (0, sleep_1.sleep)(this.insightsConfig.compactionBatchDelayMilliseconds, runState.signal);
121
+ }
122
+ catch {
123
+ }
124
+ }
125
+ async compactRawToHour() {
126
+ const batchQuery = this.insightsRawRepository.getRawInsightsBatchQuery(this.insightsConfig.compactionBatchSize);
127
+ return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
128
+ sourceBatchQuery: batchQuery,
129
+ sourceTableName: this.insightsRawRepository.metadata.tableName,
130
+ periodUnitToCompactInto: 'hour',
131
+ });
132
+ }
133
+ async compactHourToDay() {
134
+ const batchQuery = this.insightsByPeriodRepository.getPeriodInsightsBatchQuery({
135
+ periodUnitToCompactFrom: 'hour',
136
+ compactionBatchSize: this.insightsConfig.compactionBatchSize,
137
+ maxAgeInDays: this.insightsConfig.compactionHourlyToDailyThresholdDays,
138
+ });
139
+ return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
140
+ sourceBatchQuery: batchQuery,
141
+ periodUnitToCompactInto: 'day',
142
+ });
143
+ }
144
+ async compactDayToWeek() {
145
+ const batchQuery = this.insightsByPeriodRepository.getPeriodInsightsBatchQuery({
146
+ periodUnitToCompactFrom: 'day',
147
+ compactionBatchSize: this.insightsConfig.compactionBatchSize,
148
+ maxAgeInDays: this.insightsConfig.compactionDailyToWeeklyThresholdDays,
149
+ });
150
+ return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
151
+ sourceBatchQuery: batchQuery,
152
+ periodUnitToCompactInto: 'week',
153
+ });
154
+ }
155
+ };
156
+ exports.InsightsCompactionService = InsightsCompactionService;
157
+ exports.InsightsCompactionService = InsightsCompactionService = __decorate([
158
+ (0, di_1.Service)(),
159
+ __metadata("design:paramtypes", [insights_by_period_repository_1.InsightsByPeriodRepository, insights_raw_repository_1.InsightsRawRepository, insights_config_1.InsightsConfig, backend_common_1.Logger])
160
+ ], InsightsCompactionService);
161
+ //# sourceMappingURL=insights-compaction.service.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"insights-compaction.service.js","sourceRoot":"","sources":["../src/insights-compaction.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wDAA6C;AAC7C,8CAAsC;AACtC,gCAAkC;AAClC,4CAAyC;AAEzC,mFAA6E;AAC7E,uEAAkE;AAClE,uDAAmD;AAkB5C,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IACrC,YACkB,0BAAsD,EACtD,qBAA4C,EAC5C,cAA8B,EAC9B,MAAc;0CAHd,0BAA0B;qCAC1B,qBAAqB;8BACrB,cAAc;sBACd,MAAM;QAEvB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IAGD,KAAK,CAAC,eAAe,CAAC,MAAmB;QACxC,MAAM,QAAQ,GAAuB;YACpC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,gBAAgB,EAAE,CAAC;YACnB,aAAa,EAAE,CAAC;YAChB,MAAM;SACN,CAAC;QAEF,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;YACrD,SAAS,EAAE,aAAa;YACxB,kBAAkB,EAAE,0CAA0C;YAC9D,iBAAiB,EAAE,CAAC,aAAa,EAAE,EAAE,CACpC,aAAa,aAAa,gCAAgC;YAC3D,YAAY,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;YAC9C,QAAQ;SACR,CAAC,CAAC;QACH,IAAI,qBAAqB;YAAE,OAAO;QAElC,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;YACrD,SAAS,EAAE,aAAa;YACxB,kBAAkB,EAAE,4CAA4C;YAChE,iBAAiB,EAAE,CAAC,aAAa,EAAE,EAAE,CACpC,aAAa,aAAa,kCAAkC;YAC7D,YAAY,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;YAC9C,QAAQ;SACR,CAAC,CAAC;QACH,IAAI,qBAAqB;YAAE,OAAO;QAElC,MAAM,IAAI,CAAC,YAAY,CAAC;YACvB,SAAS,EAAE,aAAa;YACxB,kBAAkB,EAAE,4CAA4C;YAChE,iBAAiB,EAAE,CAAC,aAAa,EAAE,EAAE,CACpC,aAAa,aAAa,kCAAkC;YAC7D,YAAY,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;YAC9C,QAAQ;SACR,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,EAC1B,SAAS,EACT,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,EACZ,QAAQ,GAOR;QACA,IAAI,qBAA6B,CAAC;QAElC,GAAG,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;YAC7D,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC9B,IAAI,CAAC,4BAA4B,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gBACnE,OAAO,IAAI,CAAC;YACb,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YACtC,qBAAqB,GAAG,MAAM,YAAY,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,CAAC,CAAC;YAE5D,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YAC5B,QAAQ,CAAC,aAAa,IAAI,qBAAqB,CAAC;YAEhD,MAAM,oBAAoB,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;YACvE,IAAI,oBAAoB,KAAK,SAAS,EAAE,CAAC;gBACxC,IAAI,CAAC,4BAA4B,CAAC,oBAAoB,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gBAC7E,OAAO,IAAI,CAAC;YACb,CAAC;YAED,MAAM,IAAI,CAAC,yBAAyB,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;QACvE,CAAC,QAAQ,qBAAqB,KAAK,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE;QAE5E,OAAO,KAAK,CAAC;IACd,CAAC;IAEO,0BAA0B,CACjC,QAA4B;QAE5B,IAAI,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC7B,OAAO,SAAS,CAAC;QAClB,CAAC;QAED,IACC,IAAI,CAAC,cAAc,CAAC,0BAA0B,GAAG,CAAC;YAClD,QAAQ,CAAC,gBAAgB,IAAI,IAAI,CAAC,cAAc,CAAC,0BAA0B,EAC1E,CAAC;YACF,OAAO,aAAa,CAAC;QACtB,CAAC;QAED,IACC,IAAI,CAAC,cAAc,CAAC,2BAA2B,GAAG,CAAC;YACnD,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,SAAS;gBAC9B,IAAI,CAAC,cAAc,CAAC,2BAA2B,GAAG,gBAAI,CAAC,OAAO,CAAC,cAAc,EAC7E,CAAC;YACF,OAAO,aAAa,CAAC;QACtB,CAAC;QAED,OAAO,SAAS,CAAC;IAClB,CAAC;IAEO,4BAA4B,CACnC,MAA4B,EAC5B,SAAiB,EACjB,QAA4B;QAE5B,MAAM,OAAO,GAAG;YACf,MAAM;YACN,SAAS;YACT,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;YAC3C,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,0BAA0B,EAAE,IAAI,CAAC,cAAc,CAAC,0BAA0B;YAC1E,2BAA2B,EAAE,IAAI,CAAC,cAAc,CAAC,2BAA2B;SAC5E,CAAC;QACF,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0DAA0D,EAAE,OAAO,CAAC,CAAC;QACxF,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kEAAkE,EAAE,OAAO,CAAC,CAAC;QAC/F,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,yBAAyB,CACtC,qBAA6B,EAC7B,QAA4B;QAE5B,IACC,qBAAqB,KAAK,IAAI,CAAC,cAAc,CAAC,mBAAmB;YACjE,IAAI,CAAC,cAAc,CAAC,gCAAgC,IAAI,CAAC,EACxD,CAAC;YACF,OAAO;QACR,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,IAAA,aAAK,EAAC,IAAI,CAAC,cAAc,CAAC,gCAAgC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpF,CAAC;QAAC,MAAM,CAAC;QAET,CAAC;IACF,CAAC;IAKD,KAAK,CAAC,gBAAgB;QAErB,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,CAAC,wBAAwB,CACrE,IAAI,CAAC,cAAc,CAAC,mBAAmB,CACvC,CAAC;QAEF,OAAO,MAAM,IAAI,CAAC,0BAA0B,CAAC,kCAAkC,CAAC;YAC/E,gBAAgB,EAAE,UAAU;YAC5B,eAAe,EAAE,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,SAAS;YAC9D,uBAAuB,EAAE,MAAM;SAC/B,CAAC,CAAC;IACJ,CAAC;IAKD,KAAK,CAAC,gBAAgB;QAErB,MAAM,UAAU,GAAG,IAAI,CAAC,0BAA0B,CAAC,2BAA2B,CAAC;YAC9E,uBAAuB,EAAE,MAAM;YAC/B,mBAAmB,EAAE,IAAI,CAAC,cAAc,CAAC,mBAAmB;YAC5D,YAAY,EAAE,IAAI,CAAC,cAAc,CAAC,oCAAoC;SACtE,CAAC,CAAC;QAEH,OAAO,MAAM,IAAI,CAAC,0BAA0B,CAAC,kCAAkC,CAAC;YAC/E,gBAAgB,EAAE,UAAU;YAC5B,uBAAuB,EAAE,KAAK;SAC9B,CAAC,CAAC;IACJ,CAAC;IAKD,KAAK,CAAC,gBAAgB;QAErB,MAAM,UAAU,GAAG,IAAI,CAAC,0BAA0B,CAAC,2BAA2B,CAAC;YAC9E,uBAAuB,EAAE,KAAK;YAC9B,mBAAmB,EAAE,IAAI,CAAC,cAAc,CAAC,mBAAmB;YAC5D,YAAY,EAAE,IAAI,CAAC,cAAc,CAAC,oCAAoC;SACtE,CAAC,CAAC;QAEH,OAAO,MAAM,IAAI,CAAC,0BAA0B,CAAC,kCAAkC,CAAC;YAC/E,gBAAgB,EAAE,UAAU;YAC5B,uBAAuB,EAAE,MAAM;SAC/B,CAAC,CAAC;IACJ,CAAC;CACD,CAAA;;oCA1MY,yBAAyB;IADrC,IAAA,YAAO,GAAE;qCAGqC,0DAA0B,EAC/B,+CAAqB,EAC5B,gCAAc,EACtB,uBAAM;GALpB,yBAAyB,CA0MrC"}
@@ -0,0 +1,14 @@
1
+ import { SystemTask } from '@n8n/decorators';
2
+ import type { SystemTaskEffects, SystemTaskPlacement, SystemTaskSchedule } from '@n8n/decorators';
3
+ import { InsightsCompactionService } from './insights-compaction.service';
4
+ import { InsightsConfig } from './insights.config';
5
+ export declare class InsightsCompactionTask implements SystemTask {
6
+ private readonly insightsConfig;
7
+ private readonly compactionService;
8
+ readonly name = "insights-compaction";
9
+ readonly schedule: SystemTaskSchedule;
10
+ readonly effects: SystemTaskEffects;
11
+ readonly placement: SystemTaskPlacement;
12
+ constructor(insightsConfig: InsightsConfig, compactionService: InsightsCompactionService);
13
+ run(signal: AbortSignal): Promise<void>;
14
+ }
@@ -0,0 +1,35 @@
1
+ "use strict";
2
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
7
+ };
8
+ var __metadata = (this && this.__metadata) || function (k, v) {
9
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.InsightsCompactionTask = void 0;
13
+ const constants_1 = require("@n8n/constants");
14
+ const decorators_1 = require("@n8n/decorators");
15
+ const insights_compaction_service_1 = require("./insights-compaction.service");
16
+ const insights_config_1 = require("./insights.config");
17
+ let InsightsCompactionTask = class InsightsCompactionTask {
18
+ constructor(insightsConfig, compactionService) {
19
+ this.insightsConfig = insightsConfig;
20
+ this.compactionService = compactionService;
21
+ this.name = 'insights-compaction';
22
+ this.schedule = (0, decorators_1.intervalFromSeconds)(this.insightsConfig.compactionIntervalMinutes * constants_1.Time.minutes.toSeconds);
23
+ this.effects = 'idempotent';
24
+ this.placement = { scope: 'cluster', durable: true };
25
+ }
26
+ async run(signal) {
27
+ await this.compactionService.compactInsights(signal);
28
+ }
29
+ };
30
+ exports.InsightsCompactionTask = InsightsCompactionTask;
31
+ exports.InsightsCompactionTask = InsightsCompactionTask = __decorate([
32
+ (0, decorators_1.SystemTask)(),
33
+ __metadata("design:paramtypes", [insights_config_1.InsightsConfig, insights_compaction_service_1.InsightsCompactionService])
34
+ ], InsightsCompactionTask);
35
+ //# sourceMappingURL=insights-compaction.task.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"insights-compaction.task.js","sourceRoot":"","sources":["../src/insights-compaction.task.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,8CAAsC;AACtC,gDAAkE;AAGlE,+EAA0E;AAC1E,uDAAmD;AAO5C,IAAM,sBAAsB,GAA5B,MAAM,sBAAsB;IAWlC,YACkB,cAA8B,EAC9B,iBAA4C;QAD5C,mBAAc,GAAd,cAAc,CAAgB;QAC9B,sBAAiB,GAAjB,iBAAiB,CAA2B;QAZrD,SAAI,GAAG,qBAAqB,CAAC;QAE7B,aAAQ,GAAuB,IAAA,gCAAmB,EAC1D,IAAI,CAAC,cAAc,CAAC,yBAAyB,GAAG,gBAAI,CAAC,OAAO,CAAC,SAAS,CACtE,CAAC;QAEO,YAAO,GAAsB,YAAY,CAAC;QAE1C,cAAS,GAAwB,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAK3E,CAAC;IAEJ,KAAK,CAAC,GAAG,CAAC,MAAmB;QAC5B,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IACtD,CAAC;CACD,CAAA;;iCAnBY,sBAAsB;IADlC,IAAA,uBAAU,GAAE;qCAasB,gCAAc,EACX,uDAAyB;GAblD,sBAAsB,CAmBlC"}
@@ -0,0 +1,8 @@
1
+ import { BaseEntity } from '@n8n/typeorm';
2
+ export declare class InsightsMetadata extends BaseEntity {
3
+ metaId: number;
4
+ workflowId: string;
5
+ projectId: string;
6
+ workflowName: string;
7
+ projectName: string;
8
+ }
@@ -0,0 +1,40 @@
1
+ "use strict";
2
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
7
+ };
8
+ var __metadata = (this && this.__metadata) || function (k, v) {
9
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.InsightsMetadata = void 0;
13
+ const typeorm_1 = require("@n8n/typeorm");
14
+ let InsightsMetadata = class InsightsMetadata extends typeorm_1.BaseEntity {
15
+ };
16
+ exports.InsightsMetadata = InsightsMetadata;
17
+ __decorate([
18
+ (0, typeorm_1.PrimaryGeneratedColumn)(),
19
+ __metadata("design:type", Number)
20
+ ], InsightsMetadata.prototype, "metaId", void 0);
21
+ __decorate([
22
+ (0, typeorm_1.Column)({ unique: true, type: 'varchar', length: 36 }),
23
+ __metadata("design:type", String)
24
+ ], InsightsMetadata.prototype, "workflowId", void 0);
25
+ __decorate([
26
+ (0, typeorm_1.Column)({ type: 'varchar', length: 36 }),
27
+ __metadata("design:type", String)
28
+ ], InsightsMetadata.prototype, "projectId", void 0);
29
+ __decorate([
30
+ (0, typeorm_1.Column)({ type: 'varchar', length: 128 }),
31
+ __metadata("design:type", String)
32
+ ], InsightsMetadata.prototype, "workflowName", void 0);
33
+ __decorate([
34
+ (0, typeorm_1.Column)({ type: 'varchar', length: 255 }),
35
+ __metadata("design:type", String)
36
+ ], InsightsMetadata.prototype, "projectName", void 0);
37
+ exports.InsightsMetadata = InsightsMetadata = __decorate([
38
+ (0, typeorm_1.Entity)()
39
+ ], InsightsMetadata);
40
+ //# sourceMappingURL=insights-metadata.entity.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"insights-metadata.entity.js","sourceRoot":"","sources":["../src/insights-metadata.entity.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,0CAAkF;AAG3E,IAAM,gBAAgB,GAAtB,MAAM,gBAAiB,SAAQ,oBAAU;CAe/C,CAAA;;AAbA;IADC,IAAA,gCAAsB,GAAE;;gDACV;AAGf;IADC,IAAA,gBAAM,EAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;;oDACnC;AAGnB;IADC,IAAA,gBAAM,EAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;;mDACtB;AAGlB;IADC,IAAA,gBAAM,EAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;;sDACpB;AAGrB;IADC,IAAA,gBAAM,EAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;;qDACrB;2BAdR,gBAAgB;IAD5B,IAAA,gBAAM,GAAE;GACI,gBAAgB,CAe5B"}
@@ -0,0 +1,7 @@
1
+ import { DataSource, Repository } from '@n8n/typeorm';
2
+ import { InsightsMetadata } from './insights-metadata.entity';
3
+ export declare class InsightsMetadataRepository extends Repository<InsightsMetadata> {
4
+ constructor(dataSource: DataSource);
5
+ upsertWorkflowMetadata(metadata: InsightsMetadata[]): Promise<void>;
6
+ findByWorkflowIds(workflowIds: string[]): Promise<InsightsMetadata[]>;
7
+ }
@@ -0,0 +1,36 @@
1
+ "use strict";
2
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
7
+ };
8
+ var __metadata = (this && this.__metadata) || function (k, v) {
9
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.InsightsMetadataRepository = void 0;
13
+ const di_1 = require("@n8n/di");
14
+ const typeorm_1 = require("@n8n/typeorm");
15
+ const insights_metadata_entity_1 = require("./insights-metadata.entity");
16
+ let InsightsMetadataRepository = class InsightsMetadataRepository extends typeorm_1.Repository {
17
+ constructor(dataSource) {
18
+ super(insights_metadata_entity_1.InsightsMetadata, dataSource.manager);
19
+ }
20
+ async upsertWorkflowMetadata(metadata) {
21
+ if (metadata.length === 0)
22
+ return;
23
+ await this.upsert(metadata, ['workflowId']);
24
+ }
25
+ async findByWorkflowIds(workflowIds) {
26
+ if (workflowIds.length === 0)
27
+ return [];
28
+ return await this.findBy({ workflowId: (0, typeorm_1.In)(workflowIds) });
29
+ }
30
+ };
31
+ exports.InsightsMetadataRepository = InsightsMetadataRepository;
32
+ exports.InsightsMetadataRepository = InsightsMetadataRepository = __decorate([
33
+ (0, di_1.Service)(),
34
+ __metadata("design:paramtypes", [typeorm_1.DataSource])
35
+ ], InsightsMetadataRepository);
36
+ //# sourceMappingURL=insights-metadata.repository.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"insights-metadata.repository.js","sourceRoot":"","sources":["../src/insights-metadata.repository.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,gCAAkC;AAClC,0CAA0D;AAE1D,yEAA8D;AAGvD,IAAM,0BAA0B,GAAhC,MAAM,0BAA2B,SAAQ,oBAA4B;IAC3E,YAAY,UAAsB;QACjC,KAAK,CAAC,2CAAgB,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,QAA4B;QACxD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAElC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,WAAqB;QAC5C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAExC,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,IAAA,YAAE,EAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;CACD,CAAA;;qCAhBY,0BAA0B;IADtC,IAAA,YAAO,GAAE;qCAEe,oBAAU;GADtB,0BAA0B,CAgBtC"}