@solidstarters/solid-core 1.2.150 → 1.2.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dtos/create-scheduled-job.dto.d.ts +2 -0
- package/dist/dtos/create-scheduled-job.dto.d.ts.map +1 -1
- package/dist/dtos/create-scheduled-job.dto.js +13 -1
- package/dist/dtos/create-scheduled-job.dto.js.map +1 -1
- package/dist/dtos/create-user.dto.js +3 -3
- package/dist/dtos/create-user.dto.js.map +1 -1
- package/dist/dtos/update-scheduled-job.dto.d.ts +2 -0
- package/dist/dtos/update-scheduled-job.dto.d.ts.map +1 -1
- package/dist/dtos/update-scheduled-job.dto.js +13 -1
- package/dist/dtos/update-scheduled-job.dto.js.map +1 -1
- package/dist/entities/scheduled-job.entity.d.ts +2 -0
- package/dist/entities/scheduled-job.entity.d.ts.map +1 -1
- package/dist/entities/scheduled-job.entity.js +9 -1
- package/dist/entities/scheduled-job.entity.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/jobs/database/trigger-mcp-client-subscriber-database.service.d.ts.map +1 -1
- package/dist/jobs/database/trigger-mcp-client-subscriber-database.service.js +4 -1
- package/dist/jobs/database/trigger-mcp-client-subscriber-database.service.js.map +1 -1
- package/dist/repository/scheduled-job.repository.d.ts +15 -0
- package/dist/repository/scheduled-job.repository.d.ts.map +1 -0
- package/dist/repository/scheduled-job.repository.js +93 -0
- package/dist/repository/scheduled-job.repository.js.map +1 -0
- package/dist/seeders/module-metadata-seeder.service.d.ts +5 -1
- package/dist/seeders/module-metadata-seeder.service.d.ts.map +1 -1
- package/dist/seeders/module-metadata-seeder.service.js +20 -2
- package/dist/seeders/module-metadata-seeder.service.js.map +1 -1
- package/dist/seeders/seed-data/solid-core-metadata.json +156 -78
- package/dist/services/ai-interaction.service.d.ts +1 -1
- package/dist/services/ai-interaction.service.d.ts.map +1 -1
- package/dist/services/ai-interaction.service.js +4 -3
- package/dist/services/ai-interaction.service.js.map +1 -1
- package/dist/services/computed-fields/entity/concat-entity-computed-field-provider.service.d.ts.map +1 -1
- package/dist/services/computed-fields/entity/concat-entity-computed-field-provider.service.js +16 -12
- package/dist/services/computed-fields/entity/concat-entity-computed-field-provider.service.js.map +1 -1
- package/dist/services/scheduled-jobs/scheduler.service.d.ts.map +1 -1
- package/dist/services/scheduled-jobs/scheduler.service.js +1 -1
- package/dist/services/scheduled-jobs/scheduler.service.js.map +1 -1
- package/dist/services/textract.service.d.ts +20 -0
- package/dist/services/textract.service.d.ts.map +1 -0
- package/dist/services/textract.service.js +199 -0
- package/dist/services/textract.service.js.map +1 -0
- package/dist/solid-core.module.d.ts.map +1 -1
- package/dist/solid-core.module.js +7 -0
- package/dist/solid-core.module.js.map +1 -1
- package/dist/subscribers/scheduled-job.subscriber.d.ts +19 -0
- package/dist/subscribers/scheduled-job.subscriber.d.ts.map +1 -0
- package/dist/subscribers/scheduled-job.subscriber.js +176 -0
- package/dist/subscribers/scheduled-job.subscriber.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -1
- package/rebuild.sh +1 -1
- package/src/dtos/create-scheduled-job.dto.ts +8 -11
- package/src/dtos/create-user.dto.ts +2 -2
- package/src/dtos/update-scheduled-job.dto.ts +8 -12
- package/src/entities/scheduled-job.entity.ts +8 -2
- package/src/index.ts +1 -0
- package/src/jobs/database/trigger-mcp-client-subscriber-database.service.ts +8 -1
- package/src/repository/scheduled-job.repository.ts +105 -0
- package/src/seeders/module-metadata-seeder.service.ts +21 -1
- package/src/seeders/seed-data/solid-core-metadata.json +156 -78
- package/src/services/ai-interaction.service.ts +4 -3
- package/src/services/computed-fields/entity/concat-entity-computed-field-provider.service.ts +28 -18
- package/src/services/scheduled-jobs/scheduler.service.ts +2 -2
- package/src/services/textract.service.ts +189 -0
- package/src/solid-core.module.ts +7 -0
- package/src/subscribers/scheduled-job.subscriber.ts +176 -0
- package/src/# computed field pending issues.md +0 -3
- package/src/services/pending_import_issues +0 -3
- package/src/services/question-data-providers/test.sql +0 -1
- package/test.json +0 -1
|
@@ -43,19 +43,20 @@ export class AiInteractionService extends CRUDService<AiInteraction> {
|
|
|
43
43
|
super(modelMetadataService, moduleMetadataService, configService, fileService, discoveryService, crudHelperService, entityManager, repo, 'aiInteraction', 'solid-core', moduleRef);
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
-
async triggerMcpClientJob(prompt: string): Promise<any> {
|
|
46
|
+
async triggerMcpClientJob(prompt: string, isAutoApply: boolean = false, threadId: string = null): Promise<any> {
|
|
47
47
|
const activeUser: ActiveUserData = this.requestContextService.getActiveUser();
|
|
48
48
|
|
|
49
49
|
const aiInteraction = await this.create({
|
|
50
50
|
userId: activeUser.sub,
|
|
51
|
-
threadId: `thread-${activeUser.sub}`,
|
|
51
|
+
threadId: threadId ? threadId : `thread-${activeUser.sub}`,
|
|
52
52
|
role: 'human',
|
|
53
53
|
message: prompt,
|
|
54
54
|
contentType: '',
|
|
55
55
|
errorMessage: '',
|
|
56
56
|
modelUsed: '',
|
|
57
57
|
responseTimeMs: 0,
|
|
58
|
-
metadata: ''
|
|
58
|
+
metadata: '',
|
|
59
|
+
isAutoApply: isAutoApply
|
|
59
60
|
});
|
|
60
61
|
const m = {
|
|
61
62
|
payload: {
|
package/src/services/computed-fields/entity/concat-entity-computed-field-provider.service.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Injectable } from "@nestjs/common";
|
|
2
|
-
import { kebabCase } from "lodash";
|
|
2
|
+
import { kebabCase, get } from "lodash";
|
|
3
3
|
import { ComputedFieldProvider } from "src/decorators/computed-field-provider.decorator";
|
|
4
4
|
import { CommonEntity } from "src/entities/common.entity";
|
|
5
5
|
import { ComputedFieldMetadata } from "src/helpers/solid-registry";
|
|
@@ -24,28 +24,38 @@ export class ConcatEntityComputedFieldProvider<T extends CommonEntity> implement
|
|
|
24
24
|
return "Computed field provider used to create fields whose value is a concatenation of other fields in the same model.";
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
async preComputeValue(triggerEntity: T, computedFieldMetadata: ComputedFieldMetadata<ConcatComputedFieldContext>){
|
|
27
|
+
async preComputeValue(triggerEntity: T, computedFieldMetadata: ComputedFieldMetadata<ConcatComputedFieldContext>) {
|
|
28
28
|
const { computedFieldValueProviderCtxt } = computedFieldMetadata;
|
|
29
|
-
const separator = computedFieldValueProviderCtxt.separator
|
|
30
|
-
const fields = computedFieldValueProviderCtxt.fields
|
|
31
|
-
const slugify = computedFieldValueProviderCtxt.slugify
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
if (
|
|
40
|
-
|
|
29
|
+
const separator = computedFieldValueProviderCtxt.separator ?? ' ';
|
|
30
|
+
const fields: string[] = computedFieldValueProviderCtxt.fields ?? [];
|
|
31
|
+
const slugify = computedFieldValueProviderCtxt.slugify ?? false;
|
|
32
|
+
|
|
33
|
+
const parts: string[] = [];
|
|
34
|
+
|
|
35
|
+
for (const fieldExpr of fields) {
|
|
36
|
+
let val = get(triggerEntity as any, fieldExpr);
|
|
37
|
+
|
|
38
|
+
// normalize to string (skip null/undefined)
|
|
39
|
+
if (val == null) continue;
|
|
40
|
+
|
|
41
|
+
if (typeof val !== 'string') {
|
|
42
|
+
val = String(val);
|
|
41
43
|
}
|
|
42
44
|
|
|
43
|
-
if (
|
|
44
|
-
|
|
45
|
+
if (slugify) {
|
|
46
|
+
val = kebabCase(val);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// ignore empty strings so you don't get stray separators
|
|
50
|
+
if (val.length > 0) {
|
|
51
|
+
parts.push(val);
|
|
45
52
|
}
|
|
46
|
-
concatenatedString += fieldVal;
|
|
47
53
|
}
|
|
48
|
-
|
|
54
|
+
|
|
55
|
+
const concatenatedString = parts.join(separator);
|
|
56
|
+
|
|
57
|
+
// set the computed field on the entity
|
|
58
|
+
(triggerEntity as any)[computedFieldMetadata.fieldName] = concatenatedString;
|
|
49
59
|
}
|
|
50
60
|
|
|
51
61
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Injectable, Logger, Inject } from '@nestjs/common';
|
|
2
2
|
import { InjectRepository } from '@nestjs/typeorm';
|
|
3
|
-
import { LessThanOrEqual, Repository } from 'typeorm';
|
|
3
|
+
import { IsNull, LessThanOrEqual, Repository } from 'typeorm';
|
|
4
4
|
|
|
5
5
|
import { ISchedulerService } from './scheduler.interface';
|
|
6
6
|
import { SolidRegistry } from 'src/helpers/solid-registry';
|
|
@@ -32,7 +32,7 @@ export class SchedulerServiceImpl implements ISchedulerService {
|
|
|
32
32
|
// Newly created jobs are also picked for examination
|
|
33
33
|
{
|
|
34
34
|
isActive: true,
|
|
35
|
-
nextRunAt:
|
|
35
|
+
nextRunAt: IsNull(),
|
|
36
36
|
},
|
|
37
37
|
],
|
|
38
38
|
});
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
// src/services/textract.service.ts
|
|
2
|
+
import { Inject, Injectable, Logger } from '@nestjs/common';
|
|
3
|
+
import { ConfigType } from '@nestjs/config';
|
|
4
|
+
import commonConfig, { AwsS3Config } from '../config/common.config';
|
|
5
|
+
import {
|
|
6
|
+
TextractClient,
|
|
7
|
+
StartDocumentTextDetectionCommand,
|
|
8
|
+
GetDocumentTextDetectionCommand,
|
|
9
|
+
StartDocumentAnalysisCommand,
|
|
10
|
+
GetDocumentAnalysisCommand,
|
|
11
|
+
Block as TextractBlock,
|
|
12
|
+
} from '@aws-sdk/client-textract';
|
|
13
|
+
import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
|
|
14
|
+
import * as fs from 'node:fs';
|
|
15
|
+
|
|
16
|
+
@Injectable()
|
|
17
|
+
export class TextractService {
|
|
18
|
+
private readonly logger = new Logger(TextractService.name);
|
|
19
|
+
private readonly textractClient: TextractClient;
|
|
20
|
+
private readonly s3Client: S3Client;
|
|
21
|
+
|
|
22
|
+
constructor(
|
|
23
|
+
@Inject(commonConfig.KEY)
|
|
24
|
+
private readonly commonConfiguration: ConfigType<typeof commonConfig>,
|
|
25
|
+
) {
|
|
26
|
+
if (!this.isValidS3Config(this.commonConfiguration.awsS3Credentials)) { return }
|
|
27
|
+
this.s3Client = new S3Client({
|
|
28
|
+
region: this.commonConfiguration.awsS3Credentials.S3_AWS_REGION_NAME,
|
|
29
|
+
credentials: {
|
|
30
|
+
accessKeyId: this.commonConfiguration.awsS3Credentials.S3_AWS_ACCESS_KEY,
|
|
31
|
+
secretAccessKey: this.commonConfiguration.awsS3Credentials.S3_AWS_SECRET_KEY,
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
this.textractClient = new TextractClient({
|
|
36
|
+
region: this.commonConfiguration.awsS3Credentials.S3_AWS_REGION_NAME,
|
|
37
|
+
credentials: {
|
|
38
|
+
accessKeyId: this.commonConfiguration.awsS3Credentials.S3_AWS_ACCESS_KEY,
|
|
39
|
+
secretAccessKey: this.commonConfiguration.awsS3Credentials.S3_AWS_SECRET_KEY,
|
|
40
|
+
},
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
private isValidS3Config(config: AwsS3Config): boolean {
|
|
45
|
+
return !!config.S3_AWS_ACCESS_KEY && !!config.S3_AWS_SECRET_KEY && !!config.S3_AWS_REGION_NAME;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async uploadToS3(localPath: string, bucket: string, key: string) {
|
|
49
|
+
const body = fs.createReadStream(localPath);
|
|
50
|
+
await this.s3Client.send(new PutObjectCommand({
|
|
51
|
+
Bucket: bucket,
|
|
52
|
+
Key: key,
|
|
53
|
+
Body: body,
|
|
54
|
+
ContentType: 'application/pdf',
|
|
55
|
+
}));
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async startTextDetection(bucket: string, key: string): Promise<string> {
|
|
59
|
+
const res = await this.textractClient.send(new StartDocumentTextDetectionCommand({
|
|
60
|
+
DocumentLocation: { S3Object: { Bucket: bucket, Name: key } },
|
|
61
|
+
}));
|
|
62
|
+
if (!res.JobId) throw new Error('Failed to start Textract job');
|
|
63
|
+
return res.JobId;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Polls Textract until SUCCEEDED and returns page chunks (with pagination handled).
|
|
68
|
+
* Returns an array where each item corresponds to one GetDocumentTextDetection response page.
|
|
69
|
+
*/
|
|
70
|
+
async getAllPages(jobId: string) {
|
|
71
|
+
// poll
|
|
72
|
+
let status = 'IN_PROGRESS';
|
|
73
|
+
while (status === 'IN_PROGRESS') {
|
|
74
|
+
await this.sleep(3000);
|
|
75
|
+
const head = await this.textractClient.send(new GetDocumentTextDetectionCommand({ JobId: jobId, MaxResults: 1 }));
|
|
76
|
+
status = head.JobStatus || 'IN_PROGRESS';
|
|
77
|
+
if (status === 'FAILED') throw new Error('Textract job failed');
|
|
78
|
+
if (status === 'SUCCEEDED') break;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// paginate
|
|
82
|
+
const pages: any[] = [];
|
|
83
|
+
let nextToken: string | undefined = undefined;
|
|
84
|
+
do {
|
|
85
|
+
const res = await this.textractClient.send(new GetDocumentTextDetectionCommand({
|
|
86
|
+
JobId: jobId,
|
|
87
|
+
NextToken: nextToken,
|
|
88
|
+
}));
|
|
89
|
+
pages.push(res);
|
|
90
|
+
nextToken = res.NextToken;
|
|
91
|
+
} while (nextToken);
|
|
92
|
+
|
|
93
|
+
return pages;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
private sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); }
|
|
97
|
+
|
|
98
|
+
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
|
99
|
+
// New set of methods used for document analysis, not just text detection
|
|
100
|
+
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
|
101
|
+
/**
|
|
102
|
+
* Start async DocumentAnalysis (better for specs: TABLES + FORMS).
|
|
103
|
+
*/
|
|
104
|
+
async startDocumentAnalysis(bucket: string, key: string, featureTypes: ('TABLES' | 'FORMS')[] = ['TABLES', 'FORMS']): Promise<string> {
|
|
105
|
+
const res = await this.textractClient.send(new StartDocumentAnalysisCommand({
|
|
106
|
+
DocumentLocation: { S3Object: { Bucket: bucket, Name: key } },
|
|
107
|
+
FeatureTypes: featureTypes,
|
|
108
|
+
}));
|
|
109
|
+
if (!res.JobId) throw new Error('Failed to start Textract analysis job');
|
|
110
|
+
return res.JobId;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Wait for analysis job to finish, then fetch ALL pages (handles NextToken).
|
|
115
|
+
* Returns a flat array of Blocks from all result pages.
|
|
116
|
+
*/
|
|
117
|
+
async getAllAnalysisBlocks(jobId: string): Promise<TextractBlock[]> {
|
|
118
|
+
// poll
|
|
119
|
+
let status = 'IN_PROGRESS';
|
|
120
|
+
while (status === 'IN_PROGRESS') {
|
|
121
|
+
await this.sleep(3000);
|
|
122
|
+
const head = await this.textractClient.send(new GetDocumentAnalysisCommand({ JobId: jobId, MaxResults: 1 }));
|
|
123
|
+
status = head.JobStatus || 'IN_PROGRESS';
|
|
124
|
+
if (status === 'FAILED') throw new Error('Textract analysis job failed');
|
|
125
|
+
if (status === 'SUCCEEDED') break;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// paginate + collect blocks
|
|
129
|
+
const blocks: TextractBlock[] = [];
|
|
130
|
+
let nextToken: string | undefined = undefined;
|
|
131
|
+
do {
|
|
132
|
+
const res = await this.textractClient.send(new GetDocumentAnalysisCommand({
|
|
133
|
+
JobId: jobId,
|
|
134
|
+
NextToken: nextToken,
|
|
135
|
+
}));
|
|
136
|
+
if (res.Blocks?.length) blocks.push(...res.Blocks);
|
|
137
|
+
nextToken = res.NextToken;
|
|
138
|
+
} while (nextToken);
|
|
139
|
+
|
|
140
|
+
return blocks;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Collate LINE blocks into page-wise plain text,
|
|
145
|
+
* sorted roughly top-to-bottom then left-to-right for better reading order.
|
|
146
|
+
*/
|
|
147
|
+
collatePageWiseTextFromBlocks(blocks: TextractBlock[]): Record<string, string> {
|
|
148
|
+
const byPage = new Map<number, TextractBlock[]>();
|
|
149
|
+
|
|
150
|
+
for (const b of blocks ?? []) {
|
|
151
|
+
if (b.BlockType === 'LINE' && b.Text) {
|
|
152
|
+
const page = (b as any).Page ?? 1;
|
|
153
|
+
if (!byPage.has(page)) byPage.set(page, []);
|
|
154
|
+
byPage.get(page)!.push(b);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const pages = Array.from(byPage.keys()).sort((a, b) => a - b);
|
|
159
|
+
const result: Record<string, string> = {};
|
|
160
|
+
|
|
161
|
+
for (const p of pages) {
|
|
162
|
+
const lines = byPage.get(p)!;
|
|
163
|
+
lines.sort((a: any, b: any) => {
|
|
164
|
+
const at = a.Geometry?.BoundingBox?.Top ?? 0;
|
|
165
|
+
const bt = b.Geometry?.BoundingBox?.Top ?? 0;
|
|
166
|
+
if (Math.abs(at - bt) > 0.002) return at - bt; // row order
|
|
167
|
+
const al = a.Geometry?.BoundingBox?.Left ?? 0;
|
|
168
|
+
const bl = b.Geometry?.BoundingBox?.Left ?? 0;
|
|
169
|
+
return al - bl; // within row
|
|
170
|
+
});
|
|
171
|
+
result[`page_${p}`] = lines.map((l: any) => l.Text).join('\n');
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return result;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Convenience: Run recommended flow end-to-end on a PDF in S3
|
|
179
|
+
* and get `{ page_1: "...", page_2: "..." }`.
|
|
180
|
+
*/
|
|
181
|
+
async analyzePdfInS3ToPageWiseText(bucket: string, key: string): Promise<Record<string, string>> {
|
|
182
|
+
this.logger.debug(`Starting DocumentAnalysis for s3://${bucket}/${key}`);
|
|
183
|
+
const jobId = await this.startDocumentAnalysis(bucket, key, ['TABLES', 'FORMS']);
|
|
184
|
+
const blocks = await this.getAllAnalysisBlocks(jobId);
|
|
185
|
+
const pageWise = this.collatePageWiseTextFromBlocks(blocks);
|
|
186
|
+
this.logger.debug(`Completed DocumentAnalysis for s3://${bucket}/${key} with ${Object.keys(pageWise).length} pages`);
|
|
187
|
+
return pageWise;
|
|
188
|
+
}
|
|
189
|
+
}
|
package/src/solid-core.module.ts
CHANGED
|
@@ -263,10 +263,13 @@ import { SolidAddFieldMcpToolResponseHandler } from './services/mcp-tool-respons
|
|
|
263
263
|
import { ViewMetadataRepository } from './repository/view-metadata.repository';
|
|
264
264
|
import { SolidCreateModelLayoutMcpToolResponseHandler } from './services/mcp-tool-response-handlers/solid-save-model-layout-mcp-tool-response-handler.service';
|
|
265
265
|
import { NoopsEntityComputedFieldProviderService } from './services/computed-fields/entity/noops-entity-computed-field-provider.service';
|
|
266
|
+
import { ScheduledJobRepository } from './repository/scheduled-job.repository';
|
|
267
|
+
import { ScheduledJobSubscriber } from './subscribers/scheduled-job.subscriber';
|
|
266
268
|
import { AlphaNumExternalIdComputationProvider } from './services/computed-fields/entity/alpha-num-external-id-computed-field-provider';
|
|
267
269
|
import { MailFactory } from './factories/mail.factory';
|
|
268
270
|
import { TwilioSMSService } from './services/sms/TwilioSMSService';
|
|
269
271
|
import { PollerService } from './services/poller.service';
|
|
272
|
+
import { TextractService } from './services/textract.service';
|
|
270
273
|
|
|
271
274
|
|
|
272
275
|
@Global()
|
|
@@ -427,6 +430,7 @@ import { PollerService } from './services/poller.service';
|
|
|
427
430
|
Reflector,
|
|
428
431
|
MetadataScanner,
|
|
429
432
|
FileService,
|
|
433
|
+
TextractService,
|
|
430
434
|
SolidRegistry,
|
|
431
435
|
SeedCommand,
|
|
432
436
|
SMTPEMailService,
|
|
@@ -559,6 +563,8 @@ import { PollerService } from './services/poller.service';
|
|
|
559
563
|
SolidAddFieldMcpToolResponseHandler,
|
|
560
564
|
ViewMetadataRepository,
|
|
561
565
|
SolidCreateModelLayoutMcpToolResponseHandler,
|
|
566
|
+
ScheduledJobRepository,
|
|
567
|
+
ScheduledJobSubscriber,
|
|
562
568
|
AlphaNumExternalIdComputationProvider,
|
|
563
569
|
MailFactory,
|
|
564
570
|
],
|
|
@@ -573,6 +579,7 @@ import { PollerService } from './services/poller.service';
|
|
|
573
579
|
CRUDService,
|
|
574
580
|
MulterModule,
|
|
575
581
|
FileService,
|
|
582
|
+
TextractService,
|
|
576
583
|
SolidRegistry,
|
|
577
584
|
SMTPEMailService,
|
|
578
585
|
ElasticEmailService,
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { Injectable, Logger } from "@nestjs/common";
|
|
2
|
+
import { InjectDataSource } from "@nestjs/typeorm";
|
|
3
|
+
import * as fs from "fs/promises";
|
|
4
|
+
import { ModuleMetadata } from "src/entities/module-metadata.entity";
|
|
5
|
+
import { ScheduledJob } from "src/entities/scheduled-job.entity";
|
|
6
|
+
import { ModuleMetadataHelperService } from "src/helpers/module-metadata-helper.service";
|
|
7
|
+
import { ScheduledJobRepository } from "src/repository/scheduled-job.repository";
|
|
8
|
+
import {
|
|
9
|
+
DataSource,
|
|
10
|
+
EntityManager,
|
|
11
|
+
EntitySubscriberInterface,
|
|
12
|
+
InsertEvent,
|
|
13
|
+
RemoveEvent,
|
|
14
|
+
UpdateEvent,
|
|
15
|
+
} from "typeorm";
|
|
16
|
+
|
|
17
|
+
@Injectable()
|
|
18
|
+
export class ScheduledJobSubscriber
|
|
19
|
+
implements EntitySubscriberInterface<ScheduledJob> {
|
|
20
|
+
private readonly logger = new Logger(ScheduledJobSubscriber.name);
|
|
21
|
+
|
|
22
|
+
/** Fields that, when changed (and only these changed), should NOT trigger metadata update. */
|
|
23
|
+
private readonly ignoredUpdateFields: Array<keyof ScheduledJob | string> = [
|
|
24
|
+
"lastRunAt",
|
|
25
|
+
"nextRunAt",
|
|
26
|
+
"updatedAt",
|
|
27
|
+
];
|
|
28
|
+
|
|
29
|
+
constructor(
|
|
30
|
+
@InjectDataSource() private readonly dataSource: DataSource,
|
|
31
|
+
private readonly moduleMetadataHelperService: ModuleMetadataHelperService,
|
|
32
|
+
private readonly scheduledJobRepo: ScheduledJobRepository
|
|
33
|
+
) {
|
|
34
|
+
this.dataSource.subscribers.push(this);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
listenTo() {
|
|
38
|
+
return ScheduledJob;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async afterInsert(event: InsertEvent<ScheduledJob>) {
|
|
42
|
+
if (!event.entity) {
|
|
43
|
+
this.logger.debug('No schedule Job entity found in the afterInsert method');
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
await this.updateMetadata(event.entity, event.queryRunner.manager);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async afterUpdate(event: UpdateEvent<ScheduledJob>) {
|
|
50
|
+
if (!event.databaseEntity) {
|
|
51
|
+
this.logger.debug('No schedule Job entity found in the afterUpdate method');
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// get hold of the changed field names
|
|
56
|
+
const changedProps = (event.updatedColumns ?? []).map((c) => c.propertyName);
|
|
57
|
+
|
|
58
|
+
// Decide whether to skip: only skip when *all* changed fields are in the ignore list
|
|
59
|
+
const onlyIgnoredChanged = changedProps.every((p) => this.ignoredUpdateFields.includes(p));
|
|
60
|
+
|
|
61
|
+
if (onlyIgnoredChanged) {
|
|
62
|
+
this.logger.debug(`Skipping metadata update for ScheduledJob#${(event.databaseEntity as ScheduledJob).id}; only ignored fields changed: ${changedProps.join(", ")}`
|
|
63
|
+
);
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
await this.updateMetadata(event.databaseEntity, event.queryRunner.manager);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async afterRemove(event: RemoveEvent<ScheduledJob>) {
|
|
71
|
+
await this.removeMetadata(event);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
private async removeMetadata(event: RemoveEvent<ScheduledJob>) {
|
|
75
|
+
const jobEntity = event.entity;
|
|
76
|
+
const moduleMetadata = jobEntity?.module;
|
|
77
|
+
|
|
78
|
+
if (!moduleMetadata) {
|
|
79
|
+
this.logger.error(
|
|
80
|
+
`Module metadata not found for scheduled job with ID ${jobEntity?.id}`
|
|
81
|
+
);
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const moduleMetadataRepo = this.dataSource.getRepository(ModuleMetadata);
|
|
86
|
+
const populatedModuleMetadata = await moduleMetadataRepo.findOne({
|
|
87
|
+
where: { id: moduleMetadata.id },
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
if (!populatedModuleMetadata) {
|
|
91
|
+
this.logger.error(
|
|
92
|
+
`Could not find ModuleMetadata with ID ${moduleMetadata.id}`
|
|
93
|
+
);
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
const filePath =
|
|
97
|
+
await this.moduleMetadataHelperService.getModuleMetadataFilePath(
|
|
98
|
+
populatedModuleMetadata.name
|
|
99
|
+
);
|
|
100
|
+
try {
|
|
101
|
+
await fs.access(filePath);
|
|
102
|
+
} catch {
|
|
103
|
+
this.logger.error(`Metadata file not found: ${filePath}`);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const metaData =
|
|
107
|
+
await this.moduleMetadataHelperService.getModuleMetadataConfiguration(
|
|
108
|
+
filePath
|
|
109
|
+
);
|
|
110
|
+
// Remove, update or insert logic
|
|
111
|
+
const jobName = jobEntity.scheduleName;
|
|
112
|
+
const existingIndex = metaData.scheduledJobs.findIndex(
|
|
113
|
+
(job) => job.scheduleName === jobName
|
|
114
|
+
);
|
|
115
|
+
if (existingIndex !== -1) {
|
|
116
|
+
metaData.scheduledJobs.splice(existingIndex, 1);
|
|
117
|
+
this.logger.log(`Removed scheduled job ${jobName} from metadata`);
|
|
118
|
+
}
|
|
119
|
+
const updatedContent = JSON.stringify(metaData, null, 2);
|
|
120
|
+
await fs.writeFile(filePath, updatedContent);
|
|
121
|
+
this.logger.log(`Updated scheduledJobs in ${filePath}`);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
private async updateMetadata(jobEntity: ScheduledJob, entityManager: EntityManager) {
|
|
125
|
+
// populate the job with its relation
|
|
126
|
+
const populatedScheduleJob = await entityManager.findOne(ScheduledJob, {
|
|
127
|
+
where: { id: jobEntity.id },
|
|
128
|
+
relations: ['module'],
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
if (!populatedScheduleJob) {
|
|
132
|
+
throw new Error(`ScheduleJob not found for id ${jobEntity.id}`);
|
|
133
|
+
}
|
|
134
|
+
const filePath =
|
|
135
|
+
await this.moduleMetadataHelperService.getModuleMetadataFilePath(
|
|
136
|
+
populatedScheduleJob.module?.name
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
try {
|
|
140
|
+
await fs.access(filePath);
|
|
141
|
+
} catch {
|
|
142
|
+
this.logger.error(`Metadata file not found: ${filePath}`);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const metaData =
|
|
147
|
+
await this.moduleMetadataHelperService.getModuleMetadataConfiguration(
|
|
148
|
+
filePath
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
// Ensure scheduledJobs exists
|
|
152
|
+
if (!metaData.scheduledJobs) {
|
|
153
|
+
metaData.scheduledJobs = [];
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Remove, update or insert logic
|
|
157
|
+
const jobName = jobEntity.scheduleName;
|
|
158
|
+
const existingIndex = metaData.scheduledJobs.findIndex(
|
|
159
|
+
(job) => job.scheduleName === jobName
|
|
160
|
+
);
|
|
161
|
+
// Insert or update job in metadata
|
|
162
|
+
const jobDto = await this.scheduledJobRepo.toDto(populatedScheduleJob as ScheduledJob);
|
|
163
|
+
const {moduleId, ...dtoToWrite} = jobDto
|
|
164
|
+
if (existingIndex !== -1) {
|
|
165
|
+
metaData.scheduledJobs[existingIndex] = dtoToWrite;
|
|
166
|
+
this.logger.log(`Updated scheduled job ${jobName} in metadata`);
|
|
167
|
+
} else {
|
|
168
|
+
metaData.scheduledJobs.push(dtoToWrite);
|
|
169
|
+
this.logger.log(`Added scheduled job ${jobName} to metadata`);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const updatedContent = JSON.stringify(metaData, null, 2);
|
|
173
|
+
await fs.writeFile(filePath, updatedContent);
|
|
174
|
+
this.logger.log(`Updated scheduledJobs in ${filePath}`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
SELECT TO_CHAR(DATE_TRUNC('month', created_at), 'Mon-YYYY') AS label,FROM public.sapphire_clientwhere created_at >= $1 GROUP BY DATE_TRUNC('month', created_at) ORDER BY DATE_TRUNC('month', created_at)
|
package/test.json
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"selectionValueType":"string","id":1367,"createdAt":"2025-06-26T23:41:13.508Z","updatedAt":"2025-06-26T23:41:13.508Z","deletedAt":null,"deletedTracker":"not-deleted","publishedAt":null,"localeName":null,"defaultEntityLocaleId":null,"name":"variableName","displayName":"Variable Name","description":null,"type":"shortText","ormType":"varchar","defaultValue":null,"regexPattern":null,"regexPatternNotMatchingErrorMsg":null,"required":true,"unique":true,"encrypt":false,"encryptionType":null,"decryptWhen":null,"index":true,"length":256,"max":null,"min":null,"private":false,"mediaTypes":null,"mediaMaxSizeKb":null,"relationType":null,"relationCoModelSingularName":null,"relationCreateInverse":false,"relationCascade":null,"relationModelModuleName":null,"relationCoModelFieldName":null,"isRelationManyToManyOwner":null,"relationFieldFixedFilter":null,"selectionDynamicProvider":null,"selectionDynamicProviderCtxt":null,"selectionStaticValues":null,"computedFieldValueProvider":null,"computedFieldValueProviderCtxt":null,"computedFieldValueType":null,"computedFieldTriggerConfig":null,"uuid":null,"isSystem":true,"isMarkedForRemoval":false,"columnName":null,"relationCoModelColumnName":null,"isUserKey":true,"relationJoinTableName":null,"enableAuditTracking":false,"isMultiSelect":false}' --fields='{"selectionValueType":"string","id":1368,"createdAt":"2025-06-26T23:41:13.515Z","updatedAt":"2025-06-26T23:41:13.515Z","deletedAt":null,"deletedTracker":"not-deleted","publishedAt":null,"localeName":null,"defaultEntityLocaleId":null,"name":"variableType","displayName":"Variable Type","description":null,"type":"selectionStatic","ormType":"varchar","defaultValue":null,"regexPattern":null,"regexPatternNotMatchingErrorMsg":null,"required":true,"unique":false,"encrypt":false,"encryptionType":null,"decryptWhen":null,"index":true,"length":256,"max":null,"min":null,"private":false,"mediaTypes":null,"mediaMaxSizeKb":null,"relationType":null,"relationCoModelSingularName":null,"relationCreateInverse":false,"relationCascade":null,"relationModelModuleName":null,"relationCoModelFieldName":null,"isRelationManyToManyOwner":null,"relationFieldFixedFilter":null,"selectionDynamicProvider":null,"selectionDynamicProviderCtxt":null,"selectionStaticValues":["date:Date","selectionStatic:Selection Static","selectionDynamic:Selection Dynamic"],"computedFieldValueProvider":null,"computedFieldValueProviderCtxt":null,"computedFieldValueType":null,"computedFieldTriggerConfig":null,"uuid":null,"isSystem":true,"isMarkedForRemoval":false,"columnName":null,"relationCoModelColumnName":null,"isUserKey":false,"relationJoinTableName":null,"enableAuditTracking":false,"isMultiSelect":false}' --fields='{"selectionValueType":"string","id":1369,"createdAt":"2025-06-26T23:41:13.521Z","updatedAt":"2025-06-26T23:41:13.521Z","deletedAt":null,"deletedTracker":"not-deleted","publishedAt":null,"localeName":null,"defaultEntityLocaleId":null,"name":"selectionStaticValues","displayName":"Selection Static Values","description":null,"type":"json","ormType":"jsonb","defaultValue":null,"regexPattern":null,"regexPatternNotMatchingErrorMsg":null,"required":false,"unique":false,"encrypt":false,"encryptionType":null,"decryptWhen":null,"index":false,"length":null,"max":null,"min":null,"private":false,"mediaTypes":null,"mediaMaxSizeKb":null,"relationType":null,"relationCoModelSingularName":null,"relationCreateInverse":false,"relationCascade":null,"relationModelModuleName":null,"relationCoModelFieldName":null,"isRelationManyToManyOwner":null,"relationFieldFixedFilter":null,"selectionDynamicProvider":null,"selectionDynamicProviderCtxt":null,"selectionStaticValues":null,"computedFieldValueProvider":null,"computedFieldValueProviderCtxt":null,"computedFieldValueType":null,"computedFieldTriggerConfig":null,"uuid":null,"isSystem":true,"isMarkedForRemoval":false,"columnName":null,"relationCoModelColumnName":null,"isUserKey":false,"relationJoinTableName":null,"enableAuditTracking":false,"isMultiSelect":false}' --fields='{"selectionValueType":"string","id":1370,"createdAt":"2025-06-26T23:41:13.527Z","updatedAt":"2025-06-26T23:41:13.527Z","deletedAt":null,"deletedTracker":"not-deleted","publishedAt":null,"localeName":null,"defaultEntityLocaleId":null,"name":"selectionDynamicSourceType","displayName":"Selection Dynamic Source Type","description":null,"type":"selectionStatic","ormType":"","defaultValue":null,"regexPattern":null,"regexPatternNotMatchingErrorMsg":null,"required":false,"unique":false,"encrypt":false,"encryptionType":null,"decryptWhen":null,"index":false,"length":256,"max":null,"min":null,"private":false,"mediaTypes":null,"mediaMaxSizeKb":null,"relationType":null,"relationCoModelSingularName":null,"relationCreateInverse":false,"relationCascade":null,"relationModelModuleName":null,"relationCoModelFieldName":null,"isRelationManyToManyOwner":null,"relationFieldFixedFilter":null,"selectionDynamicProvider":null,"selectionDynamicProviderCtxt":null,"selectionStaticValues":["sql:SQL","provider:Provider"],"computedFieldValueProvider":null,"computedFieldValueProviderCtxt":null,"computedFieldValueType":null,"computedFieldTriggerConfig":null,"uuid":null,"isSystem":true,"isMarkedForRemoval":false,"columnName":null,"relationCoModelColumnName":null,"isUserKey":false,"relationJoinTableName":null,"enableAuditTracking":false,"isMultiSelect":false}' --fields='{"selectionValueType":"string","id":1371,"createdAt":"2025-06-26T23:41:13.535Z","updatedAt":"2025-06-26T23:41:13.535Z","deletedAt":null,"deletedTracker":"not-deleted","publishedAt":null,"localeName":null,"defaultEntityLocaleId":null,"name":"selectionDynamicSQL","displayName":"Selection Dynamic SQL","description":"SQL query to fetch the data for this variable when it is rendered at runtime. This is only applicable when selectionDynamicSourceType is set to SQL.","type":"longText","ormType":"text","defaultValue":null,"regexPattern":null,"regexPatternNotMatchingErrorMsg":null,"required":false,"unique":false,"encrypt":false,"encryptionType":null,"decryptWhen":null,"index":false,"length":null,"max":null,"min":null,"private":false,"mediaTypes":null,"mediaMaxSizeKb":null,"relationType":null,"relationCoModelSingularName":null,"relationCreateInverse":false,"relationCascade":null,"relationModelModuleName":null,"relationCoModelFieldName":null,"isRelationManyToManyOwner":null,"relationFieldFixedFilter":null,"selectionDynamicProvider":null,"selectionDynamicProviderCtxt":null,"selectionStaticValues":null,"computedFieldValueProvider":null,"computedFieldValueProviderCtxt":null,"computedFieldValueType":null,"computedFieldTriggerConfig":null,"uuid":null,"isSystem":true,"isMarkedForRemoval":false,"columnName":null,"relationCoModelColumnName":null,"isUserKey":false,"relationJoinTableName":null,"enableAuditTracking":false,"isMultiSelect":false}' --fields='{"selectionValueType":"string","id":1372,"createdAt":"2025-06-26T23:41:13.541Z","updatedAt":"2025-06-26T23:41:13.541Z","deletedAt":null,"deletedTracker":"not-deleted","publishedAt":null,"localeName":null,"defaultEntityLocaleId":null,"name":"selectionDynamicProviderName","displayName":"Selection Dynamic Provider Name","description":"This is only applicable when selectionDynamicSourceType is set to provider. It allows the user to select any pre-existing SelectionDynamicProvider implementation used to fetch a dynamic dropdown of values to choose from when this variable is presented to the user.","type":"selectionDynamic","ormType":"varchar","defaultValue":null,"regexPattern":null,"regexPatternNotMatchingErrorMsg":null,"required":false,"unique":false,"encrypt":false,"encryptionType":null,"decryptWhen":null,"index":false,"length":256,"max":null,"min":null,"private":false,"mediaTypes":null,"mediaMaxSizeKb":null,"relationType":null,"relationCoModelSingularName":null,"relationCreateInverse":false,"relationCascade":null,"relationModelModuleName":null,"relationCoModelFieldName":null,"isRelationManyToManyOwner":null,"relationFieldFixedFilter":null,"selectionDynamicProvider":"SelectionDynamicProvider","selectionDynamicProviderCtxt":"{}","selectionStaticValues":null,"computedFieldValueProvider":null,"computedFieldValueProviderCtxt":null,"computedFieldValueType":null,"computedFieldTriggerConfig":null,"uuid":null,"isSystem":true,"isMarkedForRemoval":false,"columnName":null,"relationCoModelColumnName":null,"isUserKey":false,"relationJoinTableName":null,"enableAuditTracking":false,"isMultiSelect":false}' --fields='{"selectionValueType":"string","id":1373,"createdAt":"2025-06-26T23:41:13.548Z","updatedAt":"2025-06-26T23:41:13.548Z","deletedAt":null,"deletedTracker":"not-deleted","publishedAt":null,"localeName":null,"defaultEntityLocaleId":null,"name":"isMultiSelect","displayName":"Is Multi Select","description":"This is relevant only for variables of type \'selectionStatic\' or \'selectionDynamic\'. When set to true, it allows the user to select multiple values from the dropdown.","type":"boolean","ormType":"boolean","defaultValue":"false","regexPattern":null,"regexPatternNotMatchingErrorMsg":null,"required":false,"unique":false,"encrypt":false,"encryptionType":null,"decryptWhen":null,"index":false,"length":null,"max":null,"min":null,"private":false,"mediaTypes":null,"mediaMaxSizeKb":null,"relationType":null,"relationCoModelSingularName":null,"relationCreateInverse":false,"relationCascade":null,"relationModelModuleName":null,"relationCoModelFieldName":null,"isRelationManyToManyOwner":null,"relationFieldFixedFilter":null,"selectionDynamicProvider":null,"selectionDynamicProviderCtxt":null,"selectionStaticValues":null,"computedFieldValueProvider":null,"computedFieldValueProviderCtxt":null,"computedFieldValueType":null,"computedFieldTriggerConfig":null,"uuid":null,"isSystem":true,"isMarkedForRemoval":false,"columnName":null,"relationCoModelColumnName":null,"isUserKey":false,"relationJoinTableName":null,"enableAuditTracking":false,"isMultiSelect":false}' --fields='{"selectionValueType":"string","id":1374,"createdAt":"2025-06-26T23:41:13.554Z","updatedAt":"2025-06-26T23:41:13.554Z","deletedAt":null,"deletedTracker":"not-deleted","publishedAt":null,"localeName":null,"defaultEntityLocaleId":null,"name":"dashboard","displayName":"Dashboard","description":"Related Dashboard Model","type":"relation","ormType":"integer","defaultValue":null,"regexPattern":null,"regexPatternNotMatchingErrorMsg":null,"required":false,"unique":false,"encrypt":false,"encryptionType":null,"decryptWhen":null,"index":false,"length":null,"max":null,"min":null,"private":false,"mediaTypes":null,"mediaMaxSizeKb":null,"relationType":"many-to-one","relationCoModelSingularName":"dashboard","relationCreateInverse":true,"relationCascade":"cascade","relationModelModuleName":"solid-core","relationCoModelFieldName":"dashboardVariables","isRelationManyToManyOwner":null,"relationFieldFixedFilter":null,"selectionDynamicProvider":null,"selectionDynamicProviderCtxt":null,"selectionStaticValues":null,"computedFieldValueProvider":null,"computedFieldValueProviderCtxt":null,"computedFieldValueType":null,"computedFieldTriggerConfig":null,"uuid":null,"isSystem":true,"isMarkedForRemoval":false,"columnName":null,"relationCoModelColumnName":null,"isUserKey":false,"relationJoinTableName":null,"enableAuditTracking":false,"isMultiSelect":false}
|