@solidxai/core 0.1.10-beta.25 → 0.1.10-beta.27

package/src/services/crud-helper.service.ts

@@ -284,8 +284,10 @@ export class CrudHelperService {
                     const value = orderOptions[key] as 'ASC' | 'DESC';
                     const field = String(key);
                     if (field.includes('.')) {
-                        const { alias, property } = this.ensureRelationPathJoined(qb, entityAlias, field.split('.'));
-                        qb.addOrderBy(`${alias}.${property}`, value);
+                        const { alias, property, created } = this.ensureRelationPathJoined(qb, entityAlias, field.split('.'));
+                        const orderColumn = `${alias}.${property}`;
+                        qb.addOrderBy(orderColumn, value);
+                        if (created) qb.addSelect(orderColumn);
                     } else {
                         qb.addOrderBy(`${entityAlias}.${field}`, value);
                     }
@@ -347,6 +349,7 @@ export class CrudHelperService {
             qb.expressionMap?.aliases?.find(a => a.metadata)?.name ||
             qb.expressionMap?.aliases?.[0]?.name;
         let parentAlias = mainAlias || rootAlias;
+        let leafJoinCreated = false;
         for (let i = 0; i < pathParts.length - 1; i++) {
             const part = pathParts[i];
             const joinProperty = `${parentAlias}.${part}`;
@@ -354,10 +357,13 @@ export class CrudHelperService {
             const joinAlias = existingAlias ?? this.sanitizeAlias(`${parentAlias}_${part}`);
             if (!existingAlias && !this.isRelationJoined(qb, joinProperty) && !this.isAliasJoined(qb, joinAlias)) {
                 qb.leftJoin(joinProperty, joinAlias);
+                leafJoinCreated = true;
+            } else {
+                leafJoinCreated = false;
             }
             parentAlias = joinAlias;
         }
-        return { alias: parentAlias, property: pathParts[pathParts.length - 1] };
+        return { alias: parentAlias, property: pathParts[pathParts.length - 1], created: leafJoinCreated };
     }
 
     private getDriver(qb: SelectQueryBuilder<any>) {
@@ -749,4 +755,76 @@ export class CrudHelperService {
         return matchingPermssions.length > 0
     }
 
+    pagedResponse<T>(offset: number | undefined, limit: number | undefined, count: number, entities: T[]) {
+        const safeLimit = limit ?? count ?? 0;
+        const safeOffset = offset ?? 0;
+        const currentPage = safeLimit ? Math.floor(safeOffset / safeLimit) + 1 : 1;
+        const totalPages = safeLimit ? Math.ceil(count / safeLimit) : 1;
+        const nextPage = safeLimit && currentPage < totalPages ? currentPage + 1 : null;
+        const prevPage = safeLimit && currentPage > 1 ? currentPage - 1 : null;
+        return {
+            meta: {
+                totalRecords: count,
+                currentPage,
+                nextPage,
+                prevPage,
+                totalPages,
+                perPage: safeLimit ? +safeLimit : 0,
+            },
+            records: entities,
+        };
+    }
+
+    async executeGroupPipeline<T>(
+        filterQb: SelectQueryBuilder<T>,
+        basicFilterDto: BasicFilterDto,
+        alias: string,
+        createQbFn: () => Promise<SelectQueryBuilder<T>>,
+        postProcessEntities?: (entities: T[]) => Promise<void>
+    ): Promise<{ meta: { totalRecords: number }; groupMeta: any[]; groupRecords: any[] }> {
+        const groupByFields = this.normalize(basicFilterDto.groupBy);
+        if (!groupByFields.length) throw new BadRequestException(ERROR_MESSAGES.INVALID_GROUP_BY_COUNT);
+
+        if (basicFilterDto.populateGroup) {
+            const hasRelationGroup = groupByFields.some(f => f.includes('.'));
+            if (hasRelationGroup) throw new BadRequestException('populateGroup is not supported when grouping on relation fields. Fetch group metadata first and retrieve records in a separate call.');
+        }
+
+        const { aliasMap: groupAliasMap, formatMap: groupFormatMap, expressionMap: groupExpressionMap } =
+            this.applyGroupBySelections(filterQb, groupByFields, alias);
+        const aggregateAliasMap = this.applyAggregates(filterQb, basicFilterDto.aggregates, alias);
+        this.applyGroupSortingAndPagination(filterQb, basicFilterDto.sort, { ...groupAliasMap, ...aggregateAliasMap }, basicFilterDto.limit, basicFilterDto.offset);
+
+        const groupByResult = await filterQb.getRawMany();
+        const totalGroups = await this.countGroups(filterQb);
+        const aggregateAliasSet = new Set(Object.values(aggregateAliasMap));
+
+        const groupMeta = [];
+        const groupRecords = [];
+
+        for (const group of groupByResult) {
+            groupMeta.push(this.createGroupMeta(group, aggregateAliasSet, groupByFields, groupAliasMap, groupFormatMap));
+
+            if (basicFilterDto.populateGroup) {
+                let groupQb = await createQbFn();
+                const { groupBy: _gb, aggregates: _agg, ...rest } = basicFilterDto;
+                const groupFilterDto: BasicFilterDto = {
+                    ...rest,
+                    ...basicFilterDto.groupFilter,
+                    groupBy: undefined,
+                    aggregates: undefined,
+                    sort: basicFilterDto.groupFilter?.sort,
+                };
+                groupQb = this.buildFilterQuery(groupQb, groupFilterDto, alias);
+                groupQb = this.buildGroupByRecordsQuery(groupQb, group, alias, groupAliasMap, aggregateAliasMap, groupExpressionMap);
+                const [entities, count] = await groupQb.getManyAndCount();
+                if (postProcessEntities) await postProcessEntities(entities);
+                const groupData = this.pagedResponse(basicFilterDto.groupFilter?.offset, basicFilterDto.groupFilter?.limit, count, entities);
+                groupRecords.push(this.createGroupRecords(group, aggregateAliasSet, groupData, groupByFields, groupAliasMap, groupFormatMap));
+            }
+        }
+
+        return { meta: { totalRecords: totalGroups }, groupMeta, groupRecords };
+    }
+
 }

package/src/services/crud.service.ts

@@ -41,7 +41,6 @@ import { SolidRegistry } from "src/helpers/solid-registry";
 import { getMediaStorageProvider } from "./mediaStorageProviders";
 import { ModelMetadataService } from "./model-metadata.service";
 import { RequestContextService } from "./request-context.service";
-import { BasicGroupFilterDto } from "src/dtos/basic-group-filters.dto";
 
 
 export class CRUDService<T extends CommonEntity> { // Add two generic value i.e Person,CreatePersonDto, so we get the proper types in our service
@@ -480,10 +479,10 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
         return false; // If it is not a partial update, then do not skip computation
     }
 
-    async find(basicFilterDto: BasicFilterDto, solidRequestContext: any = {}) {
+    async find(basicFilterDto: BasicFilterDto, solidRequestContext: any = {}): Promise<any> {
         const alias = 'entity';
         // Extract the required keys from the input query
-        let { limit, offset, populateMedia, populateGroup, groupFilter } = basicFilterDto;
+        let { limit, offset, populateMedia } = basicFilterDto;
         const populateUserIdFields = this.crudHelperService.extractUserIdFieldsFromPopulate(basicFilterDto.populate);
 
         const { singularName, internationalisation, draftPublishWorkflow } = await this.loadModel();
@@ -502,41 +501,19 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
         // Create above query on pincode table using query builder
         var qb: SelectQueryBuilder<T> = await this.repo.createSecurityRuleAwareQueryBuilder(alias)
 
-        if (basicFilterDto.groupBy) {
+        if (basicFilterDto.groupBy?.length) {
             const groupFilterQb = (internationalisation && draftPublishWorkflow)
                 ? this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias, internationalisation, draftPublishWorkflow, this.moduleRef, FilterCombinator.AND, false, false)
                 : this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias, undefined, undefined, undefined, FilterCombinator.AND, false, false);
 
-            const groupByFields = this.crudHelperService.normalize(basicFilterDto.groupBy);
-            if (!groupByFields.length) {
-                throw new BadRequestException(ERROR_MESSAGES.INVALID_GROUP_BY_COUNT);
-            }
-
-            if (basicFilterDto.populateGroup) {
-                const hasRelationGroup = groupByFields.some(field => field.includes('.'));
-                if (hasRelationGroup) {
-                    throw new BadRequestException('populateGroup is not supported when grouping on relation fields. Fetch group metadata first and retrieve records in a separate call.');
+            return this.crudHelperService.executeGroupPipeline(
+                groupFilterQb, basicFilterDto, alias,
+                () => this.repo.createSecurityRuleAwareQueryBuilder(alias),
+                async (entities) => {
+                    if (populateUserIdFields?.length) await this.handlePopulateUserIdFields(populateUserIdFields, entities);
+                    if (populateMedia?.length) await this.handlePopulateMedia(populateMedia, entities);
                 }
-            }
-
-            const { aliasMap: groupAliasMap, formatMap: groupFormatMap, expressionMap: groupExpressionMap } = this.crudHelperService.applyGroupBySelections(groupFilterQb, groupByFields, alias);
-            const aggregateAliasMap = this.crudHelperService.applyAggregates(groupFilterQb, basicFilterDto.aggregates, alias);
-            const sortAliasMap = { ...groupAliasMap, ...aggregateAliasMap };
-            this.crudHelperService.applyGroupSortingAndPagination(groupFilterQb, basicFilterDto.sort, sortAliasMap, limit, offset);
-
-            const groupByResult = await groupFilterQb.getRawMany();
-            const totalGroups = await this.crudHelperService.countGroups(groupFilterQb);
-
-            const groupByFieldsOrdered = this.crudHelperService.normalize(basicFilterDto.groupBy || []);
-            const { groupMeta, groupRecords } = await this.handleGroupFind(groupByResult, groupFilter, populateGroup, alias, populateUserIdFields, populateMedia, basicFilterDto, groupAliasMap, aggregateAliasMap, groupByFieldsOrdered, groupFormatMap, groupExpressionMap);
-
-            return {
-                meta: {
-                    "totalRecords": totalGroups
-                },
-                groupMeta,
-                groupRecords,
-            }
+            );
         }
         else {
             qb = (internationalisation && draftPublishWorkflow)
@@ -566,78 +543,8 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
         return this.wrapFindResponse(offset, limit, count, entities);
     }
 
-    private async handleGroupFind(
-        groupByResult: any[],
-        groupFilter: BasicGroupFilterDto | undefined,
-        populateGroup: boolean,
-        alias: string,
-        populateUserIdFields: UserIdFields[],
-        populateMedia: string[],
-        baseFilterDto: BasicFilterDto,
-        groupAliasMap: Record<string, string>,
-        aggregateAliasMap: Record<string, string>,
-        groupByFieldsOrdered: string[],
-        groupFormatMap: Record<string, string | undefined>,
-        groupExpressionMap: Record<string, string>
-    ) {
-        const groupMeta = [];
-        const groupRecords = [];
-        const aggregateAliasSet = new Set(Object.values(aggregateAliasMap));
-        // For each group, get the records and the count
-        for (const group of groupByResult) {
-            if (populateGroup) {
-                let groupByQb: SelectQueryBuilder<T> = await this.repo.createSecurityRuleAwareQueryBuilder(alias);
-                const groupFilterDto: BasicFilterDto = {
-                    ...baseFilterDto,
-                    ...groupFilter,
-                    groupBy: undefined,
-                    aggregates: undefined,
-                    // Only use explicit groupFilter.sort for record ordering; group-level sorts can contain
-                    // group expressions (e.g. createdAt:day) that are invalid on record queries.
-                    sort: groupFilter?.sort,
-                };
-                groupByQb = this.crudHelperService.buildFilterQuery(groupByQb, groupFilterDto, alias);
-                groupByQb = this.crudHelperService.buildGroupByRecordsQuery(groupByQb, group, alias, groupAliasMap, aggregateAliasMap, groupExpressionMap);
-                const [entities, count] = await groupByQb.getManyAndCount();
-
-                // Populate the entity with the userId fields
-                if (populateUserIdFields && populateUserIdFields.length > 0) {
-                    await this.handlePopulateUserIdFields(populateUserIdFields, entities);
-                }
-
-                // Populate the entity with the media
-                if (populateMedia && populateMedia.length > 0) {
-                    await this.handlePopulateMedia(populateMedia, entities);
-                }
-                const groupData = this.wrapFindResponse(groupFilter?.offset, groupFilter?.limit, count, entities);
-                groupRecords.push(this.crudHelperService.createGroupRecords(group, aggregateAliasSet, groupData, groupByFieldsOrdered, groupAliasMap, groupFormatMap));
-            }
-            groupMeta.push(this.crudHelperService.createGroupMeta(group, aggregateAliasSet, groupByFieldsOrdered, groupAliasMap, groupFormatMap));
-        }
-        return { groupMeta, groupRecords };
-    }
-
     private wrapFindResponse(offset: number | undefined, limit: number | undefined, count: number, entities: T[]) {
-        const safeLimit = limit ?? count ?? 0;
-        const safeOffset = offset ?? 0;
-        const currentPage = safeLimit ? Math.floor(safeOffset / safeLimit) + 1 : 1;
-        const totalPages = safeLimit ? Math.ceil(count / safeLimit) : 1;
-
-        const nextPage = safeLimit && currentPage < totalPages ? currentPage + 1 : null;
-        const prevPage = safeLimit && currentPage > 1 ? currentPage - 1 : null;
-
-        const r = {
-            meta: {
-                totalRecords: count,
-                currentPage: currentPage,
-                nextPage: nextPage,
-                prevPage: prevPage,
-                totalPages: totalPages,
-                perPage: safeLimit ? +safeLimit : 0,
-            },
-            records: entities
-        };
-        return r;
+        return this.crudHelperService.pagedResponse(offset, limit, count, entities);
     }
 
     // entities is an array of T

package/src/services/dashboard-providers/README.md

@@ -38,7 +38,7 @@ Not persisted:
 
 At runtime, metadata is read from file paths resolved by `ModuleMetadataHelperService`:
 
-1. `module-metadata/<module-name>/<module-name>-metadata.json`
+1. `src/<module-name>/metadata/<module-name>-metadata.json`
 2. fallback (solid-core local run): `src/seeders/seed-data/solid-core-metadata.json`
 3. fallback (consuming project): `node_modules/@solidxai/core/src/seeders/seed-data/solid-core-metadata.json`
 

package/src/services/import-transaction.service.ts

@@ -27,6 +27,7 @@ import { getUserExcludedFields } from 'src/helpers/user-helper';
 import { ActiveUserData } from 'src/interfaces/active-user-data.interface';
 import { upperFirst, camelCase } from 'lodash';
 import { classify } from '../helpers/string.helper';
+import type { SolidCoreSetting } from './settings/default-settings-provider.service';
 
 interface ImportTemplateFileInfo {
   stream: NodeJS.ReadableStream;
@@ -180,11 +181,20 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
     // Replace modelMetadata.fields with combined (child + parent) fields
     // modelMetadata.fields = allFields;
 
+    const dateFieldFormat =
+      (this.settingService.getConfigValue<SolidCoreSetting>('dateFormat') as string | null) ??
+      'YYYY-MM-DD';
+    const dateTimeFieldFormat =
+      (this.settingService.getConfigValue<SolidCoreSetting>('dateTimeFormat') as string | null) ??
+      'YYYY-MM-DD HH:mm:ss';
+
     // Create the standard import instructions
     const standardInstructions: StandardImportInstructionsResponseDto = {
       requiredFields: [],
       dateFields: [],
+      dateFieldFormat,
       dateTimeFields: [],
+      dateTimeFieldFormat,
       numberFields: [],
       emailFields: [],
       regexFields: [],
@@ -792,4 +802,4 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
     const relatedRecordsIds = relatedRecordsResult.records.map(record => record.id);
     return relatedRecordsIds;
   }
-}
+}

package/src/services/mediaStorageProviders/file-s3-storage-provider.ts

@@ -21,8 +21,43 @@ export class FileS3StorageProvider<T> implements MediaStorageProvider<T> {
         readonly mediaRepository: MediaRepository,
     ) { }
 
-    storeStreams(streamPairs: [Readable, string][], entity: T, mediaFieldMetadata: FieldMetadata): Promise<Media[]> {
-        throw new Error("Method not implemented.");
+    async storeStreams(streamPairs: [Readable, string][], entity: T, mediaFieldMetadata: FieldMetadata): Promise<Media[]> {
+        const isSupportedEntity = entity instanceof CommonEntity
+            || entity instanceof LegacyCommonEntityWithExistingId
+            || entity instanceof LegacyCommonEntityWithGeneratedId;
+        if (!isSupportedEntity) {
+            throw new Error("Entity must be an instance of CommonEntity, LegacyCommonEntityWithExistingId or LegacyCommonEntityWithGeneratedId");
+        }
+        const result: Media[] = [];
+        const storageProvider = mediaFieldMetadata.mediaStorageProvider;
+        const region = this.getEffectiveRegion(storageProvider.region);
+
+        for (const [stream, fileName] of streamPairs) {
+            const bucketName = storageProvider.bucketName;
+
+            // Buffer the stream so we can get the byte count and upload in one pass
+            const chunks: Buffer[] = [];
+            for await (const chunk of stream) {
+                chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+            }
+            const fileData = Buffer.concat(chunks);
+            const fileSize = fileData.length;
+
+            await this.s3FileService.write(`${bucketName}:${fileName}`, fileData, { region });
+
+            const mediaEntity = await this.mediaRepository.createMedia({
+                // @ts-ignore
+                entityId: entity.id,
+                modelMetadataId: mediaFieldMetadata.model.id,
+                relativeUri: fileName,
+                fileSize,
+                mediaStorageProviderMetadataId: mediaFieldMetadata.mediaStorageProvider.id,
+                fieldMetadataId: mediaFieldMetadata.id
+            }) as unknown as Media;
+            result.push(mediaEntity);
+            this.logger.debug(`Stored media with`, mediaEntity);
+        }
+        return result;
     }
 
     async retrieve(entity: T, mediaFieldMetadata: FieldMetadata): Promise<Media[]> {

package/src/services/mediaStorageProviders/file-storage-provider.ts

@@ -8,6 +8,7 @@ import { MediaRepository } from "src/repository/media.repository";
 import { DiskFileService } from "src/services/file";
 import { Readable } from "stream";
 import * as path from "path";
+import * as fs from "fs";
 import { SettingService } from "../setting.service";
 import { DEFAULT_MEDIA_FILE_STORAGE_DIR } from "src/services/settings/default-settings-provider.service";
 import type { SolidCoreSetting } from "src/services/settings/default-settings-provider.service";
@@ -80,12 +81,15 @@ export class FileStorageProvider<T> implements MediaStorageProvider<T> {
         for (const pair of streamPairs) {
             const stream = pair[0];
             const fileName = pair[1];
-            await this.fileService.writeStream(this.getFullFilePath(fileName), stream);
+            const fullPath = this.getFullFilePath(fileName);
+            await this.fileService.writeStream(fullPath, stream);
+            const { size: fileSize } = await fs.promises.stat(fullPath);
             const mediaEntity = await this.mediaRepository.createMedia({
                 //@ts-ignore
                 entityId: entity.id,
                 modelMetadataId: mediaFieldMetadata.model.id,
                 relativeUri: fileName,
+                fileSize,
                 mediaStorageProviderMetadataId: mediaFieldMetadata.mediaStorageProvider.id,
                 fieldMetadataId: mediaFieldMetadata.id
             }) as unknown as Media;