@solidxai/core 0.1.10-beta.25 → 0.1.10-beta.26

package/src/services/crud-helper.service.ts

@@ -749,4 +749,76 @@ export class CrudHelperService {
         return matchingPermssions.length > 0
     }
 
+    pagedResponse<T>(offset: number | undefined, limit: number | undefined, count: number, entities: T[]) {
+        const safeLimit = limit ?? count ?? 0;
+        const safeOffset = offset ?? 0;
+        const currentPage = safeLimit ? Math.floor(safeOffset / safeLimit) + 1 : 1;
+        const totalPages = safeLimit ? Math.ceil(count / safeLimit) : 1;
+        const nextPage = safeLimit && currentPage < totalPages ? currentPage + 1 : null;
+        const prevPage = safeLimit && currentPage > 1 ? currentPage - 1 : null;
+        return {
+            meta: {
+                totalRecords: count,
+                currentPage,
+                nextPage,
+                prevPage,
+                totalPages,
+                perPage: safeLimit ? +safeLimit : 0,
+            },
+            records: entities,
+        };
+    }
+
+    async executeGroupPipeline<T>(
+        filterQb: SelectQueryBuilder<T>,
+        basicFilterDto: BasicFilterDto,
+        alias: string,
+        createQbFn: () => Promise<SelectQueryBuilder<T>>,
+        postProcessEntities?: (entities: T[]) => Promise<void>
+    ): Promise<{ meta: { totalRecords: number }; groupMeta: any[]; groupRecords: any[] }> {
+        const groupByFields = this.normalize(basicFilterDto.groupBy);
+        if (!groupByFields.length) throw new BadRequestException(ERROR_MESSAGES.INVALID_GROUP_BY_COUNT);
+
+        if (basicFilterDto.populateGroup) {
+            const hasRelationGroup = groupByFields.some(f => f.includes('.'));
+            if (hasRelationGroup) throw new BadRequestException('populateGroup is not supported when grouping on relation fields. Fetch group metadata first and retrieve records in a separate call.');
+        }
+
+        const { aliasMap: groupAliasMap, formatMap: groupFormatMap, expressionMap: groupExpressionMap } =
+            this.applyGroupBySelections(filterQb, groupByFields, alias);
+        const aggregateAliasMap = this.applyAggregates(filterQb, basicFilterDto.aggregates, alias);
+        this.applyGroupSortingAndPagination(filterQb, basicFilterDto.sort, { ...groupAliasMap, ...aggregateAliasMap }, basicFilterDto.limit, basicFilterDto.offset);
+
+        const groupByResult = await filterQb.getRawMany();
+        const totalGroups = await this.countGroups(filterQb);
+        const aggregateAliasSet = new Set(Object.values(aggregateAliasMap));
+
+        const groupMeta = [];
+        const groupRecords = [];
+
+        for (const group of groupByResult) {
+            groupMeta.push(this.createGroupMeta(group, aggregateAliasSet, groupByFields, groupAliasMap, groupFormatMap));
+
+            if (basicFilterDto.populateGroup) {
+                let groupQb = await createQbFn();
+                const { groupBy: _gb, aggregates: _agg, ...rest } = basicFilterDto;
+                const groupFilterDto: BasicFilterDto = {
+                    ...rest,
+                    ...basicFilterDto.groupFilter,
+                    groupBy: undefined,
+                    aggregates: undefined,
+                    sort: basicFilterDto.groupFilter?.sort,
+                };
+                groupQb = this.buildFilterQuery(groupQb, groupFilterDto, alias);
+                groupQb = this.buildGroupByRecordsQuery(groupQb, group, alias, groupAliasMap, aggregateAliasMap, groupExpressionMap);
+                const [entities, count] = await groupQb.getManyAndCount();
+                if (postProcessEntities) await postProcessEntities(entities);
+                const groupData = this.pagedResponse(basicFilterDto.groupFilter?.offset, basicFilterDto.groupFilter?.limit, count, entities);
+                groupRecords.push(this.createGroupRecords(group, aggregateAliasSet, groupData, groupByFields, groupAliasMap, groupFormatMap));
+            }
+        }
+
+        return { meta: { totalRecords: totalGroups }, groupMeta, groupRecords };
+    }
+
 }

package/src/services/crud.service.ts

@@ -41,7 +41,6 @@ import { SolidRegistry } from "src/helpers/solid-registry";
 import { getMediaStorageProvider } from "./mediaStorageProviders";
 import { ModelMetadataService } from "./model-metadata.service";
 import { RequestContextService } from "./request-context.service";
-import { BasicGroupFilterDto } from "src/dtos/basic-group-filters.dto";
 
 
 export class CRUDService<T extends CommonEntity> { // Add two generic value i.e Person,CreatePersonDto, so we get the proper types in our service
@@ -480,10 +479,10 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
         return false; // If it is not a partial update, then do not skip computation
     }
 
-    async find(basicFilterDto: BasicFilterDto, solidRequestContext: any = {}) {
+    async find(basicFilterDto: BasicFilterDto, solidRequestContext: any = {}): Promise<any> {
         const alias = 'entity';
         // Extract the required keys from the input query
-        let { limit, offset, populateMedia, populateGroup, groupFilter } = basicFilterDto;
+        let { limit, offset, populateMedia } = basicFilterDto;
         const populateUserIdFields = this.crudHelperService.extractUserIdFieldsFromPopulate(basicFilterDto.populate);
 
         const { singularName, internationalisation, draftPublishWorkflow } = await this.loadModel();
@@ -502,41 +501,19 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
         // Create above query on pincode table using query builder
         var qb: SelectQueryBuilder<T> = await this.repo.createSecurityRuleAwareQueryBuilder(alias)
 
-        if (basicFilterDto.groupBy) {
+        if (basicFilterDto.groupBy?.length) {
             const groupFilterQb = (internationalisation && draftPublishWorkflow)
                 ? this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias, internationalisation, draftPublishWorkflow, this.moduleRef, FilterCombinator.AND, false, false)
                 : this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias, undefined, undefined, undefined, FilterCombinator.AND, false, false);
 
-            const groupByFields = this.crudHelperService.normalize(basicFilterDto.groupBy);
-            if (!groupByFields.length) {
-                throw new BadRequestException(ERROR_MESSAGES.INVALID_GROUP_BY_COUNT);
-            }
-
-            if (basicFilterDto.populateGroup) {
-                const hasRelationGroup = groupByFields.some(field => field.includes('.'));
-                if (hasRelationGroup) {
-                    throw new BadRequestException('populateGroup is not supported when grouping on relation fields. Fetch group metadata first and retrieve records in a separate call.');
+            return this.crudHelperService.executeGroupPipeline(
+                groupFilterQb, basicFilterDto, alias,
+                () => this.repo.createSecurityRuleAwareQueryBuilder(alias),
+                async (entities) => {
+                    if (populateUserIdFields?.length) await this.handlePopulateUserIdFields(populateUserIdFields, entities);
+                    if (populateMedia?.length) await this.handlePopulateMedia(populateMedia, entities);
                 }
-            }
-
-            const { aliasMap: groupAliasMap, formatMap: groupFormatMap, expressionMap: groupExpressionMap } = this.crudHelperService.applyGroupBySelections(groupFilterQb, groupByFields, alias);
-            const aggregateAliasMap = this.crudHelperService.applyAggregates(groupFilterQb, basicFilterDto.aggregates, alias);
-            const sortAliasMap = { ...groupAliasMap, ...aggregateAliasMap };
-            this.crudHelperService.applyGroupSortingAndPagination(groupFilterQb, basicFilterDto.sort, sortAliasMap, limit, offset);
-
-            const groupByResult = await groupFilterQb.getRawMany();
-            const totalGroups = await this.crudHelperService.countGroups(groupFilterQb);
-
-            const groupByFieldsOrdered = this.crudHelperService.normalize(basicFilterDto.groupBy || []);
-            const { groupMeta, groupRecords } = await this.handleGroupFind(groupByResult, groupFilter, populateGroup, alias, populateUserIdFields, populateMedia, basicFilterDto, groupAliasMap, aggregateAliasMap, groupByFieldsOrdered, groupFormatMap, groupExpressionMap);
-
-            return {
-                meta: {
-                    "totalRecords": totalGroups
-                },
-                groupMeta,
-                groupRecords,
-            }
+            );
         }
         else {
             qb = (internationalisation && draftPublishWorkflow)
@@ -566,78 +543,8 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
         return this.wrapFindResponse(offset, limit, count, entities);
     }
 
-    private async handleGroupFind(
-        groupByResult: any[],
-        groupFilter: BasicGroupFilterDto | undefined,
-        populateGroup: boolean,
-        alias: string,
-        populateUserIdFields: UserIdFields[],
-        populateMedia: string[],
-        baseFilterDto: BasicFilterDto,
-        groupAliasMap: Record<string, string>,
-        aggregateAliasMap: Record<string, string>,
-        groupByFieldsOrdered: string[],
-        groupFormatMap: Record<string, string | undefined>,
-        groupExpressionMap: Record<string, string>
-    ) {
-        const groupMeta = [];
-        const groupRecords = [];
-        const aggregateAliasSet = new Set(Object.values(aggregateAliasMap));
-        // For each group, get the records and the count
-        for (const group of groupByResult) {
-            if (populateGroup) {
-                let groupByQb: SelectQueryBuilder<T> = await this.repo.createSecurityRuleAwareQueryBuilder(alias);
-                const groupFilterDto: BasicFilterDto = {
-                    ...baseFilterDto,
-                    ...groupFilter,
-                    groupBy: undefined,
-                    aggregates: undefined,
-                    // Only use explicit groupFilter.sort for record ordering; group-level sorts can contain
-                    // group expressions (e.g. createdAt:day) that are invalid on record queries.
-                    sort: groupFilter?.sort,
-                };
-                groupByQb = this.crudHelperService.buildFilterQuery(groupByQb, groupFilterDto, alias);
-                groupByQb = this.crudHelperService.buildGroupByRecordsQuery(groupByQb, group, alias, groupAliasMap, aggregateAliasMap, groupExpressionMap);
-                const [entities, count] = await groupByQb.getManyAndCount();
-
-                // Populate the entity with the userId fields
-                if (populateUserIdFields && populateUserIdFields.length > 0) {
-                    await this.handlePopulateUserIdFields(populateUserIdFields, entities);
-                }
-
-                // Populate the entity with the media
-                if (populateMedia && populateMedia.length > 0) {
-                    await this.handlePopulateMedia(populateMedia, entities);
-                }
-                const groupData = this.wrapFindResponse(groupFilter?.offset, groupFilter?.limit, count, entities);
-                groupRecords.push(this.crudHelperService.createGroupRecords(group, aggregateAliasSet, groupData, groupByFieldsOrdered, groupAliasMap, groupFormatMap));
-            }
-            groupMeta.push(this.crudHelperService.createGroupMeta(group, aggregateAliasSet, groupByFieldsOrdered, groupAliasMap, groupFormatMap));
-        }
-        return { groupMeta, groupRecords };
-    }
-
     private wrapFindResponse(offset: number | undefined, limit: number | undefined, count: number, entities: T[]) {
-        const safeLimit = limit ?? count ?? 0;
-        const safeOffset = offset ?? 0;
-        const currentPage = safeLimit ? Math.floor(safeOffset / safeLimit) + 1 : 1;
-        const totalPages = safeLimit ? Math.ceil(count / safeLimit) : 1;
-
-        const nextPage = safeLimit && currentPage < totalPages ? currentPage + 1 : null;
-        const prevPage = safeLimit && currentPage > 1 ? currentPage - 1 : null;
-
-        const r = {
-            meta: {
-                totalRecords: count,
-                currentPage: currentPage,
-                nextPage: nextPage,
-                prevPage: prevPage,
-                totalPages: totalPages,
-                perPage: safeLimit ? +safeLimit : 0,
-            },
-            records: entities
-        };
-        return r;
+        return this.crudHelperService.pagedResponse(offset, limit, count, entities);
     }
 
     // entities is an array of T

package/src/services/dashboard-providers/README.md

@@ -38,7 +38,7 @@ Not persisted:
 
 At runtime, metadata is read from file paths resolved by `ModuleMetadataHelperService`:
 
-1. `module-metadata/<module-name>/<module-name>-metadata.json`
+1. `src/<module-name>/metadata/<module-name>-metadata.json`
 2. fallback (solid-core local run): `src/seeders/seed-data/solid-core-metadata.json`
 3. fallback (consuming project): `node_modules/@solidxai/core/src/seeders/seed-data/solid-core-metadata.json`
 

package/src/services/genai/ingest-metadata.service.ts

@@ -97,14 +97,14 @@ export class IngestMetadataService {
         for (let i = 0; i < enabledModules.length; i++) {
             const enabledModule = enabledModules[i];
             const fileName = `${enabledModule}-metadata.json`;
-            const enabledModuleSeedFile = `module-metadata/${enabledModule}/${fileName}`;
+            const enabledModuleSeedFile = `src/${enabledModule}/metadata/${fileName}`;
             const fullPath = path.join(process.cwd(), enabledModuleSeedFile);
             const overallMetadata: any = JSON.parse(fs.readFileSync(fullPath, 'utf-8').toString());
 
             const moduleMetadata: CreateModuleMetadataDto = overallMetadata.moduleMetadata;
 
             // Manage all the ingestion info file paths...
-            const enabledModulIngestionInfoFile = `module-metadata/${enabledModule}/genai/${enabledModule}-ingested-info.json`;
+            const enabledModulIngestionInfoFile = `src/${enabledModule}/metadata/genai/${enabledModule}-ingested-info.json`;
             const enabledModulIngestionInfoFullPath = path.join(process.cwd(), enabledModulIngestionInfoFile);
             const ingestionInfo: ModuleRAGIngestionInfo = fs.existsSync(enabledModulIngestionInfoFullPath) ? JSON.parse(fs.readFileSync(enabledModulIngestionInfoFullPath, 'utf-8').toString()) : {};
             const enabledModulIngestionInfoDir = path.dirname(enabledModulIngestionInfoFullPath);
@@ -795,4 +795,4 @@ ${JSON.stringify(model)}
         this.logger.log(`[OK] Ingested field ${moduleName}.${modelName}.${fieldName} → id=${newId}, hash=${schemaHash.slice(0, 8)}…`);
         return newId;
     }
-}
+}

package/src/services/mediaStorageProviders/file-s3-storage-provider.ts

@@ -21,8 +21,43 @@ export class FileS3StorageProvider<T> implements MediaStorageProvider<T> {
         readonly mediaRepository: MediaRepository,
     ) { }
 
-    storeStreams(streamPairs: [Readable, string][], entity: T, mediaFieldMetadata: FieldMetadata): Promise<Media[]> {
-        throw new Error("Method not implemented.");
+    async storeStreams(streamPairs: [Readable, string][], entity: T, mediaFieldMetadata: FieldMetadata): Promise<Media[]> {
+        const isSupportedEntity = entity instanceof CommonEntity
+            || entity instanceof LegacyCommonEntityWithExistingId
+            || entity instanceof LegacyCommonEntityWithGeneratedId;
+        if (!isSupportedEntity) {
+            throw new Error("Entity must be an instance of CommonEntity, LegacyCommonEntityWithExistingId or LegacyCommonEntityWithGeneratedId");
+        }
+        const result: Media[] = [];
+        const storageProvider = mediaFieldMetadata.mediaStorageProvider;
+        const region = this.getEffectiveRegion(storageProvider.region);
+
+        for (const [stream, fileName] of streamPairs) {
+            const bucketName = storageProvider.bucketName;
+
+            // Buffer the stream so we can get the byte count and upload in one pass
+            const chunks: Buffer[] = [];
+            for await (const chunk of stream) {
+                chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+            }
+            const fileData = Buffer.concat(chunks);
+            const fileSize = fileData.length;
+
+            await this.s3FileService.write(`${bucketName}:${fileName}`, fileData, { region });
+
+            const mediaEntity = await this.mediaRepository.createMedia({
+                // @ts-ignore
+                entityId: entity.id,
+                modelMetadataId: mediaFieldMetadata.model.id,
+                relativeUri: fileName,
+                fileSize,
+                mediaStorageProviderMetadataId: mediaFieldMetadata.mediaStorageProvider.id,
+                fieldMetadataId: mediaFieldMetadata.id
+            }) as unknown as Media;
+            result.push(mediaEntity);
+            this.logger.debug(`Stored media with`, mediaEntity);
+        }
+        return result;
     }
 
     async retrieve(entity: T, mediaFieldMetadata: FieldMetadata): Promise<Media[]> {

package/src/services/mediaStorageProviders/file-storage-provider.ts

@@ -8,6 +8,7 @@ import { MediaRepository } from "src/repository/media.repository";
 import { DiskFileService } from "src/services/file";
 import { Readable } from "stream";
 import * as path from "path";
+import * as fs from "fs";
 import { SettingService } from "../setting.service";
 import { DEFAULT_MEDIA_FILE_STORAGE_DIR } from "src/services/settings/default-settings-provider.service";
 import type { SolidCoreSetting } from "src/services/settings/default-settings-provider.service";
@@ -80,12 +81,15 @@ export class FileStorageProvider<T> implements MediaStorageProvider<T> {
         for (const pair of streamPairs) {
             const stream = pair[0];
             const fileName = pair[1];
-            await this.fileService.writeStream(this.getFullFilePath(fileName), stream);
+            const fullPath = this.getFullFilePath(fileName);
+            await this.fileService.writeStream(fullPath, stream);
+            const { size: fileSize } = await fs.promises.stat(fullPath);
             const mediaEntity = await this.mediaRepository.createMedia({
                 //@ts-ignore
                 entityId: entity.id,
                 modelMetadataId: mediaFieldMetadata.model.id,
                 relativeUri: fileName,
+                fileSize,
                 mediaStorageProviderMetadataId: mediaFieldMetadata.mediaStorageProvider.id,
                 fieldMetadataId: mediaFieldMetadata.id
             }) as unknown as Media;