@solidxai/core 0.1.10 → 0.1.11-beta.0

package/src/controllers/datasource-introspection.controller.ts

@@ -0,0 +1,58 @@
+import { Body, Controller, Get, Param, ParseIntPipe, Post, Query } from "@nestjs/common";
+import { ApiBearerAuth, ApiTags } from "@nestjs/swagger";
+import { DatasourceIntrospectionMappingDto } from "../dtos/datasource-introspection-mapping.dto";
+import { DatasourceIntrospectionService } from "../services/datasource-introspection.service";
+
+@Controller("datasource-introspection")
+@ApiTags("Solid Core")
+export class DatasourceIntrospectionController {
+    constructor(
+        private readonly datasourceIntrospectionService: DatasourceIntrospectionService,
+    ) { }
+
+    @ApiBearerAuth("jwt")
+    @Get("modules/:moduleId/bootstrap")
+    getBootstrap(
+        @Param("moduleId", ParseIntPipe) moduleId: number,
+    ) {
+        return this.datasourceIntrospectionService.getBootstrap(moduleId);
+    }
+
+    @ApiBearerAuth("jwt")
+    @Get("modules/:moduleId/tables")
+    getTables(
+        @Param("moduleId", ParseIntPipe) moduleId: number,
+        @Query("datasource") datasourceName: string,
+    ) {
+        return this.datasourceIntrospectionService.getTables(moduleId, datasourceName);
+    }
+
+    @ApiBearerAuth("jwt")
+    @Get("modules/:moduleId/table-detail")
+    getTableDetail(
+        @Param("moduleId", ParseIntPipe) moduleId: number,
+        @Query("datasource") datasourceName: string,
+        @Query("table") tableName: string,
+        @Query("schema") schema?: string,
+    ) {
+        return this.datasourceIntrospectionService.getTableDetail(moduleId, datasourceName, tableName, schema);
+    }
+
+    @ApiBearerAuth("jwt")
+    @Post("modules/:moduleId/mapping-preview")
+    previewMapping(
+        @Param("moduleId", ParseIntPipe) moduleId: number,
+        @Body() mappingDto: DatasourceIntrospectionMappingDto,
+    ) {
+        return this.datasourceIntrospectionService.previewMapping(moduleId, mappingDto);
+    }
+
+    @ApiBearerAuth("jwt")
+    @Post("modules/:moduleId/apply-mapping")
+    applyMapping(
+        @Param("moduleId", ParseIntPipe) moduleId: number,
+        @Body() mappingDto: DatasourceIntrospectionMappingDto,
+    ) {
+        return this.datasourceIntrospectionService.applyMapping(moduleId, mappingDto);
+    }
+}

package/src/controllers/datasource-management.controller.ts

@@ -0,0 +1,24 @@
+import { Body, Controller, Get, Post } from "@nestjs/common";
+import { ApiBearerAuth, ApiTags } from "@nestjs/swagger";
+import { CreateDatasourceManagementDto } from "../dtos/create-datasource-management.dto";
+import { DatasourceManagementService } from "../services/datasource-management.service";
+
+@Controller("datasource-management")
+@ApiTags("Solid Core")
+export class DatasourceManagementController {
+    constructor(
+        private readonly datasourceManagementService: DatasourceManagementService,
+    ) { }
+
+    @ApiBearerAuth("jwt")
+    @Get()
+    findMany() {
+        return this.datasourceManagementService.findMany();
+    }
+
+    @ApiBearerAuth("jwt")
+    @Post()
+    create(@Body() createDto: CreateDatasourceManagementDto) {
+        return this.datasourceManagementService.create(createDto);
+    }
+}

package/src/dtos/create-datasource-management.dto.ts

@@ -0,0 +1,139 @@
+import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger";
+import { Type } from "class-transformer";
+import {
+    IsBoolean,
+    IsIn,
+    IsInt,
+    IsOptional,
+    IsString,
+    Matches,
+    Max,
+    Min,
+} from "class-validator";
+import { DatasourceType } from "./create-model-metadata.dto";
+
+export class CreateDatasourceManagementDto {
+    @ApiProperty({ description: "Logical datasource key used in generated filenames and env prefixes." })
+    @IsString()
+    @Matches(/^[a-z][a-z0-9-]*$/, {
+        message: "Datasource name must start with a lower-case letter and use only lower-case letters, numbers, or hyphens.",
+    })
+    name: string;
+
+    @ApiPropertyOptional({ description: "Human friendly label shown in the admin interface." })
+    @IsOptional()
+    @IsString()
+    displayName?: string;
+
+    @ApiProperty({ enum: [DatasourceType.postgres, DatasourceType.mysql, DatasourceType.mssql] })
+    @IsIn([DatasourceType.postgres, DatasourceType.mysql, DatasourceType.mssql])
+    type: DatasourceType;
+
+    @ApiProperty()
+    @IsString()
+    host: string;
+
+    @ApiProperty()
+    @Type(() => Number)
+    @IsInt()
+    @Min(1)
+    @Max(65535)
+    port: number;
+
+    @ApiProperty()
+    @IsString()
+    database: string;
+
+    @ApiProperty()
+    @IsString()
+    username: string;
+
+    @ApiProperty()
+    @IsString()
+    password: string;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Boolean)
+    @IsBoolean()
+    synchronize?: boolean;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Boolean)
+    @IsBoolean()
+    logging?: boolean;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Boolean)
+    @IsBoolean()
+    ssl?: boolean;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Boolean)
+    @IsBoolean()
+    sslRejectUnauthorized?: boolean;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Number)
+    @IsInt()
+    @Min(0)
+    poolMax?: number;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Number)
+    @IsInt()
+    @Min(0)
+    connectionTimeoutMs?: number;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Number)
+    @IsInt()
+    @Min(0)
+    idleTimeoutMs?: number;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Number)
+    @IsInt()
+    @Min(0)
+    statementTimeoutMs?: number;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Number)
+    @IsInt()
+    @Min(0)
+    idleInTxTimeoutMs?: number;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Number)
+    @IsInt()
+    @Min(0)
+    retryAttempts?: number;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Number)
+    @IsInt()
+    @Min(0)
+    retryDelayMs?: number;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Boolean)
+    @IsBoolean()
+    encrypt?: boolean;
+
+    @ApiPropertyOptional()
+    @IsOptional()
+    @Type(() => Boolean)
+    @IsBoolean()
+    trustServerCertificate?: boolean;
+}

package/src/dtos/datasource-introspection-mapping.dto.ts

@@ -0,0 +1,182 @@
+import { ApiProperty } from "@nestjs/swagger";
+import { Type } from "class-transformer";
+import {
+    IsArray,
+    IsBoolean,
+    IsEnum,
+    IsInt,
+    IsObject,
+    IsOptional,
+    IsString,
+    ValidateNested,
+} from "class-validator";
+import { OrmType, SolidFieldType } from "./create-field-metadata.dto";
+import { LegacyTableType } from "src/enums/legacy-table-type.enum";
+
+export class DatasourceIntrospectionMappingColumnDto {
+    @ApiProperty()
+    @IsString()
+    columnName: string;
+
+    @ApiProperty()
+    @IsString()
+    fieldName: string;
+
+    @ApiProperty()
+    @IsString()
+    dataType: string;
+
+    @ApiProperty({ enum: SolidFieldType })
+    @IsEnum(SolidFieldType)
+    solidFieldType: SolidFieldType;
+
+    @ApiProperty()
+    @IsString()
+    ormType: OrmType;
+
+    @ApiProperty()
+    @IsBoolean()
+    nullable: boolean;
+
+    @ApiProperty({ required: false, nullable: true })
+    @IsOptional()
+    defaultValue?: string | null;
+
+    @ApiProperty({ required: false, nullable: true })
+    @IsOptional()
+    @IsInt()
+    characterMaximumLength?: number | null;
+
+    @ApiProperty({ required: false, nullable: true })
+    @IsOptional()
+    @IsInt()
+    numericPrecision?: number | null;
+
+    @ApiProperty({ required: false, nullable: true })
+    @IsOptional()
+    @IsInt()
+    numericScale?: number | null;
+
+    @ApiProperty({ required: false, nullable: true })
+    @IsOptional()
+    @IsInt()
+    ordinalPosition?: number | null;
+
+    @ApiProperty()
+    @IsBoolean()
+    isAutoGenerated: boolean;
+
+    @ApiProperty()
+    @IsBoolean()
+    handledBySuperclass: boolean;
+
+    @ApiProperty({ required: false, nullable: true })
+    @IsOptional()
+    @IsString()
+    superclassFieldName?: string | null;
+
+    @ApiProperty({ required: false, nullable: true })
+    @IsOptional()
+    @IsInt()
+    fieldId?: number | null;
+
+    @ApiProperty()
+    @IsBoolean()
+    include: boolean;
+
+    @ApiProperty()
+    @IsBoolean()
+    matched: boolean;
+
+    @ApiProperty()
+    @IsBoolean()
+    isPrimaryKey: boolean;
+}
+
+export class DatasourceIntrospectionMappingDto {
+    @ApiProperty()
+    @IsBoolean()
+    mapped: boolean;
+
+    @ApiProperty({ required: false, nullable: true })
+    @IsOptional()
+    @IsInt()
+    modelId?: number | null;
+
+    @ApiProperty()
+    @IsString()
+    singularName: string;
+
+    @ApiProperty()
+    @IsString()
+    pluralName: string;
+
+    @ApiProperty()
+    @IsString()
+    displayName: string;
+
+    @ApiProperty()
+    @IsString()
+    description: string;
+
+    @ApiProperty()
+    @IsString()
+    tableName: string;
+
+    @ApiProperty({ required: false, nullable: true })
+    @IsOptional()
+    @IsString()
+    schema?: string | null;
+
+    @ApiProperty()
+    @IsString()
+    dataSource: string;
+
+    @ApiProperty()
+    @IsString()
+    dataSourceType: string;
+
+    @ApiProperty({ enum: LegacyTableType })
+    @IsEnum(LegacyTableType)
+    legacyTableType: LegacyTableType;
+
+    @ApiProperty()
+    @IsString()
+    baseClassName: string;
+
+    @ApiProperty()
+    @IsString()
+    userKeyField: string;
+
+    @ApiProperty()
+    @IsBoolean()
+    enableAuditTracking: boolean;
+
+    @ApiProperty()
+    @IsBoolean()
+    enableSoftDelete: boolean;
+
+    @ApiProperty()
+    @IsBoolean()
+    draftPublishWorkflow: boolean;
+
+    @ApiProperty()
+    @IsBoolean()
+    internationalisation: boolean;
+
+    @ApiProperty({ required: false, nullable: true })
+    @IsOptional()
+    @IsInt()
+    plannedMigrationTimestamp?: number | null;
+
+    @ApiProperty({ type: () => [DatasourceIntrospectionMappingColumnDto] })
+    @IsArray()
+    @ValidateNested({ each: true })
+    @Type(() => DatasourceIntrospectionMappingColumnDto)
+    columns: DatasourceIntrospectionMappingColumnDto[];
+
+    @ApiProperty({ required: false, type: "object" })
+    @IsOptional()
+    @IsObject()
+    reviewedModel?: Record<string, any>;
+}

package/src/index.ts

@@ -62,6 +62,7 @@ export * from './dtos/create-view-metadata.dto'
 export * from './dtos/create-chatter-message.dto'
 export * from './dtos/create-chatter-message-details.dto'
 export * from './dtos/create-locale.dto'
+export * from './dtos/create-datasource-management.dto'
 export * from './dtos/fetch-roles.dto'
 export * from './dtos/initiate-forgot-password.dto'
 export * from './dtos/mutate-role-permissions.dto'
@@ -334,7 +335,11 @@ export * from './services/whatsapp/GupshupOtpWhatsappService'
 export * from './services/setting.service'
 export * from './services/encryption.service'
 export * from './services/info.service'
+export * from './services/datasource-management.service'
+export * from './services/datasource-introspection.service'
 export * from './controllers/info.controller'
+export * from './controllers/datasource-management.controller'
+export * from './controllers/datasource-introspection.controller'
 export * from './controllers/module-metadata-explorer.controller'
 export * from './controllers/module-package.controller'
 export * from './services/settings/default-settings-provider.service'

package/src/interfaces/datasource-introspection-provider.interface.ts

@@ -0,0 +1,28 @@
+import { DataSource } from "typeorm";
+
+export type DatasourceIntrospectionTableRef = {
+    tableSchema: string | null;
+    tableName: string;
+};
+
+export type DatasourceIntrospectionColumn = {
+    tableSchema: string | null;
+    tableName: string;
+    columnName: string;
+    dataType: string;
+    isAutoGenerated: boolean;
+    isNullable: boolean;
+    defaultValue: string | null;
+    characterMaximumLength: number | null;
+    numericPrecision: number | null;
+    numericScale: number | null;
+    ordinalPosition: number | null;
+};
+
+export interface DatasourceIntrospectionProvider {
+    readonly type: string;
+    supports(type: string): boolean;
+    listTables(dataSource: DataSource): Promise<DatasourceIntrospectionTableRef[]>;
+    listColumns(dataSource: DataSource): Promise<DatasourceIntrospectionColumn[]>;
+    getTableDefinition(dataSource: DataSource, table: DatasourceIntrospectionTableRef): Promise<string | null>;
+}

package/src/seeders/seed-data/solid-core-metadata.json

@@ -5381,6 +5381,19 @@
       "moduleUserKey": "solid-core",
       "modelUserKey": "fieldMetadata"
     },
+    {
+      "displayName": "Datasource Management",
+      "name": "datasource-management-action",
+      "type": "custom",
+      "domain": "",
+      "context": "",
+      "customComponent": "/admin/core/solid-core/datasources",
+      "customIsModal": true,
+      "serverEndpoint": "",
+      "viewUserKey": "",
+      "moduleUserKey": "solid-core",
+      "modelUserKey": "moduleMetadata"
+    },
     {
       "displayName": "Field Metadata Tree Action",
       "name": "fieldMetadata-tree-action",
@@ -6105,6 +6118,14 @@
       "moduleUserKey": "solid-core",
       "parentMenuItemUserKey": "appBuilder-menu-item"
     },
+    {
+      "displayName": "Datasources",
+      "name": "datasource-management-menu-item",
+      "sequenceNumber": 4,
+      "actionUserKey": "datasource-management-action",
+      "moduleUserKey": "solid-core",
+      "parentMenuItemUserKey": "appBuilder-menu-item"
+    },
     {
       "displayName": "Menu Item",
       "name": "menuItemMetadata-menu-item",
@@ -14737,4 +14758,4 @@
     }
   ],
   "savedFilters": []
-}
+}

package/src/services/authentication.service.ts

@@ -309,7 +309,7 @@ export class AuthenticationService {
     if (signUpDto.roles && signUpDto.roles.length > 0) {
       for (let i = 0; i < signUpDto.roles.length; i++) {
         const roleName = signUpDto.roles[i];
-        await this.roleMetadataService.findRoleByName(roleName);
+        await this.roleMetadataService.assertRoleExistsByName(roleName);
       }
     }
     user.username = signUpDto.username;

package/src/services/dashboard-providers/mq-dashboard-latency-trend-provider.service.ts

@@ -2,7 +2,13 @@ import { Injectable } from "@nestjs/common";
 import { DashboardWidgetDataProvider } from "src/decorators/dashboard-widget-data-provider.decorator";
 import { MqMessageRepository } from "src/repository/mq-message.repository";
 import { IDashboardWidgetDataProvider, IDashboardWidgetDataProviderContext, IDashboardWidgetDataResponseEnvelope } from "src/interfaces";
-import { applyMqDashboardFilters, normalizeBucket, toNumber } from "src/services/dashboard-providers/mq-dashboard-provider-utils";
+import {
+    applyMqDashboardFilters,
+    buildMqDashboardBucketExpression,
+    normalizeBucket,
+    normalizeMqDashboardBucketValue,
+    toNumber,
+} from "src/services/dashboard-providers/mq-dashboard-provider-utils";
 
 @DashboardWidgetDataProvider()
 @Injectable()
@@ -29,7 +35,7 @@ export class MqDashboardLatencyTrendProvider implements IDashboardWidgetDataProv
         applyMqDashboardFilters(qb, ctxt.variables ?? {});
         qb.andWhere("mqMessage.elapsedMillis IS NOT NULL");
 
-        const bucketExpr = `DATE_TRUNC('${bucket}', mqMessage.createdAt)`;
+        const bucketExpr = buildMqDashboardBucketExpression(qb, bucket, "mqMessage.createdAt");
         const rows = await qb
             .select(bucketExpr, "bucket")
             .addSelect("AVG(mqMessage.elapsedMillis)", "value")
@@ -37,7 +43,7 @@ export class MqDashboardLatencyTrendProvider implements IDashboardWidgetDataProv
             .orderBy("bucket", "ASC")
             .getRawMany<{ bucket: string; value: string | number }>();
 
-        const categories = rows.map((r) => new Date(r.bucket).toISOString());
+        const categories = rows.map((r) => normalizeMqDashboardBucketValue(r.bucket));
         const values = rows.map((r) => Number(toNumber(r.value).toFixed(2)));
 
         return {
@@ -55,4 +61,3 @@ export class MqDashboardLatencyTrendProvider implements IDashboardWidgetDataProv
         };
     }
 }
-

package/src/services/dashboard-providers/mq-dashboard-messages-over-time-provider.service.ts

@@ -2,7 +2,13 @@ import { Injectable } from "@nestjs/common";
 import { DashboardWidgetDataProvider } from "src/decorators/dashboard-widget-data-provider.decorator";
 import { MqMessageRepository } from "src/repository/mq-message.repository";
 import { IDashboardWidgetDataProvider, IDashboardWidgetDataProviderContext, IDashboardWidgetDataResponseEnvelope } from "src/interfaces";
-import { applyMqDashboardFilters, normalizeBucket, toNumber } from "src/services/dashboard-providers/mq-dashboard-provider-utils";
+import {
+    applyMqDashboardFilters,
+    buildMqDashboardBucketExpression,
+    normalizeBucket,
+    normalizeMqDashboardBucketValue,
+    toNumber,
+} from "src/services/dashboard-providers/mq-dashboard-provider-utils";
 
 @DashboardWidgetDataProvider()
 @Injectable()
@@ -31,7 +37,7 @@ export class MqDashboardMessagesOverTimeProvider implements IDashboardWidgetData
         const qb = await this.mqMessageRepository.createSecurityRuleAwareQueryBuilder("mqMessage");
         applyMqDashboardFilters(qb, ctxt.variables ?? {});
 
-        const bucketExpr = `DATE_TRUNC('${bucket}', mqMessage.createdAt)`;
+        const bucketExpr = buildMqDashboardBucketExpression(qb, bucket, "mqMessage.createdAt");
         const rows = await qb
             .select(bucketExpr, "bucket")
             .addSelect("mqMessage.stage", "stage")
@@ -43,7 +49,7 @@ export class MqDashboardMessagesOverTimeProvider implements IDashboardWidgetData
 
         const bucketToStageMap = new Map<string, Record<string, number>>();
         for (const row of rows) {
-            const key = new Date(row.bucket).toISOString();
+            const key = normalizeMqDashboardBucketValue(row.bucket);
             const entry = bucketToStageMap.get(key) ?? {};
             entry[row.stage] = (entry[row.stage] ?? 0) + toNumber(row.count, 0);
             bucketToStageMap.set(key, entry);
@@ -76,4 +82,3 @@ export class MqDashboardMessagesOverTimeProvider implements IDashboardWidgetData
         };
     }
 }
-

package/src/services/dashboard-providers/mq-dashboard-provider-utils.ts

@@ -54,6 +54,59 @@ export function toNumber(value: any, fallback = 0): number {
     return Number.isFinite(n) ? n : fallback;
 }
 
+export function buildMqDashboardBucketExpression(
+    qb: SelectQueryBuilder<MqMessage>,
+    bucket: 'hour' | 'day' | 'week' | 'month',
+    columnExpr: string,
+): string {
+    const driver = `${qb.connection.options.type ?? ''}`.toLowerCase();
+
+    switch (driver) {
+        case 'postgres':
+        case 'cockroachdb':
+            return `DATE_TRUNC('${bucket}', ${columnExpr})`;
+        case 'mysql':
+        case 'mariadb':
+            switch (bucket) {
+                case 'hour':
+                    return `DATE_FORMAT(${columnExpr}, '%Y-%m-%d %H:00:00')`;
+                case 'day':
+                    return `DATE_FORMAT(${columnExpr}, '%Y-%m-%d 00:00:00')`;
+                case 'week':
+                    return `STR_TO_DATE(DATE_FORMAT(${columnExpr}, '%x-%v-1'), '%x-%v-%w')`;
+                case 'month':
+                    return `DATE_FORMAT(${columnExpr}, '%Y-%m-01 00:00:00')`;
+                default:
+                    throw new Error(`Unsupported bucket ${bucket} for driver ${driver}`);
+            }
+        case 'mssql':
+        case 'sqlserver':
+            switch (bucket) {
+                case 'hour':
+                    return `DATEADD(hour, DATEDIFF(hour, 0, ${columnExpr}), 0)`;
+                case 'day':
+                    return `DATEADD(day, DATEDIFF(day, 0, ${columnExpr}), 0)`;
+                case 'week':
+                    return `DATEADD(week, DATEDIFF(week, 0, ${columnExpr}), 0)`;
+                case 'month':
+                    return `DATEFROMPARTS(YEAR(${columnExpr}), MONTH(${columnExpr}), 1)`;
+                default:
+                    throw new Error(`Unsupported bucket ${bucket} for driver ${driver}`);
+            }
+        default:
+            throw new Error(`Unsupported driver ${driver} for mq dashboard bucket expressions.`);
+    }
+}
+
+export function normalizeMqDashboardBucketValue(value: any): string {
+    if (!value) return '';
+    const parsed = value instanceof Date ? value : new Date(value);
+    if (Number.isNaN(parsed.getTime())) {
+        return `${value}`;
+    }
+    return parsed.toISOString();
+}
+
 
 function applyDateFilter(qb: SelectQueryBuilder<MqMessage>, dateVariable: any): void {
     const range = resolveDateRange(dateVariable);
@@ -88,6 +141,35 @@ function applyStringOrArrayFilter(
     qb.andWhere(`${qualifiedColumn} IN (:...${paramKey})`, { [paramKey]: values });
 }
 
+export function applyCaseInsensitiveContainsFilter<T>(
+    qb: SelectQueryBuilder<T>,
+    qualifiedColumn: string,
+    query: string | undefined,
+    paramKey: string,
+) {
+    if (!query) {
+        return;
+    }
+
+    const driver = `${qb.connection.options.type ?? ''}`.toLowerCase();
+    const value = `%${query}%`;
+
+    switch (driver) {
+        case 'postgres':
+        case 'cockroachdb':
+            qb.andWhere(`${qualifiedColumn} ILIKE :${paramKey}`, { [paramKey]: value });
+            return;
+        case 'mysql':
+        case 'mariadb':
+        case 'mssql':
+        case 'sqlserver':
+            qb.andWhere(`LOWER(${qualifiedColumn}) LIKE LOWER(:${paramKey})`, { [paramKey]: value });
+            return;
+        default:
+            qb.andWhere(`${qualifiedColumn} LIKE :${paramKey}`, { [paramKey]: value });
+    }
+}
+
 function resolveDateRange(dateVariable: any): { from?: Date; to?: Date } {
     if (!dateVariable) return {};