@solidxai/core 0.1.11-beta.4 → 0.1.11-beta.5

package/src/services/module-metadata.service.ts

@@ -26,6 +26,7 @@ import {
 } from '../helpers/schematic.service';
 import { CommandService } from '../helpers/command.service';
 import { SolidRegistry } from '../helpers/solid-registry';
+import { startNodemonHeartbeat } from '../helpers/nodemon-heartbeat';
 import { CodeGenerationOptions, ModuleMetadataConfiguration } from '../interfaces';
 import { CrudHelperService } from './crud-helper.service';
 import { ModelMetadataService } from './model-metadata.service';
@@ -534,6 +535,40 @@ export class ModuleMetadataService {
   @DisallowInProduction()
   async generateCodeViaCtl(moduleId: number): Promise<string> {
     const module = await this.findOne(moduleId);
+    const isEmbedded = process.env.DEFAULT_DATABASE_DRIVER === 'pglite';
+
+    // When using an embedded PGlite database, the single-connection limit means
+    // a spawned `solid refresh-module` subprocess cannot get its own DB connection
+    // while the API server is already holding it. Fall back to in-process code
+    // generation (which reuses the existing connection) and only spawn solidctl
+    // for the UI module scaffold (which does not need a database connection).
+    if (isEmbedded) {
+      this.logger.log('Embedded database detected — generating API code in-process');
+      // Prevent nodemon from restarting the server while schematics write files.
+      // The heartbeat touches a watched file every 1.5 s (nodemon delay is 2 s),
+      // keeping the delay from expiring during DB queries between schematic runs.
+      const heartbeat = startNodemonHeartbeat(path.join(process.cwd(), 'src'));
+      try {
+        const apiOutput = await this.generateCode({
+          moduleUserKey: module.name,
+          dryRun: false,
+        });
+        let uiOutput = '';
+        try {
+          uiOutput = await this.commandService.executeCommandWithArgs({
+            command: 'npx',
+            args: ['@solidxai/solidctl@latest', 'generate', 'ui-module', `--name=${module.name}`],
+            cwd: path.join(process.cwd(), '..'),
+          });
+        } catch (error) {
+          this.logger.warn('UI module scaffold failed (solidctl may be outdated); API code was generated successfully.');
+        }
+        return `${apiOutput}\n${uiOutput}`;
+      } finally {
+        heartbeat.stop();
+      }
+    }
+
     return this.commandService.executeCommandWithArgs({
       command: 'npx',
       args: ['@solidxai/solidctl@latest', 'generate', 'module', `--name=${module.name}`],

package/src/services/removed-field-migration.service.ts

@@ -0,0 +1,334 @@
+import * as fs from "fs/promises";
+import { BadRequestException, Injectable, Logger, NotFoundException } from "@nestjs/common";
+import { ModuleRef } from "@nestjs/core";
+import { getDataSourceToken } from "@nestjs/typeorm";
+import { kebabCase, snakeCase } from "lodash";
+import * as path from "path";
+import { ERROR_MESSAGES } from "src/constants/error-messages";
+import { RelationType, SolidFieldType } from "src/dtos/create-field-metadata.dto";
+import { FieldMetadata } from "src/entities/field-metadata.entity";
+import { ModelMetadata } from "src/entities/model-metadata.entity";
+import { ModuleMetadataHelperService } from "src/helpers/module-metadata-helper.service";
+import { classify } from "src/helpers/string.helper";
+import { FieldMetadataRepository } from "src/repository/field-metadata.repository";
+import { ModelMetadataRepository } from "src/repository/model-metadata.repository";
+import { DataSource, EntityMetadata, QueryRunner, Table } from "typeorm";
+
+export interface RemovedFieldMigrationResult {
+    dryRun: boolean;
+    modelName: string;
+    operations: string[];
+    removedFieldNames: string[];
+}
+
+@Injectable()
+export class RemovedFieldMigrationService {
+    constructor(
+        private readonly modelMetadataRepo: ModelMetadataRepository,
+        private readonly fieldMetadataRepo: FieldMetadataRepository,
+        private readonly moduleMetadataHelperService: ModuleMetadataHelperService,
+        private readonly moduleRef: ModuleRef,
+    ) { }
+
+    private readonly logger = new Logger(RemovedFieldMigrationService.name);
+
+    // Cleans fields marked for removal by updating schema state and metadata for a single model.
+    async migrateMarkedFields(modelUserKey: string, dryRun: boolean = false): Promise<RemovedFieldMigrationResult> {
+        if (!modelUserKey) {
+            throw new BadRequestException("Model name is required");
+        }
+
+        const model = await this.modelMetadataRepo.findOne({
+            where: { singularName: modelUserKey },
+            relations: { fields: true, module: true },
+        });
+
+        if (!model) {
+            throw new NotFoundException(ERROR_MESSAGES.MODEL_NOT_FOUND(modelUserKey));
+        }
+
+        const fieldsForRemoval = model.fields.filter((field) => field.isMarkedForRemoval);
+        const operations: string[] = [];
+
+        if (fieldsForRemoval.length === 0) {
+            const message = `No fields marked for removal were found for model "${model.singularName}".`;
+            this.logger.log(message);
+            operations.push(message);
+            return {
+                dryRun,
+                modelName: model.singularName,
+                operations,
+                removedFieldNames: [],
+            };
+        }
+
+        const dataSource = await this.resolveDataSource(model.dataSource);
+        const entityMetadata = this.resolveEntityMetadata(dataSource, model);
+        const queryRunner = dataSource.createQueryRunner();
+
+        try {
+            await queryRunner.connect();
+
+            if (!dryRun) {
+                await queryRunner.startTransaction();
+            }
+
+            for (const field of fieldsForRemoval) {
+                await this.cleanupMarkedField({
+                    field,
+                    model,
+                    entityMetadata,
+                    queryRunner,
+                    dryRun,
+                    operations,
+                });
+            }
+
+            if (!dryRun) {
+                await queryRunner.commitTransaction();
+            }
+        } catch (error) {
+            if (!dryRun && queryRunner.isTransactionActive) {
+                await queryRunner.rollbackTransaction();
+            }
+            throw error;
+        } finally {
+            await queryRunner.release();
+        }
+
+        return {
+            dryRun,
+            modelName: model.singularName,
+            operations,
+            removedFieldNames: fieldsForRemoval.map((field) => field.name),
+        };
+    }
+
+    private async cleanupMarkedField(params: { field: FieldMetadata; model: ModelMetadata; entityMetadata?: EntityMetadata; queryRunner: QueryRunner; dryRun: boolean; operations: string[]; }): Promise<void> {
+        const { field, model, entityMetadata, queryRunner, dryRun, operations } = params;
+        const relationMetadata = entityMetadata?.relations.find((relation) => relation.propertyName === field.name);
+        const resolvedTableName = entityMetadata?.tableName || model.tableName;
+
+        if (field.type !== SolidFieldType.relation) {
+            const columnCandidates = this.buildColumnCandidates(field, entityMetadata, relationMetadata);
+            await this.dropColumnsForField(resolvedTableName, field, columnCandidates, queryRunner, dryRun, operations);
+            return;
+        }
+
+        if (field.relationType === RelationType.manyToOne) {
+            const columnCandidates = this.buildColumnCandidates(field, entityMetadata, relationMetadata);
+            await this.dropColumnsForField(resolvedTableName, field, columnCandidates, queryRunner, dryRun, operations);
+            return;
+        }
+
+        if (field.relationType === RelationType.manyTomany) {
+            const joinTableName = relationMetadata?.junctionEntityMetadata?.tableName || field.relationJoinTableName;
+            const ownsJoinTable = relationMetadata?.isOwning || field.isRelationManyToManyOwner;
+
+            if (!ownsJoinTable) {
+                operations.push(`No direct database cleanup required for inverse many-to-many field "${field.name}".`);
+                return;
+            }
+
+            if (!joinTableName) {
+                operations.push(`Skipping join-table cleanup for "${field.name}" because no join table name could be resolved.`);
+                return;
+            }
+
+            await this.dropJoinTable(joinTableName, field, queryRunner, dryRun, operations);
+            return;
+        }
+
+        operations.push(`No direct database cleanup required for relation field "${field.name}" with type "${field.relationType}".`);
+    }
+
+    private async dropColumnsForField(tableName: string, field: FieldMetadata, columnCandidates: string[], queryRunner: QueryRunner, dryRun: boolean, operations: string[],): Promise<void> {
+        if (!tableName) {
+            operations.push(`Skipping field "${field.name}" because the model table name could not be resolved.`);
+            return;
+        }
+
+        if (columnCandidates.length === 0) {
+            operations.push(`Skipping field "${field.name}" because no column candidates could be resolved.`);
+            return;
+        }
+
+        const handledColumns = new Set<string>();
+        let droppedAnyColumn = false;
+
+        for (const columnName of columnCandidates) {
+            if (!columnName || handledColumns.has(columnName)) {
+                continue;
+            }
+
+            handledColumns.add(columnName);
+            const table = await this.loadTable(queryRunner, tableName);
+            const column = table?.columns.find((tableColumn) => tableColumn.name === columnName);
+
+            if (!column) {
+                continue;
+            }
+
+            droppedAnyColumn = true;
+            await this.dropColumnArtifacts(table, columnName, queryRunner, dryRun, operations);
+        }
+
+        if (!droppedAnyColumn) {
+            operations.push(`No database column found for field "${field.name}" on table "${tableName}". Metadata cleanup will still proceed.`);
+        }
+    }
+
+    private async dropColumnArtifacts(table: Table, columnName: string, queryRunner: QueryRunner, dryRun: boolean, operations: string[],): Promise<void> {
+        for (const foreignKey of table.foreignKeys.filter((item) => item.columnNames.includes(columnName))) {
+            operations.push(`Drop foreign key "${foreignKey.name}" on "${table.name}.${columnName}"`);
+            if (!dryRun) {
+                await queryRunner.dropForeignKey(table, foreignKey);
+            }
+        }
+
+        for (const uniqueConstraint of table.uniques.filter((item) => item.columnNames.includes(columnName))) {
+            operations.push(`Drop unique constraint "${uniqueConstraint.name}" on "${table.name}.${columnName}"`);
+            if (!dryRun) {
+                await queryRunner.dropUniqueConstraint(table, uniqueConstraint);
+            }
+        }
+
+        for (const index of table.indices.filter((item) => item.columnNames.includes(columnName))) {
+            operations.push(`Drop index "${index.name}" on "${table.name}.${columnName}"`);
+            if (!dryRun) {
+                await queryRunner.dropIndex(table, index);
+            }
+        }
+
+        operations.push(`Drop column "${table.name}.${columnName}"`);
+        if (!dryRun) {
+            await queryRunner.dropColumn(table, columnName);
+        }
+    }
+
+    private async dropJoinTable(joinTableName: string, field: FieldMetadata, queryRunner: QueryRunner, dryRun: boolean, operations: string[]): Promise<void> {
+        const table = await this.loadTable(queryRunner, joinTableName);
+        if (!table) {
+            operations.push(`Join table "${joinTableName}" for field "${field.name}" does not exist. Metadata cleanup will still proceed.`);
+            return;
+        }
+
+        operations.push(`Drop join table "${joinTableName}" for field "${field.name}"`);
+        if (!dryRun) {
+            await queryRunner.dropTable(joinTableName);
+        }
+    }
+
+    private buildColumnCandidates(field: FieldMetadata, entityMetadata?: EntityMetadata, relationMetadata?: EntityMetadata["relations"][number]): string[] {
+        const columnCandidates = new Set<string>();
+
+        relationMetadata?.joinColumns?.forEach((column) => {
+            if (column.databaseName) {
+                columnCandidates.add(column.databaseName);
+            }
+        });
+
+        entityMetadata?.columns
+            .filter((column) => column.propertyName === field.name)
+            .forEach((column) => {
+                if (column.databaseName) {
+                    columnCandidates.add(column.databaseName);
+                }
+            });
+
+        if (field.columnName) {
+            columnCandidates.add(field.columnName);
+        }
+
+        if (field.relationCoModelColumnName) {
+            columnCandidates.add(field.relationCoModelColumnName);
+        }
+
+        if (field.type === SolidFieldType.relation && field.relationType === RelationType.manyToOne) {
+            columnCandidates.add(`${snakeCase(field.name)}_id`);
+        } else {
+            columnCandidates.add(snakeCase(field.name));
+        }
+
+        return [...columnCandidates].filter(Boolean);
+    }
+
+    private async resolveDataSource(dataSourceName?: string): Promise<DataSource> {
+        const normalizedDataSourceName = dataSourceName && dataSourceName !== "default" ? dataSourceName : undefined;
+        const token = normalizedDataSourceName ? getDataSourceToken(normalizedDataSourceName) : getDataSourceToken();
+        let dataSource: DataSource | undefined;
+
+        try {
+            dataSource = this.moduleRef.get<DataSource>(token, { strict: false });
+        } catch (error: any) {
+            throw new NotFoundException(`Datasource "${normalizedDataSourceName ?? "default"}" could not be resolved: ${error?.message ?? error}`);
+        }
+
+        if (!dataSource) {
+            throw new NotFoundException(`Datasource "${normalizedDataSourceName ?? "default"}" could not be resolved.`);
+        }
+
+        if (!dataSource.isInitialized) {
+            await dataSource.initialize();
+        }
+
+        return dataSource;
+    }
+
+    private resolveEntityMetadata(dataSource: DataSource, model: ModelMetadata): EntityMetadata | undefined {
+        const candidates = [classify(model.singularName), model.singularName, model.tableName].filter(Boolean);
+
+        for (const candidate of candidates) {
+            try {
+                return dataSource.getMetadata(candidate);
+            } catch {
+                // Try the next candidate.
+            }
+        }
+
+        return dataSource.entityMetadatas.find((metadata) => metadata.tableName === model.tableName);
+    }
+
+    private async resolveMetadataFilePath(moduleName: string): Promise<string> {
+        const defaultPath = await this.moduleMetadataHelperService.getModuleMetadataFilePath(moduleName);
+        if (await this.fileExists(defaultPath)) {
+            return defaultPath;
+        }
+
+        const dashModuleName = kebabCase(moduleName);
+        const moduleMetadataFilePath = path.resolve(
+            process.cwd(),
+            "module-metadata",
+            dashModuleName,
+            `${dashModuleName}-metadata.json`,
+        );
+
+        if (await this.fileExists(moduleMetadataFilePath)) {
+            return moduleMetadataFilePath;
+        }
+
+        return defaultPath;
+    }
+
+    private async fileExists(filePath: string): Promise<boolean> {
+        try {
+            await fs.access(filePath);
+            return true;
+        } catch {
+            return false;
+        }
+    }
+
+    private async loadTable(queryRunner: QueryRunner, tableName: string): Promise<Table | undefined> {
+        if (!tableName) {
+            return undefined;
+        }
+
+        const hasTable = await queryRunner.hasTable(tableName);
+        if (!hasTable) {
+            return undefined;
+        }
+
+        return queryRunner.getTable(tableName) ?? undefined;
+    }
+}

package/src/solid-core.module.ts

@@ -13,6 +13,7 @@ import {
 import { MulterModule } from "@nestjs/platform-express";
 import { TypeOrmModule } from "@nestjs/typeorm";
 import { RemoveFieldsCommand } from "./commands/remove-fields.command";
+import { MigrateRemovedFieldsCommand } from "./commands/migrate-removed-fields.command";
 import { FieldMetadataController } from "./controllers/field-metadata.controller";
 import { DashboardController } from "./controllers/dashboard.controller";
 import { MediaStorageProviderMetadataController } from "./controllers/media-storage-provider-metadata.controller";
@@ -47,6 +48,7 @@ import { ListOfValuesService } from "./services/list-of-values.service";
 import { MediaStorageProviderMetadataService } from "./services/media-storage-provider-metadata.service";
 import { MediaService } from "./services/media.service";
 import { ModelMetadataService } from "./services/model-metadata.service";
+import { RemovedFieldMigrationService } from "./services/removed-field-migration.service";
 import { ModuleMetadataExplorerService } from "./services/module-metadata-explorer.service";
 import { ModuleMetadataService } from "./services/module-metadata.service";
 import { ModulePackageService } from "./services/module-package.service";
@@ -545,6 +547,7 @@ import { PostgresDatasourceIntrospectionProviderService } from "./services/datas
     FieldMetadataService,
     DashboardRuntimeService,
     RemoveFieldsCommand,
+    MigrateRemovedFieldsCommand,
     RefreshModelCommand,
     RefreshModuleCommand,
     InfoCommand,
@@ -791,6 +794,7 @@ import { PostgresDatasourceIntrospectionProviderService } from "./services/datas
     UserViewMetadataRepository,
     ModelMetadataRepository,
     ModuleMetadataRepository,
+    RemovedFieldMigrationService,
     ActionMetadataRepository,
     MediaStorageProviderMetadataRepository,
     FixturesService,

package/src/subscribers/audit.subscriber.ts

@@ -1,12 +1,14 @@
 import { Injectable, Logger, Scope } from '@nestjs/common';
+import { ModelMetadataHelperService } from 'src/helpers/model-metadata-helper.service';
 import { lowerFirst } from 'src/helpers/string.helper';
 import { SolidRegistry } from 'src/helpers/solid-registry';
 import { DataSource, EntityMetadata, EntitySubscriberInterface, InsertEvent, RemoveEvent, UpdateEvent } from 'typeorm';
 import { AuditQueuePayload } from 'src/interfaces';
 import { RequestContextService } from 'src/services/request-context.service';
 import { PublisherFactory } from 'src/services/queues/publisher-factory.service';
+const AUDIT_BEFORE_SNAPSHOT = '__auditBeforeSnapshot';
 
-@Injectable({scope: Scope.TRANSIENT})
+@Injectable({ scope: Scope.TRANSIENT })
 export class AuditSubscriber implements EntitySubscriberInterface {
     private readonly logger = new Logger(AuditSubscriber.name);
     private dataSource: DataSource;
@@ -14,6 +16,7 @@ export class AuditSubscriber implements EntitySubscriberInterface {
         private readonly publisherFactory: PublisherFactory<AuditQueuePayload>,
         private readonly solidRegistry: SolidRegistry,
         private readonly requestContextService: RequestContextService,
+        private readonly modelMetadataHelperService: ModelMetadataHelperService,
     ) { }
 
     bindToDataSource(dataSource: DataSource) {
@@ -53,20 +56,82 @@ export class AuditSubscriber implements EntitySubscriberInterface {
 
     async afterUpdate(event: UpdateEvent<any>) {
         if (!this.shouldTrackAudit(event.metadata)) return;
+        const entityId = event.entity?.id ?? event.databaseEntity?.id ?? (event as any).entityId ?? null;
+        let before = event.databaseEntity ?? null;
+        let after = event.entity ?? null;
+        const updatedColumnNames = (event.updatedColumns ?? []).map(c => c.propertyName);
+
+        const auditResult = await this.prepareManyToManyAuditUpdateSnapshot(event, entityId, updatedColumnNames);
+
+        if (auditResult) {
+            before = auditResult.before;
+            after = auditResult.after;
+        }
+
         this.enqueue(event, {
             eventType: 'update',
             modelName: event.metadata.name,
-            entityId: event.entity?.id ?? null,
+            entityId: entityId,
             occurredAt: new Date().toISOString(),
-            after: event.entity ?? null,
+            after: after,
             // databaseEntity is only populated when the entity was fetched first (save() path).
             // QueryBuilder update() leaves this undefined; postAuditMessageOnUpdate guards for it.
-            before: event.databaseEntity ?? null,
-            updatedColumnNames: (event.updatedColumns ?? []).map(c => c.propertyName),
+            before: before,
+            updatedColumnNames: updatedColumnNames,
             userId: this.activeUserId(),
         });
     }
 
+    /**
+     * Resolves before/after snapshots for audit-enabled many-to-many relations
+     * and updates changed relation field names for audit tracking.
+     */
+    private async prepareManyToManyAuditUpdateSnapshot(event: UpdateEvent<any>, entityId: number | null, updatedColumnNames: string[]): Promise<{ before: any; after: any } | null> {
+
+        const auditRelationFields = (await this.modelMetadataHelperService.loadFieldHierarchy(lowerFirst(event.metadata.name))).filter(field =>
+            field.enableAuditTracking &&
+            field.type === 'relation' &&
+            field.relationType !== 'one-to-many'
+        );
+
+        if (!entityId || auditRelationFields.length === 0) {
+            return null;
+        }
+
+        const relations: Record<string, boolean> = {};
+
+        auditRelationFields.forEach(field => {
+            relations[field.name] = true;
+        });
+
+        const relationBefore = event.entity?.[AUDIT_BEFORE_SNAPSHOT] ?? null;
+
+        const relationAfter = await event.manager.getRepository(event.metadata.target as any).findOne({
+            where: { id: entityId } as any,
+            relations: relations as any,
+        });
+
+        if (relationBefore && relationAfter) {
+            auditRelationFields.forEach(field => {
+                const oldIds = Array.isArray(relationBefore[field.name])
+                    ? relationBefore[field.name].map(item => item.id).sort()
+                    : [];
+
+                const newIds = Array.isArray(relationAfter[field.name])
+                    ? relationAfter[field.name].map(item => item.id).sort()
+                    : [];
+
+                if ((oldIds.length !== newIds.length || JSON.stringify(oldIds) !== JSON.stringify(newIds))
+                    && !updatedColumnNames.includes(field.name)) {
+                    updatedColumnNames.push(field.name);
+                }
+            });
+        }
+        return {
+            before: relationBefore ?? event.databaseEntity ?? null,
+            after: relationAfter ?? event.entity ?? null,
+        };
+    }
     async afterRemove(event: RemoveEvent<any>) {
         if (!this.shouldTrackAudit(event.metadata)) return;
         this.enqueue(event, {

package/.claude/settings.local.json

@@ -1,16 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(node -e:*)",
-      "WebFetch(domain:docs.solidxai.com)",
-      "WebFetch(domain:github.com)",
-      "Read(//Users/oswald/projects/Solid_Starters/solidctl/**)",
-      "Read(//Users/oswald/projects/Solid_Starters/**)",
-      "Bash(find /Users/oswald/projects/Solid_Starters/solid-core-module -type d -name migration* -o -name *database*)",
-      "Bash(node verify-imports.js)"
-    ],
-    "additionalDirectories": [
-      "/Users/oswald/projects/Solid_Starters/solidctl"
-    ]
-  }
-}

package/CLAUDE.md

@@ -1,26 +0,0 @@
-# This is the SolidX API project.
-This is included as a dependency in consuming projects.
-
-# SolidX UI
-# The corresponding UI project is at the below path:
- /Users/oswald/projects/Solid_Starters/solid-core-ui
-
-# SolidX Docs
-# The documentation project is at the below path:
-/Users/oswald/projects/solidxai-docs
-
-# SolidX Agent
-# The agent project which has the skills for helping users configure, build and use SolidX is at the below path:
-/Users/oswald/projects/Solid_Starters/agent
-
-# SolidX Control Plane Commands
-# The control plane commands for managing SolidX instances are in the below path:
-/Users/oswald/projects/Solid_Starters/solidctl
-
-# Testing reference project 
-# This is the reference library management project that is used for testing and development of SolidX features. It is located at the below path:
-/Users/oswald/projects/Solid_Starters/solid-library-management
-
-# Code Builder project
-# This is the project that contains the code builder which generates code based on user input and templates. It is located at the below path:
-/Users/oswald/projects/Solid_Starters/solid-code-builder

package/src/services/1.js

@@ -1,6 +0,0 @@
-1. Do i need to create a storeStreams method for aws service too?
-    - Handle later
-2. queues handling -> if queues is enabled by default, i.e triggerExport(exportTransactionEntity.id).
-    - startExport should either return the data or return the transaction id
-3. How to handle scenarios wherein, nested related exist.(do i need to only get the userkey)
-    - show the userKey