insforge 0.3.3 → 1.3.0

package/backend/src/api/routes/{secrets.ts → secrets/index.routes.ts}

@@ -1,12 +1,13 @@
 import { Router, Response, NextFunction } from 'express';
-import { SecretsService } from '@/core/secrets/secrets.js';
-import { verifyAdmin, AuthRequest } from '@/api/middleware/auth.js';
-import { AuditService } from '@/core/logs/audit.js';
-import { AppError } from '@/api/middleware/error.js';
+import { SecretService } from '@/services/secrets/secret.service.js';
+import { verifyAdmin, AuthRequest } from '@/api/middlewares/auth.js';
+import { AuditService } from '@/services/logs/audit.service.js';
+import { AppError } from '@/api/middlewares/error.js';
 import { ERROR_CODES } from '@/types/error-constants.js';
+import { successResponse } from '@/utils/response.js';
 
 const router = Router();
-const secretsService = new SecretsService();
+const secretService = SecretService.getInstance();
 const auditService = AuditService.getInstance();
 
 /**
@@ -15,8 +16,8 @@ const auditService = AuditService.getInstance();
  */
 router.get('/', verifyAdmin, async (_req: AuthRequest, res: Response, next: NextFunction) => {
   try {
-    const secrets = await secretsService.listSecrets();
-    res.json({ secrets });
+    const secrets = await secretService.listSecrets();
+    successResponse(res, { secrets });
   } catch (error) {
     next(error);
   }
@@ -29,7 +30,7 @@ router.get('/', verifyAdmin, async (_req: AuthRequest, res: Response, next: Next
 router.get('/:key', verifyAdmin, async (req: AuthRequest, res: Response, next: NextFunction) => {
   try {
     const { key } = req.params;
-    const value = await secretsService.getSecretByKey(key);
+    const value = await secretService.getSecretByKey(key);
 
     if (value === null) {
       throw new AppError(`Secret not found: ${key}`, 404, ERROR_CODES.NOT_FOUND);
@@ -44,7 +45,7 @@ router.get('/:key', verifyAdmin, async (req: AuthRequest, res: Response, next: N
       ip_address: req.ip,
     });
 
-    res.json({ key, value });
+    successResponse(res, { key, value });
   } catch (error) {
     next(error);
   }
@@ -73,12 +74,12 @@ router.post('/', verifyAdmin, async (req: AuthRequest, res: Response, next: Next
     }
 
     // Check if secret already exists
-    const existing = await secretsService.getSecretByKey(key);
+    const existing = await secretService.getSecretByKey(key);
     if (existing !== null) {
       throw new AppError(`Secret already exists: ${key}`, 409, ERROR_CODES.INVALID_INPUT);
     }
 
-    const result = await secretsService.createSecret({
+    const result = await secretService.createSecret({
       key,
       value,
       isReserved: isReserved || false,
@@ -94,11 +95,15 @@ router.post('/', verifyAdmin, async (req: AuthRequest, res: Response, next: Next
       ip_address: req.ip,
     });
 
-    res.status(201).json({
-      success: true,
-      message: `Secret ${key} has been created successfully`,
-      id: result.id,
-    });
+    successResponse(
+      res,
+      {
+        success: true,
+        message: `Secret ${key} has been created successfully`,
+        id: result.id,
+      },
+      201
+    );
   } catch (error) {
     next(error);
   }
@@ -114,14 +119,14 @@ router.put('/:key', verifyAdmin, async (req: AuthRequest, res: Response, next: N
     const { value, isActive, isReserved, expiresAt } = req.body;
 
     // Get existing secret
-    const secrets = await secretsService.listSecrets();
+    const secrets = await secretService.listSecrets();
     const secret = secrets.find((s) => s.key === key);
 
     if (!secret) {
       throw new AppError(`Secret not found: ${key}`, 404, ERROR_CODES.NOT_FOUND);
     }
 
-    const success = await secretsService.updateSecret(secret.id, {
+    const success = await secretService.updateSecret(secret.id, {
       value,
       isActive,
       isReserved,
@@ -141,7 +146,7 @@ router.put('/:key', verifyAdmin, async (req: AuthRequest, res: Response, next: N
       ip_address: req.ip,
     });
 
-    res.json({
+    successResponse(res, {
       success: true,
       message: `Secret ${key} has been updated successfully`,
     });
@@ -159,7 +164,7 @@ router.delete('/:key', verifyAdmin, async (req: AuthRequest, res: Response, next
     const { key } = req.params;
 
     // Get existing secret
-    const secrets = await secretsService.listSecrets();
+    const secrets = await secretService.listSecrets();
     const secret = secrets.find((s) => s.key === key);
 
     if (!secret) {
@@ -172,7 +177,7 @@ router.delete('/:key', verifyAdmin, async (req: AuthRequest, res: Response, next
     }
 
     // Mark as inactive instead of hard delete
-    const success = await secretsService.updateSecret(secret.id, { isActive: false });
+    const success = await secretService.updateSecret(secret.id, { isActive: false });
 
     if (!success) {
       throw new AppError(`Failed to delete secret: ${key}`, 500, ERROR_CODES.INTERNAL_ERROR);
@@ -187,7 +192,7 @@ router.delete('/:key', verifyAdmin, async (req: AuthRequest, res: Response, next
       ip_address: req.ip,
     });
 
-    res.json({
+    successResponse(res, {
       success: true,
       message: `Secret ${key} has been deleted successfully`,
     });

package/backend/src/api/routes/{storage.ts → storage/index.routes.ts}

@@ -1,20 +1,14 @@
 import { Router, Request, Response, NextFunction } from 'express';
-import path from 'path';
-import { verifyAdmin, AuthRequest, verifyUser } from '@/api/middleware/auth.js';
-import { AppError } from '@/api/middleware/error.js';
-import { StorageService } from '@/core/storage/storage.js';
-import { DatabaseManager } from '@/core/database/manager.js';
+import { verifyAdmin, AuthRequest, verifyUser } from '@/api/middlewares/auth.js';
+import { AppError } from '@/api/middlewares/error.js';
+import { StorageService } from '@/services/storage/storage.service.js';
 import { successResponse } from '@/utils/response.js';
-import { upload, handleUploadError } from '@/api/middleware/upload.js';
+import { upload, handleUploadError } from '@/api/middlewares/upload.js';
 import { ERROR_CODES } from '@/types/error-constants.js';
-import {
-  StorageBucketSchema,
-  createBucketRequestSchema,
-  updateBucketRequestSchema,
-} from '@insforge/shared-schemas';
-import { SocketService } from '@/core/socket/socket';
-import { DataUpdateResourceType, ServerEvents } from '@/core/socket/types';
-import { AuditService } from '@/core/logs/audit.js';
+import { createBucketRequestSchema, updateBucketRequestSchema } from '@insforge/shared-schemas';
+import { SocketManager } from '@/infra/socket/socket.manager.js';
+import { DataUpdateResourceType, ServerEvents } from '@/types/socket.js';
+import { AuditService } from '@/services/logs/audit.service.js';
 
 const router = Router();
 const auditService = AuditService.getInstance();
@@ -40,24 +34,19 @@ const conditionalAuth = async (req: Request, res: Response, next: NextFunction)
   return verifyUser(req, res, next);
 };
 
-// GET /api/storage/buckets - List all buckets (requires auth)
+// GET /api/storage/buckets - List all buckets (requires admin)
 router.get('/buckets', verifyAdmin, async (req: AuthRequest, res: Response, next: NextFunction) => {
   try {
-    const db = DatabaseManager.getInstance().getDb();
+    const storageService = StorageService.getInstance();
+    const buckets = await storageService.listBuckets();
 
-    // Get all buckets with their metadata from _storage_buckets table
-    const buckets = (await db
-      .prepare('SELECT name, public, created_at FROM _storage_buckets ORDER BY name')
-      .all()) as StorageBucketSchema[];
-
-    // Traditional REST: return array directly
     successResponse(res, buckets);
   } catch (error) {
     next(error);
   }
 });
 
-// POST /api/storage/buckets - Create a new bucket (requires auth)
+// POST /api/storage/buckets - Create a new bucket (requires admin)
 router.post(
   '/buckets',
   verifyAdmin,
@@ -89,10 +78,13 @@ router.post(
         ip_address: req.ip,
       });
 
-      const socket = SocketService.getInstance();
-      socket.broadcastToRoom('role:project_admin', ServerEvents.DATA_UPDATE, {
-        resource: DataUpdateResourceType.STORAGE_SCHEMA,
-      });
+      const socket = SocketManager.getInstance();
+      socket.broadcastToRoom(
+        'role:project_admin',
+        ServerEvents.DATA_UPDATE,
+        { resource: DataUpdateResourceType.BUCKETS },
+        'system'
+      );
 
       const accessInfo = isPublic
         ? 'This is a PUBLIC bucket - objects can be accessed without authentication.'
@@ -160,13 +152,13 @@ router.patch(
         ip_address: req.ip,
       });
 
-      const socket = SocketService.getInstance();
-      socket.broadcastToRoom('role:project_admin', ServerEvents.DATA_UPDATE, {
-        resource: DataUpdateResourceType.BUCKET_SCHEMA,
-        data: {
-          name: bucketName,
-        },
-      });
+      const socket = SocketManager.getInstance();
+      socket.broadcastToRoom(
+        'role:project_admin',
+        ServerEvents.DATA_UPDATE,
+        { resource: DataUpdateResourceType.BUCKETS, data: { bucketName } },
+        'system'
+      );
 
       const accessInfo = isPublic
         ? 'Bucket is now PUBLIC - objects can be accessed without authentication.'
@@ -287,17 +279,11 @@ router.post(
         throw new AppError('File is required', 400, ERROR_CODES.STORAGE_INVALID_PARAMETER);
       }
 
-      // Generate a unique key for the object
-      const timestamp = Date.now();
-      const randomStr = Math.random().toString(36).substring(2, 8);
-      const fileExt = req.file.originalname ? path.extname(req.file.originalname) : '';
-      const baseName = req.file.originalname
-        ? path.basename(req.file.originalname, fileExt)
-        : 'file';
-      const sanitizedBaseName = baseName.replace(/[^a-zA-Z0-9-_]/g, '-').substring(0, 32);
-      const objectKey = `${sanitizedBaseName}-${timestamp}-${randomStr}${fileExt}`;
-
       const storageService = StorageService.getInstance();
+
+      // Generate a unique key for the object using service
+      const objectKey = storageService.generateObjectKey(req.file.originalname);
+
       const storedFile = await storageService.putObject(
         bucketName,
         objectKey,
@@ -339,12 +325,10 @@ router.get(
       }
 
       const storageService = StorageService.getInstance();
-      const expiresIn = (await storageService.isBucketPublic(bucketName)) ? 0 : 3600;
-      const strategy = await storageService.getDownloadStrategy(
-        bucketName,
-        objectKey,
-        Number(expiresIn)
-      );
+
+      // Get download strategy (service auto-calculates expiry based on bucket visibility)
+      const strategy = await storageService.getDownloadStrategy(bucketName, objectKey);
+
       if (strategy.method === 'presigned') {
         return res.redirect(strategy.url);
       }
@@ -397,10 +381,13 @@ router.delete(
         ip_address: req.ip,
       });
 
-      const socket = SocketService.getInstance();
-      socket.broadcastToRoom('role:project_admin', ServerEvents.DATA_UPDATE, {
-        resource: DataUpdateResourceType.STORAGE_SCHEMA,
-      });
+      const socket = SocketManager.getInstance();
+      socket.broadcastToRoom(
+        'role:project_admin',
+        ServerEvents.DATA_UPDATE,
+        { resource: DataUpdateResourceType.BUCKETS },
+        'system'
+      );
 
       successResponse(
         res,
@@ -432,7 +419,12 @@ router.delete(
 
       // Delete specific object
       const storageService = StorageService.getInstance();
-      const deleted = await storageService.deleteObject(bucketName, objectKey, req.user?.id);
+      const deleted = await storageService.deleteObject(
+        bucketName,
+        objectKey,
+        req.user?.id,
+        req.user?.role === 'project_admin'
+      );
 
       if (!deleted) {
         throw new AppError('Object not found', 404, ERROR_CODES.NOT_FOUND);
@@ -525,14 +517,9 @@ router.post(
   async (req: AuthRequest | Request, res: Response, next: NextFunction) => {
     try {
       const { bucketName, objectKey } = req.params;
-      const { expiresIn = 3600 } = req.body;
 
       const storageService = StorageService.getInstance();
-      const strategy = await storageService.getDownloadStrategy(
-        bucketName,
-        objectKey,
-        Number(expiresIn)
-      );
+      const strategy = await storageService.getDownloadStrategy(bucketName, objectKey);
 
       successResponse(res, strategy);
     } catch (error) {

package/backend/src/api/routes/usage/index.routes.ts

@@ -0,0 +1,91 @@
+import { Router, NextFunction, Response } from 'express';
+import {
+  verifyCloudBackend,
+  verifyApiKey,
+  verifyAdmin,
+  AuthRequest,
+} from '@/api/middlewares/auth.js';
+import { SocketManager } from '@/infra/socket/socket.manager.js';
+import { ServerEvents } from '@/types/socket.js';
+import { UsageService } from '@/services/usage/usage.service.js';
+import { successResponse } from '@/utils/response.js';
+import { AppError } from '@/api/middlewares/error.js';
+import { ERROR_CODES } from '@/types/error-constants.js';
+
+export const usageRouter = Router();
+const usageService = UsageService.getInstance();
+
+// Create MCP tool usage record
+usageRouter.post(
+  '/mcp',
+  verifyApiKey,
+  async (req: AuthRequest, res: Response, next: NextFunction) => {
+    try {
+      const { tool_name, success = true } = req.body;
+
+      if (!tool_name) {
+        throw new AppError('tool_name is required', 400, ERROR_CODES.INVALID_INPUT);
+      }
+
+      // Create MCP usage record via service
+      const result = await usageService.recordMCPUsage(tool_name, success);
+
+      // Broadcast MCP tool usage to frontend via socket
+      const socketService = SocketManager.getInstance();
+
+      socketService.broadcastToRoom(
+        'role:project_admin',
+        ServerEvents.MCP_CONNECTED,
+        { tool_name, created_at: result.created_at },
+        'system'
+      );
+
+      successResponse(res, { success: true });
+    } catch (error) {
+      next(error);
+    }
+  }
+);
+
+// Get MCP usage records
+usageRouter.get(
+  '/mcp',
+  verifyAdmin,
+  async (req: AuthRequest, res: Response, next: NextFunction) => {
+    try {
+      const { limit = '5', success = 'true' } = req.query;
+
+      // Get MCP usage records via service
+      const records = await usageService.getMCPUsage(parseInt(limit as string), success === 'true');
+
+      successResponse(res, { records });
+    } catch (error) {
+      next(error);
+    }
+  }
+);
+
+// Get usage statistics (called by cloud backend)
+usageRouter.get(
+  '/stats',
+  verifyCloudBackend,
+  async (req: AuthRequest, res: Response, next: NextFunction) => {
+    try {
+      const { start_date, end_date } = req.query;
+
+      if (!start_date || !end_date) {
+        throw new AppError('start_date and end_date are required', 400, ERROR_CODES.INVALID_INPUT);
+      }
+
+      // Get usage statistics via service
+      const stats = await usageService.getUsageStats(
+        new Date(start_date as string),
+        new Date(end_date as string)
+      );
+
+      successResponse(res, stats);
+    } catch (error) {
+      next(error);
+    }
+  }
+);

package/backend/src/infra/config/app.config.ts

@@ -0,0 +1,51 @@
+export interface AppConfig {
+  app: {
+    port: number;
+    jwtSecret: string;
+    apiKey: string;
+    logLevel: string;
+  };
+  database: {
+    host: string;
+    port: number;
+    username: string;
+    password: string;
+    databaseName: string;
+  };
+  cloud: {
+    storageBucket: string;
+    instanceProfile: string;
+    apiHost: string;
+    projectId: string;
+    appKey: string;
+    cloudFrontUrl: string;
+    cloudFrontKeyPairId: string;
+    cloudFrontPrivateKey: string;
+  };
+}
+
+export const config: AppConfig = {
+  app: {
+    port: parseInt(process.env.PORT || '3000', 10),
+    jwtSecret: process.env.JWT_SECRET || 'your_jwt_secret',
+    apiKey: process.env.ACCESS_API_KEY || 'your_api_key',
+    logLevel: process.env.LOG_LEVEL || 'info',
+  },
+  database: {
+    host: process.env.POSTGRES_HOST || 'localhost',
+    port: parseInt(process.env.POSTGRES_PORT || '5432', 10),
+    username: process.env.POSTGRES_USERNAME || 'user',
+    password: process.env.POSTGRES_PASSWORD || 'password',
+    databaseName: process.env.POSTGRES_NAME || 'insforge',
+  },
+  cloud: {
+    storageBucket: process.env.AWS_S3_BUCKET || 'insforge-test-bucket',
+    instanceProfile: process.env.AWS_INSTANCE_PROFILE_NAME || 'insforge-instance-profile',
+    apiHost: process.env.CLOUD_API_HOST || 'https://api.insforge.dev',
+    projectId: process.env.PROJECT_ID || 'local',
+    appKey: process.env.APP_KEY || 'default-app-key',
+    cloudFrontUrl: process.env.AWS_CLOUDFRONT_URL || '',
+    cloudFrontKeyPairId: process.env.AWS_CLOUDFRONT_KEY_PAIR_ID || '',
+    cloudFrontPrivateKey: process.env.AWS_CLOUDFRONT_PRIVATE_KEY || '',
+  },
+};

package/backend/src/infra/database/database.manager.ts

@@ -0,0 +1,182 @@
+import { Pool, Client } from 'pg';
+import path from 'path';
+import fs from 'fs/promises';
+import { fileURLToPath } from 'url';
+import { DatabaseMetadataSchema } from '@insforge/shared-schemas';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+export class DatabaseManager {
+  private static instance: DatabaseManager;
+  private pool!: Pool;
+  private dataDir: string;
+
+  private constructor() {
+    this.dataDir = process.env.DATABASE_DIR || path.join(__dirname, '../../data');
+  }
+
+  static getInstance(): DatabaseManager {
+    if (!DatabaseManager.instance) {
+      DatabaseManager.instance = new DatabaseManager();
+    }
+    return DatabaseManager.instance;
+  }
+
+  async initialize(): Promise<void> {
+    await fs.mkdir(this.dataDir, { recursive: true });
+
+    // PostgreSQL connection configuration
+    this.pool = new Pool({
+      host: process.env.POSTGRES_HOST || 'localhost',
+      port: parseInt(process.env.POSTGRES_PORT || '5432'),
+      database: process.env.POSTGRES_DB || 'insforge',
+      user: process.env.POSTGRES_USER || 'postgres',
+      password: process.env.POSTGRES_PASSWORD || 'postgres',
+      max: 20,
+      idleTimeoutMillis: 30000,
+      connectionTimeoutMillis: 2000,
+    });
+
+    const client = await this.pool.connect();
+    await client.query('BEGIN');
+
+    // Note: Schema migrations are now handled by node-pg-migrate
+    // Run: npm run migrate:up
+
+    await client.query('COMMIT');
+  }
+
+  static async getColumnTypeMap(tableName: string): Promise<Record<string, string>> {
+    const instance = DatabaseManager.getInstance();
+    const client = await instance.pool.connect();
+    try {
+      const result = await client.query(
+        `SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = 'public' AND table_name = $1`,
+        [tableName]
+      );
+      const map: Record<string, string> = {};
+      for (const row of result.rows) {
+        map[row.column_name] = row.data_type;
+      }
+      return map;
+    } finally {
+      client.release();
+    }
+  }
+
+  async getUserTables(): Promise<string[]> {
+    const client = await this.pool.connect();
+    try {
+      const result = await client.query(
+        `
+          SELECT table_name as name
+          FROM information_schema.tables
+          WHERE table_schema = 'public'
+          AND table_type = 'BASE TABLE'
+          AND (table_name NOT LIKE '\\_%')
+          ORDER BY table_name
+        `
+      );
+      return result.rows.map((row: { name: string }) => row.name);
+    } finally {
+      client.release();
+    }
+  }
+
+  async getMetadata(): Promise<DatabaseMetadataSchema> {
+    const client = await this.pool.connect();
+    try {
+      // Fetch all tables, database size, and record counts in parallel
+      const [allTables, databaseSize, countResults] = await Promise.all([
+        this.getUserTables(),
+        this.getDatabaseSizeInGB(),
+        // Get all counts in a single query using UNION ALL
+        (async () => {
+          try {
+            const tablesResult = await client.query(
+              `
+              SELECT table_name as name
+              FROM information_schema.tables
+              WHERE table_schema = 'public'
+              AND table_type = 'BASE TABLE'
+              AND (table_name NOT LIKE '\\_%')
+              ORDER BY table_name
+            `
+            );
+            const tableNames = tablesResult.rows.map((row: { name: string }) => row.name);
+
+            if (tableNames.length === 0) {
+              return [];
+            }
+
+            // Build a UNION ALL query to get all counts in one query
+            const unionQuery = tableNames
+              .map(
+                (tableName) =>
+                  `SELECT '${tableName.replace(/'/g, "''")}' as table_name, COUNT(*) as count FROM "${tableName}"`
+              )
+              .join(' UNION ALL ');
+
+            const result = await client.query(unionQuery);
+            return result.rows as { table_name: string; count: number }[];
+          } catch {
+            return [];
+          }
+        })(),
+      ]);
+
+      // Map the count results to a lookup object
+      const countMap = new Map(countResults.map((r) => [r.table_name, Number(r.count)]));
+
+      const tableMetadatas = allTables.map((tableName) => ({
+        tableName,
+        recordCount: countMap.get(tableName) || 0,
+      }));
+
+      return {
+        tables: tableMetadatas,
+        totalSizeInGB: databaseSize,
+        hint: 'To retrieve detailed schema information for a specific table, call the get-table-schema tool with the table name.',
+      };
+    } finally {
+      client.release();
+    }
+  }
+
+  async getDatabaseSizeInGB(): Promise<number> {
+    const client = await this.pool.connect();
+    try {
+      // Query PostgreSQL for database size
+      const result = await client.query(`SELECT pg_database_size(current_database()) as size`);
+
+      // PostgreSQL returns size in bytes, convert to GB
+      return (result.rows[0]?.size || 0) / (1024 * 1024 * 1024);
+    } catch {
+      return 0;
+    } finally {
+      client.release();
+    }
+  }
+
+  getPool(): Pool {
+    return this.pool;
+  }
+
+  /**
+   * Create a dedicated client for operations that can't use pooled connections (e.g., LISTEN/NOTIFY)
+   */
+  createClient(): Client {
+    return new Client({
+      host: process.env.POSTGRES_HOST || 'localhost',
+      port: parseInt(process.env.POSTGRES_PORT || '5432'),
+      database: process.env.POSTGRES_DB || 'insforge',
+      user: process.env.POSTGRES_USER || 'postgres',
+      password: process.env.POSTGRES_PASSWORD || 'postgres',
+    });
+  }
+
+  async close(): Promise<void> {
+    await this.pool.end();
+  }
+}