insforge 0.3.3 → 1.2.10

package/backend/src/providers/storage/s3.provider.ts

@@ -0,0 +1,313 @@
+import {
+  S3Client,
+  PutObjectCommand,
+  GetObjectCommand,
+  DeleteObjectCommand,
+  ListObjectsV2Command,
+  DeleteObjectsCommand,
+  HeadObjectCommand,
+} from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { createPresignedPost } from '@aws-sdk/s3-presigned-post';
+import { getSignedUrl as getCloudFrontSignedUrl } from '@aws-sdk/cloudfront-signer';
+import { UploadStrategyResponse, DownloadStrategyResponse } from '@insforge/shared-schemas';
+import { StorageProvider } from './base.provider.js';
+import logger from '@/utils/logger.js';
+
+const ONE_HOUR_IN_SECONDS = 3600;
+const DEFAULT_MAX_UPLOAD_SIZE_BYTES = 10485760;
+const SEVEN_DAYS_IN_SECONDS = 604800;
+
+/**
+ * S3 storage implementation
+ */
+export class S3StorageProvider implements StorageProvider {
+  private s3Client: S3Client | null = null;
+
+  constructor(
+    private s3Bucket: string,
+    private appKey: string,
+    private region: string = 'us-east-2'
+  ) {}
+
+  initialize(): void {
+    // Use explicit AWS credentials if provided (local dev or self hosting)
+    // Otherwise, use IAM role credentials (EC2 production)
+    const s3Config: {
+      region: string;
+      credentials?: { accessKeyId: string; secretAccessKey: string };
+    } = {
+      region: this.region,
+    };
+
+    if (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) {
+      s3Config.credentials = {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+      };
+    }
+
+    this.s3Client = new S3Client(s3Config);
+  }
+
+  private getS3Key(bucket: string, key: string): string {
+    return `${this.appKey}/${bucket}/${key}`;
+  }
+
+  async putObject(bucket: string, key: string, file: Express.Multer.File): Promise<void> {
+    if (!this.s3Client) {
+      throw new Error('S3 client not initialized');
+    }
+    const s3Key = this.getS3Key(bucket, key);
+
+    const command = new PutObjectCommand({
+      Bucket: this.s3Bucket,
+      Key: s3Key,
+      Body: file.buffer,
+      ContentType: file.mimetype || 'application/octet-stream',
+    });
+
+    try {
+      await this.s3Client.send(command);
+    } catch (error) {
+      logger.error('S3 Upload error', {
+        error: error instanceof Error ? error.message : String(error),
+        bucket,
+        key: s3Key,
+      });
+      throw error;
+    }
+  }
+
+  async getObject(bucket: string, key: string): Promise<Buffer | null> {
+    if (!this.s3Client) {
+      throw new Error('S3 client not initialized');
+    }
+    try {
+      const command = new GetObjectCommand({
+        Bucket: this.s3Bucket,
+        Key: this.getS3Key(bucket, key),
+      });
+      const response = await this.s3Client.send(command);
+      const chunks: Uint8Array[] = [];
+      // Type assertion for readable stream
+      const body = response.Body as AsyncIterable<Uint8Array>;
+      for await (const chunk of body) {
+        chunks.push(chunk);
+      }
+      return Buffer.concat(chunks);
+    } catch {
+      return null;
+    }
+  }
+
+  async deleteObject(bucket: string, key: string): Promise<void> {
+    if (!this.s3Client) {
+      throw new Error('S3 client not initialized');
+    }
+    const command = new DeleteObjectCommand({
+      Bucket: this.s3Bucket,
+      Key: this.getS3Key(bucket, key),
+    });
+    await this.s3Client.send(command);
+  }
+
+  async createBucket(_bucket: string): Promise<void> {
+    // In S3 with multi-tenant, we don't create actual buckets
+    // We just use folders under the app key
+  }
+
+  async deleteBucket(bucket: string): Promise<void> {
+    if (!this.s3Client) {
+      throw new Error('S3 client not initialized');
+    }
+    // List and delete all objects in the "bucket" (folder)
+    const prefix = `${this.appKey}/${bucket}/`;
+
+    let continuationToken: string | undefined;
+    do {
+      const listCommand = new ListObjectsV2Command({
+        Bucket: this.s3Bucket,
+        Prefix: prefix,
+        ContinuationToken: continuationToken,
+      });
+      const listResponse = await this.s3Client.send(listCommand);
+
+      if (listResponse.Contents && listResponse.Contents.length) {
+        const deleteCommand = new DeleteObjectsCommand({
+          Bucket: this.s3Bucket,
+          Delete: {
+            Objects: listResponse.Contents.filter((obj) => obj.Key !== undefined).map((obj) => ({
+              Key: obj.Key as string,
+            })),
+          },
+        });
+        await this.s3Client.send(deleteCommand);
+      }
+
+      continuationToken = listResponse.NextContinuationToken;
+    } while (continuationToken);
+  }
+
+  // S3 supports presigned URLs
+  supportsPresignedUrls(): boolean {
+    return true;
+  }
+
+  async getUploadStrategy(
+    bucket: string,
+    key: string,
+    metadata: { contentType?: string; size?: number }
+  ): Promise<UploadStrategyResponse> {
+    if (!this.s3Client) {
+      throw new Error('S3 client not initialized');
+    }
+
+    const s3Key = this.getS3Key(bucket, key);
+    const expiresIn = ONE_HOUR_IN_SECONDS; // 1 hour
+
+    try {
+      // Generate presigned POST URL for multipart form upload
+      const { url, fields } = await createPresignedPost(this.s3Client, {
+        Bucket: this.s3Bucket,
+        Key: s3Key,
+        Conditions: [
+          ['content-length-range', 0, metadata.size || DEFAULT_MAX_UPLOAD_SIZE_BYTES], // Max 10MB by default
+        ],
+        Expires: expiresIn,
+      });
+
+      return {
+        method: 'presigned',
+        uploadUrl: url,
+        fields,
+        key,
+        confirmRequired: true,
+        confirmUrl: `/api/storage/buckets/${bucket}/objects/${encodeURIComponent(key)}/confirm-upload`,
+        expiresAt: new Date(Date.now() + expiresIn * 1000),
+      };
+    } catch (error) {
+      logger.error('Failed to generate presigned upload URL', {
+        error: error instanceof Error ? error.message : String(error),
+        bucket,
+        key,
+      });
+      throw error;
+    }
+  }
+
+  async getDownloadStrategy(
+    bucket: string,
+    key: string,
+    expiresIn: number = ONE_HOUR_IN_SECONDS,
+    isPublic: boolean = false
+  ): Promise<DownloadStrategyResponse> {
+    if (!this.s3Client) {
+      throw new Error('S3 client not initialized');
+    }
+
+    const s3Key = this.getS3Key(bucket, key);
+    // Public files get longer expiration (7 days), private files get shorter (1 hour default)
+    const actualExpiresIn = isPublic ? SEVEN_DAYS_IN_SECONDS : expiresIn; // 604800 = 7 days
+    const cloudFrontUrl = process.env.AWS_CLOUDFRONT_URL;
+
+    try {
+      // If CloudFront URL is configured, use CloudFront for downloads
+      if (cloudFrontUrl) {
+        const cloudFrontKeyPairId = process.env.AWS_CLOUDFRONT_KEY_PAIR_ID;
+        const cloudFrontPrivateKey = process.env.AWS_CLOUDFRONT_PRIVATE_KEY;
+
+        if (!cloudFrontKeyPairId || !cloudFrontPrivateKey) {
+          logger.warn(
+            'CloudFront URL configured but missing key pair ID or private key, falling back to S3'
+          );
+        } else {
+          try {
+            // Generate CloudFront signed URL
+            // IMPORTANT: URL-encode the S3 key to match what CloudFront receives
+            // This ensures the signature matches for files with spaces, parentheses, etc.
+            const encodedS3Key = s3Key
+              .split('/')
+              .map((segment) => encodeURIComponent(segment))
+              .join('/');
+            const cloudFrontObjectUrl = `${cloudFrontUrl.replace(/\/$/, '')}/${encodedS3Key}`;
+
+            // Convert escaped newlines to actual newlines in the private key
+            const formattedPrivateKey = cloudFrontPrivateKey.replace(/\\n/g, '\n');
+
+            // dateLessThan can be string | number | Date - using Date object directly
+            const dateLessThan = new Date(Date.now() + actualExpiresIn * 1000);
+
+            const signedUrl = getCloudFrontSignedUrl({
+              url: cloudFrontObjectUrl,
+              keyPairId: cloudFrontKeyPairId,
+              privateKey: formattedPrivateKey,
+              dateLessThan,
+            });
+
+            logger.info('CloudFront signed URL generated successfully.');
+
+            return {
+              method: 'presigned',
+              url: signedUrl,
+              expiresAt: dateLessThan,
+            };
+          } catch (cfError) {
+            logger.error('Failed to generate CloudFront signed URL, falling back to S3', {
+              error: cfError instanceof Error ? cfError.message : String(cfError),
+              bucket,
+              key,
+            });
+            // Fall through to S3 signed URL generation
+          }
+        }
+      }
+
+      // Note: isPublic here refers to the application-level setting,
+      // not the actual S3 bucket policy. In a multi-tenant setup,
+      // we're using a single S3 bucket with folder-based isolation,
+      // so we always use presigned URLs for security.
+      // The "public" setting only affects the URL expiration time.
+
+      // Always generate presigned URL for security in multi-tenant environment
+      const command = new GetObjectCommand({
+        Bucket: this.s3Bucket,
+        Key: s3Key,
+      });
+
+      const url = await getSignedUrl(this.s3Client, command, { expiresIn: actualExpiresIn });
+
+      return {
+        method: 'presigned',
+        url,
+        expiresAt: new Date(Date.now() + actualExpiresIn * 1000),
+      };
+    } catch (error) {
+      logger.error('Failed to generate download URL', {
+        error: error instanceof Error ? error.message : String(error),
+        bucket,
+        key,
+      });
+      throw error;
+    }
+  }
+
+  async verifyObjectExists(bucket: string, key: string): Promise<boolean> {
+    if (!this.s3Client) {
+      throw new Error('S3 client not initialized');
+    }
+
+    const s3Key = this.getS3Key(bucket, key);
+
+    try {
+      const command = new HeadObjectCommand({
+        Bucket: this.s3Bucket,
+        Key: s3Key,
+      });
+      await this.s3Client.send(command);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}

package/backend/src/server.ts

@@ -5,29 +5,25 @@ import dotenv from 'dotenv';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import fs from 'fs';
-import authRouter from '@/api/routes/auth.js';
-import { databaseTablesRouter } from '@/api/routes/database.tables.js';
-import { databaseRecordsRouter } from '@/api/routes/database.records.js';
-import databaseAdvanceRouter from '@/api/routes/database.advance.js';
-import { storageRouter } from '@/api/routes/storage.js';
-import { metadataRouter } from '@/api/routes/metadata.js';
-import { logsRouter } from '@/api/routes/logs.js';
-import { docsRouter } from '@/api/routes/docs.js';
-import functionsRouter from '@/api/routes/functions.js';
-import secretsRouter from '@/api/routes/secrets.js';
-import { usageRouter } from '@/api/routes/usage.js';
-import { openAPIRouter } from '@/api/routes/openapi.js';
-import { agentDocsRouter } from '@/api/routes/agent.js';
-import { aiRouter } from '@/api/routes/ai.js';
-import { errorMiddleware } from '@/api/middleware/error.js';
+import authRouter from '@/api/routes/auth/index.routes.js';
+import databaseRouter from '@/api/routes/database/index.routes.js';
+import { storageRouter } from '@/api/routes/storage/index.routes.js';
+import { metadataRouter } from '@/api/routes/metadata/index.routes.js';
+import { logsRouter } from '@/api/routes/logs/index.routes.js';
+import { docsRouter } from '@/api/routes/docs/index.routes.js';
+import functionsRouter from '@/api/routes/functions/index.routes.js';
+import secretsRouter from '@/api/routes/secrets/index.routes.js';
+import { usageRouter } from '@/api/routes/usage/index.routes.js';
+import { aiRouter } from '@/api/routes/ai/index.routes.js';
+import { errorMiddleware } from '@/api/middlewares/error.js';
 import fetch, { HeadersInit } from 'node-fetch';
-import { DatabaseManager } from '@/core/database/manager.js';
-import { AnalyticsManager } from '@/core/logs/analytics.js';
-import { StorageService } from '@/core/storage/storage.js';
-import { SocketService } from '@/core/socket/socket.js';
+import { DatabaseManager } from '@/infra/database/database.manager.js';
+import { LogService } from '@/services/logs/log.service.js';
+import { StorageService } from '@/services/storage/storage.service.js';
+import { SocketManager } from '@/infra/socket/socket.manager.js';
 import { seedBackend } from '@/utils/seed.js';
 import logger from '@/utils/logger.js';
-import { isProduction } from './utils/environment';
+import { isProduction } from './utils/environment.js';
 import packageJson from '../../package.json';
 
 const __filename = fileURLToPath(import.meta.url);
@@ -51,11 +47,9 @@ export async function createApp() {
   const storageService = StorageService.getInstance();
   await storageService.initialize(); // create data/storage
 
-  // Metadata is now handled by individual modules on-demand
-
-  // Initialize analytics service
-  const analyticsManager = AnalyticsManager.getInstance();
-  await analyticsManager.initialize(); // connect to _insforge database
+  // Initialize logs service
+  const logService = LogService.getInstance();
+  await logService.initialize(); // connect to CloudWatch
 
   const app = express();
 
@@ -87,25 +81,47 @@ export async function createApp() {
     let responseSize = 0;
 
     // Override send method
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    res.send = function (data: any) {
-      if (data) {
-        responseSize = Buffer.byteLength(typeof data === 'string' ? data : JSON.stringify(data));
+    res.send = function (
+      data: string | Buffer | Record<string, unknown> | unknown[] | number | boolean
+    ) {
+      if (data !== undefined && data !== null) {
+        if (typeof data === 'string') {
+          responseSize = Buffer.byteLength(data);
+        } else if (Buffer.isBuffer(data)) {
+          responseSize = data.length;
+        } else if (typeof data === 'number' || typeof data === 'boolean') {
+          responseSize = Buffer.byteLength(String(data));
+        } else {
+          try {
+            responseSize = Buffer.byteLength(JSON.stringify(data));
+          } catch {
+            // Handle circular references or unstringifiable objects
+            responseSize = 0;
+          }
+        }
       }
       return originalSend.call(this, data);
     };
+
     // Override json method
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    res.json = function (data: any) {
-      if (data) {
-        responseSize = Buffer.byteLength(JSON.stringify(data));
+    res.json = function (
+      data: Record<string, unknown> | unknown[] | string | number | boolean | null
+    ) {
+      if (data !== undefined) {
+        try {
+          responseSize = Buffer.byteLength(JSON.stringify(data));
+        } catch {
+          // Handle circular references or unstringifiable objects
+          responseSize = 0;
+        }
       }
       return originalJson.call(this, data);
     };
+
     // Log after response is finished
     res.on('finish', () => {
-      // Skip logging for analytics endpoints to avoid infinite loops
-      if (req.path.includes('/analytics/')) {
+      // Skip logging for logs endpoints to avoid infinite loops
+      if (req.path.includes('/logs/')) {
         return;
       }
 
@@ -121,6 +137,7 @@ export async function createApp() {
         timestamp: new Date().toISOString(),
       });
     });
+
     next();
   });
 
@@ -142,11 +159,9 @@ export async function createApp() {
     });
   });
 
-  // Mount auth routes
+  // Mount all routes
   apiRouter.use('/auth', authRouter);
-  apiRouter.use('/database/tables', databaseTablesRouter);
-  apiRouter.use('/database/records', databaseRecordsRouter);
-  apiRouter.use('/database/advance', databaseAdvanceRouter);
+  apiRouter.use('/database', databaseRouter);
   apiRouter.use('/storage', storageRouter);
   apiRouter.use('/metadata', metadataRouter);
   apiRouter.use('/logs', logsRouter);
@@ -154,37 +169,11 @@ export async function createApp() {
   apiRouter.use('/functions', functionsRouter);
   apiRouter.use('/secrets', secretsRouter);
   apiRouter.use('/usage', usageRouter);
-  apiRouter.use('/openapi', openAPIRouter);
-  apiRouter.use('/agent-docs', agentDocsRouter);
   apiRouter.use('/ai', aiRouter);
 
   // Mount all API routes under /api prefix
   app.use('/api', apiRouter);
 
-  // Add direct OpenAPI route at /openapi
-  app.get('/openapi', async (_req: Request, res: Response) => {
-    try {
-      const { OpenAPIService } = await import('@/core/documentation/openapi.js');
-      const openAPIService = OpenAPIService.getInstance();
-      const openAPIDocument = await openAPIService.generateOpenAPIDocument();
-      res.json(openAPIDocument);
-    } catch {
-      res.status(500).json({ error: 'Failed to generate OpenAPI document' });
-    }
-  });
-
-  // Add direct AI agent documentation route at /agent-docs
-  app.get('/agent-docs', async (_req: Request, res: Response) => {
-    try {
-      const { AgentAPIDocService } = await import('@/core/documentation/agent.js');
-      const agentAPIDocService = AgentAPIDocService.getInstance();
-      const agentDocs = await agentAPIDocService.generateAgentDocumentation();
-      res.json(agentDocs);
-    } catch {
-      res.status(500).json({ error: 'Failed to generate agent API documentation' });
-    }
-  });
-
   // Proxy function execution to Deno runtime
   app.all('/functions/:slug', async (req: Request, res: Response) => {
     try {
@@ -224,19 +213,26 @@ export async function createApp() {
     }
   });
 
-  // Always try to serve frontend if it exists
-  const frontendPath = path.join(__dirname, 'frontend');
+  // Serve auth app
+  const authAppPath = path.join(__dirname, 'auth');
+  if (fs.existsSync(authAppPath)) {
+    app.use('/auth', express.static(authAppPath));
+    app.get('/auth*', (_req: Request, res: Response) => {
+      res.sendFile(path.join(authAppPath, 'index.html'));
+    });
+  } else if (!isProduction()) {
+    const authAppUrl = process.env.AUTH_APP_URL || 'http://localhost:7132';
+    logger.info('Auth app not built, proxying to development server', { authAppUrl });
+  }
 
-  // Check if frontend build exists
+  // Serve main frontend if it exists
+  const frontendPath = path.join(__dirname, 'frontend');
   if (fs.existsSync(frontendPath)) {
+    app.use(express.static(frontendPath, { index: false }));
     // Catch all handler for SPA routes
-    app.get('/cloud*', (_req: Request, res: Response) => {
-      res.sendFile(path.join(frontendPath, 'index.html'));
-    });
-    app.get('/dashboard*', (_req: Request, res: Response) => {
+    app.get(['/cloud*', '/dashboard*'], (_req: Request, res: Response) => {
       res.sendFile(path.join(frontendPath, 'index.html'));
     });
-    app.use(express.static(frontendPath));
   } else {
     // Catch-all for 404 errors - Traditional REST format
     app.use('*', (req: Request, res: Response) => {
@@ -266,7 +262,7 @@ async function initializeServer() {
     });
 
     // Initialize Socket.IO service
-    const socketService = SocketService.getInstance();
+    const socketService = SocketManager.getInstance();
     socketService.initialize(server);
   } catch (error) {
     logger.error('Failed to initialize server', {

package/backend/src/{core/ai/config.ts → services/ai/ai-config.service.ts}

@@ -1,11 +1,21 @@
 import { Pool } from 'pg';
-import { DatabaseManager } from '@/core/database/manager.js';
+import { DatabaseManager } from '@/infra/database/database.manager.js';
 import logger from '@/utils/logger.js';
 import { AIConfigurationSchema, AIConfigurationWithUsageSchema } from '@insforge/shared-schemas';
 
 export class AIConfigService {
+  private static instance: AIConfigService;
   private pool: Pool | null = null;
 
+  private constructor() {}
+
+  public static getInstance(): AIConfigService {
+    if (!AIConfigService.instance) {
+      AIConfigService.instance = new AIConfigService();
+    }
+    return AIConfigService.instance;
+  }
+
   private getPool(): Pool {
     if (!this.pool) {
       this.pool = DatabaseManager.getInstance().getPool();
@@ -20,9 +30,8 @@ export class AIConfigService {
     modelId: string,
     systemPrompt?: string
   ): Promise<{ id: string }> {
-    const client = await this.getPool().connect();
     try {
-      const result = await client.query(
+      const result = await this.getPool().query(
         `INSERT INTO _ai_configs (input_modality, output_modality, provider, model_id, system_prompt)
          VALUES ($1, $2, $3, $4, $5)
          RETURNING id`,
@@ -34,17 +43,14 @@ export class AIConfigService {
     } catch (error) {
       logger.error('Failed to create AI configuration', { error });
       throw new Error('Failed to create AI configuration');
-    } finally {
-      client.release();
     }
   }
 
   async findAll(): Promise<AIConfigurationWithUsageSchema[]> {
-    const client = await this.getPool().connect();
     try {
       // Use a single query with aggregation to get configs with usage stats
-      const result = await client.query(
-        `SELECT 
+      const result = await this.getPool().query(
+        `SELECT
           c.id,
           c.input_modality as "inputModality",
           c.output_modality as "outputModality",
@@ -63,7 +69,7 @@ export class AIConfigService {
       );
 
       return result.rows.map((row) => ({
-        id: row.id,
+        id: row.id, // UUID
         inputModality: row.inputModality,
         outputModality: row.outputModality,
         provider: row.provider,
@@ -80,16 +86,13 @@ export class AIConfigService {
     } catch (error) {
       logger.error('Failed to fetch AI configurations with usage', { error });
       throw new Error('Failed to fetch AI configurations');
-    } finally {
-      client.release();
     }
   }
 
   async update(id: string, systemPrompt: string | null): Promise<boolean> {
-    const client = await this.getPool().connect();
     try {
-      const result = await client.query(
-        `UPDATE _ai_configs 
+      const result = await this.getPool().query(
+        `UPDATE _ai_configs
          SET system_prompt = $1, updated_at = NOW()
          WHERE id = $2`,
         [systemPrompt, id]
@@ -103,15 +106,12 @@ export class AIConfigService {
     } catch (error) {
       logger.error('Failed to update AI configuration', { error, id });
       throw new Error('Failed to update AI configuration');
-    } finally {
-      client.release();
     }
   }
 
   async delete(id: string): Promise<boolean> {
-    const client = await this.getPool().connect();
     try {
-      const result = await client.query('DELETE FROM _ai_configs WHERE id = $1', [id]);
+      const result = await this.getPool().query('DELETE FROM _ai_configs WHERE id = $1', [id]);
 
       const success = (result.rowCount ?? 0) > 0;
       if (success) {
@@ -121,15 +121,12 @@ export class AIConfigService {
     } catch (error) {
       logger.error('Failed to delete AI configuration', { error, id });
       throw new Error('Failed to delete AI configuration');
-    } finally {
-      client.release();
     }
   }
 
   async findByModelId(modelId: string): Promise<AIConfigurationSchema | null> {
-    const client = await this.getPool().connect();
     try {
-      const result = await client.query(
+      const result = await this.getPool().query(
         `SELECT id, input_modality as "inputModality", output_modality as "outputModality", provider, model_id as "modelId", system_prompt as "systemPrompt", created_at, updated_at
          FROM _ai_configs
          WHERE model_id = $1`,
@@ -155,8 +152,6 @@ export class AIConfigService {
         modelId,
       });
       throw new Error('Failed to fetch AI configuration');
-    } finally {
-      client.release();
     }
   }