insforge 0.3.1

package/backend/src/api/routes/database.records.ts

@@ -0,0 +1,246 @@
+import { Router, Response, NextFunction } from 'express';
+import axios from 'axios';
+import http from 'http';
+import https from 'https';
+import { AuthRequest, extractApiKey } from '@/api/middleware/auth.js';
+import { DatabaseManager } from '@/core/database/manager.js';
+import { AppError } from '@/api/middleware/error.js';
+import { ERROR_CODES } from '@/types/error-constants.js';
+import { validateTableName } from '@/utils/validations.js';
+import { DatabaseRecord } from '@/types/database.js';
+import { successResponse } from '@/utils/response.js';
+import logger from '@/utils/logger.js';
+import { SecretsService } from '@/core/secrets/secrets.js';
+import { AuthService } from '@/core/auth/auth.js';
+
+const router = Router();
+const authService = AuthService.getInstance();
+const secretService = new SecretsService();
+const postgrestUrl = process.env.POSTGREST_BASE_URL || 'http://localhost:5430';
+
+// Create a dedicated HTTP agent with connection pooling for PostgREST
+// Optimized connection pool for Docker network communication
+const httpAgent = new http.Agent({
+  keepAlive: true,
+  keepAliveMsecs: 5000, // Shorter for Docker network
+  maxSockets: 20, // Reduced for stability
+  maxFreeSockets: 5,
+  timeout: 10000, // Match axios timeout
+});
+
+const httpsAgent = new https.Agent({
+  keepAlive: true,
+  keepAliveMsecs: 5000,
+  maxSockets: 20,
+  maxFreeSockets: 5,
+  timeout: 10000,
+});
+
+// Create axios instance with optimized configuration for PostgREST
+const postgrestAxios = axios.create({
+  httpAgent,
+  httpsAgent,
+  timeout: 10000, // Increased timeout for stability
+  maxRedirects: 0,
+  // Additional connection stability options
+  headers: {
+    Connection: 'keep-alive',
+    'Keep-Alive': 'timeout=5, max=10',
+  },
+});
+
+// Generate admin token once and reuse
+// If user request with api key, this token should be added automatically.
+const adminToken = authService.generateToken({
+  sub: 'project-admin-with-api-key',
+  email: 'project-admin@email.com',
+  role: 'project_admin',
+});
+
+// anonymous users can access the database, postgREST does not require authentication, however we seed to unwrap session token for better auth, thus
+// we need to verify user token below.
+// router.use(verifyUserOrApiKey);
+
+/**
+ * Forward database requests to PostgREST
+ */
+const forwardToPostgrest = async (req: AuthRequest, res: Response, next: NextFunction) => {
+  const { tableName } = req.params;
+  const wildcardPath = req.params[0] || '';
+
+  // Build the target URL early so it's available in error handling
+  const targetPath = wildcardPath ? `/${tableName}/${wildcardPath}` : `/${tableName}`;
+  const targetUrl = `${postgrestUrl}${targetPath}`;
+
+  try {
+    // Validate table name with operation type
+    const method = req.method.toUpperCase();
+
+    try {
+      validateTableName(tableName);
+    } catch (error) {
+      if (error instanceof AppError) {
+        throw error;
+      }
+      throw new AppError('Invalid table name', 400, ERROR_CODES.INVALID_INPUT);
+    }
+
+    // Process request body for POST/PATCH/PUT operations
+    if (['POST', 'PATCH', 'PUT'].includes(method) && req.body && typeof req.body === 'object') {
+      const columnTypeMap = await DatabaseManager.getColumnTypeMap(tableName);
+      if (Array.isArray(req.body)) {
+        req.body = req.body.map((item) => {
+          if (item && typeof item === 'object') {
+            const filtered: DatabaseRecord = {};
+            for (const key in item) {
+              if (columnTypeMap[key] !== 'text' && item[key] === '') {
+                continue;
+              }
+              filtered[key] = item[key];
+            }
+            return filtered;
+          }
+          return item;
+        });
+      } else {
+        const body = req.body as DatabaseRecord;
+        for (const key in body) {
+          if (columnTypeMap[key] === 'uuid' && body[key] === '') {
+            delete body[key];
+          }
+        }
+      }
+    }
+
+    // Forward the request
+    const axiosConfig: {
+      method: string;
+      url: string;
+      params: unknown;
+      headers: Record<string, string | string[] | undefined>;
+      data?: unknown;
+    } = {
+      method: req.method,
+      url: targetUrl,
+      params: req.query,
+      headers: {
+        ...req.headers,
+        host: undefined, // Remove host header
+        'content-length': undefined, // Let axios calculate
+      },
+    };
+
+    // Check for API key using shared logic
+    const apiKey = extractApiKey(req);
+
+    // If we have an API key, verify it and use admin token for PostgREST
+    if (apiKey) {
+      const isValid = await secretService.verifyApiKey(apiKey);
+      if (isValid) {
+        axiosConfig.headers.authorization = `Bearer ${adminToken}`;
+      }
+    }
+
+    // Add body for methods that support it
+    if (['POST', 'PUT', 'PATCH'].includes(req.method)) {
+      axiosConfig.data = req.body;
+    }
+
+    // Enhanced retry logic with improved error handling
+    let response;
+    let lastError;
+    const maxRetries = 3; // Increased retries for connection resets
+
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        response = await postgrestAxios(axiosConfig);
+        break; // Success, exit retry loop
+      } catch (error) {
+        lastError = error;
+
+        // Retry on network errors (ECONNRESET, ECONNREFUSED, timeout) but not HTTP errors
+        const shouldRetry = axios.isAxiosError(error) && !error.response && attempt < maxRetries;
+
+        if (shouldRetry) {
+          logger.warn(`PostgREST request failed, retrying (attempt ${attempt}/${maxRetries})`, {
+            url: targetUrl,
+            errorCode: error.code,
+            message: error.message,
+          });
+
+          // Enhanced exponential backoff: 200ms, 500ms, 1000ms
+          const backoffDelay = Math.min(200 * Math.pow(2.5, attempt - 1), 1000);
+          await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+        } else {
+          throw error; // Don't retry on HTTP errors or last attempt
+        }
+      }
+    }
+
+    if (!response) {
+      throw lastError || new Error('Failed to get response from PostgREST');
+    }
+
+    // Forward response headers
+    Object.entries(response.headers).forEach(([key, value]) => {
+      const keyLower = key.toLowerCase();
+      if (
+        keyLower !== 'content-length' &&
+        keyLower !== 'transfer-encoding' &&
+        keyLower !== 'connection' &&
+        keyLower !== 'content-encoding'
+      ) {
+        res.setHeader(key, value);
+      }
+    });
+
+    // Handle empty responses
+    let responseData = response.data;
+    if (
+      response.data === undefined ||
+      (typeof response.data === 'string' && response.data.trim() === '')
+    ) {
+      responseData = [];
+    }
+
+    successResponse(res, responseData, response.status);
+  } catch (error) {
+    if (axios.isAxiosError(error)) {
+      // Log more detailed error information
+      logger.error('PostgREST request failed', {
+        url: targetUrl,
+        method: req.method,
+        error: {
+          code: error.code,
+          message: error.message,
+          response: error.response?.data,
+          responseStatus: error.response?.status,
+        },
+      });
+
+      // Forward PostgREST errors
+      if (error.response) {
+        res.status(error.response.status).json(error.response.data);
+      } else {
+        // Network error - connection refused, DNS failure, etc.
+        const errorMessage =
+          error.code === 'ECONNREFUSED'
+            ? 'PostgREST connection refused'
+            : error.code === 'ENOTFOUND'
+              ? 'PostgREST service not found'
+              : 'Database service unavailable';
+
+        next(new AppError(errorMessage, 503, ERROR_CODES.INTERNAL_ERROR));
+      }
+    } else {
+      logger.error('Unexpected error in database route', { error });
+      next(error);
+    }
+  }
+};
+
+// Forward all database operations to PostgREST
+router.all('/:tableName', forwardToPostgrest);
+router.all('/:tableName/*', forwardToPostgrest);
+
+export { router as databaseRecordsRouter };

package/backend/src/api/routes/database.tables.ts

@@ -0,0 +1,161 @@
+import { Router, Response, NextFunction } from 'express';
+import { verifyAdmin, verifyUser, AuthRequest } from '@/api/middleware/auth.js';
+import { DatabaseTableService } from '@/core/database/table.js';
+import { successResponse } from '@/utils/response.js';
+import { AppError } from '@/api/middleware/error.js';
+import { ERROR_CODES } from '@/types/error-constants.js';
+import { createTableRequestSchema, updateTableSchemaRequestSchema } from '@insforge/shared-schemas';
+import { SocketService } from '@/core/socket/socket';
+import { DataUpdateResourceType, ServerEvents } from '@/core/socket/types';
+import { AuditService } from '@/core/logs/audit';
+
+const router = Router();
+const tableService = new DatabaseTableService();
+const auditService = AuditService.getInstance();
+
+// All table routes accept either JWT token or API key authentication
+// router.use(verifyAdmin);
+
+// List all tables
+router.get('/', verifyUser, async (_req: AuthRequest, res: Response, next: NextFunction) => {
+  try {
+    const tables = await tableService.listTables();
+    successResponse(res, tables);
+  } catch (error) {
+    next(error);
+  }
+});
+
+// Create a new table
+router.post('/', verifyAdmin, async (req: AuthRequest, res: Response, next: NextFunction) => {
+  try {
+    const validation = createTableRequestSchema.safeParse(req.body);
+    if (!validation.success) {
+      throw new AppError(
+        validation.error.issues.map((e) => `${e.path.join('.')}: ${e.message}`).join(', '),
+        400,
+        ERROR_CODES.INVALID_INPUT,
+        'Please check the request body, it must conform with the CreateTableRequest schema.'
+      );
+    }
+
+    const { tableName, columns, rlsEnabled } = validation.data;
+    const result = await tableService.createTable(tableName, columns, rlsEnabled);
+
+    // Log audit for table creation
+    await auditService.log({
+      actor: req.user?.email || 'api-key',
+      action: 'CREATE_TABLE',
+      module: 'DATABASE',
+      details: {
+        tableName,
+        columns,
+        rlsEnabled,
+      },
+      ip_address: req.ip,
+    });
+
+    const socket = SocketService.getInstance();
+    socket.broadcastToRoom('role:project_admin', ServerEvents.DATA_UPDATE, {
+      resource: DataUpdateResourceType.DATABASE_SCHEMA,
+    });
+    successResponse(res, result, 201);
+  } catch (error) {
+    next(error);
+  }
+});
+
+// Get table schema
+router.get(
+  '/:tableName/schema',
+  verifyAdmin,
+  async (req: AuthRequest, res: Response, next: NextFunction) => {
+    try {
+      const { tableName } = req.params;
+      const schema = await tableService.getTableSchema(tableName);
+      successResponse(res, schema);
+    } catch (error) {
+      next(error);
+    }
+  }
+);
+
+// Update table schema
+router.patch(
+  '/:tableName/schema',
+  verifyAdmin,
+  async (req: AuthRequest, res: Response, next: NextFunction) => {
+    try {
+      const { tableName } = req.params;
+
+      const validation = updateTableSchemaRequestSchema.safeParse(req.body);
+      if (!validation.success) {
+        throw new AppError(
+          validation.error.issues.map((e) => `${e.path.join('.')}: ${e.message}`).join(', '),
+          400,
+          ERROR_CODES.INVALID_INPUT,
+          'Please check the request body, it must conform with the UpdateTableRequest schema.'
+        );
+      }
+
+      const operations = validation.data;
+      const result = await tableService.updateTableSchema(tableName, operations);
+
+      // Log audit for table schema update
+      await auditService.log({
+        actor: req.user?.email || 'api-key',
+        action: 'UPDATE_TABLE',
+        module: 'DATABASE',
+        details: {
+          tableName,
+          operations,
+        },
+        ip_address: req.ip,
+      });
+
+      const socket = SocketService.getInstance();
+      socket.broadcastToRoom('role:project_admin', ServerEvents.DATA_UPDATE, {
+        resource: DataUpdateResourceType.TABLE_SCHEMA,
+        data: {
+          name: tableName,
+        },
+      });
+      successResponse(res, result);
+    } catch (error) {
+      next(error);
+    }
+  }
+);
+
+// Delete a table
+router.delete(
+  '/:tableName',
+  verifyAdmin,
+  async (req: AuthRequest, res: Response, next: NextFunction) => {
+    try {
+      const { tableName } = req.params;
+      const result = await tableService.deleteTable(tableName);
+
+      // Log audit for table deletion
+      await auditService.log({
+        actor: req.user?.email || 'api-key',
+        action: 'DELETE_TABLE',
+        module: 'DATABASE',
+        details: {
+          tableName,
+        },
+        ip_address: req.ip,
+      });
+
+      const socket = SocketService.getInstance();
+      socket.broadcastToRoom('role:project_admin', ServerEvents.DATA_UPDATE, {
+        resource: DataUpdateResourceType.DATABASE_SCHEMA,
+      });
+      successResponse(res, result);
+    } catch (error) {
+      next(error);
+    }
+  }
+);
+
+export { router as databaseTablesRouter };

package/backend/src/api/routes/docs.ts

@@ -0,0 +1,66 @@
+import { Router } from 'express';
+import { readFile } from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { successResponse, errorResponse } from '@/utils/response.js';
+import { ERROR_CODES } from '@/types/error-constants.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const router = Router();
+
+// Define available documentation files
+const DOCS_MAP: Record<string, string> = {
+  instructions: 'insforge-instructions-sdk.md',
+  'db-api': 'insforge-db-sdk.md',
+  'auth-api': 'insforge-auth-sdk.md',
+  'storage-api': 'insforge-storage-sdk.md',
+  debug: 'insforge-debug-sdk.md',
+  project: 'insforge-project-sdk.md',
+};
+
+// GET /api/docs/:docType - Get specific documentation
+router.get('/:docType', async (req, res, next) => {
+  try {
+    const { docType } = req.params;
+
+    // Validate doc type
+    const docFileName = DOCS_MAP[docType];
+    if (!docFileName) {
+      return errorResponse(res, ERROR_CODES.NOT_FOUND, 'Documentation not found', 404);
+    }
+
+    // Read the documentation file
+    // PROJECT_ROOT is set in the docker-compose.yml file to point to the InsForge directory
+    const projectRoot = process.env.PROJECT_ROOT || path.resolve(__dirname, '../../../..');
+    const filePath = path.join(projectRoot, 'docs', docFileName);
+    const content = await readFile(filePath, 'utf-8');
+
+    // Traditional REST: return documentation directly
+    return successResponse(res, {
+      type: docType,
+      content,
+    });
+  } catch (error) {
+    // If file doesn't exist or other error
+    if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+      return errorResponse(res, ERROR_CODES.NOT_FOUND, 'Documentation file not found', 404);
+    }
+    next(error);
+  }
+});
+
+// GET /api/docs - List available documentation
+router.get('/', (_req, res) => {
+  const available = Object.keys(DOCS_MAP).map((key) => ({
+    type: key,
+    filename: DOCS_MAP[key],
+    endpoint: `/api/docs/${key}`,
+  }));
+
+  // Traditional REST: return list directly
+  return successResponse(res, available);
+});
+
+export { router as docsRouter };

package/backend/src/api/routes/functions.ts

@@ -0,0 +1,183 @@
+import { Router, Response } from 'express';
+import { AuthRequest, verifyAdmin } from '@/api/middleware/auth.js';
+import { FunctionsService } from '@/core/functions/functions.js';
+import { AuditService } from '@/core/logs/audit.js';
+import { AppError } from '@/api/middleware/error.js';
+import logger from '@/utils/logger.js';
+import { functionUploadRequestSchema, functionUpdateRequestSchema } from '@insforge/shared-schemas';
+
+const router = Router();
+const functionsService = FunctionsService.getInstance();
+const auditService = AuditService.getInstance();
+
+/**
+ * GET /api/functions
+ * List all edge functions
+ */
+router.get('/', verifyAdmin, async (req: AuthRequest, res: Response) => {
+  try {
+    const result = await functionsService.listFunctions();
+    res.json(result);
+  } catch {
+    res.status(500).json({ error: 'Failed to list functions' });
+  }
+});
+
+/**
+ * GET /api/functions/:slug
+ * Get specific function details including code
+ */
+router.get('/:slug', verifyAdmin, async (req: AuthRequest, res: Response) => {
+  try {
+    const { slug } = req.params;
+    const func = await functionsService.getFunction(slug);
+
+    if (!func) {
+      return res.status(404).json({ error: 'Function not found' });
+    }
+
+    res.json(func);
+  } catch {
+    res.status(500).json({ error: 'Failed to get function' });
+  }
+});
+
+/**
+ * POST /api/functions
+ * Create a new function
+ */
+router.post('/', verifyAdmin, async (req: AuthRequest, res: Response) => {
+  try {
+    const validation = functionUploadRequestSchema.safeParse(req.body);
+    if (!validation.success) {
+      return res.status(400).json({
+        error: 'Invalid request',
+        details: validation.error.issues,
+      });
+    }
+
+    const created = await functionsService.createFunction(validation.data);
+
+    // Log audit event
+    logger.info(`Function ${created.name} (${created.slug}) created by ${req.user?.email}`);
+    await auditService.log({
+      actor: req.user?.email || 'api-key',
+      action: 'CREATE_FUNCTION',
+      module: 'FUNCTIONS',
+      details: {
+        functionId: created.id,
+        slug: created.slug,
+        name: created.name,
+        status: created.status,
+      },
+      ip_address: req.ip,
+    });
+
+    res.status(201).json({
+      success: true,
+      function: created,
+    });
+  } catch (error) {
+    if (error instanceof AppError) {
+      return res.status(error.statusCode).json({
+        error: error.code,
+        message: error.message,
+      });
+    }
+
+    res.status(500).json({
+      error: 'INTERNAL_ERROR',
+      message: error instanceof Error ? error.message : 'Failed to create function',
+    });
+  }
+});
+
+/**
+ * PUT /api/functions/:slug
+ * Update an existing function
+ */
+router.put('/:slug', verifyAdmin, async (req: AuthRequest, res: Response) => {
+  try {
+    const { slug } = req.params;
+    const validation = functionUpdateRequestSchema.safeParse(req.body);
+
+    if (!validation.success) {
+      return res.status(400).json({
+        error: 'Invalid request',
+        details: validation.error.issues,
+      });
+    }
+
+    const updated = await functionsService.updateFunction(slug, validation.data);
+
+    if (!updated) {
+      return res.status(404).json({ error: 'Function not found' });
+    }
+
+    // Log audit event
+    logger.info(`Function ${slug} updated by ${req.user?.email}`);
+    await auditService.log({
+      actor: req.user?.email || 'api-key',
+      action: 'UPDATE_FUNCTION',
+      module: 'FUNCTIONS',
+      details: {
+        slug,
+        changes: validation.data,
+      },
+      ip_address: req.ip,
+    });
+
+    res.json({
+      success: true,
+      function: updated,
+    });
+  } catch (error) {
+    if (error instanceof AppError) {
+      return res.status(error.statusCode).json({
+        error: error.code,
+        message: error.message,
+      });
+    }
+
+    res.status(500).json({
+      error: 'INTERNAL_ERROR',
+      message: error instanceof Error ? error.message : 'Failed to update function',
+    });
+  }
+});
+
+/**
+ * DELETE /api/functions/:slug
+ * Delete a function
+ */
+router.delete('/:slug', verifyAdmin, async (req: AuthRequest, res: Response) => {
+  try {
+    const { slug } = req.params;
+    const deleted = await functionsService.deleteFunction(slug);
+
+    if (!deleted) {
+      return res.status(404).json({ error: 'Function not found' });
+    }
+
+    // Log audit event
+    logger.info(`Function ${slug} deleted by ${req.user?.email}`);
+    await auditService.log({
+      actor: req.user?.email || 'api-key',
+      action: 'DELETE_FUNCTION',
+      module: 'FUNCTIONS',
+      details: {
+        slug,
+      },
+      ip_address: req.ip,
+    });
+
+    res.json({
+      success: true,
+      message: `Function ${slug} deleted successfully`,
+    });
+  } catch {
+    res.status(500).json({ error: 'Failed to delete function' });
+  }
+});
+
+export default router;