insforge 0.3.1

package/backend/migrations/010_modify-ai-config-modalities.sql

@@ -0,0 +1,93 @@
+-- Migration: 010 - Modify AI configurations table to support input/output modalities
+-- This migration modifies the _ai_configs table to:
+-- 1. Add new columns: input_modality and output_modality (TEXT arrays)
+-- 2. Migrate existing modality data to input_modality
+-- 3. Set default output_modality based on existing modality
+-- 4. Drop the old modality column
+
+DO $$
+BEGIN
+    -- Add new columns for input and output modalities
+    ALTER TABLE _ai_configs 
+    ADD COLUMN IF NOT EXISTS input_modality TEXT[] DEFAULT '{text}';
+    
+    ALTER TABLE _ai_configs 
+    ADD COLUMN IF NOT EXISTS output_modality TEXT[] DEFAULT '{text}';
+
+    -- Check if modality column exists and migrate data if it does
+    IF EXISTS (
+        SELECT 1
+        FROM information_schema.columns
+        WHERE table_schema = 'public'
+        AND table_name = '_ai_configs'
+        AND column_name = 'modality'
+    ) THEN
+        -- Migrate existing modality data to input_modality
+        -- For most cases, we'll set input_modality to the existing modality
+        -- and output_modality to the same value, only supporting text and image
+        UPDATE _ai_configs
+        SET
+            input_modality = CASE
+                WHEN modality = 'multi' THEN '{text,image}'::TEXT[]
+                WHEN modality = 'image' THEN '{text,image}'::TEXT[]
+                ELSE ARRAY[modality]::TEXT[]
+            END,
+            output_modality = CASE
+                WHEN modality = 'multi' THEN '{text,image}'::TEXT[]
+                WHEN modality = 'text' THEN '{text}'::TEXT[]
+                WHEN modality = 'image' THEN '{text,image}'::TEXT[]
+                ELSE '{text}'::TEXT[]
+            END
+        WHERE input_modality = '{text}' OR input_modality IS NULL;
+    END IF;
+
+    -- Make the new columns NOT NULL after migration
+    ALTER TABLE _ai_configs 
+    ALTER COLUMN input_modality SET NOT NULL;
+    
+    ALTER TABLE _ai_configs 
+    ALTER COLUMN output_modality SET NOT NULL;
+
+    -- Drop the old modality column
+    ALTER TABLE _ai_configs 
+    DROP COLUMN IF EXISTS modality;
+
+    -- Create indexes for the new TEXT array columns for better query performance
+    CREATE INDEX IF NOT EXISTS idx_ai_configs_input_modality ON _ai_configs USING GIN (input_modality);
+    CREATE INDEX IF NOT EXISTS idx_ai_configs_output_modality ON _ai_configs USING GIN (output_modality);
+
+    -- Drop existing constraints if they exist, then add them
+    ALTER TABLE _ai_configs
+    DROP CONSTRAINT IF EXISTS check_input_modality_not_empty;
+
+    ALTER TABLE _ai_configs
+    ADD CONSTRAINT check_input_modality_not_empty
+    CHECK (array_length(input_modality, 1) > 0);
+
+    ALTER TABLE _ai_configs
+    DROP CONSTRAINT IF EXISTS check_output_modality_not_empty;
+
+    ALTER TABLE _ai_configs
+    ADD CONSTRAINT check_output_modality_not_empty
+    CHECK (array_length(output_modality, 1) > 0);
+
+    -- Drop existing constraints if they exist, then add them
+    ALTER TABLE _ai_configs
+    DROP CONSTRAINT IF EXISTS check_input_modality_valid;
+
+    ALTER TABLE _ai_configs
+    ADD CONSTRAINT check_input_modality_valid
+    CHECK (
+        input_modality <@ '{text,image}'::TEXT[]
+    );
+
+    ALTER TABLE _ai_configs
+    DROP CONSTRAINT IF EXISTS check_output_modality_valid;
+
+    ALTER TABLE _ai_configs
+    ADD CONSTRAINT check_output_modality_valid
+    CHECK (
+        output_modality <@ '{text,image}'::TEXT[]
+    );
+
+END $$;

package/backend/migrations/011_refactor-secrets-table.sql

@@ -0,0 +1,15 @@
+-- Migration: 011 - Drop function secrets table and update main secrets table
+-- This migration is part of the refactoring to unify all secrets management
+
+-- 1. Drop the _function_secrets table (replaced by main _secrets table)
+DROP TRIGGER IF EXISTS update__function_secrets_updated_at ON _function_secrets;
+DROP INDEX IF EXISTS idx_function_secrets_key;
+DROP TABLE IF EXISTS _function_secrets;
+
+-- 2. Add is_reserved column to _secrets table
+ALTER TABLE _secrets 
+ADD COLUMN IF NOT EXISTS is_reserved BOOLEAN DEFAULT FALSE;
+
+-- 3. Rename name column to key
+ALTER TABLE _secrets 
+RENAME COLUMN name TO key;

package/backend/migrations/012_add-storage-uploaded-by.sql

@@ -0,0 +1,8 @@
+-- Migration: 012 - Add uploaded_by column to _storage table
+-- This migration adds a foreign key relationship to track which account uploaded each file
+
+ALTER TABLE _storage
+ADD COLUMN uploaded_by UUID REFERENCES _accounts(id) ON DELETE SET NULL;
+
+-- Create an index for better query performance when filtering by uploader
+CREATE INDEX IF NOT EXISTS idx_storage_uploaded_by ON _storage(uploaded_by);

package/backend/package.json

@@ -0,0 +1,75 @@
+{
+  "name": "insforge-backend",
+  "version": "1.0.0",
+  "author": "Insforge",
+  "description": "Open source backend-as-a-service with PostgreSQL and MCP integration",
+  "type": "module",
+  "main": "../dist/server.js",
+  "scripts": {
+    "start": "node ../dist/server.js",
+    "dev": "tsx watch src/server.ts",
+    "build": "tsup",
+    "clean": "rimraf dist",
+    "prebuild": "npm run clean",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "test:ui": "vitest --ui",
+    "test:e2e": "./tests/run-all-tests.sh",
+    "migrate:up": "node-pg-migrate up --migrations-table _migrations",
+    "migrate:down": "node-pg-migrate down --migrations-table _migrations",
+    "migrate:create": "node-pg-migrate create --migrations-table _migrations",
+    "migrate:redo": "node-pg-migrate redo --migrations-table _migrations",
+    "migrate:up:local": "dotenv -e ../.env -- node-pg-migrate up --migrations-table _migrations",
+    "migrate:down:local": "dotenv -e ../.env -- node-pg-migrate down --migrations-table _migrations"
+  },
+  "keywords": [
+    "backend-as-a-service",
+    "postgresql",
+    "jwt"
+  ],
+  "dependencies": {
+    "@asteasolutions/zod-to-openapi": "^7.3.4",
+    "@aws-sdk/client-cloudwatch-logs": "^3.713.0",
+    "@aws-sdk/client-s3": "^3.713.0",
+    "@aws-sdk/s3-presigned-post": "^3.879.0",
+    "@aws-sdk/s3-request-presigner": "^3.879.0",
+    "@databases/split-sql-query": "^1.0.4",
+    "@databases/sql": "^3.3.0",
+    "@types/multer": "^1.4.13",
+    "@types/node-pg-migrate": "^2.3.1",
+    "@types/pg": "^8.15.4",
+    "@types/socket.io": "^3.0.2",
+    "axios": "^1.11.0",
+    "bcryptjs": "^3.0.2",
+    "cors": "^2.8.5",
+    "csv-parse": "^6.1.0",
+    "dotenv": "^16.4.5",
+    "express": "^4.19.2",
+    "express-rate-limit": "^7.1.5",
+    "google-auth-library": "^10.1.0",
+    "jose": "^6.0.12",
+    "jsonwebtoken": "^9.0.2",
+    "multer": "^2.0.2",
+    "node-fetch": "^3.3.2",
+    "node-pg-migrate": "^8.0.3",
+    "openai": "^5.19.1",
+    "pg": "^8.16.3",
+    "pg-format": "^1.0.4",
+    "socket.io": "^4.8.1",
+    "winston": "^3.17.0",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/cors": "^2.8.17",
+    "@types/express": "^4.17.21",
+    "@types/jsonwebtoken": "^9.0.9",
+    "@types/node": "^20.11.24",
+    "@types/pg-format": "^1.0.5",
+    "dotenv-cli": "^10.0.0",
+    "tsup": "^8.5.0",
+    "tsx": "^4.7.1",
+    "typescript": "^5.3.3",
+    "vitest": "^3.2.4"
+  }
+}

package/backend/src/api/middleware/auth.ts

@@ -0,0 +1,240 @@
+import { Request, Response, NextFunction } from 'express';
+import { AuthService } from '@/core/auth/auth.js';
+import { AppError } from './error.js';
+import { ERROR_CODES, NEXT_ACTION } from '@/types/error-constants.js';
+import { verifyCloudToken } from '@/utils/cloud-token.js';
+import { SecretsService } from '@/core/secrets/secrets.js';
+
+export interface AuthRequest extends Request {
+  user?: {
+    id: string;
+    email: string;
+    role: string;
+  };
+  authenticated?: boolean;
+  apiKey?: string;
+  projectId?: string;
+}
+
+const authService = AuthService.getInstance();
+const secretService = new SecretsService();
+
+// Helper function to extract Bearer token
+function extractBearerToken(authHeader: string | undefined): string | null {
+  if (!authHeader || !authHeader.startsWith('Bearer ')) {
+    return null;
+  }
+  return authHeader.substring(7);
+}
+
+// Helper function to extract API key from request
+// Checks both Bearer token (if starts with 'ik_') and x-api-key header
+export function extractApiKey(req: AuthRequest): string | null {
+  const bearerToken = extractBearerToken(req.headers.authorization);
+  if (bearerToken && bearerToken.startsWith('ik_')) {
+    return bearerToken;
+  }
+
+  // Fall back to x-api-key header for backward compatibility
+  if (req.headers['x-api-key']) {
+    return req.headers['x-api-key'] as string;
+  }
+
+  return null;
+}
+
+// Helper function to set user on request
+function setRequestUser(req: AuthRequest, payload: { sub: string; email: string; role: string }) {
+  req.user = {
+    id: payload.sub,
+    email: payload.email,
+    role: payload.role,
+  };
+}
+
+/**
+ * Verifies user authentication (accepts both user and admin tokens)
+ */
+export async function verifyUser(req: AuthRequest, res: Response, next: NextFunction) {
+  const apiKey = extractApiKey(req);
+  if (apiKey) {
+    return verifyApiKey(req, res, next);
+  }
+
+  // Use the main verifyToken that handles JWT authentication
+  return verifyToken(req, res, next);
+}
+
+/**
+ * Verifies admin authentication (requires admin token)
+ */
+export async function verifyAdmin(req: AuthRequest, res: Response, next: NextFunction) {
+  const apiKey = extractApiKey(req);
+  if (apiKey) {
+    return verifyApiKey(req, res, next);
+  }
+
+  try {
+    const token = extractBearerToken(req.headers.authorization);
+    if (!token) {
+      throw new AppError(
+        'No admin token provided',
+        401,
+        ERROR_CODES.AUTH_INVALID_CREDENTIALS,
+        NEXT_ACTION.CHECK_TOKEN
+      );
+    }
+
+    // For admin, we use JWT tokens
+    const payload = authService.verifyToken(token);
+
+    if (payload.role !== 'project_admin') {
+      throw new AppError(
+        'Admin access required',
+        403,
+        ERROR_CODES.AUTH_UNAUTHORIZED,
+        NEXT_ACTION.CHECK_ADMIN_TOKEN
+      );
+    }
+
+    setRequestUser(req, payload);
+    next();
+  } catch (error) {
+    if (error instanceof AppError) {
+      next(error);
+    } else {
+      next(
+        new AppError(
+          'Invalid admin token',
+          401,
+          ERROR_CODES.AUTH_INVALID_CREDENTIALS,
+          NEXT_ACTION.CHECK_ADMIN_TOKEN
+        )
+      );
+    }
+  }
+}
+
+/**
+ * Verifies API key authentication
+ * Accepts API key via Authorization: Bearer header or x-api-key header (backward compatibility)
+ */
+export async function verifyApiKey(req: AuthRequest, _res: Response, next: NextFunction) {
+  try {
+    // Extract API key from request using helper
+    const apiKey = extractApiKey(req);
+
+    if (!apiKey) {
+      throw new AppError(
+        'No API key provided',
+        401,
+        ERROR_CODES.AUTH_INVALID_API_KEY,
+        NEXT_ACTION.CHECK_API_KEY
+      );
+    }
+
+    const isValid = await secretService.verifyApiKey(apiKey);
+    if (!isValid) {
+      throw new AppError(
+        'Invalid API key',
+        401,
+        ERROR_CODES.AUTH_INVALID_API_KEY,
+        NEXT_ACTION.CHECK_API_KEY
+      );
+    }
+    req.authenticated = true;
+    req.apiKey = apiKey;
+    next();
+  } catch (error) {
+    next(error);
+  }
+}
+
+/**
+ * Core token verification middleware that handles JWT tokens
+ * Sets req.user with the authenticated user information
+ */
+export function verifyToken(req: AuthRequest, _res: Response, next: NextFunction) {
+  try {
+    const token = extractBearerToken(req.headers.authorization);
+    if (!token) {
+      throw new AppError(
+        'No token provided',
+        401,
+        ERROR_CODES.AUTH_INVALID_CREDENTIALS,
+        NEXT_ACTION.CHECK_TOKEN
+      );
+    }
+
+    // Verify JWT token
+    const payload = authService.verifyToken(token);
+
+    // Validate token has a role
+    if (!payload.role) {
+      throw new AppError(
+        'Invalid token: missing role',
+        401,
+        ERROR_CODES.AUTH_INVALID_CREDENTIALS,
+        NEXT_ACTION.CHECK_TOKEN
+      );
+    }
+
+    // Set user info on request
+    setRequestUser(req, payload);
+
+    next();
+  } catch (error) {
+    if (error instanceof AppError) {
+      next(error);
+    } else {
+      next(
+        new AppError(
+          'Invalid token',
+          401,
+          ERROR_CODES.AUTH_INVALID_CREDENTIALS,
+          NEXT_ACTION.CHECK_TOKEN
+        )
+      );
+    }
+  }
+}
+
+/**
+ * Verifies JWT token from cloud backend (api.insforge.dev)
+ * Validates signature using JWKS and checks project_id claim
+ */
+export async function verifyCloudBackend(req: AuthRequest, _res: Response, next: NextFunction) {
+  try {
+    const token = extractBearerToken(req.headers.authorization);
+    if (!token) {
+      throw new AppError(
+        'No authorization token provided',
+        401,
+        ERROR_CODES.AUTH_INVALID_CREDENTIALS,
+        NEXT_ACTION.CHECK_TOKEN
+      );
+    }
+
+    // Use helper function to verify cloud token
+    const { projectId } = await verifyCloudToken(token);
+
+    // Set project_id on request for use in route handlers
+    req.projectId = projectId;
+    req.authenticated = true;
+
+    next();
+  } catch (error) {
+    if (error instanceof AppError) {
+      next(error);
+    } else {
+      next(
+        new AppError(
+          'Invalid cloud backend token',
+          401,
+          ERROR_CODES.AUTH_INVALID_CREDENTIALS,
+          NEXT_ACTION.CHECK_TOKEN
+        )
+      );
+    }
+  }
+}

package/backend/src/api/middleware/error.ts

@@ -0,0 +1,231 @@
+import { Request, Response, NextFunction } from 'express';
+import { DatabaseError } from 'pg';
+import { errorResponse } from '@/utils/response.js';
+import { ERROR_CODES, NEXT_ACTION } from '@/types/error-constants.js';
+import logger from '@/utils/logger.js';
+
+export class AppError extends Error {
+  constructor(
+    public message: string,
+    public statusCode: number = 500,
+    public code: string,
+    public nextActions?: string
+  ) {
+    super(message);
+    this.name = 'AppError';
+  }
+}
+
+// PostgreSQL error code handlers
+const POSTGRES_ERROR_HANDLERS: Record<
+  string,
+  (err: DatabaseError) => {
+    code: string;
+    message: string;
+    statusCode: number;
+    nextActions?: string;
+  }
+> = {
+  // Integrity constraint violations
+  '23505': (err) => {
+    // unique_violation
+    const detail = err.detail || '';
+    const fieldMatch = detail.match(/Key \(([\w_]+)\)=/);
+    const fieldName = fieldMatch ? fieldMatch[1] : 'field';
+    return {
+      code: ERROR_CODES.ALREADY_EXISTS,
+      message: err.message,
+      statusCode: 409,
+      nextActions: NEXT_ACTION.CHECK_UNIQUE_FIELD(fieldName),
+    };
+  },
+  '23503': (err) => {
+    // foreign_key_violation
+    return {
+      code: ERROR_CODES.DATABASE_CONSTRAINT_VIOLATION,
+      message: err.message,
+      statusCode: 400,
+      nextActions: NEXT_ACTION.CHECK_REFERENCE_EXISTS,
+    };
+  },
+  '23502': (err) => {
+    // not_null_violation
+    const column = err.column || '';
+    return {
+      code: ERROR_CODES.MISSING_FIELD,
+      message: err.message,
+      statusCode: 400,
+      nextActions: NEXT_ACTION.FILL_REQUIRED_FIELD(column),
+    };
+  },
+  '42P01': (err) => ({
+    // undefined_table
+    code: ERROR_CODES.DATABASE_VALIDATION_ERROR,
+    message: err.message,
+    statusCode: 400,
+    nextActions: NEXT_ACTION.CHECK_TABLE_EXISTS,
+  }),
+  '42701': (err) => {
+    // duplicate_column
+    const message = err.message || '';
+    const columnMatch = message.match(/column "([^"]+)"/);
+    const columnName = columnMatch ? columnMatch[1] : '';
+    return {
+      code: ERROR_CODES.DATABASE_VALIDATION_ERROR,
+      message: err.message,
+      statusCode: 400,
+      nextActions: NEXT_ACTION.REMOVE_DUPLICATE_COLUMN(columnName),
+    };
+  },
+  '42703': (err) => ({
+    // undefined_column
+    code: ERROR_CODES.DATABASE_VALIDATION_ERROR,
+    message: err.message,
+    statusCode: 400,
+    nextActions: NEXT_ACTION.CHECK_COLUMN_EXISTS,
+  }),
+  '42830': (err) => ({
+    // invalid_foreign_key
+    code: ERROR_CODES.DATABASE_VALIDATION_ERROR,
+    message: err.message,
+    statusCode: 400,
+    nextActions: NEXT_ACTION.CHECK_UNIQUE_CONSTRAINT,
+  }),
+  '42804': (err) => ({
+    // datatype_mismatch
+    code: ERROR_CODES.DATABASE_VALIDATION_ERROR,
+    message: err.message,
+    statusCode: 400,
+    nextActions: NEXT_ACTION.CHECK_DATATYPE_MATCH,
+  }),
+};
+
+// Handle database-specific errors
+function handleDatabaseError(
+  err: DatabaseError
+): { code: string; message: string; statusCode: number; nextActions?: string } | null {
+  // Check PostgreSQL error codes
+  if (err.code && POSTGRES_ERROR_HANDLERS[err.code]) {
+    return POSTGRES_ERROR_HANDLERS[err.code](err);
+  }
+
+  return null;
+}
+
+// Generic error-like object that could have various properties
+interface ErrorLike {
+  message?: string;
+  status?: number;
+  statusCode?: number;
+  type?: string;
+  expose?: boolean;
+  body?: unknown;
+}
+
+// Single type guard for all error-like objects
+function isErrorObject(err: unknown): err is ErrorLike {
+  return typeof err === 'object' && err !== null;
+}
+
+// Helper to safely get numeric status
+function getErrorStatus(err: unknown): number | undefined {
+  if (!isErrorObject(err)) {
+    return undefined;
+  }
+  if (typeof err.status === 'number') {
+    return err.status;
+  }
+  if (typeof err.statusCode === 'number') {
+    return err.statusCode;
+  }
+  return undefined;
+}
+
+export function errorMiddleware(err: unknown, _req: Request, res: Response, _next: NextFunction) {
+  // Only log non-authentication errors or unexpected errors
+  if (!(err instanceof AppError && err.statusCode === 401)) {
+    logger.error('Error occurred', {
+      error: err instanceof Error ? err.message : String(err),
+      stack: err instanceof Error ? err.stack : undefined,
+    });
+  }
+
+  // Handle known AppError instances
+  if (err instanceof AppError) {
+    const errorCode = err.code || getErrorCode(err.statusCode);
+    return errorResponse(res, errorCode, err.message, err.statusCode, err.nextActions);
+  }
+
+  // Handle SyntaxError from JSON.parse
+  if (err instanceof SyntaxError && err.message.includes('JSON')) {
+    return errorResponse(
+      res,
+      ERROR_CODES.INVALID_INPUT,
+      err.message,
+      400,
+      'Please ensure your request body contains valid JSON'
+    );
+  }
+
+  // Handle PostgreSQL database errors
+  if (err instanceof DatabaseError) {
+    const dbError = handleDatabaseError(err);
+    if (dbError) {
+      return errorResponse(
+        res,
+        dbError.code,
+        dbError.message,
+        dbError.statusCode,
+        dbError.nextActions
+      );
+    }
+  }
+
+  // For all other errors, check if it's an object we can work with
+  if (!isErrorObject(err)) {
+    return errorResponse(res, ERROR_CODES.INTERNAL_ERROR, 'Internal server error', 500);
+  }
+
+  // Handle JSON parsing errors from body-parser
+  if (err.type === 'entity.parse.failed' && err.status === 400) {
+    return errorResponse(
+      res,
+      ERROR_CODES.INVALID_INPUT,
+      err.message || 'Invalid JSON in request body',
+      400,
+      'Please ensure your request body contains valid JSON'
+    );
+  }
+
+  // Get the status code from either status or statusCode property
+  const status = getErrorStatus(err);
+  // Handle client errors (4xx)
+  if (status && status >= 400 && status < 500) {
+    const errorCode = getErrorCode(status);
+    const message = err.message || 'Client error';
+    const body = err.expose ? err.body : undefined;
+    return errorResponse(res, errorCode, message, status, body as string | undefined);
+  }
+
+  // Default internal error with optional message
+  const message = err.message || 'Internal server error';
+  return errorResponse(res, ERROR_CODES.INTERNAL_ERROR, message, 500);
+}
+
+// Helper to map status codes to error codes
+function getErrorCode(statusCode: number): string {
+  switch (statusCode) {
+    case 400:
+      return ERROR_CODES.INVALID_INPUT;
+    case 401:
+      return ERROR_CODES.AUTH_UNAUTHORIZED;
+    case 403:
+      return ERROR_CODES.FORBIDDEN;
+    case 404:
+      return ERROR_CODES.NOT_FOUND;
+    case 409:
+      return ERROR_CODES.ALREADY_EXISTS;
+    default:
+      return ERROR_CODES.INTERNAL_ERROR;
+  }
+}