insforge 0.3.1

package/backend/src/server.ts

@@ -0,0 +1,288 @@
+import express, { Request, Response, NextFunction } from 'express';
+import cors from 'cors';
+import rateLimit from 'express-rate-limit';
+import dotenv from 'dotenv';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import fs from 'fs';
+import authRouter from '@/api/routes/auth.js';
+import { databaseTablesRouter } from '@/api/routes/database.tables.js';
+import { databaseRecordsRouter } from '@/api/routes/database.records.js';
+import databaseAdvanceRouter from '@/api/routes/database.advance.js';
+import { storageRouter } from '@/api/routes/storage.js';
+import { metadataRouter } from '@/api/routes/metadata.js';
+import { logsRouter } from '@/api/routes/logs.js';
+import { docsRouter } from '@/api/routes/docs.js';
+import functionsRouter from '@/api/routes/functions.js';
+import secretsRouter from '@/api/routes/secrets.js';
+import { usageRouter } from '@/api/routes/usage.js';
+import { openAPIRouter } from '@/api/routes/openapi.js';
+import { agentDocsRouter } from '@/api/routes/agent.js';
+import { aiRouter } from '@/api/routes/ai.js';
+import { errorMiddleware } from '@/api/middleware/error.js';
+import fetch, { HeadersInit } from 'node-fetch';
+import { DatabaseManager } from '@/core/database/manager.js';
+import { AnalyticsManager } from '@/core/logs/analytics.js';
+import { StorageService } from '@/core/storage/storage.js';
+import { SocketService } from '@/core/socket/socket.js';
+import { seedBackend } from '@/utils/seed.js';
+import logger from '@/utils/logger.js';
+import { isProduction } from './utils/environment';
+import packageJson from '../../package.json';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Load .env file from the root directory (parent of backend)
+const envPath = path.resolve(__dirname, '../../.env');
+if (fs.existsSync(envPath)) {
+  dotenv.config({ path: envPath });
+} else {
+  // Fallback to default behavior (looks in current working directory)
+  dotenv.config();
+}
+
+export async function createApp() {
+  // Initialize database first
+  const dbManager = DatabaseManager.getInstance();
+  await dbManager.initialize(); // create data/app.db
+
+  // Initialize storage service
+  const storageService = StorageService.getInstance();
+  await storageService.initialize(); // create data/storage
+
+  // Metadata is now handled by individual modules on-demand
+
+  // Initialize analytics service
+  const analyticsManager = AnalyticsManager.getInstance();
+  await analyticsManager.initialize(); // connect to _insforge database
+
+  const app = express();
+
+  // Enable trust proxy setting for rate limiting behind proxies/load balancers
+  app.set('trust proxy', true);
+
+  const limiter = rateLimit({
+    windowMs: 15 * 60 * 1000,
+    max: 1000,
+    message: 'Too many requests from this IP',
+  });
+
+  // Basic middleware
+  app.use(
+    cors({
+      origin: true, // Allow all origins (matches Better Auth's trustedOrigins: ['*'])
+      credentials: true, // Allow cookies/credentials
+    })
+  );
+  if (isProduction()) {
+    app.use(limiter);
+  }
+  app.use((req: Request, res: Response, next: NextFunction) => {
+    const startTime = Date.now();
+    const originalSend = res.send;
+    const originalJson = res.json;
+
+    // Track response size
+    let responseSize = 0;
+
+    // Override send method
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    res.send = function (data: any) {
+      if (data) {
+        responseSize = Buffer.byteLength(typeof data === 'string' ? data : JSON.stringify(data));
+      }
+      return originalSend.call(this, data);
+    };
+    // Override json method
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    res.json = function (data: any) {
+      if (data) {
+        responseSize = Buffer.byteLength(JSON.stringify(data));
+      }
+      return originalJson.call(this, data);
+    };
+    // Log after response is finished
+    res.on('finish', () => {
+      // Skip logging for analytics endpoints to avoid infinite loops
+      if (req.path.includes('/analytics/')) {
+        return;
+      }
+
+      const duration = Date.now() - startTime;
+      logger.info('HTTP Request', {
+        method: req.method,
+        path: req.path,
+        status: res.statusCode,
+        size: responseSize,
+        duration: `${duration}ms`,
+        ip: req.ip || req.socket.remoteAddress,
+        userAgent: req.headers['user-agent'],
+        timestamp: new Date().toISOString(),
+      });
+    });
+    next();
+  });
+
+  // Apply JSON middleware
+  app.use(express.json({ limit: '100mb' }));
+  app.use(express.urlencoded({ extended: true, limit: '10mb' }));
+
+  // Create API router and mount all API routes under /api
+  const apiRouter = express.Router();
+
+  apiRouter.get('/health', (_req: Request, res: Response) => {
+    // Traditional REST: return data directly
+    const version = packageJson.version;
+    res.json({
+      status: 'ok',
+      version,
+      service: 'Insforge OSS Backend',
+      timestamp: new Date().toISOString(),
+    });
+  });
+
+  // Mount auth routes
+  apiRouter.use('/auth', authRouter);
+  apiRouter.use('/database/tables', databaseTablesRouter);
+  apiRouter.use('/database/records', databaseRecordsRouter);
+  apiRouter.use('/database/advance', databaseAdvanceRouter);
+  apiRouter.use('/storage', storageRouter);
+  apiRouter.use('/metadata', metadataRouter);
+  apiRouter.use('/logs', logsRouter);
+  apiRouter.use('/docs', docsRouter);
+  apiRouter.use('/functions', functionsRouter);
+  apiRouter.use('/secrets', secretsRouter);
+  apiRouter.use('/usage', usageRouter);
+  apiRouter.use('/openapi', openAPIRouter);
+  apiRouter.use('/agent-docs', agentDocsRouter);
+  apiRouter.use('/ai', aiRouter);
+
+  // Mount all API routes under /api prefix
+  app.use('/api', apiRouter);
+
+  // Add direct OpenAPI route at /openapi
+  app.get('/openapi', async (_req: Request, res: Response) => {
+    try {
+      const { OpenAPIService } = await import('@/core/documentation/openapi.js');
+      const openAPIService = OpenAPIService.getInstance();
+      const openAPIDocument = await openAPIService.generateOpenAPIDocument();
+      res.json(openAPIDocument);
+    } catch {
+      res.status(500).json({ error: 'Failed to generate OpenAPI document' });
+    }
+  });
+
+  // Add direct AI agent documentation route at /agent-docs
+  app.get('/agent-docs', async (_req: Request, res: Response) => {
+    try {
+      const { AgentAPIDocService } = await import('@/core/documentation/agent.js');
+      const agentAPIDocService = AgentAPIDocService.getInstance();
+      const agentDocs = await agentAPIDocService.generateAgentDocumentation();
+      res.json(agentDocs);
+    } catch {
+      res.status(500).json({ error: 'Failed to generate agent API documentation' });
+    }
+  });
+
+  // Proxy function execution to Deno runtime
+  app.all('/functions/:slug', async (req: Request, res: Response) => {
+    try {
+      const { slug } = req.params;
+      const denoUrl = process.env.DENO_RUNTIME_URL || 'http://localhost:7133';
+
+      // Simple direct proxy - just pass everything through
+      const response = await fetch(
+        `${denoUrl}/${slug}${req.url.includes('?') ? req.url.substring(req.url.indexOf('?')) : ''}`,
+        {
+          method: req.method,
+          headers: req.headers as HeadersInit,
+          body: ['GET', 'HEAD'].includes(req.method) ? undefined : JSON.stringify(req.body),
+        }
+      );
+
+      // Get response text
+      const responseText = await response.text();
+
+      res
+        .status(response.status)
+        .set('Content-Type', response.headers.get('content-type') || 'application/json')
+        .set('Access-Control-Allow-Origin', '*')
+        .send(responseText);
+    } catch (error) {
+      logger.error('Failed to proxy to Deno runtime', {
+        error: error instanceof Error ? error.message : String(error),
+        stack: error instanceof Error ? error.stack : undefined,
+        slug: req.params.slug,
+      });
+
+      // Return the actual error from Deno or connection error
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      res.status(502).json({
+        error: errorMessage,
+      });
+    }
+  });
+
+  // Always try to serve frontend if it exists
+  const frontendPath = path.join(__dirname, 'frontend');
+
+  // Check if frontend build exists
+  if (fs.existsSync(frontendPath)) {
+    // Catch all handler for SPA routes
+    app.get('/cloud*', (_req: Request, res: Response) => {
+      res.sendFile(path.join(frontendPath, 'index.html'));
+    });
+    app.get('/dashboard*', (_req: Request, res: Response) => {
+      res.sendFile(path.join(frontendPath, 'index.html'));
+    });
+    app.use(express.static(frontendPath));
+  } else {
+    // Catch-all for 404 errors - Traditional REST format
+    app.use('*', (req: Request, res: Response) => {
+      res.status(404).json({
+        error: 'NOT_FOUND',
+        message: `Endpoint ${req.originalUrl} not found`,
+        statusCode: 404,
+        nextActions: 'Please check the API documentation for available endpoints',
+      });
+    });
+  }
+
+  app.use(errorMiddleware);
+  await seedBackend();
+
+  return app;
+}
+
+// Use PORT from environment variable, fallback to 7130
+const PORT = parseInt(process.env.PORT || '7130');
+
+async function initializeServer() {
+  try {
+    const app = await createApp();
+    const server = app.listen(PORT, () => {
+      logger.info(`Backend API service listening on port ${PORT}`);
+    });
+
+    // Initialize Socket.IO service
+    const socketService = SocketService.getInstance();
+    socketService.initialize(server);
+  } catch (error) {
+    logger.error('Failed to initialize server', {
+      error: error instanceof Error ? error.message : String(error),
+      stack: error instanceof Error ? error.stack : undefined,
+    });
+    process.exit(1);
+  }
+}
+
+void initializeServer();
+
+function cleanup() {
+  logger.info('Shutting down gracefully...');
+  process.exit(0);
+}
+
+process.on('SIGINT', cleanup);
+process.on('SIGTERM', cleanup);

package/backend/src/types/ai.ts

@@ -0,0 +1,46 @@
+// Additional types that are backend-specific or internal
+
+import { ChatCompletionRequest } from '@insforge/shared-schemas';
+
+export type ChatCompletionOptions = Omit<ChatCompletionRequest, 'stream' | 'messages'>;
+
+export interface OpenRouterImageMessage {
+  type: 'image_url';
+  image_url: {
+    url: string; // Can be a direct URL or data:image base64 URL
+  };
+}
+
+// ============= OpenRouter API Types =============
+
+export interface RawOpenRouterModel {
+  id: string;
+  name: string;
+  created: number;
+  description?: string;
+  architecture?: {
+    input_modalities: string[];
+    output_modalities: string[];
+    tokenizer: string;
+    instruct_type: string;
+  };
+  topProvider?: {
+    is_moderated: boolean;
+    context_length: number;
+    max_completion_tokens: number;
+  };
+  pricing: {
+    prompt: string;
+    completion: string;
+    image?: string;
+    request?: string;
+    web_search?: string;
+    internal_reasoning?: string;
+    input_cache_read?: string;
+    input_cache_write?: string;
+  };
+  context_length: number;
+  max_completion_tokens?: number;
+  per_request_limits?: Record<string, unknown>;
+  supported_parameters?: string[];
+}

package/backend/src/types/auth.ts

@@ -0,0 +1,90 @@
+// Type definitions for database user records
+export interface UserRecord {
+  id: string;
+  email: string;
+  name: string;
+  email_verified: boolean;
+  created_at: string;
+  updated_at: string;
+  password: string | null;
+  providers: string | null;
+}
+
+// OAuth provider data from external providers
+export interface OAuthIdentityData {
+  sub?: string; // Subject identifier from provider
+  email?: string;
+  email_verified?: boolean;
+  name?: string;
+  given_name?: string;
+  family_name?: string;
+  picture?: string;
+  locale?: string;
+  [key: string]: string | boolean | number | undefined;
+}
+
+// Auth database record
+export interface AuthRecord {
+  id: string;
+  email: string;
+  password_hash: string;
+  created_at: string;
+  updated_at: string;
+}
+
+// Identity record for OAuth providers
+export interface IdentifiesRecord {
+  auth_id: string;
+  provider?: string;
+  provider_id?: string;
+  identity_data?: OAuthIdentityData;
+  email?: string;
+  last_login_at?: string;
+  created_at: string;
+  updated_at: string;
+}
+
+// Superuser auth record
+export interface SuperUserAuthRecord {
+  id: string;
+  email: string;
+  password_hash: string;
+  created_at: string;
+  updated_at: string;
+}
+
+// Superuser profile record
+export interface SuperUserProfileRecord {
+  id: string;
+  auth_id: string;
+  name: string;
+  created_at: string;
+  updated_at: string;
+}
+
+// Type definitions for OAuth providers
+export interface GoogleUserInfo {
+  sub: string;
+  email: string;
+  email_verified?: boolean;
+  name?: string;
+  picture?: string;
+  given_name?: string;
+  family_name?: string;
+  locale?: string;
+}
+
+export interface GitHubUserInfo {
+  id: number;
+  login: string;
+  name?: string;
+  email?: string;
+  avatar_url?: string;
+}
+
+export interface GitHubEmailInfo {
+  email: string;
+  primary: boolean;
+  verified: boolean;
+  visibility?: string;
+}

package/backend/src/types/database.ts

@@ -0,0 +1,136 @@
+// Type definitions for database schema management
+import { ColumnType, ColumnSchema, ForeignKeySchema } from '@insforge/shared-schemas';
+
+// Database metadata format returned by getDatabaseMetadata
+export interface DatabaseMetadata {
+  tables: Record<
+    string,
+    {
+      columns: ColumnSchema[];
+      record_count: number;
+    }
+  >;
+}
+
+export interface ColumnTypeInfo {
+  type: ColumnType;
+  sqlType: string;
+  defaultValue: string | null;
+  description: string;
+  icon?: string;
+  validation?: {
+    min?: number;
+    max?: number;
+    pattern?: string;
+  };
+}
+
+// Map frontend types to PostgreSQL types with metadata
+export const COLUMN_TYPES: Record<ColumnType, ColumnTypeInfo> = {
+  [ColumnType.STRING]: {
+    type: ColumnType.STRING,
+    sqlType: 'TEXT',
+    defaultValue: null,
+    description: 'Text up to 255 characters',
+    icon: 'text',
+  },
+  [ColumnType.DATE]: {
+    type: ColumnType.DATE,
+    sqlType: 'DATE',
+    defaultValue: 'now()',
+    description: 'Date',
+    icon: 'calendar',
+  },
+  [ColumnType.DATETIME]: {
+    type: ColumnType.DATETIME,
+    sqlType: 'TIMESTAMPTZ',
+    defaultValue: 'now()',
+    description: 'Date and time with timezone',
+    icon: 'calendar',
+  },
+  [ColumnType.INTEGER]: {
+    type: ColumnType.INTEGER,
+    sqlType: 'INTEGER',
+    defaultValue: null,
+    description: 'Whole numbers',
+    icon: 'hash',
+  },
+  [ColumnType.FLOAT]: {
+    type: ColumnType.FLOAT,
+    sqlType: 'DOUBLE PRECISION',
+    defaultValue: null,
+    description: 'Decimal numbers with double precision',
+    icon: 'percent',
+  },
+  [ColumnType.BOOLEAN]: {
+    type: ColumnType.BOOLEAN,
+    sqlType: 'BOOLEAN',
+    defaultValue: 'false',
+    description: 'True/false values',
+    icon: 'toggle',
+  },
+  [ColumnType.UUID]: {
+    type: ColumnType.UUID,
+    sqlType: 'UUID',
+    defaultValue: 'gen_random_uuid()',
+    description: 'Unique identifier (auto-generated)',
+    icon: 'fingerprint',
+  },
+  [ColumnType.JSON]: {
+    type: ColumnType.JSON,
+    sqlType: 'JSONB',
+    defaultValue: null,
+    description: 'Structured JSON data with indexing support',
+    icon: 'code',
+  },
+};
+
+export interface ForeignKeyDefinition {
+  column: string;
+  references_table: string;
+  references_column: string;
+  on_delete?: 'CASCADE' | 'SET NULL' | 'RESTRICT' | 'NO ACTION';
+  on_update?: 'CASCADE' | 'SET NULL' | 'RESTRICT' | 'NO ACTION';
+}
+
+// Type definition for foreign key information
+export type ForeignKeyInfo = ForeignKeySchema & {
+  constraint_name: string;
+};
+
+// Type definition for foreign key row from database
+export interface ForeignKeyRow {
+  constraint_name: string;
+  from_column: string;
+  foreign_table: string;
+  foreign_column: string;
+  on_delete: string;
+  on_update: string;
+}
+
+// Type definition for column information from database
+export interface ColumnInfo {
+  column_name: string;
+  data_type: string;
+  is_nullable: string;
+  column_default: string | null;
+  character_maximum_length: number | null;
+}
+
+// Type definition for primary key information
+export interface PrimaryKeyInfo {
+  column_name: string;
+}
+
+// Valid types for database column values
+export type DatabaseValue =
+  | string
+  | number
+  | boolean
+  | null
+  | Date
+  | DatabaseValue[]
+  | { [key: string]: DatabaseValue }; // jsonB type
+
+// Generic database record type - used when we don't know the exact table structure
+export type DatabaseRecord = Record<string, DatabaseValue>;

package/backend/src/types/error-constants.ts

@@ -0,0 +1,86 @@
+// Common types and constants used across the application
+
+// Error codes - what went wrong
+export enum ERROR_CODES {
+  // AUTH module
+  AUTH_INVALID_EMAIL = 'AUTH_INVALID_EMAIL',
+  AUTH_WEAK_PASSWORD = 'AUTH_WEAK_PASSWORD',
+  AUTH_INVALID_CREDENTIALS = 'AUTH_INVALID_CREDENTIALS',
+  AUTH_INVALID_API_KEY = 'AUTH_INVALID_API_KEY',
+  AUTH_EMAIL_EXISTS = 'AUTH_EMAIL_EXISTS',
+  AUTH_OAUTH_CONFIG_ERROR = 'AUTH_OAUTH_CONFIG_ERROR',
+  AUTH_UNSUPPORTED_PROVIDER = 'AUTH_UNSUPPORTED_PROVIDER',
+  AUTH_TOKEN_EXPIRED = 'AUTH_TOKEN_EXPIRED',
+  AUTH_UNAUTHORIZED = 'AUTH_UNAUTHORIZED',
+  AUTH_NEED_VERIFICATION = 'AUTH_NEED_VERIFICATION',
+
+  // DATABASE module
+  DATABASE_INVALID_PARAMETER = 'DATABASE_INVALID_PARAMETER',
+  DATABASE_VALIDATION_ERROR = 'DATABASE_VALIDATION_ERROR',
+  DATABASE_CONSTRAINT_VIOLATION = 'DATABASE_CONSTRAINT_VIOLATION',
+  DATABASE_NOT_FOUND = 'DATABASE_NOT_FOUND',
+  DATABASE_DUPLICATE = 'DATABASE_DUPLICATE',
+  DATABASE_PERMISSION_DENIED = 'DATABASE_PERMISSION_DENIED',
+  DATABASE_INTERNAL_ERROR = 'DATABASE_INTERNAL_ERROR',
+  DATABASE_FORBIDDEN = 'DATABASE_FORBIDDEN',
+
+  // STORAGE module
+  STORAGE_INVALID_PARAMETER = 'STORAGE_INVALID_PARAMETER',
+  STORAGE_INVALID_FILE_TYPE = 'STORAGE_INVALID_FILE_TYPE',
+  STORAGE_INSUFFICIENT_QUOTA = 'STORAGE_INSUFFICIENT_QUOTA',
+  STORAGE_NOT_FOUND = 'STORAGE_NOT_FOUND',
+  STORAGE_PERMISSION_DENIED = 'STORAGE_PERMISSION_DENIED',
+
+  // REALTIME module
+  REALTIME_CONNECTION_FAILED = 'REALTIME_CONNECTION_FAILED',
+  REALTIME_UNAUTHORIZED = 'REALTIME_UNAUTHORIZED',
+  REALTIME_INVALID_EVENT = 'REALTIME_INVALID_EVENT',
+
+  // AI module
+  AI_INVALID_API_KEY = 'AI_INVALID_API_KEY',
+
+  // Billing module
+  BILLING_INSUFFICIENT_BALANCE = 'BILLING_INSUFFICIENT_BALANCE',
+
+  // General
+  MISSING_FIELD = 'MISSING_FIELD',
+  ALREADY_EXISTS = 'ALREADY_EXISTS',
+  INVALID_INPUT = 'INVALID_INPUT',
+  NOT_FOUND = 'NOT_FOUND',
+  UNKNOWN_ERROR = 'UNKNOWN_ERROR',
+  INTERNAL_ERROR = 'INTERNAL_ERROR',
+  TOO_MANY_REQUESTS = 'TOO_MANY_REQUESTS',
+  FORBIDDEN = 'FORBIDDEN',
+}
+
+// Next actions - what the user should do
+export const NEXT_ACTION = {
+  // Authentication next actions
+  CHECK_TOKEN: 'Check the token is valid or login to get a new token.',
+  CHECK_ADMIN_TOKEN: 'Check the admin token is valid or login as admin to get a new token.',
+  CHECK_API_KEY: 'Check the API key is valid or generate a new API key.',
+
+  // Database next actions
+  CHECK_UNIQUE_FIELD: (field: string) =>
+    `A record with this field(${field}) already exists. You can query the existing record by using the query tool and then try again.`,
+  CHECK_TABLE_EXISTS:
+    'The resource you are trying to access does not exist. Please check the table with get metadata tool and try again.',
+  FILL_REQUIRED_FIELD: (field: string) =>
+    `The ${field} field is required and cannot be empty. Please fill in a value and try again.`,
+  CHECK_REFERENCE_EXISTS:
+    'The referenced record does not exist. Please check the reference with get metadata tool and try again.',
+
+  // Schema validation next actions
+  CHECK_COLUMN_EXISTS:
+    'Check the column name spelling and verify it exists in the table using GET /api/database/tables/{table}/schema',
+  CHECK_UNIQUE_CONSTRAINT:
+    'Ensure the referenced column has a unique constraint or is a primary key',
+  CHECK_DATATYPE_MATCH:
+    'Ensure the foreign key column and the referenced column have the same data type.',
+  REMOVE_DUPLICATE_COLUMN: (column: string) =>
+    `Remove the duplicate "${column}" column definition. Each column name must be unique within a table.`,
+
+  // Add more next actions as needed
+} as const;
+
+export type NextActionKey = keyof typeof NEXT_ACTION;

package/backend/src/types/logs.ts

@@ -0,0 +1,47 @@
+// Audit log types
+export interface AuditLogEntry {
+  actor: string;
+  action: string;
+  module: string;
+  details?: Record<string, unknown>;
+  ip_address?: string;
+}
+
+export interface AuditLogQuery {
+  actor?: string;
+  action?: string;
+  module?: string;
+  start_date?: Date;
+  end_date?: Date;
+  limit?: number;
+  offset?: number;
+}
+
+// Types for Logflare analytics logs
+export interface LogSource {
+  id: number;
+  name: string;
+  token: string;
+}
+
+export interface AnalyticsLogRecord {
+  id: string;
+  event_message: string;
+  timestamp: string;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  body: Record<string, any>;
+}
+
+export interface AnalyticsLogResponse {
+  source: string;
+  logs: AnalyticsLogRecord[];
+  total: number;
+  page: number;
+  pageSize: number;
+}
+
+export interface LogSourceStats {
+  source: string;
+  count: number;
+  lastActivity: string;
+}