insforge 0.3.2 → 1.2.10

package/backend/src/services/usage/usage.service.ts

@@ -0,0 +1,149 @@
+import { Pool } from 'pg';
+import { DatabaseManager } from '@/infra/database/database.manager.js';
+import logger from '@/utils/logger.js';
+
+export interface AIUsageByModel {
+  model: string;
+  total_input_tokens: number;
+  total_output_tokens: number;
+  total_images: number;
+}
+
+export interface UsageStats {
+  mcp_usage_count: number;
+  database_size_bytes: number;
+  storage_size_bytes: number;
+  ai_usage_by_model: AIUsageByModel[];
+}
+
+export interface MCPUsageRecord {
+  tool_name: string;
+  success: boolean;
+  created_at: string;
+}
+
+/**
+ * UsageService - Handles usage tracking and statistics
+ * Business logic layer for MCP usage, system resource tracking, and AI usage
+ */
+export class UsageService {
+  private static instance: UsageService;
+  private pool: Pool | null = null;
+
+  private constructor() {
+    logger.info('UsageService initialized');
+  }
+
+  private getPool(): Pool {
+    if (!this.pool) {
+      this.pool = DatabaseManager.getInstance().getPool();
+    }
+    return this.pool;
+  }
+
+  public static getInstance(): UsageService {
+    if (!UsageService.instance) {
+      UsageService.instance = new UsageService();
+    }
+    return UsageService.instance;
+  }
+
+  /**
+   * Record MCP tool usage
+   */
+  async recordMCPUsage(toolName: string, success: boolean = true): Promise<{ created_at: string }> {
+    try {
+      const result = await this.getPool().query(
+        `INSERT INTO _mcp_usage (tool_name, success)
+         VALUES ($1, $2)
+         RETURNING created_at`,
+        [toolName, success]
+      );
+
+      logger.info('MCP usage recorded', { toolName, success });
+      return { created_at: result.rows[0].created_at };
+    } catch (error) {
+      logger.error('Failed to record MCP usage', { error, toolName });
+      throw new Error('Failed to record MCP usage');
+    }
+  }
+
+  /**
+   * Get recent MCP usage records
+   */
+  async getMCPUsage(limit: number = 5, success: boolean = true): Promise<MCPUsageRecord[]> {
+    try {
+      const result = await this.getPool().query(
+        `SELECT tool_name, success, created_at
+         FROM _mcp_usage
+         WHERE success = $1
+         ORDER BY created_at DESC
+         LIMIT $2`,
+        [success, limit]
+      );
+
+      return result.rows as MCPUsageRecord[];
+    } catch (error) {
+      logger.error('Failed to get MCP usage', { error });
+      throw new Error('Failed to get MCP usage');
+    }
+  }
+
+  /**
+   * Get comprehensive usage statistics for a date range
+   * Returns MCP usage, database size, storage size, and AI usage by model
+   */
+  async getUsageStats(startDate: Date, endDate: Date): Promise<UsageStats> {
+    try {
+      // Get MCP tool usage count
+      const mcpResult = await this.getPool().query(
+        `SELECT COUNT(*) as count
+         FROM _mcp_usage
+         WHERE success = true
+           AND created_at >= $1
+           AND created_at < $2`,
+        [startDate, endDate]
+      );
+
+      // Get database size
+      const dbSizeResult = await this.getPool().query(
+        `SELECT pg_database_size(current_database()) as size`
+      );
+
+      // Get total storage size
+      const storageResult = await this.getPool().query(
+        `SELECT COALESCE(SUM(size), 0) as total_size FROM _storage`
+      );
+
+      // Get AI usage breakdown by model (only billable metrics)
+      const aiUsageByModel = await this.getPool().query(
+        `SELECT
+          COALESCE(u.model_id, c.model_id) as model,
+          COALESCE(SUM(u.input_tokens), 0) as total_input_tokens,
+          COALESCE(SUM(u.output_tokens), 0) as total_output_tokens,
+          COALESCE(SUM(u.image_count), 0) as total_images
+        FROM _ai_usage u
+        LEFT JOIN _ai_configs c ON u.config_id = c.id
+        WHERE u.created_at >= $1 AND u.created_at < $2
+        GROUP BY COALESCE(u.model_id, c.model_id)
+        ORDER BY (COALESCE(SUM(u.input_tokens), 0) + COALESCE(SUM(u.output_tokens), 0)) DESC`,
+        [startDate, endDate]
+      );
+
+      return {
+        mcp_usage_count: parseInt(mcpResult.rows[0]?.count || '0'),
+        database_size_bytes: parseInt(dbSizeResult.rows[0]?.size || '0'),
+        storage_size_bytes: parseInt(storageResult.rows[0]?.total_size || '0'),
+        ai_usage_by_model: aiUsageByModel.rows.map((row) => ({
+          model: (row.model as string) || 'unknown',
+          total_input_tokens: parseInt(String(row.total_input_tokens || '0')),
+          total_output_tokens: parseInt(String(row.total_output_tokens || '0')),
+          total_images: parseInt(String(row.total_images || '0')),
+        })),
+      };
+    } catch (error) {
+      logger.error('Failed to get usage stats', { error });
+      throw new Error('Failed to get usage stats');
+    }
+  }
+}

package/backend/src/types/auth.ts

@@ -6,8 +6,8 @@ export interface UserRecord {
   email_verified: boolean;
   created_at: string;
   updated_at: string;
-  password: string | null;
-  providers: string | null;
+  password?: string;
+  providers?: string;
 }
 
 // OAuth provider data from external providers
@@ -88,3 +88,67 @@ export interface GitHubEmailInfo {
   verified: boolean;
   visibility?: string;
 }
+
+export interface MicrosoftUserInfo {
+  id: string;
+  email: string;
+  name: string;
+  avatar_url?: string;
+}
+export interface DiscordUserInfo {
+  id: string;
+  username: string;
+  email?: string;
+  avatar?: string;
+}
+
+export interface LinkedInUserInfo {
+  sub: string;
+  email: string;
+  email_verified?: boolean;
+  name?: string;
+  picture?: string;
+  given_name?: string;
+  family_name?: string;
+  locale?: string;
+}
+
+export interface FacebookUserInfo {
+  id: string;
+  email: string;
+  name?: string;
+  picture?: {
+    data?: {
+      url?: string;
+    };
+  };
+  first_name?: string;
+  last_name?: string;
+}
+
+export interface XUserInfo {
+  id: string;
+  name: string;
+  username?: string;
+  profile_image_url?: string;
+  verified?: boolean;
+  created_at?: string;
+}
+
+// Generic OAuth user data returned by provider services
+export interface OAuthUserData {
+  provider: string;
+  providerId: string;
+  email: string;
+  userName: string;
+  avatarUrl: string;
+  identityData:
+    | GoogleUserInfo
+    | GitHubUserInfo
+    | DiscordUserInfo
+    | LinkedInUserInfo
+    | FacebookUserInfo
+    | MicrosoftUserInfo
+    | XUserInfo
+    | Record<string, unknown>;
+}

package/backend/src/types/email.ts

@@ -0,0 +1,8 @@
+/**
+ * Email template types supported by email providers
+ */
+export type EmailTemplate =
+  | 'email-verification-code' // Numeric OTP for email verification
+  | 'email-verification-link' // Magic link for email verification
+  | 'reset-password-code' // Numeric OTP for password reset
+  | 'reset-password-link'; // Magic link for password reset

package/backend/src/types/error-constants.ts

@@ -39,6 +39,9 @@ export enum ERROR_CODES {
   // AI module
   AI_INVALID_API_KEY = 'AI_INVALID_API_KEY',
 
+  // LOGS module
+  LOGS_AWS_NOT_CONFIGURED = 'LOGS_AWS_NOT_CONFIGURED',
+
   // Billing module
   BILLING_INSUFFICIENT_BALANCE = 'BILLING_INSUFFICIENT_BALANCE',
 
@@ -51,6 +54,7 @@ export enum ERROR_CODES {
   INTERNAL_ERROR = 'INTERNAL_ERROR',
   TOO_MANY_REQUESTS = 'TOO_MANY_REQUESTS',
   FORBIDDEN = 'FORBIDDEN',
+  RATE_LIMITED = 'RATE_LIMITED',
 }
 
 // Next actions - what the user should do

package/backend/src/types/logs.ts

@@ -16,32 +16,3 @@ export interface AuditLogQuery {
   limit?: number;
   offset?: number;
 }
-
-// Types for Logflare analytics logs
-export interface LogSource {
-  id: number;
-  name: string;
-  token: string;
-}
-
-export interface AnalyticsLogRecord {
-  id: string;
-  event_message: string;
-  timestamp: string;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  body: Record<string, any>;
-}
-
-export interface AnalyticsLogResponse {
-  source: string;
-  logs: AnalyticsLogRecord[];
-  total: number;
-  page: number;
-  pageSize: number;
-}
-
-export interface LogSourceStats {
-  source: string;
-  count: number;
-  lastActivity: string;
-}

package/backend/src/{core/socket/types.ts → types/socket.ts}

@@ -41,12 +41,11 @@ export interface NotificationPayload {
 }
 
 export enum DataUpdateResourceType {
-  METADATA = 'metadata',
-  DATABASE_SCHEMA = 'database_schema',
-  TABLE_SCHEMA = 'table_schema',
-  STORAGE_SCHEMA = 'storage_schema',
-  BUCKET_SCHEMA = 'bucket_schema',
-  AUTH_SCHEMA = 'auth_schmea',
+  DATABASE = 'database',
+  USERS = 'users',
+  RECORDS = 'records',
+  BUCKETS = 'buckets',
+  FUNCTIONS = 'functions',
 }
 
 export interface DataUpdatePayload {

package/backend/src/utils/environment.ts

@@ -7,9 +7,7 @@
  * Currently checks for AWS instance profile, but can be extended for other cloud providers
  */
 export function isCloudEnvironment(): boolean {
-  return !!(
-    process.env.AWS_INSTANCE_PROFILE_NAME && process.env.AWS_INSTANCE_PROFILE_NAME.trim().length > 0
-  );
+  return !!(process.env.AWS_INSTANCE_PROFILE_NAME && process.env.AWS_INSTANCE_PROFILE_NAME.trim());
 }
 
 /**
@@ -33,3 +31,11 @@ export function isDevelopment(): boolean {
 export function isProduction(): boolean {
   return process.env.NODE_ENV === 'production';
 }
+
+/**
+ * Get the API base URL from environment variable or default to localhost
+ * @returns The API base URL
+ */
+export function getApiBaseUrl(): string {
+  return process.env.API_BASE_URL || 'http://localhost:7130';
+}

package/backend/src/utils/logger.ts

@@ -1,13 +1,31 @@
 import winston from 'winston';
+import path from 'path';
 
-const logger = winston.createLogger({
+const logsDir = process.env.LOGS_DIR || path.join(process.cwd(), 'logs');
+
+export const logger = winston.createLogger({
   level: process.env.LOG_LEVEL || 'info',
   format: winston.format.combine(
     winston.format.timestamp(),
     winston.format.errors({ stack: true }),
     winston.format.json()
   ),
-  transports: [new winston.transports.Console()],
+  transports: [
+    new winston.transports.Console(),
+    new winston.transports.File({
+      filename: path.join(logsDir, 'insforge.logs.jsonl'),
+      format: winston.format.printf((info) => {
+        const { timestamp, level, message, ...metadata } = info;
+        return JSON.stringify({
+          id: `${Date.now()}-${Math.random()}`,
+          timestamp,
+          message,
+          level,
+          metadata,
+        });
+      }),
+    }),
+  ],
 });
 
 export default logger;

package/backend/src/utils/seed.ts

@@ -1,9 +1,12 @@
-import { DatabaseManager } from '@/core/database/manager.js';
-import { AIConfigService } from '@/core/ai/config.js';
+import { DatabaseManager } from '@/infra/database/database.manager.js';
+import { TokenManager } from '@/infra/security/token.manager.js';
+import { AIConfigService } from '@/services/ai/ai-config.service.js';
 import { isCloudEnvironment } from '@/utils/environment.js';
 import logger from '@/utils/logger.js';
-import { SecretsService } from '@/core/secrets/secrets';
-import { OAuthConfigService } from '@/core/auth/oauth.js';
+import { SecretService } from '@/services/secrets/secret.service.js';
+import { OAuthConfigService } from '@/services/auth/oauth-config.service.js';
+import { OAuthProvidersSchema } from '@insforge/shared-schemas';
+import { AuthConfigService } from '@/services/auth/auth-config.service.js';
 
 /**
  * Validates admin credentials are configured
@@ -26,12 +29,12 @@ async function seedDefaultAIConfigs(): Promise<void> {
     return;
   }
 
-  const aiConfigService = new AIConfigService();
+  const aiConfigService = AIConfigService.getInstance();
 
   // Check if AI configs already exist
   const existingConfigs = await aiConfigService.findAll();
 
-  if (existingConfigs.length > 0) {
+  if (existingConfigs.length) {
     return;
   }
 
@@ -47,39 +50,92 @@ async function seedDefaultAIConfigs(): Promise<void> {
     ['text', 'image'],
     ['text', 'image'],
     'openrouter',
-    'google/gemini-2.5-flash-image-preview'
+    'google/gemini-3-pro-image-preview'
   );
 
   logger.info('✅ Default AI models configured (cloud environment)');
 }
 
 /**
- * Seeds default OAuth configurations for Google and GitHub
+ * Seeds default auth configuration for cloud environments
+ * Enables email verification with code-based verification method
+ * Only inserts config if table is empty (first startup, never configured)
+ */
+async function seedDefaultAuthConfig(): Promise<void> {
+  const dbManager = DatabaseManager.getInstance();
+  const pool = dbManager.getPool();
+  const client = await pool.connect();
+
+  try {
+    const result = await client.query('SELECT COUNT(*) as count FROM _auth_configs');
+    const hasConfig = result.rows.length > 0 && Number(result.rows[0].count) > 0;
+
+    if (hasConfig) {
+      const authConfigService = AuthConfigService.getInstance();
+      const currentConfig = await authConfigService.getAuthConfig();
+      logger.info(
+        '✅ Email verification configured:',
+        currentConfig.requireEmailVerification ? 'enabled' : 'disabled'
+      );
+      return;
+    }
+
+    // Table is empty - this is first startup, insert default cloud configuration
+    // Note: Migration 016 will add verify_email_method, reset_password_method, sign_in_redirect_to
+    // so we only insert fields that exist in migration 015
+    await client.query(
+      `INSERT INTO _auth_configs (
+        require_email_verification,
+        password_min_length,
+        require_number,
+        require_lowercase,
+        require_uppercase,
+        require_special_char
+      ) VALUES ($1, $2, $3, $4, $5, $6)
+       ON CONFLICT DO NOTHING`,
+      [
+        isCloudEnvironment(), // Enable email verification for cloud
+        6, // password_min_length
+        false, // require_number
+        false, // require_lowercase
+        false, // require_uppercase
+        false, // require_special_char
+      ]
+    );
+
+    logger.info('✅ Email verification enabled (cloud environment)');
+  } catch (error) {
+    logger.error('Failed to seed default auth config', {
+      error: error instanceof Error ? error.message : String(error),
+    });
+    // Don't throw - this is not critical for app startup
+  } finally {
+    client.release();
+  }
+}
+
+/**
+ * Seeds default OAuth configurations for supported providers
  */
 async function seedDefaultOAuthConfigs(): Promise<void> {
-  const oauthService = OAuthConfigService.getInstance();
+  const oauthConfigService = OAuthConfigService.getInstance();
 
   try {
     // Check if OAuth configs already exist
-    const existingConfigs = await oauthService.getAllConfigs();
+    const existingConfigs = await oauthConfigService.getAllConfigs();
     const existingProviders = existingConfigs.map((config) => config.provider.toLowerCase());
 
-    // Seed Google OAuth config if not exists
-    if (!existingProviders.includes('google')) {
-      await oauthService.createConfig({
-        provider: 'google',
-        useSharedKey: true,
-      });
-      logger.info('✅ Default Google OAuth config created');
-    }
+    // Default providers to seed
+    const defaultProviders: OAuthProvidersSchema[] = ['google', 'github'];
 
-    // Seed GitHub OAuth config if not exists
-    if (!existingProviders.includes('github')) {
-      await oauthService.createConfig({
-        provider: 'github',
-        useSharedKey: true,
-      });
-      logger.info('✅ Default GitHub OAuth config created');
+    for (const provider of defaultProviders) {
+      if (!existingProviders.includes(provider)) {
+        await oauthConfigService.createConfig({
+          provider,
+          useSharedKey: true,
+        });
+        logger.info(`✅ Default ${provider} OAuth config created`);
+      }
     }
   } catch (error) {
     logger.warn('Failed to seed OAuth configs', {
@@ -93,41 +149,59 @@ async function seedDefaultOAuthConfigs(): Promise<void> {
  * Seeds OAuth configurations from local environment variables
  */
 async function seedLocalOAuthConfigs(): Promise<void> {
-  const oauthService = OAuthConfigService.getInstance();
+  const oauthConfigService = OAuthConfigService.getInstance();
 
   try {
     // Check if OAuth configs already exist
-    const existingConfigs = await oauthService.getAllConfigs();
+    const existingConfigs = await oauthConfigService.getAllConfigs();
     const existingProviders = existingConfigs.map((config) => config.provider.toLowerCase());
 
-    // Seed Google OAuth config from environment variables if credentials exist
-    const googleClientId = process.env.GOOGLE_CLIENT_ID;
-    const googleClientSecret = process.env.GOOGLE_CLIENT_SECRET;
-
-    if (googleClientId && googleClientSecret && !existingProviders.includes('google')) {
-      await oauthService.createConfig({
+    // Environment variable mappings for OAuth providers
+    const envMappings: Array<{
+      provider: OAuthProvidersSchema;
+      clientIdEnv: string;
+      clientSecretEnv: string;
+    }> = [
+      {
         provider: 'google',
-        clientId: googleClientId,
-        clientSecret: googleClientSecret,
-        scopes: ['openid', 'email', 'profile'],
-        useSharedKey: false,
-      });
-      logger.info('✅ Google OAuth config loaded from environment variables');
-    }
+        clientIdEnv: 'GOOGLE_CLIENT_ID',
+        clientSecretEnv: 'GOOGLE_CLIENT_SECRET',
+      },
+      {
+        provider: 'github',
+        clientIdEnv: 'GITHUB_CLIENT_ID',
+        clientSecretEnv: 'GITHUB_CLIENT_SECRET',
+      },
+      {
+        provider: 'discord',
+        clientIdEnv: 'DISCORD_CLIENT_ID',
+        clientSecretEnv: 'DISCORD_CLIENT_SECRET',
+      },
+      {
+        provider: 'linkedin',
+        clientIdEnv: 'LINKEDIN_CLIENT_ID',
+        clientSecretEnv: 'LINKEDIN_CLIENT_SECRET',
+      },
+      {
+        provider: 'microsoft',
+        clientIdEnv: 'MICROSOFT_CLIENT_ID',
+        clientSecretEnv: 'MICROSOFT_CLIENT_SECRET',
+      },
+    ];
 
-    // Seed GitHub OAuth config from environment variables if credentials exist
-    const githubClientId = process.env.GITHUB_CLIENT_ID;
-    const githubClientSecret = process.env.GITHUB_CLIENT_SECRET;
+    for (const { provider, clientIdEnv, clientSecretEnv } of envMappings) {
+      const clientId = process.env[clientIdEnv];
+      const clientSecret = process.env[clientSecretEnv];
 
-    if (githubClientId && githubClientSecret && !existingProviders.includes('github')) {
-      await oauthService.createConfig({
-        provider: 'github',
-        clientId: githubClientId,
-        clientSecret: githubClientSecret,
-        scopes: ['user:email'],
-        useSharedKey: false,
-      });
-      logger.info('✅ GitHub OAuth config loaded from environment variables');
+      if (clientId && clientSecret && !existingProviders.includes(provider)) {
+        await oauthConfigService.createConfig({
+          provider,
+          clientId,
+          clientSecret,
+          useSharedKey: false,
+        });
+        logger.info(`✅ ${provider} OAuth config loaded from environment variables`);
+      }
     }
   } catch (error) {
     logger.warn('Failed to seed local OAuth configs', {
@@ -138,7 +212,8 @@ async function seedLocalOAuthConfigs(): Promise<void> {
 
 // Create api key, admin user, and default AI configs
 export async function seedBackend(): Promise<void> {
-  const secretService = new SecretsService();
+  const secretService = SecretService.getInstance();
+
   const dbManager = DatabaseManager.getInstance();
 
   const adminEmail = process.env.ADMIN_EMAIL || 'admin@example.com';
@@ -163,13 +238,16 @@ export async function seedBackend(): Promise<void> {
     });
     // Database connection info is already logged above
 
-    if (tables.length > 0) {
+    if (tables.length) {
       logger.info(`✅ Found ${tables.length} user tables`);
     }
 
     // seed AI configs for cloud environment
     await seedDefaultAIConfigs();
 
+    // Enable email verification in cloud environment
+    await seedDefaultAuthConfig();
+
     // add default OAuth configs in Cloud hosting
     if (isCloudEnvironment()) {
       await seedDefaultOAuthConfigs();
@@ -180,15 +258,30 @@ export async function seedBackend(): Promise<void> {
     // Initialize reserved secrets for edge functions
     // Add INSFORGE_INTERNAL_URL for Deno-to-backend container communication
     const insforgInternalUrl = 'http://insforge:7130';
-    const existingSecret = await secretService.getSecretByKey('INSFORGE_INTERNAL_URL');
+    const existingInternalUrlSecret = await secretService.getSecretByKey('INSFORGE_INTERNAL_URL');
 
-    if (existingSecret === null) {
+    if (existingInternalUrlSecret === null) {
       await secretService.createSecret({
         key: 'INSFORGE_INTERNAL_URL',
         isReserved: true,
         value: insforgInternalUrl,
       });
-      logger.info('✅ System secrets initialized');
+      logger.info('✅ INSFORGE_INTERNAL_URL secret initialized');
+    }
+
+    // Add ANON_KEY for public edge function access
+    const existingAnonKeySecret = await secretService.getSecretByKey('ANON_KEY');
+
+    if (existingAnonKeySecret === null) {
+      const tokenManager = TokenManager.getInstance();
+      const anonToken = tokenManager.generateAnonToken();
+
+      await secretService.createSecret({
+        key: 'ANON_KEY',
+        isReserved: true,
+        value: anonToken,
+      });
+      logger.info('✅ ANON_KEY secret initialized');
     }
 
     logger.info(`API key generated: ${apiKey}`);

package/backend/src/utils/sql-parser.ts

@@ -49,7 +49,7 @@ export function parseSQLStatements(sqlText: string): string[] {
           .replace(/--.*$/gm, '') // Remove line comments
           .replace(/\/\*[\s\S]*?\*\//g, '') // Remove block comments
           .trim();
-        return withoutComments.length > 0;
+        return withoutComments.length;
       });
 
     logger.debug(`Parsed ${statements.length} SQL statements from input`);