insforge 0.3.3 → 1.3.0

package/backend/src/services/usage/usage.service.ts

@@ -0,0 +1,149 @@
+import { Pool } from 'pg';
+import { DatabaseManager } from '@/infra/database/database.manager.js';
+import logger from '@/utils/logger.js';
+
+export interface AIUsageByModel {
+  model: string;
+  total_input_tokens: number;
+  total_output_tokens: number;
+  total_images: number;
+}
+
+export interface UsageStats {
+  mcp_usage_count: number;
+  database_size_bytes: number;
+  storage_size_bytes: number;
+  ai_usage_by_model: AIUsageByModel[];
+}
+
+export interface MCPUsageRecord {
+  tool_name: string;
+  success: boolean;
+  created_at: string;
+}
+
+/**
+ * UsageService - Handles usage tracking and statistics
+ * Business logic layer for MCP usage, system resource tracking, and AI usage
+ */
+export class UsageService {
+  private static instance: UsageService;
+  private pool: Pool | null = null;
+
+  private constructor() {
+    logger.info('UsageService initialized');
+  }
+
+  private getPool(): Pool {
+    if (!this.pool) {
+      this.pool = DatabaseManager.getInstance().getPool();
+    }
+    return this.pool;
+  }
+
+  public static getInstance(): UsageService {
+    if (!UsageService.instance) {
+      UsageService.instance = new UsageService();
+    }
+    return UsageService.instance;
+  }
+
+  /**
+   * Record MCP tool usage
+   */
+  async recordMCPUsage(toolName: string, success: boolean = true): Promise<{ created_at: string }> {
+    try {
+      const result = await this.getPool().query(
+        `INSERT INTO _mcp_usage (tool_name, success)
+         VALUES ($1, $2)
+         RETURNING created_at`,
+        [toolName, success]
+      );
+
+      logger.info('MCP usage recorded', { toolName, success });
+      return { created_at: result.rows[0].created_at };
+    } catch (error) {
+      logger.error('Failed to record MCP usage', { error, toolName });
+      throw new Error('Failed to record MCP usage');
+    }
+  }
+
+  /**
+   * Get recent MCP usage records
+   */
+  async getMCPUsage(limit: number = 5, success: boolean = true): Promise<MCPUsageRecord[]> {
+    try {
+      const result = await this.getPool().query(
+        `SELECT tool_name, success, created_at
+         FROM _mcp_usage
+         WHERE success = $1
+         ORDER BY created_at DESC
+         LIMIT $2`,
+        [success, limit]
+      );
+
+      return result.rows as MCPUsageRecord[];
+    } catch (error) {
+      logger.error('Failed to get MCP usage', { error });
+      throw new Error('Failed to get MCP usage');
+    }
+  }
+
+  /**
+   * Get comprehensive usage statistics for a date range
+   * Returns MCP usage, database size, storage size, and AI usage by model
+   */
+  async getUsageStats(startDate: Date, endDate: Date): Promise<UsageStats> {
+    try {
+      // Get MCP tool usage count
+      const mcpResult = await this.getPool().query(
+        `SELECT COUNT(*) as count
+         FROM _mcp_usage
+         WHERE success = true
+           AND created_at >= $1
+           AND created_at < $2`,
+        [startDate, endDate]
+      );
+
+      // Get database size
+      const dbSizeResult = await this.getPool().query(
+        `SELECT pg_database_size(current_database()) as size`
+      );
+
+      // Get total storage size
+      const storageResult = await this.getPool().query(
+        `SELECT COALESCE(SUM(size), 0) as total_size FROM _storage`
+      );
+
+      // Get AI usage breakdown by model (only billable metrics)
+      const aiUsageByModel = await this.getPool().query(
+        `SELECT
+          COALESCE(u.model_id, c.model_id) as model,
+          COALESCE(SUM(u.input_tokens), 0) as total_input_tokens,
+          COALESCE(SUM(u.output_tokens), 0) as total_output_tokens,
+          COALESCE(SUM(u.image_count), 0) as total_images
+        FROM _ai_usage u
+        LEFT JOIN _ai_configs c ON u.config_id = c.id
+        WHERE u.created_at >= $1 AND u.created_at < $2
+        GROUP BY COALESCE(u.model_id, c.model_id)
+        ORDER BY (COALESCE(SUM(u.input_tokens), 0) + COALESCE(SUM(u.output_tokens), 0)) DESC`,
+        [startDate, endDate]
+      );
+
+      return {
+        mcp_usage_count: parseInt(mcpResult.rows[0]?.count || '0'),
+        database_size_bytes: parseInt(dbSizeResult.rows[0]?.size || '0'),
+        storage_size_bytes: parseInt(storageResult.rows[0]?.total_size || '0'),
+        ai_usage_by_model: aiUsageByModel.rows.map((row) => ({
+          model: (row.model as string) || 'unknown',
+          total_input_tokens: parseInt(String(row.total_input_tokens || '0')),
+          total_output_tokens: parseInt(String(row.total_output_tokens || '0')),
+          total_images: parseInt(String(row.total_images || '0')),
+        })),
+      };
+    } catch (error) {
+      logger.error('Failed to get usage stats', { error });
+      throw new Error('Failed to get usage stats');
+    }
+  }
+}

package/backend/src/types/auth.ts

@@ -6,8 +6,8 @@ export interface UserRecord {
   email_verified: boolean;
   created_at: string;
   updated_at: string;
-  password: string | null;
-  providers: string | null;
+  password?: string;
+  providers?: string;
 }
 
 // OAuth provider data from external providers
@@ -88,3 +88,78 @@ export interface GitHubEmailInfo {
   verified: boolean;
   visibility?: string;
 }
+
+export interface MicrosoftUserInfo {
+  id: string;
+  email: string;
+  name: string;
+  avatar_url?: string;
+}
+export interface DiscordUserInfo {
+  id: string;
+  username: string;
+  email?: string;
+  avatar?: string;
+}
+
+export interface LinkedInUserInfo {
+  sub: string;
+  email: string;
+  email_verified?: boolean;
+  name?: string;
+  picture?: string;
+  given_name?: string;
+  family_name?: string;
+  locale?: string;
+}
+
+export interface FacebookUserInfo {
+  id: string;
+  email: string;
+  name?: string;
+  picture?: {
+    data?: {
+      url?: string;
+    };
+  };
+  first_name?: string;
+  last_name?: string;
+}
+
+export interface XUserInfo {
+  id: string;
+  name: string;
+  username?: string;
+  profile_image_url?: string;
+  verified?: boolean;
+  created_at?: string;
+}
+
+export interface AppleUserInfo {
+  sub: string;
+  email: string;
+  email_verified?: boolean;
+  is_private_email?: boolean;
+  name?: string;
+  given_name?: string;
+  family_name?: string;
+}
+
+// Generic OAuth user data returned by provider services
+export interface OAuthUserData {
+  provider: string;
+  providerId: string;
+  email: string;
+  userName: string;
+  avatarUrl: string;
+  identityData:
+    | GoogleUserInfo
+    | GitHubUserInfo
+    | DiscordUserInfo
+    | LinkedInUserInfo
+    | FacebookUserInfo
+    | MicrosoftUserInfo
+    | XUserInfo
+    | AppleUserInfo
+    | Record<string, unknown>;
+}

package/backend/src/types/email.ts

@@ -0,0 +1,8 @@
+/**
+ * Email template types supported by email providers
+ */
+export type EmailTemplate =
+  | 'email-verification-code' // Numeric OTP for email verification
+  | 'email-verification-link' // Magic link for email verification
+  | 'reset-password-code' // Numeric OTP for password reset
+  | 'reset-password-link'; // Magic link for password reset

package/backend/src/types/error-constants.ts

@@ -39,6 +39,9 @@ export enum ERROR_CODES {
   // AI module
   AI_INVALID_API_KEY = 'AI_INVALID_API_KEY',
 
+  // LOGS module
+  LOGS_AWS_NOT_CONFIGURED = 'LOGS_AWS_NOT_CONFIGURED',
+
   // Billing module
   BILLING_INSUFFICIENT_BALANCE = 'BILLING_INSUFFICIENT_BALANCE',
 
@@ -51,6 +54,7 @@ export enum ERROR_CODES {
   INTERNAL_ERROR = 'INTERNAL_ERROR',
   TOO_MANY_REQUESTS = 'TOO_MANY_REQUESTS',
   FORBIDDEN = 'FORBIDDEN',
+  RATE_LIMITED = 'RATE_LIMITED',
 }
 
 // Next actions - what the user should do

package/backend/src/types/logs.ts

@@ -16,32 +16,3 @@ export interface AuditLogQuery {
   limit?: number;
   offset?: number;
 }
-
-// Types for Logflare analytics logs
-export interface LogSource {
-  id: number;
-  name: string;
-  token: string;
-}
-
-export interface AnalyticsLogRecord {
-  id: string;
-  event_message: string;
-  timestamp: string;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  body: Record<string, any>;
-}
-
-export interface AnalyticsLogResponse {
-  source: string;
-  logs: AnalyticsLogRecord[];
-  total: number;
-  page: number;
-  pageSize: number;
-}
-
-export interface LogSourceStats {
-  source: string;
-  count: number;
-  lastActivity: string;
-}

package/backend/src/types/realtime.ts

@@ -0,0 +1,18 @@
+/**
+ * Realtime feature types - Backend-only types
+ *
+ * Shared types should be imported directly from @insforge/shared-schemas.
+ */
+
+// ============================================================================
+// Backend-Only Types (Internal Use)
+// ============================================================================
+
+/**
+ * Delivery statistics after message processing
+ */
+export interface DeliveryResult {
+  wsAudienceCount: number;
+  whAudienceCount: number;
+  whDeliveredCount: number;
+}

package/backend/src/{core/socket/types.ts → types/socket.ts}

@@ -10,24 +10,18 @@ export enum ServerEvents {
   NOTIFICATION = 'notification',
   DATA_UPDATE = 'data:update',
   MCP_CONNECTED = 'mcp:connected',
+  // Realtime events
+  REALTIME_ERROR = 'realtime:error',
 }
 
 /**
  * Client-to-Server events
  */
 export enum ClientEvents {
-  SUBSCRIBE = 'subscribe',
-  UNSUBSCRIBE = 'unsubscribe',
-}
-
-/**
- * Generic message interface
- */
-export interface SocketMessage<T = unknown> {
-  type: string;
-  payload?: T;
-  timestamp: number;
-  id?: string;
+  // Realtime events
+  REALTIME_SUBSCRIBE = 'realtime:subscribe',
+  REALTIME_UNSUBSCRIBE = 'realtime:unsubscribe',
+  REALTIME_PUBLISH = 'realtime:publish',
 }
 
 /**
@@ -41,30 +35,11 @@ export interface NotificationPayload {
 }
 
 export enum DataUpdateResourceType {
-  METADATA = 'metadata',
-  DATABASE_SCHEMA = 'database_schema',
-  TABLE_SCHEMA = 'table_schema',
-  STORAGE_SCHEMA = 'storage_schema',
-  BUCKET_SCHEMA = 'bucket_schema',
-  AUTH_SCHEMA = 'auth_schmea',
-}
-
-export interface DataUpdatePayload {
-  resource: DataUpdateResourceType;
-  action: 'created' | 'updated' | 'deleted';
-  data: unknown;
-}
-
-/**
- * Client event payloads
- */
-export interface SubscribePayload {
-  channel: string;
-  filters?: Record<string, unknown>;
-}
-
-export interface UnsubscribePayload {
-  channel: string;
+  DATABASE = 'database',
+  USERS = 'users',
+  BUCKETS = 'buckets',
+  FUNCTIONS = 'functions',
+  REALTIME = 'realtime',
 }
 
 /**

package/backend/src/utils/cookies.ts

@@ -0,0 +1,35 @@
+import { Response } from 'express';
+import { isProduction } from './environment';
+
+/**
+ * Cookie names
+ */
+export const REFRESH_TOKEN_COOKIE_NAME = 'insforge_refresh_token';
+
+/**
+ * Set an auth cookie on response
+ * @param name - Cookie name
+ * @param value - Cookie value
+ */
+export function setAuthCookie(res: Response, name: string, value: string): void {
+  res.cookie(name, value, {
+    httpOnly: true,
+    secure: isProduction(),
+    sameSite: 'none',
+    path: '/api/auth',
+    maxAge: 7 * 24 * 60 * 60 * 1000, // 7 days
+  });
+}
+
+/**
+ * Clear an auth cookie on response
+ * IMPORTANT: Must use the same options (especially path) as when setting the cookie
+ */
+export function clearAuthCookie(res: Response, name: string): void {
+  res.clearCookie(name, {
+    httpOnly: true,
+    secure: isProduction(),
+    sameSite: 'none',
+    path: '/api/auth',
+  });
+}

package/backend/src/utils/environment.ts

@@ -7,9 +7,7 @@
  * Currently checks for AWS instance profile, but can be extended for other cloud providers
  */
 export function isCloudEnvironment(): boolean {
-  return !!(
-    process.env.AWS_INSTANCE_PROFILE_NAME && process.env.AWS_INSTANCE_PROFILE_NAME.trim().length > 0
-  );
+  return !!(process.env.AWS_INSTANCE_PROFILE_NAME && process.env.AWS_INSTANCE_PROFILE_NAME.trim());
 }
 
 /**
@@ -33,3 +31,11 @@ export function isDevelopment(): boolean {
 export function isProduction(): boolean {
   return process.env.NODE_ENV === 'production';
 }
+
+/**
+ * Get the API base URL from environment variable or default to localhost
+ * @returns The API base URL
+ */
+export function getApiBaseUrl(): string {
+  return process.env.API_BASE_URL || 'http://localhost:7130';
+}

package/backend/src/utils/logger.ts

@@ -1,13 +1,31 @@
 import winston from 'winston';
+import path from 'path';
 
-const logger = winston.createLogger({
+const logsDir = process.env.LOGS_DIR || path.join(process.cwd(), 'logs');
+
+export const logger = winston.createLogger({
   level: process.env.LOG_LEVEL || 'info',
   format: winston.format.combine(
     winston.format.timestamp(),
     winston.format.errors({ stack: true }),
     winston.format.json()
   ),
-  transports: [new winston.transports.Console()],
+  transports: [
+    new winston.transports.Console(),
+    new winston.transports.File({
+      filename: path.join(logsDir, 'insforge.logs.jsonl'),
+      format: winston.format.printf((info) => {
+        const { timestamp, level, message, ...metadata } = info;
+        return JSON.stringify({
+          id: `${Date.now()}-${Math.random()}`,
+          timestamp,
+          message,
+          level,
+          metadata,
+        });
+      }),
+    }),
+  ],
 });
 
 export default logger;

package/backend/src/utils/s3-config-loader.ts

@@ -0,0 +1,64 @@
+import { S3Client, GetObjectCommand } from '@aws-sdk/client-s3';
+import logger from '@/utils/logger.js';
+
+// TODO: make these configurable in env variables in cloud backend
+const CONFIG_BUCKET = process.env.AWS_CONFIG_BUCKET || 'insforge-config';
+const CONFIG_REGION = process.env.AWS_CONFIG_REGION || 'us-east-2';
+
+let s3Client: S3Client | null = null;
+
+/**
+ * Get or create S3 client for config loading
+ */
+function getS3Client(): S3Client {
+  if (s3Client) {
+    return s3Client;
+  }
+
+  const s3Config: {
+    region: string;
+    credentials?: { accessKeyId: string; secretAccessKey: string };
+  } = {
+    region: CONFIG_REGION,
+  };
+
+  // Use explicit credentials if provided, otherwise IAM role
+  if (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) {
+    s3Config.credentials = {
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+    };
+  }
+
+  s3Client = new S3Client(s3Config);
+  return s3Client;
+}
+
+/**
+ * Fetches a JSON config file from the S3 config bucket
+ * @param key - The S3 object key (e.g., 'default-ai-models.json')
+ * @returns Parsed JSON content or null if fetch fails
+ */
+export async function fetchS3Config<T>(key: string): Promise<T | null> {
+  try {
+    const command = new GetObjectCommand({
+      Bucket: CONFIG_BUCKET,
+      Key: key,
+    });
+
+    const response = await getS3Client().send(command);
+    const body = await response.Body?.transformToString();
+
+    if (!body) {
+      logger.warn(`Empty config file from S3: ${key}`);
+      return null;
+    }
+
+    return JSON.parse(body) as T;
+  } catch (error) {
+    logger.warn(`Failed to fetch config from S3: ${key}`, {
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return null;
+  }
+}