midway-fatcms 0.0.1-beta.1

package/src/service/proxyapi/ProxyApiLoadService.ts

@@ -0,0 +1,165 @@
+import { Inject, Provide } from '@midwayjs/core';
+import { Context } from '@midwayjs/koa';
+import * as _ from 'lodash';
+import { BaseService } from '../base/BaseService';
+import { getRouteTrie, RouteTrie } from './RouteTrie';
+import { RouteHandler } from './RouteHandler';
+import { CurdProService } from '../curd/CurdProService';
+import {SystemDbName, SystemDbType, SystemTables} from '../../models/SystemTables';
+import { KeysOfSimpleSQL } from '../../libs/crud-pro/models/keys';
+import { IProxyApiEntity, IUpstreamInfo } from '../../models/SystemEntities';
+import { parseJsonObject } from '../../libs/utils/functions';
+import { BALANCE_STRATEGY } from './ProxyApiUtils';
+import { WeightedRandom } from './WeightedRandom';
+import { WeightedRoundRobin } from './WeightedRoundRobin';
+import { IScheduleService } from '../../interface';
+import { WorkbenchService } from '../WorkbenchService';
+import { CommonException } from '../../libs/crud-pro/exceptions';
+
+function toUpstreamInfo(config_content: any): IUpstreamInfo {
+  const obj = parseJsonObject(config_content);
+  if (!obj) {
+    return {
+      balanceStrategy: BALANCE_STRATEGY.ROUND_ROBIN,
+      targetList: [],
+    };
+  }
+  const targetList = _.get(obj, 'data.targetList', []);
+  const balanceStrategy = _.get(obj, 'data.baseInfo.balanceStrategy');
+  if (Array.isArray(targetList)) {
+    const targetListParsed = targetList
+      .map((item: any) => {
+        return {
+          origin: item.origin,
+          weight: item.weight || 1,
+        };
+      })
+      .filter(item => {
+        const origin: string = item?.origin || '';
+        return origin.startsWith('http://') || origin.startsWith('https://');
+      });
+
+    return {
+      balanceStrategy,
+      targetList: targetListParsed,
+    };
+  }
+
+  return {
+    balanceStrategy: BALANCE_STRATEGY.ROUND_ROBIN,
+    targetList: [],
+  };
+}
+
+/**
+ * 接口代理。 数据加载
+ */
+@Provide()
+export class ProxyApiLoadService extends BaseService implements IScheduleService {
+  @Inject()
+  protected ctx: Context;
+
+  @Inject()
+  private curdProService: CurdProService;
+
+  @Inject()
+  private workbenchService: WorkbenchService;
+
+  /**
+   * 创建路由树。刷新路由缓存
+   * @param workbench_code
+   */
+  public async buildRouteTrie(workbench_code: string): Promise<RouteTrie> {
+    const upstreamMap: Record<string, IUpstreamInfo> = await this.buildUpstreamMap();
+    const routeTrie = getRouteTrie(workbench_code);
+    routeTrie.clearRoute();
+    const entities = await this.loadProxyApiEntity(workbench_code, upstreamMap);
+    for (let i = 0; i < entities.length; i++) {
+      const entity = entities[i];
+      if (entity.status === 1) {
+        const path_prefix2 = '/' + entity.upstream_code + entity.path_prefix;
+        routeTrie.addRoute(path_prefix2, new RouteHandler(entity));
+      }
+    }
+    return routeTrie;
+  }
+
+  private async buildUpstreamMap() {
+    //       biz_tag={'upstream'}
+    const res = await this.curdProService.executeCrudByCfg(
+      {
+        condition: {
+          biz_tag: 'upstream',
+        },
+      },
+      {
+        sqlTable: SystemTables.sys_configs,
+        sqlDatabase: SystemDbName,
+        sqlDdType: SystemDbType,
+        sqlSimpleName: KeysOfSimpleSQL.SIMPLE_QUERY,
+      }
+    );
+    const upstreamMap: Record<string, IUpstreamInfo> = {};
+    const rows = res.getResRows() as any[];
+    for (let i = 0; i < rows.length; i++) {
+      const row = rows[i];
+      const { config_code, config_content } = row || {};
+      upstreamMap[config_code] = toUpstreamInfo(config_content);
+    }
+    return upstreamMap;
+  }
+
+  private async loadProxyApiEntity(workbench_code: string, upstreamMap: Record<string, IUpstreamInfo>): Promise<IProxyApiEntity[]> {
+    const res = await this.curdProService.executeCrudByCfg(
+      {
+        condition: {},
+      },
+      {
+        sqlTable: SystemTables.sys_proxyapi,
+        sqlDatabase: SystemDbName,
+        sqlDdType: SystemDbType,
+        sqlSimpleName: KeysOfSimpleSQL.SIMPLE_QUERY,
+      }
+    );
+
+    const rows0 = res.getResRows() as IProxyApiEntity[];
+
+    const rows = rows0.filter(s => {
+      if (!s.workbench_code_array) {
+        throw new CommonException('IS_SUPPORT_CURRENT_WORKBENCH', '未配置支持的站点字段');
+      }
+      return this.workbenchService.isMatchWorkbenchCode(s.workbench_code_array, workbench_code);
+    });
+
+    for (let i = 0; i < rows.length; i++) {
+      const row = rows[i];
+      const upstream_code = row.upstream_code;
+      this.addUpstreamInfo(row, upstream_code, upstreamMap);
+    }
+    return rows;
+  }
+
+  private addUpstreamInfo(row: IProxyApiEntity, upstream_code: string, upstreamMap: Record<string, IUpstreamInfo>) {
+    const upstreamInfo = upstreamMap[upstream_code];
+    if (!upstreamInfo) {
+      return;
+    }
+    row.targetIndex = 0;
+    row.targetList = upstreamInfo.targetList || [];
+    row.balanceStrategy = upstreamInfo.balanceStrategy || BALANCE_STRATEGY.ROUND_ROBIN;
+    row.weightedRandom = new WeightedRandom(upstreamInfo.targetList || []);
+    row.weightedRoundRobin = new WeightedRoundRobin(upstreamInfo.targetList || []);
+  }
+
+  public async runBySchedule(): Promise<any> {
+    const workbenchList = await this.workbenchService.getAllWorkbenchInfoList();
+    for (let i = 0; i < workbenchList.length; i++) {
+      const workbenchInfo = workbenchList[i];
+      try {
+        await this.buildRouteTrie(workbenchInfo.workbench_code);
+      } catch (e) {
+        this.logError('ProxyApiLoadService_runBySchedule', e);
+      }
+    }
+  }
+}

package/src/service/proxyapi/ProxyApiService.ts

@@ -0,0 +1,262 @@
+import { Inject, Provide } from '@midwayjs/core';
+import { Context } from '@midwayjs/koa';
+import * as _ from 'lodash';
+import { getRouteTrie } from './RouteTrie';
+import { RouteHandler } from './RouteHandler';
+import { Exceptions } from '../../libs/crud-pro/exceptions';
+import { BizException } from '../../models/devops';
+import { CurdProService } from '../curd/CurdProService';
+import { IProxyApiEntity } from '../../models/SystemEntities';
+import axios, { AxiosResponse, AxiosResponseHeaders, RawAxiosResponseHeaders, ResponseType } from 'axios';
+import { getDebugProxyTarget, isEnableDebug } from '../../libs/utils/fatcms-request';
+import { BALANCE_STRATEGY, stringHashMod } from './ProxyApiUtils';
+import { WeightedRandom } from './WeightedRandom';
+import { WeightedRoundRobin } from './WeightedRoundRobin';
+import { ProxyApiLoadService } from './ProxyApiLoadService';
+import { API_BASE_TYPE, ApiBaseService } from '../base/ApiBaseService';
+
+/**
+ * 接口代理
+ */
+@Provide()
+export class ProxyApiService extends ApiBaseService {
+  @Inject()
+  protected ctx: Context;
+
+  @Inject()
+  protected curdProService: CurdProService;
+
+  @Inject()
+  protected proxyApiLoadService: ProxyApiLoadService;
+
+  /**
+   * 转发请求
+   * @param current_workbench_code 当前站点
+   * @param upstream_path : 形如/[upstream_code]/[api_path]
+   */
+  public async handleProxyRequest(current_workbench_code: string, upstream_path: string) {
+    const routeTrie = getRouteTrie(current_workbench_code);
+    const ss = routeTrie.findLongestMatch(upstream_path);
+    if (!ss) {
+      throw new BizException('路径没有匹配到', Exceptions.CFG_NOT_FOUND);
+    }
+    const handler: RouteHandler = _.get(ss, 'handler');
+    const cfgInfo = handler?.cfgInfo;
+    if (!cfgInfo) {
+      throw new BizException('路径配置没有匹配到', Exceptions.CFG_NOT_FOUND);
+    }
+    return this._handleProxyRequestForCfg(cfgInfo, upstream_path);
+  }
+
+  /**
+   * 转发请求
+   * @param proxyApiEntity
+   * @param upstream_path
+   * @private
+   */
+  private async _handleProxyRequestForCfg(proxyApiEntity: IProxyApiEntity, upstream_path: string) {
+    await super.beforeCheckApiAccessibility(API_BASE_TYPE.PROXY_API, proxyApiEntity);
+
+    const targetList = proxyApiEntity.targetList;
+    if (!Array.isArray(targetList) || targetList.length === 0) {
+      throw new BizException('目标服务器找不到', Exceptions.CFG_NOT_FOUND);
+    }
+
+    const ctx = this.ctx;
+    const targetBaseUrl = this.selectTargetByBalanceStrategy(proxyApiEntity, ctx);
+    if (!targetBaseUrl) {
+      throw new BizException('目标服务器找不到', Exceptions.CFG_NOT_FOUND);
+    }
+
+    const newPath = this.toNewPathByMapping(upstream_path, proxyApiEntity);
+
+    const targetUrl = new URL(newPath, targetBaseUrl);
+    targetUrl.search = new URLSearchParams(ctx.query as any).toString();
+    // const response_type: ResponseType = (proxyApiEntity.response_type || 'stream') as any;
+    const response_type: ResponseType = 'stream';
+
+    try {
+      const headers = {
+        ...ctx.headers,
+        host: targetUrl.host, // 设置正确的 host
+        'content-length': ctx.get('content-length'),
+      };
+
+      this.setRequestHeaders(headers, proxyApiEntity);
+
+      const response = await axios({
+        method: ctx.method,
+        url: targetUrl.toString(),
+        data: ctx.request.body,
+        headers: headers,
+        responseType: response_type,
+        timeout: proxyApiEntity.timeout_ms || 5 * 60 * 1000,
+        validateStatus: () => true,
+      });
+      ctx.status = response.status;
+      const resHeaders = this.toResHeaders(response.headers);
+      ctx.set({
+        ...resHeaders,
+        'cache-control': 'no-store', // 禁用缓存
+        'x-proxy-server': 'fatcms-server', // 添加代理标识
+      });
+      if (response_type === 'stream') {
+        return this.handleResponseTypeStream(response, ctx);
+      } else if (response_type === 'json') {
+        return this.handleResponseTypeJSON(response, ctx);
+      } else {
+        this.ctx.body = response.data;
+        return null;
+      }
+    } catch (e) {
+      ctx.contextLogger.error('Proxy request failed ', e);
+      ctx.status = 500;
+      ctx.body = { success: false, message: 'Proxy request failed => ' + e.message };
+    }
+    return null;
+  }
+
+  /**
+   * 处理JSON响应
+   * @param response
+   * @param ctx
+   * @private
+   */
+  private handleResponseTypeJSON(response: AxiosResponse, ctx: Context) {
+    return Promise.resolve(response.data);
+  }
+
+  /**
+   * 处理stream响应
+   * @param response
+   * @param ctx
+   * @private
+   */
+  private handleResponseTypeStream(response: AxiosResponse, ctx: Context) {
+    // 流式传输处理
+    return new Promise<void>((resolve, reject) => {
+      response.data
+        .on('data', chunk => {
+          ctx.res.write(chunk);
+        })
+        .on('end', () => {
+          ctx.res.end();
+          resolve();
+        })
+        .on('error', err => {
+          ctx.res.destroy(err);
+          reject(err);
+        });
+
+      // 处理客户端断开连接
+      ctx.req.on('close', () => {
+        response.data.destroy();
+        reject(new Error('Client disconnected'));
+      });
+    });
+  }
+
+  /**
+   * 获取响应的headers数据
+   * @param headers
+   * @private
+   */
+  private toResHeaders(headers: RawAxiosResponseHeaders | AxiosResponseHeaders): any {
+    if (!headers) {
+      return {};
+    }
+    if (typeof headers.toJSON === 'function') {
+      return headers.toJSON(false);
+    }
+    return headers;
+  }
+
+  /**
+   * 获取转发到目的服务器的路径
+   * @param upstream_path
+   * @param proxyApiEntity
+   * @private
+   */
+  private toNewPathByMapping(upstream_path: string, proxyApiEntity: IProxyApiEntity) {
+    const result = upstream_path.substring(proxyApiEntity.upstream_code.length + 1);
+    this.logInfo('toNewPathByMapping', { upstream_path, result });
+    return result;
+  }
+
+  /**
+   * 根据负载均衡策略，选择目标服务器
+   * @param proxyApiEntity
+   * @param ctx
+   * @private
+   */
+  private selectTargetByBalanceStrategy(proxyApiEntity: IProxyApiEntity, ctx: Context): string {
+    const targetList = proxyApiEntity.targetList;
+    const strategy = proxyApiEntity.balanceStrategy;
+
+    if (isEnableDebug(ctx)) {
+      // 在调试模式下，可以强制指定某台机器。
+      const targetOrigin: string = getDebugProxyTarget(ctx);
+      if (typeof targetOrigin === 'string' && targetOrigin.startsWith('http')) {
+        return targetOrigin;
+      }
+    }
+
+    if (!targetList || targetList.length === 0) {
+      return null;
+    }
+
+    if (strategy === BALANCE_STRATEGY.WEIGHTED_RANDOM) {
+      const weightedRandom: WeightedRandom = proxyApiEntity.weightedRandom;
+      const ss = weightedRandom.next();
+      return ss ? ss.origin : null;
+    } else if (strategy === BALANCE_STRATEGY.WEIGHTED_ROUND_ROBIN) {
+      const weightedRoundRobin: WeightedRoundRobin = proxyApiEntity.weightedRoundRobin;
+      const ss = weightedRoundRobin.next();
+      return ss ? ss.origin : null;
+    } else if (strategy === BALANCE_STRATEGY.USER_SESSION_HASH) {
+      // 根据用户sessionId hash
+      const sessionId = ctx.userSession.getSessionValue('sessionId') || Math.random();
+      proxyApiEntity.targetIndex = stringHashMod(`${sessionId}`, targetList.length);
+    } else if (strategy === BALANCE_STRATEGY.USER_ACCOUNT_HASH) {
+      // 根据用户accountId hash
+      const accountId = ctx.userSession.getSessionValue('accountId') || Math.random();
+      proxyApiEntity.targetIndex = stringHashMod(`${accountId}`, targetList.length);
+    } else if (strategy === BALANCE_STRATEGY.IP_HASH) {
+      // 根据用户IP地址hash
+      const ip = ctx.headers['x-real-ip'] || ctx.headers['x-forwarded-for'] || Math.random();
+      proxyApiEntity.targetIndex = stringHashMod(`${ip}`, targetList.length);
+    } else if (strategy === BALANCE_STRATEGY.RANDOM) {
+      // 随机
+      proxyApiEntity.targetIndex = stringHashMod(`${Math.random()}`, targetList.length);
+    } else {
+      // 默认，轮训
+      proxyApiEntity.targetIndex = (proxyApiEntity.targetIndex + 1) % targetList.length;
+    }
+
+    const ss = targetList[proxyApiEntity.targetIndex];
+    return ss ? ss.origin : null;
+  }
+
+  private setRequestHeaders(headers: any, proxyApiEntity: IProxyApiEntity) {
+    const ctx = this.ctx;
+
+    if (proxyApiEntity.pass_real_ip) {
+      headers['x-real-ip'] = ctx.headers['x-real-ip'];
+      headers['x-forwarded-for'] = ctx.headers['x-forwarded-for'];
+    }
+
+    if (proxyApiEntity.user_context) {
+      const isLogin = this.ctx.userSession.isLogin();
+      const sessionInfo = this.ctx.userSession.getSessionInfo();
+      headers['x-fatcms-user'] = JSON.stringify({ ...sessionInfo, isLogin });
+    }
+
+    if (proxyApiEntity.change_host && proxyApiEntity.change_host.length > 0) {
+      headers.host = proxyApiEntity.change_host;
+    }
+
+    if (proxyApiEntity.change_origin && proxyApiEntity.change_origin.length > 0) {
+      headers.origin = proxyApiEntity.change_origin;
+    }
+  }
+}

package/src/service/proxyapi/ProxyApiUtils.ts

@@ -0,0 +1,32 @@
+import * as md5 from 'md5';
+
+export const BALANCE_STRATEGY = {
+  ROUND_ROBIN: 'ROUND_ROBIN', //轮询
+  WEIGHTED_ROUND_ROBIN: 'WEIGHTED_ROUND_ROBIN', //加权轮询
+  RANDOM: 'RANDOM', // 随机
+  WEIGHTED_RANDOM: 'WEIGHTED_RANDOM', // 加权随机
+  IP_HASH: 'IP_HASH', // 源地址哈希
+  USER_ACCOUNT_HASH: 'USER_ACCOUNT_HASH', //  根据用户accountId hash
+  USER_SESSION_HASH: 'USER_SESSION_HASH', //  根据用户sessionId hash
+};
+
+/**
+ * 字符串Hash之后取模
+ * @param str
+ * @param mod
+ */
+export function stringHashMod(str: string, mod: number): number {
+  if (!str) {
+    throw new Error('stringHashMod str is required');
+  }
+  const md5Str = md5(str);
+  let sum = 0;
+  for (let i = 0; i < md5Str.length; i++) {
+    const md5StrElement = md5Str.charCodeAt(i);
+    sum = sum + md5StrElement;
+    if (sum >= Number.MAX_SAFE_INTEGER - 10000000) {
+      sum = sum % 10000000;
+    }
+  }
+  return sum % mod;
+}

package/src/service/proxyapi/RouteHandler.ts

@@ -0,0 +1,8 @@
+class RouteHandler {
+  cfgInfo: any;
+  constructor(cfgInfo: any) {
+    this.cfgInfo = cfgInfo;
+  }
+}
+
+export { RouteHandler };

package/src/service/proxyapi/RouteTrie.ts

@@ -0,0 +1,74 @@
+import { RouteHandler } from './RouteHandler';
+
+class RouteTrieNode {
+  children: Record<string, RouteTrieNode>;
+  path: string;
+  handler: RouteHandler;
+  constructor() {
+    this.children = {}; // 子节点字典
+    this.handler = null; // 路由处理程序
+    this.path = ''; // 完整路由路径
+  }
+}
+
+class RouteTrie {
+  private root: RouteTrieNode;
+  private routeCount: 0;
+  constructor() {
+    this.root = new RouteTrieNode();
+    this.routeCount = 0;
+  }
+
+  addRoute(path: string, handler: RouteHandler) {
+    const parts = path.split('/').slice(1); // 去除首部空字符串
+    let node = this.root;
+    parts.forEach(part => {
+      if (!node.children[part]) {
+        node.children[part] = new RouteTrieNode();
+      }
+      node = node.children[part];
+    });
+    // 保存路由处理函数到终点节点
+    node.handler = handler;
+    node.path = path;
+    this.routeCount++;
+  }
+
+  clearRoute() {
+    this.routeCount = 0;
+    this.root = new RouteTrieNode();
+  }
+
+  getRouteCount() {
+    return this.routeCount;
+  }
+
+  findLongestMatch(path: string) {
+    const parts = path.split('/').slice(1);
+    let node = this.root;
+    let bestMatch = null;
+
+    parts.forEach(part => {
+      if (node.children[part]) {
+        node = node.children[part];
+        if (node.handler) {
+          bestMatch = {
+            path: node.path,
+            handler: node.handler,
+          };
+        }
+      }
+    });
+    return bestMatch;
+  }
+}
+
+const ROUTE_TRIE_MANAGER: Record<string, RouteTrie> = {};
+
+function getRouteTrie(domainCode: string): RouteTrie {
+  if (!ROUTE_TRIE_MANAGER[domainCode]) {
+    ROUTE_TRIE_MANAGER[domainCode] = new RouteTrie();
+  }
+  return ROUTE_TRIE_MANAGER[domainCode];
+}
+export { RouteTrie, getRouteTrie };

package/src/service/proxyapi/WeightedRandom.ts

@@ -0,0 +1,37 @@
+import { IUpstreamItem } from '../../models/SystemEntities';
+
+class WeightedRandom {
+  private readonly items: IUpstreamItem[];
+  private readonly totalWeight: number;
+  private readonly cumulativeWeights: number[];
+  constructor(items: IUpstreamItem[]) {
+    // 预处理累积权重
+    this.cumulativeWeights = [];
+    let runningTotal = 0;
+
+    for (const item of items) {
+      if (typeof item.weight !== 'number' || item.weight <= 0) {
+        throw new Error('All items must have positive weight');
+      }
+      runningTotal += item.weight;
+      this.cumulativeWeights.push(runningTotal);
+    }
+
+    this.totalWeight = runningTotal;
+    this.items = items;
+  }
+
+  next(): IUpstreamItem {
+    // 生成[0, totalWeight)区间的随机数
+    const random = Math.random() * this.totalWeight;
+    // 线性查找命中区间
+    for (let i = 0; i < this.cumulativeWeights.length; i++) {
+      if (random < this.cumulativeWeights[i]) {
+        return this.items[i];
+      }
+    }
+    return this.items[0]; // 兜底返回第一个元素
+  }
+}
+
+export { WeightedRandom };

package/src/service/proxyapi/WeightedRoundRobin.ts

@@ -0,0 +1,44 @@
+import { IUpstreamItem } from '../../models/SystemEntities';
+
+interface IUpstreamItemExt extends IUpstreamItem {
+  currentWeight: number;
+  effectiveWeight: number;
+}
+
+class WeightedRoundRobin {
+  private readonly totalWeight: number;
+  private servers: IUpstreamItemExt[];
+  constructor(servers: IUpstreamItem[]) {
+    this.servers = servers.map(server => ({
+      ...server,
+      currentWeight: 0, // 动态权重初始化为0
+      effectiveWeight: server.weight, // 有效权重（可动态调整）
+    }));
+    this.totalWeight = servers.reduce((sum, server) => sum + server.weight, 0);
+  }
+
+  next(): IUpstreamItem {
+    let selected: IUpstreamItemExt = null;
+    let maxCurrentWeight = 0;
+
+    // 1. 遍历所有服务器，动态增加权重并选择当前权重最大的
+    this.servers.forEach(server => {
+      server.currentWeight += server.effectiveWeight; // 增加动态权重
+
+      if (server.currentWeight > maxCurrentWeight) {
+        maxCurrentWeight = server.currentWeight;
+        selected = server;
+      }
+    });
+
+    // 2. 被选中的服务器减去总权重（实现平滑分配）
+    if (selected) {
+      selected.currentWeight -= this.totalWeight;
+      return selected;
+    }
+
+    return this.servers[0]; // 兜底返回第一个元素
+  }
+}
+
+export { WeightedRoundRobin };