create-ssnode-cli 0.2.4 → 0.2.5

package/templates/backend/src/database/slow-sql.logger.ts

@@ -0,0 +1,239 @@
+import { performance } from 'perf_hooks';
+import { Logger } from 'typeorm';
+import { HostMetricsSampler, HostSnapshot } from './host-metrics.sampler';
+import { JsonlDailyWriter, localTs } from '../common/jsonl-daily-writer';
+import { sqlFingerprint, hostTension } from './sql-fingerprint';
+
+/**
+ * 慢SQL探针 Logger(《慢SQL探针与SQL优化路径方案》§3~§5)
+ *
+ * 挂载在 TypeORM Logger 层(非 better-sqlite3 专属驱动层)——better-sqlite3 与 mysql
+ * 两驱动的 QueryRunner 都内置同一钩子:maxQueryExecutionTime 超时 → logQuerySlow。
+ * 切 MySQL 时本文件零改动,只在 MySQL options 侧严禁顺手开 enableQueryTimeout
+ * (那会把 maxQueryExecutionTime 变成 server 端杀查询的超时)。
+ *
+ * 记录落独立 JSONL(data/slowsql/slowsql-YYYY-MM-DD.jsonl),非应用日志——是调试
+ * 期间的证据链,供 slowsql-report.mjs 后期分析。三类记录:
+ *   sql    慢查询样本(内联最近宿主快照 host 字段;《后端日志体系整改方案》§4.2 起
+ *          同指纹每日仅前 5 条落全量,其余只进 stats 计数——影响大的就是那几条,
+ *          原始证据过量无意义)
+ *   error  查询报错(先 console.error 保留现 yaml logging:[error] 行为,再落盘)
+ *   lag    事件循环漂移(500ms 心跳测漂移,内联宿主快照)
+ *   host   宿主基线(默认 60s 一条,提供「慢时刻 vs 平时」对比基准)
+ *   stats  每小时汇总(计数含 boot 迁移期,口径为 boot 基线不剔除;含指纹 top 段——
+ *          计数全量、样本限量,报告侧 top 表以计数为准)
+ *
+ * 写盘纪律:长驻 append 流(每日一开,换日轮转;2026-09-24 改造,防故障风暴时
+ * 同步写给事件循环加码);每查询开销仅内存计数。
+ * 日志器自身永不抛错(在查询路径上,抛错即拖死业务)。
+ */
+
+export interface SlowSqlLoggerOptions {
+  /** 落盘目录(通常 <数据库 dir>/slowsql,随 data volume 挂载) */
+  dir: string;
+  /** 慢查询阈值 ms(同时作为 TypeORM maxQueryExecutionTime) */
+  thresholdMs: number;
+  /** 事件循环漂移记录阈值 ms */
+  lagMs: number;
+  /** 宿主基线落盘周期 s */
+  hostSampleSec: number;
+  /** 保留天数 */
+  retainDays: number;
+  /** 单文件上限 MB(超限滚动瘦身,留新去旧) */
+  maxFileMb: number;
+  /** 查询报错是否同时 console.error(对齐原 yaml database.logging 的 [error] 语义) */
+  errorConsole: boolean;
+}
+
+/** params 截断:每参数 200 字符、最多 10 个(防大批量参数撑爆行/敏感原文外泄) */
+function truncateParams(params?: any[]): any[] | undefined {
+  if (!Array.isArray(params) || params.length === 0) return undefined;
+  return params.slice(0, 10).map((p) => {
+    if (p === null || p === undefined) return p;
+    const text =
+      typeof p === 'string' ? p : (() => { try { return JSON.stringify(p); } catch { return String(p); } })();
+    return text.length > 200 ? `${text.slice(0, 200)}…` : text;
+  });
+}
+
+export class SlowSqlLogger implements Logger {
+  /** 同指纹每日采样上限:细节样本够定位即可,之后仅计数(§4.2) */
+  private static readonly SAMPLE_PER_FP_PER_DAY = 5;
+
+  private readonly opts: SlowSqlLoggerOptions;
+  private readonly host: HostMetricsSampler;
+  private readonly writer: JsonlDailyWriter;
+
+  // 每小时计数(queries 含 boot 迁移期,口径为 boot 基线)
+  private queries = 0;
+  private slowCount = 0;
+  private slowTotalMs = 0;
+  private maxMs = 0;
+
+  // 指纹聚合(自上次 stats flush 清零;计数全量,与采样与否无关)
+  private fpCounters = new Map<string, { count: number; totalMs: number; maxMs: number; tension: number }>();
+  // 指纹当日已采样数(换日重置配额)
+  private fpSampled = new Map<string, number>();
+  private fpDay = '';
+
+  private lagTimer: NodeJS.Timeout | null = null;
+  private lagLast = performance.now();
+  private lagReady = false;
+  private statsTimer: NodeJS.Timeout | null = null;
+  private hostTimer: NodeJS.Timeout | null = null;
+
+  constructor(opts: SlowSqlLoggerOptions) {
+    this.opts = opts;
+    // 磁盘水位观测路径 = 慢SQL落盘目录(随 data volume 持久化,即容量风险所在卷)
+    this.host = new HostMetricsSampler(6, this.opts.dir).start(10_000);
+    // 写盘四件套(按日切/继承长度/超限滚动瘦身/每日清理)已抽 JsonlDailyWriter 与 apilog 共用
+    this.writer = new JsonlDailyWriter({
+      dir: this.opts.dir,
+      prefix: 'slowsql',
+      retainDays: this.opts.retainDays,
+      maxFileMb: this.opts.maxFileMb
+    });
+    this.startTimers();
+    this.writer.cleanupExpired();
+  }
+
+  /** TypeORM DataSource 关闭时停掉采样/心跳(测试与脚本场景防悬挂) */
+  stop(): void {
+    this.host.stop();
+    for (const t of [this.lagTimer, this.statsTimer, this.hostTimer]) {
+      if (t) clearInterval(t);
+    }
+    this.lagTimer = this.statsTimer = this.hostTimer = null;
+    this.writer.stop();
+  }
+
+  // ── TypeORM Logger 接口 ──────────────────────────────────────────────
+
+  /** 每条查询都被无条件调用:仅内存计数,零 I/O */
+  logQuery(_query: string, _parameters?: any[], _queryRunner?: any): void {
+    this.queries += 1;
+  }
+
+  /** 慢查询(超 maxQueryExecutionTime):全量计数 + 每指纹每日前 5 条落细节样本 */
+  logQuerySlow(time: number, query: string, parameters?: any[], _queryRunner?: any): void {
+    this.slowCount += 1;
+    this.slowTotalMs += time;
+    if (time > this.maxMs) this.maxMs = time;
+
+    // 指纹计数器:每条慢查询都计(样本限量只影响落盘,不影响计数口径)
+    const fp = sqlFingerprint(query);
+    let c = this.fpCounters.get(fp);
+    if (!c) {
+      c = { count: 0, totalMs: 0, maxMs: 0, tension: 0 };
+      this.fpCounters.set(fp, c);
+    }
+    c.count += 1;
+    c.totalMs += time;
+    if (time > c.maxMs) c.maxMs = time;
+    if (hostTension(this.host.latest())) c.tension += 1;
+
+    // 换日重置采样配额
+    const today = localTs().slice(0, 10);
+    if (today !== this.fpDay) {
+      this.fpDay = today;
+      this.fpSampled.clear();
+    }
+    const sampled = this.fpSampled.get(fp) ?? 0;
+    if (sampled < SlowSqlLogger.SAMPLE_PER_FP_PER_DAY) {
+      this.fpSampled.set(fp, sampled + 1);
+      this.write({
+        type: 'sql',
+        ts: localTs(),
+        fp,
+        ms: time,
+        sql: query,
+        params: truncateParams(parameters),
+        host: this.host.latest()
+      });
+    }
+  }
+
+  /** 查询报错:先 console(保留原 logging:[error] 行为)再落盘 */
+  logQueryError(error: string | Error, query: string, parameters?: any[], _queryRunner?: any): void {
+    const message = error instanceof Error ? error.message : String(error);
+    if (this.opts.errorConsole) {
+      console.error(`[typeorm] query failed: ${query.slice(0, 200)} -- ${message}`);
+    }
+    this.write({ type: 'error', ts: localTs(), sql: query, params: truncateParams(parameters), message });
+  }
+
+  // 与现 yaml「只记 error」语义一致,以下静默
+  logSchemaBuild(_message: string): void {}
+  logMigration(_message: string): void {}
+  log(_level: 'info' | 'log' | 'warn', _message: any): void {}
+
+  // ── 定时器 ──────────────────────────────────────────────────────────
+
+  private startTimers(): void {
+    // 事件循环 lag 心跳:500ms 预期间隔,漂移超 lagMs 落一条 lag 记录(内联宿主快照)。
+    // 环形采样器首轮数据未就绪,首拍只校准不判定。
+    this.lagTimer = setInterval(() => {
+      const now = performance.now();
+      const drift = now - this.lagLast - 500;
+      this.lagLast = now;
+      if (!this.lagReady) {
+        this.lagReady = true;
+        return;
+      }
+      if (drift > this.opts.lagMs) {
+        this.write({ type: 'lag', ts: localTs(), ms: Math.round(drift), host: this.host.latest() });
+      }
+    }, 500);
+    this.lagTimer.unref();
+
+    // 每小时汇总
+    this.statsTimer = setInterval(() => this.flushStats(), 3_600_000);
+    this.statsTimer.unref();
+
+    // 宿主基线(慢时刻 vs 平时的对比基准)
+    this.hostTimer = setInterval(
+      () => this.write({ type: 'host', ts: localTs(), ...this.host.latest() }),
+      Math.max(10, this.opts.hostSampleSec) * 1000
+    );
+    this.hostTimer.unref();
+
+    // 保留清理:每日一次(unref,不阻退出)
+    this.writer.startDailyCleanup();
+  }
+
+  /** 落一条 stats 并清零(暴露公开以便验收/测试手动触发) */
+  flushStats(): void {
+    if (this.queries === 0 && this.slowCount === 0 && this.writer.droppedCount === 0) return;
+    const now = new Date();
+    const windowStart = new Date(now.getTime() - 3_600_000);
+    // 指纹 top 段(计数全量;样本在 sql 记录,报告侧 join 后计数为准、样本供下钻)
+    const top = [...this.fpCounters.entries()]
+      .map(([fp, c]) => ({ fp, ...c }))
+      .sort((a, b) => b.totalMs - a.totalMs)
+      .slice(0, 20);
+    this.write({
+      type: 'stats',
+      window: localTs(windowStart),
+      queries: this.queries,
+      slowCount: this.slowCount,
+      slowTotalMs: this.slowTotalMs,
+      maxMs: this.maxMs,
+      dropped: this.writer.droppedCount,
+      top
+    });
+    this.queries = 0;
+    this.slowCount = 0;
+    this.slowTotalMs = 0;
+    this.maxMs = 0;
+    this.writer.droppedCount = 0;
+    // 计数器随窗口清零;采样配额(fpSampled)按日控制,不清
+    this.fpCounters.clear();
+  }
+
+  // ── JSONL 写盘 ─────────────────────────────────────────────────────
+
+  /** 异步入队(长驻流;按日切文件,超上限/内存背压丢条计数) */
+  private write(record: Record<string, unknown>): void {
+    this.writer.write(record);
+  }
+}

package/templates/backend/src/database/sql-fingerprint.ts

@@ -0,0 +1,32 @@
+/**
+ * SQL 指纹(《后端日志体系整改方案》§4.2,自 scripts/slowsql-report.mjs 收敛为唯一实现)
+ *
+ * TypeORM 传参走占位符,文本已参数化,只需折叠空白;再兜底洗字面量。
+ * 截断 300 字符:超长 IN 列表折叠后仍有数千 '?',既撑爆 stats 记录也无区分度;
+ * 探针侧(stats.top 汇总键)与样本记录(fp 字段)共用同一切断口径,报告脚本才能 join。
+ * (报告脚本保留一份旧实现兜底 legacy 无 fp 字段的记录,语义一致。)
+ */
+export function sqlFingerprint(sql: string): string {
+  return sql
+    .replace(/\s+/g, ' ')
+    .replace(/'[^']*'/g, "'?'")
+    .replace(/\b\d+(\.\d+)?\b/g, '?')
+    .trim()
+    .slice(0, 300);
+}
+
+/**
+ * 宿主紧张判定(与报告脚本 §5.2 同口径):
+ * iowait/steal 超阈 或 PSI io/cpu 超阈 → 该慢时刻「可能是宿主资源问题而非 SQL 问题」。
+ */
+export function hostTension(h: {
+  iowaitPct?: number;
+  stealPct?: number;
+  psiIo?: number;
+  psiCpu?: number;
+} | null): boolean {
+  if (!h) return false;
+  if ((h.iowaitPct ?? 0) > 5 || (h.stealPct ?? 0) > 5) return true;
+  if ((h.psiIo ?? 0) > 10 || (h.psiCpu ?? 0) > 20) return true;
+  return false;
+}

package/templates/backend/src/database/sqlite.config.ts

@@ -2,6 +2,7 @@ import { ConfigService } from '@nestjs/config';
 import type { AppConfig } from '../config/config.interface';
 import * as path from 'path';
 import { Init1740000000000 } from './migrations/0001-init.migration';
+import { SlowSqlLogger } from './slow-sql.logger';
 
 /**
  * 应用 PRAGMA：在驱动 createDatabaseConnection 后、连接被业务用之前调用。
@@ -54,10 +55,30 @@ export const buildTypeOrmOptions = (
   // 文件名按环境定死(不外配):prod=template.prod.sqlite / dev=template.dev.sqlite,
   // 与运维脚本 ops/sqlite/config.sh、ops 探针约定一致。仅目录(dir/DB_DIR)可外配。
   const dbFilename = isProd ? 'template.prod.sqlite' : 'template.dev.sqlite';
+  // 慢SQL探针(《慢SQL探针与SQL优化路径方案》):挂 TypeORM Logger 层,驱动无关——
+  // better-sqlite3 与 mysql 两驱动的 QueryRunner 都内置同一钩子(maxQueryExecutionTime
+  // 超时 → logQuerySlow),将来切 MySQL 本文件探针段零改动。
+  const slowSql = config.get<AppConfig['slowSql']>('slowSql');
+  const errorConsole =
+    db.logging === true || (Array.isArray(db.logging) && db.logging.includes('error'));
+  const slowSqlLogger = new SlowSqlLogger({
+    dir: path.resolve(process.cwd(), slowSql.dir),
+    thresholdMs: slowSql.thresholdMs,
+    lagMs: slowSql.lagMs,
+    hostSampleSec: slowSql.hostSampleSec,
+    retainDays: slowSql.retainDays,
+    maxFileMb: slowSql.maxFileMb,
+    errorConsole
+  });
   return {
     type: 'better-sqlite3' as const,
     database: path.join(db.dir, dbFilename),
     synchronize: isProd ? false : db.synchronize,
+    // 慢SQL探针:logger 实例替换内建 logger(logging 选项随之失效;
+    // errorConsole 已按原 database.logging 语义在 SlowSqlLogger 内保留)。
+    // maxQueryExecutionTime 是驱动无关顶层 option,超时触发 logQuerySlow。
+    logger: slowSqlLogger,
+    maxQueryExecutionTime: slowSql.thresholdMs,
     logging: db.logging as any,
     autoLoadEntities: true,
     entities: [path.join(__dirname, '..', 'entities', '*.{ts,js}')],

package/templates/backend/src/main.ts

@@ -3,6 +3,8 @@ import { ClassSerializerInterceptor, Logger, ValidationPipe } from '@nestjs/comm
 import { ConfigService } from '@nestjs/config';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import { json, urlencoded } from 'express';
+import { randomUUID } from 'crypto';
+import { performance } from 'perf_hooks';
 import * as helmet from 'helmet';
 import { NestExpressApplication } from '@nestjs/platform-express';
 import { join, isAbsolute } from 'path';
@@ -40,6 +42,17 @@ async function bootstrap() {
   if (logLevel === 'debug') loggerLevels.add('debug');
   app.useLogger(new Logger());
 
+  // traceId 中间件(《后端日志体系整改方案》§2.4):挂 req + 回传 X-Trace-Id 响应头,
+  // 联调时用户报一个 traceId、grep 一次定位全链路。必须在 guard 之前(普通拦截器
+  // 看不到 guard 阶段抛的 401/429,而那正是要登记的失败);
+  // traceStart 同时作慢请求计时起点(含 guard/pipe 耗时,口径比拦截器更完整)。
+  app.use((req: any, res: any, next: any) => {
+    req.traceId = randomUUID().slice(0, 8);
+    req.traceStart = performance.now();
+    res.setHeader('X-Trace-Id', req.traceId);
+    next();
+  });
+
   // 全局前缀 /api(注意:静态资源目录 useStaticAssets 在前缀设置之前挂载,不受 /api 前缀影响)
   app.setGlobalPrefix(cfg.get<AppConfig['app']>('app').globalPrefix, {
     // 静态资源接口排除前缀
@@ -66,8 +79,17 @@ async function bootstrap() {
   );
   app.useGlobalInterceptors(new ClassSerializerInterceptor(app.get(Reflector)));
 
-  // 请求体大小上限 2MB(文件上传走 multipart 不受此限)
-  app.use(json({ limit: '2mb' }));
+  // 请求体大小上限 2MB(文件上传走 multipart 不受此限)。
+  // verify 钩子捕获原文 buffer:apilog 登记失败请求时用 rawBody 保真解析参数
+  // (body 已被管道消费/校验器改写时,原文才是真实入参)。
+  app.use(
+    json({
+      limit: '2mb',
+      verify: (req: any, _res, buf) => {
+        if (Buffer.isBuffer(buf)) req.rawBody = buf;
+      }
+    })
+  );
   app.use(urlencoded({ extended: true, limit: '2mb' }));
 
   // 安全响应头(X-Frame-Options/X-Content-Type-Options/HSTS 等)。