befly 3.61.0 → 3.63.0

package/apis/tongJi/dailyStats.js

@@ -1,136 +1,120 @@
-import { getDailyDateRangeList, getDailyDateYmdNumber, getDailyMonthStartDate, getDailyRecentDateList, getDailyStatsDayMembersKey, getDailyStatsDayProductMembersKey, getDailyWeekStartDate } from "./_daily.js";
+import { getTongJiDateYmdNumber, getTongJiMonthStartDate, getTongJiRecentDateList } from "./_tongJi.js";
 
 const DAILY_STATS_DAY_LIMIT = 30;
+const DAILY_STATS_HISTORY_CACHE_TTL_SECONDS = 2 * 24 * 60 * 60;
 
-async function getRedisSetCount(befly, key) {
-    const count = await befly.redis.scard(key);
+function getPreviousMonthDateRange(reportDate) {
+    const year = Math.floor(reportDate / 10000);
+    const month = Math.floor((reportDate % 10000) / 100);
+    const previousYear = month === 1 ? year - 1 : year;
+    const previousMonth = month === 1 ? 12 : month - 1;
+    const lastDay = new Date(Date.UTC(year, month - 1, 0)).getUTCDate();
 
-    return Number.isFinite(Number(count)) ? Number(count) : 0;
+    return [previousYear * 10000 + previousMonth * 100 + 1, previousYear * 10000 + previousMonth * 100 + lastDay];
 }
 
-function getDayMembersKey(reportDate, productCode) {
-    return productCode ? getDailyStatsDayProductMembersKey(reportDate, productCode) : getDailyStatsDayMembersKey(reportDate);
-}
-
-async function getDayCount(befly, reportDate, productCode) {
-    return {
-        reportDate: reportDate,
-        count: await getRedisSetCount(befly, getDayMembersKey(reportDate, productCode))
-    };
-}
+async function loadHistory(befly, reportDate, dateStats) {
+    const cacheKey = `daily:stats:history:${reportDate}`;
 
-async function sumDayCounts(befly, dateList, productCode) {
-    const counts = await Promise.all(dateList.map((date) => getRedisSetCount(befly, getDayMembersKey(date, productCode))));
+    if (befly.redis) {
+        const cached = await befly.redis.getObject(cacheKey);
 
-    return counts.reduce((sum, count) => sum + count, 0);
-}
+        if (cached) {
+            return cached;
+        }
+    }
 
-async function getDayCounts(befly, dateList, productCode) {
-    return Promise.all(dateList.map((date) => getDayCount(befly, date, productCode)));
-}
+    if (!befly.mysql) {
+        return {};
+    }
 
-function buildDateRangeSummary(startDate, endDate, count) {
-    return {
-        startDate: startDate,
-        endDate: endDate,
-        count: count
-    };
-}
+    const res = await befly.mysql.execute("SELECT product_code as productCode, report_date as reportDate, COUNT(*) as count FROM befly_daily_report WHERE state > 0 AND report_date BETWEEN ? AND ? GROUP BY product_code, report_date", [dateStats.previousMonthStartDate, dateStats.yesterdayDate]);
+    const history = {};
+
+    for (const item of res.data || []) {
+        const code = String(item.productCode || "");
+        const date = Number(item.reportDate);
+        const count = Number(item.count) || 0;
+        const stats = history[code] || {
+            yesterday: 0,
+            dayBeforeYesterday: 0,
+            month: 0,
+            lastMonth: 0
+        };
 
-function buildDaySummary(reportDate, count) {
-    return {
-        reportDate: reportDate,
-        count: count
-    };
-}
+        if (date === dateStats.yesterdayDate) {
+            stats.yesterday = count;
+        }
+        if (date === dateStats.dayBeforeYesterdayDate) {
+            stats.dayBeforeYesterday = count;
+        }
+        if (date >= dateStats.monthStartDate && date <= dateStats.yesterdayDate) {
+            stats.month += count;
+        }
+        if (date >= dateStats.previousMonthStartDate && date <= dateStats.previousMonthEndDate) {
+            stats.lastMonth += count;
+        }
 
-async function getDailyStatsSummary(befly, params) {
-    const { reportDate, monthStartDate, weekStartDate, recentDateList, recent7DateList, productCode } = params;
-    const timeZone = befly.config?.tz;
-    const weekDateList = getDailyDateRangeList(weekStartDate, reportDate, timeZone);
-    const monthDateList = recentDateList.filter((date) => date >= monthStartDate);
-
-    const [today, yesterday, dayBeforeYesterday, week, month, recent7, recent30, days] = await Promise.all([
-        getDayCount(befly, reportDate, productCode),
-        getDayCount(befly, recentDateList[recentDateList.length - 2] || reportDate, productCode),
-        getDayCount(befly, recentDateList[recentDateList.length - 3] || recentDateList[recentDateList.length - 2] || reportDate, productCode),
-        sumDayCounts(befly, weekDateList, productCode).then((count) => buildDateRangeSummary(weekStartDate, reportDate, count)),
-        sumDayCounts(befly, monthDateList, productCode).then((count) => buildDateRangeSummary(monthStartDate, reportDate, count)),
-        sumDayCounts(befly, recent7DateList, productCode).then((count) => buildDateRangeSummary(recent7DateList[0], reportDate, count)),
-        sumDayCounts(befly, recentDateList, productCode).then((count) => buildDateRangeSummary(recentDateList[0], reportDate, count)),
-        getDayCounts(befly, recentDateList, productCode)
-    ]);
+        history[code] = stats;
+    }
 
-    return {
-        today: today,
-        yesterday: yesterday,
-        dayBeforeYesterday: dayBeforeYesterday,
-        week: week,
-        month: month,
-        recent7: recent7,
-        recent30: recent30,
-        days: days
-    };
-}
+    if (befly.redis) {
+        await befly.redis.setObject(cacheKey, history, DAILY_STATS_HISTORY_CACHE_TTL_SECONDS);
+    }
 
-function buildEmptyDays(recentDateList) {
-    return recentDateList.map((reportDate) => {
-        return {
-            reportDate: reportDate,
-            count: 0
-        };
-    });
+    return history;
 }
 
-function buildEmptySummary(reportDate, monthStartDate, weekStartDate, recentDateList, recent7DateList) {
-    const days = buildEmptyDays(recentDateList);
-    const yesterdayDate = recentDateList[recentDateList.length - 2] || reportDate;
-    const dayBeforeYesterdayDate = recentDateList[recentDateList.length - 3] || yesterdayDate;
+async function getTodayCount(befly, reportDate, productCode) {
+    if (!befly.redis) {
+        return 0;
+    }
 
-    return {
-        today: buildDaySummary(reportDate, 0),
-        yesterday: buildDaySummary(yesterdayDate, 0),
-        dayBeforeYesterday: buildDaySummary(dayBeforeYesterdayDate, 0),
-        week: buildDateRangeSummary(weekStartDate, reportDate, 0),
-        month: buildDateRangeSummary(monthStartDate, reportDate, 0),
-        recent7: buildDateRangeSummary(recent7DateList[0], reportDate, 0),
-        recent30: buildDateRangeSummary(recentDateList[0], reportDate, 0),
-        days: days
-    };
+    return Number(await befly.redis.scard(`daily:day:${reportDate}:members:${productCode}`)) || 0;
 }
 
-function buildProductSummary(item, summary) {
+function buildProductSummary(item, stats, todayCount) {
     return {
         key: item.code,
         productName: item.name,
         productCode: item.code,
-        today: summary.today.count,
-        yesterday: summary.yesterday.count,
-        dayBeforeYesterday: summary.dayBeforeYesterday.count,
-        week: summary.week.count,
-        month: summary.month.count,
-        recent7: summary.recent7.count,
-        recent30: summary.recent30.count,
-        days: summary.days
+        today: todayCount,
+        yesterday: stats.yesterday,
+        dayBeforeYesterday: stats.dayBeforeYesterday,
+        month: stats.month,
+        lastMonth: stats.lastMonth
     };
 }
 
-async function buildDailyStatsProducts(befly, params, projectLists) {
-    const summaries = await Promise.all(
-        projectLists.map((item) => {
-            return getDailyStatsSummary(befly, { ...params, productCode: item.code }).then((summary) => buildProductSummary(item, summary));
+async function buildDailyStatsProducts(befly, projectLists, history, dateStats) {
+    const products = await Promise.all(
+        projectLists.map(async (item) => {
+            const stats = history[item.code] || { yesterday: 0, dayBeforeYesterday: 0, month: 0, lastMonth: 0 };
+            const todayCount = await getTodayCount(befly, dateStats.reportDate, item.code);
+
+            return buildProductSummary(item, stats, todayCount);
         })
     );
 
-    return summaries.toSorted((a, b) => {
-        if (b.recent30 !== a.recent30) {
-            return b.recent30 - a.recent30;
+    return products.toSorted((a, b) => {
+        if (b.today !== a.today) {
+            return b.today - a.today;
         }
 
-        return String(a.productName).localeCompare(String(b.productName), "zh-CN");
+        return a.productName.localeCompare(b.productName, "zh-CN");
     });
 }
 
+function buildFullSummary(stats, dateStats, todayCount) {
+    return {
+        today: { reportDate: dateStats.reportDate, count: todayCount },
+        yesterday: { reportDate: dateStats.yesterdayDate, count: stats.yesterday },
+        dayBeforeYesterday: { reportDate: dateStats.dayBeforeYesterdayDate, count: stats.dayBeforeYesterday },
+        month: { startDate: dateStats.monthStartDate, endDate: dateStats.yesterdayDate, count: stats.month },
+        lastMonth: { startDate: dateStats.previousMonthStartDate, endDate: dateStats.previousMonthEndDate, count: stats.lastMonth }
+    };
+}
+
 export default {
     name: "获取日在线统计",
     method: "POST",
@@ -142,36 +126,25 @@ export default {
     required: [],
     handler: async (befly, ctx) => {
         const now = Date.now();
-        const timeZone = befly.config?.tz;
-        const reportDate = getDailyDateYmdNumber(now, timeZone);
-        const monthStartDate = getDailyMonthStartDate(now, timeZone);
-        const weekStartDate = getDailyWeekStartDate(now, timeZone);
-        const recentDateList = getDailyRecentDateList(now, DAILY_STATS_DAY_LIMIT, timeZone);
-        const recent7DateList = getDailyRecentDateList(now, 7, timeZone);
-        const productCode = ctx?.body?.productCode ?? "";
+        const reportDate = getTongJiDateYmdNumber(now, befly.config?.tz);
+        const monthStartDate = getTongJiMonthStartDate(now, befly.config?.tz);
+        const [previousMonthStartDate, previousMonthEndDate] = getPreviousMonthDateRange(reportDate);
+        const recentDateList = getTongJiRecentDateList(now, DAILY_STATS_DAY_LIMIT, befly.config?.tz);
+        const productCode = ctx.body?.productCode ?? "";
         const projectLists = productCode ? [] : befly.config?.projectLists || [];
-        const params = {
+        const dateStats = {
             reportDate: reportDate,
+            yesterdayDate: recentDateList[recentDateList.length - 2],
+            dayBeforeYesterdayDate: recentDateList[recentDateList.length - 3],
             monthStartDate: monthStartDate,
-            weekStartDate: weekStartDate,
-            recentDateList: recentDateList,
-            recent7DateList: recent7DateList,
-            productCode: productCode
+            previousMonthStartDate: previousMonthStartDate,
+            previousMonthEndDate: previousMonthEndDate,
+            recentDateList: recentDateList
         };
-
-        if (!befly.redis) {
-            const summary = buildEmptySummary(reportDate, monthStartDate, weekStartDate, recentDateList, recent7DateList);
-            const products = projectLists.map((item) => buildProductSummary(item, summary));
-
-            return befly.tool.Yes("获取成功", {
-                queryTime: now,
-                ...summary,
-                products: products
-            });
-        }
-
-        const summary = await getDailyStatsSummary(befly, params);
-        const products = await buildDailyStatsProducts(befly, params, projectLists);
+        const history = await loadHistory(befly, reportDate, dateStats);
+        const todayCount = productCode ? await getTodayCount(befly, reportDate, productCode) : 0;
+        const summary = productCode ? buildFullSummary(history[productCode] || { yesterday: 0, dayBeforeYesterday: 0, month: 0, lastMonth: 0 }, dateStats, todayCount) : null;
+        const products = await buildDailyStatsProducts(befly, projectLists, history, dateStats);
 
         return befly.tool.Yes("获取成功", {
             queryTime: now,

package/apis/tongJi/dailyStatsDistribution.js

@@ -0,0 +1,86 @@
+import { snakeCase } from "#befly/utils/util.js";
+
+import { getTongJiDateYmdNumber, getTongJiMonthStartDate, getTongJiRecentDateList } from "./_tongJi.js";
+
+const DISTRIBUTION_DIMENSIONS = ["clientType", "os", "brand", "model", "productVersion"];
+const DAILY_STATS_DISTRIBUTION_DAY_LIMIT = 30;
+
+function getDateRangeBounds(dateRange, now, timeZone) {
+    const reportDate = getTongJiDateYmdNumber(now, timeZone);
+    const recentDateList = getTongJiRecentDateList(now, DAILY_STATS_DISTRIBUTION_DAY_LIMIT, timeZone);
+
+    if (dateRange === "today") {
+        return [reportDate, reportDate];
+    }
+
+    if (dateRange === "yesterday") {
+        const yesterdayDate = recentDateList[recentDateList.length - 2];
+
+        return [yesterdayDate, yesterdayDate];
+    }
+
+    if (dateRange === "dayBeforeYesterday") {
+        const dayBeforeYesterdayDate = recentDateList[recentDateList.length - 3];
+
+        return [dayBeforeYesterdayDate, dayBeforeYesterdayDate];
+    }
+
+    if (dateRange === "recent30") {
+        return [recentDateList[0], recentDateList[recentDateList.length - 1]];
+    }
+
+    const monthStartDate = getTongJiMonthStartDate(now, timeZone);
+
+    return [monthStartDate, reportDate];
+}
+
+async function queryDimensionDistribution(befly, dimension, startDate, endDate, productCode) {
+    const snakeField = snakeCase(dimension);
+    const result = await befly.mysql.execute(`SELECT ${snakeField} as name, COUNT(*) as count FROM befly_daily_report WHERE state > 0 AND report_date BETWEEN ? AND ? AND product_code = ? GROUP BY ${snakeField} ORDER BY count DESC`, [startDate, endDate, productCode]);
+
+    return (result.data || []).map((item) => {
+        return {
+            name: String(item.name || "未知"),
+            count: Number(item.count) || 0
+        };
+    });
+}
+
+function buildEmptyDistribution() {
+    const result = {};
+
+    for (const dimension of DISTRIBUTION_DIMENSIONS) {
+        result[dimension] = [];
+    }
+
+    return result;
+}
+
+export default {
+    name: "获取日在线分布统计",
+    method: "POST",
+    body: "none",
+    auth: true,
+    fields: {
+        productCode: { name: "产品代号", input: "string", min: 1, max: 100 },
+        dateRange: { name: "日期范围", input: "enum", check: "today|yesterday|dayBeforeYesterday|recent30|month", min: 1, max: 20 }
+    },
+    required: ["productCode", "dateRange"],
+    handler: async (befly, ctx) => {
+        const productCode = ctx.body?.productCode ?? "";
+        const dateRange = ctx.body?.dateRange ?? "";
+
+        if (!befly.mysql) {
+            return befly.tool.Yes("获取成功", buildEmptyDistribution());
+        }
+
+        const [startDate, endDate] = getDateRangeBounds(dateRange, Date.now(), befly.config?.tz);
+        const result = {};
+
+        for (const dimension of DISTRIBUTION_DIMENSIONS) {
+            result[dimension] = await queryDimensionDistribution(befly, dimension, startDate, endDate, productCode);
+        }
+
+        return befly.tool.Yes("获取成功", result);
+    }
+};

package/apis/tongJi/errorList.js

@@ -1,9 +1,9 @@
 import { queryFields } from "#befly/apis/_apis.js";
 import errorReportTable from "#befly/tables/errorReport.json";
 
-const NUMBER_FIELDS = ["reportTime", "firstReportTime", "bucketTime", "bucketDate", "hitCount"];
-const FILTER_FIELDS = ["errorType", "source", "productName", "productCode", "productVersion", "deviceType", "browserName", "osName"];
-const KEYWORD_FIELDS = ["pagePath", "pageName", "message", "errorType", "productName", "productCode", "productVersion", "browserName", "osName", "deviceType", "deviceVendor", "deviceModel"];
+const NUMBER_FIELDS = ["reportTime", "firstReportTime", "bucketTime", "bucketDate"];
+const FILTER_FIELDS = ["errorType", "productName", "productCode", "productVersion"];
+const KEYWORD_FIELDS = ["pagePath", "pageName", "message", "errorType", "productName", "productCode", "productVersion"];
 
 export default {
     name: "获取错误报告列表",
@@ -13,13 +13,9 @@ export default {
     fields: {
         ...queryFields,
         errorType: errorReportTable.errorType,
-        source: errorReportTable.source,
         productName: errorReportTable.productName,
         productCode: errorReportTable.productCode,
-        productVersion: errorReportTable.productVersion,
-        deviceType: errorReportTable.deviceType,
-        browserName: errorReportTable.browserName,
-        osName: errorReportTable.osName
+        productVersion: errorReportTable.productVersion
     },
     required: [],
     handler: async (befly, ctx) => {
@@ -65,7 +61,7 @@ export default {
             const row = Object.assign({}, item);
 
             for (const field of NUMBER_FIELDS) {
-                row[field] = Number(item[field] || 0) || (field === "hitCount" ? 1 : 0);
+                row[field] = Number(item[field]) || 0;
             }
 
             return row;

package/apis/tongJi/errorReport.js

@@ -1,43 +1,27 @@
+import { Logger } from "#befly/lib/logger.js";
 import errorReportTable from "#befly/tables/errorReport.json";
 import { getDateYmdNumber, getTimeBucketStart } from "#befly/utils/datetime.js";
-import { parseUserAgent } from "#befly/utils/userAgent.js";
-
-import { expireRedisKeys, getErrorStatsDayBucketCountKey, getErrorStatsDayBucketsKey, getErrorStatsDayTypeCountKey, getErrorStatsDayTypesKey, getErrorStatsPeriodCountKey, getTongJiMonthStartDate, getTongJiWeekStartDate } from "./_tongJi.js";
+import { isValidPositiveInt } from "#befly/utils/is.js";
 
 const ERROR_STATS_BUCKET_MS = 30 * 60 * 1000;
-const ERROR_STATS_REDIS_TTL_SECONDS = 45 * 24 * 60 * 60;
-
-function getErrorReportUaData(ctx) {
-    let userAgent = "";
+const ERROR_STATS_REDIS_TTL_SECONDS = 24 * 60 * 60;
 
-    if (typeof ctx.req?.headers?.get === "function") {
-        userAgent = ctx.req.headers.get("user-agent") || "";
-    } else if (typeof ctx.headers?.get === "function") {
-        userAgent = ctx.headers.get("user-agent") || "";
+function extractErrorMessage(message) {
+    if (typeof message !== "string" || message.length === 0) {
+        return "";
     }
 
-    return parseUserAgent(userAgent);
-}
-
-async function updateErrorStatsRedis(befly, now, bucketDate, bucketTime, errorType) {
-    if (!befly.redis) {
-        return;
-    }
+    const lines = message.split(/\r?\n/);
 
-    const weekStartDate = getTongJiWeekStartDate(now);
-    const monthStartDate = getTongJiMonthStartDate(now);
-    const safeErrorType = String(errorType || "unknown");
-    const dayCountKey = getErrorStatsPeriodCountKey("day", bucketDate);
-    const weekCountKey = getErrorStatsPeriodCountKey("week", weekStartDate);
-    const monthCountKey = getErrorStatsPeriodCountKey("month", monthStartDate);
-    const dayBucketsKey = getErrorStatsDayBucketsKey(bucketDate);
-    const dayBucketCountKey = getErrorStatsDayBucketCountKey(bucketDate, bucketTime);
-    const dayTypesKey = getErrorStatsDayTypesKey(bucketDate);
-    const dayTypeCountKey = getErrorStatsDayTypeCountKey(bucketDate, safeErrorType);
+    for (const line of lines) {
+        const trimmed = line.trim();
 
-    await Promise.all([befly.redis.incr(dayCountKey), befly.redis.incr(weekCountKey), befly.redis.incr(monthCountKey), befly.redis.sadd(dayBucketsKey, [String(bucketTime)]), befly.redis.incr(dayBucketCountKey), befly.redis.sadd(dayTypesKey, [safeErrorType]), befly.redis.incr(dayTypeCountKey)]);
+        if (trimmed.length > 0) {
+            return trimmed;
+        }
+    }
 
-    await expireRedisKeys(befly, [dayCountKey, weekCountKey, monthCountKey, dayBucketsKey, dayBucketCountKey, dayTypesKey, dayTypeCountKey], ERROR_STATS_REDIS_TTL_SECONDS);
+    return "";
 }
 
 export default {
@@ -46,9 +30,10 @@ export default {
     body: "none",
     auth: false,
     fields: {
+        clientType: errorReportTable.clientType,
+        clientId: errorReportTable.clientId,
         pagePath: errorReportTable.pagePath,
         pageName: errorReportTable.pageName,
-        source: errorReportTable.source,
         productName: errorReportTable.productName,
         productCode: errorReportTable.productCode,
         productVersion: errorReportTable.productVersion,
@@ -56,33 +41,85 @@ export default {
         message: errorReportTable.message,
         detail: errorReportTable.detail
     },
-    required: [],
+    required: ["clientType"],
     handler: async (befly, ctx) => {
         const now = Date.now();
         const bucketTime = getTimeBucketStart(now, ERROR_STATS_BUCKET_MS);
         const bucketDate = getDateYmdNumber(now);
-        const uaData = getErrorReportUaData(ctx);
 
-        await befly.mysql.insData({
-            table: "beflyErrorReport",
-            data: {
-                ...ctx.body,
-                ...uaData,
+        const body = ctx.body || {};
+        const productCode = body.productCode || "";
+        const rawMessage = body.message || "";
+        const extractedMessage = extractErrorMessage(rawMessage);
+
+        let memberMeta;
+
+        if (isValidPositiveInt(ctx?.userId)) {
+            memberMeta = { type: "user", value: `user:${ctx.userId}:${productCode}`, userId: ctx.userId };
+        } else {
+            const clientId = body.clientId || ctx?.ip || "";
+
+            if (clientId) {
+                memberMeta = { type: "client", value: `client:${clientId}:${productCode}`, clientId: clientId };
+            } else {
+                memberMeta = { type: "ip", value: `ip:${ctx?.ip || "unknown"}:${productCode}`, ip: ctx?.ip || "unknown" };
+            }
+        }
+
+        const dedupMember = [body.errorType, extractedMessage, productCode, memberMeta.value].map((item) => encodeURIComponent(String(item || ""))).join(":");
+        let first = false;
+
+        if (befly.redis) {
+            const key = `error:day:${bucketDate}:dedup:${productCode}`;
+            const added = await befly.redis.sadd(key, [dedupMember]);
+
+            if (added > 0) {
+                await befly.redis.expire(key, ERROR_STATS_REDIS_TTL_SECONDS);
+                first = true;
+            }
+        }
+
+        let stored = false;
+
+        if (first && befly.mysql) {
+            const userId = isValidPositiveInt(ctx.userId) ? ctx.userId : null;
+
+            const data = {
+                clientType: body.clientType,
+                clientId: memberMeta.clientId || body.clientId || null,
+                pagePath: body.pagePath || null,
+                pageName: body.pageName || null,
+                productName: body.productName || null,
+                productCode: body.productCode || null,
+                productVersion: body.productVersion || null,
+                errorType: body.errorType || null,
+                message: rawMessage || null,
+                extractedMessage: extractedMessage || null,
+                detail: body.detail || null,
+                userId: userId,
+                memberType: memberMeta.type,
+                member: memberMeta.value,
+                ip: ctx?.ip || null,
                 reportTime: now,
                 firstReportTime: now,
+                lastReportTime: now,
                 bucketTime: bucketTime,
-                bucketDate: bucketDate,
-                hitCount: 1
-            }
-        });
+                bucketDate: bucketDate
+            };
 
-        await updateErrorStatsRedis(befly, now, bucketDate, bucketTime, ctx.body.errorType || "unknown");
+            try {
+                await befly.mysql.insData({ table: "errorReport", data: data });
+                stored = true;
+            } catch (error) {
+                Logger.error("errorReport 插入失败", error);
+            }
+        }
 
         return befly.tool.Yes("上报成功", {
             reportTime: now,
             bucketTime: bucketTime,
             bucketDate: bucketDate,
-            stored: true
+            stored: stored
         });
     }
 };