befly 3.24.18 → 3.24.19

package/apis/tongJi/errorStats.js

@@ -1,6 +1,13 @@
 import { getDateYmdNumber } from "#root/utils/datetime.js";
 
 const ERROR_STATS_DAY_LIMIT = 30;
+const ERROR_STATS_FALLBACK_COUNT_KEY = "stats:fallback:errorStats:count";
+const ERROR_STATS_FALLBACK_DAILY_COUNT_KEY_PREFIX = "stats:fallback:errorStats:count:day:";
+const ERROR_STATS_FALLBACK_TTL_SECONDS = 7 * 24 * 60 * 60;
+
+function getErrorStatsFallbackDailyCountKey(bucketDate) {
+    return `${ERROR_STATS_FALLBACK_DAILY_COUNT_KEY_PREFIX}${bucketDate}`;
+}
 
 function toNumber(value) {
     const num = Number(value);
@@ -35,6 +42,144 @@ function getErrorStatsRecentDateList(now = Date.now(), limit = ERROR_STATS_DAY_L
     return list;
 }
 
+function getErrorStatsPeriodCountKey(periodType, periodValue) {
+    return `error:${periodType}:${periodValue}:count`;
+}
+
+function getErrorStatsDayBucketsKey(bucketDate) {
+    return `error:day:${bucketDate}:buckets`;
+}
+
+function getErrorStatsDayBucketCountKey(bucketDate, bucketTime) {
+    return `error:day:${bucketDate}:bucket:${bucketTime}:count`;
+}
+
+function getErrorStatsDayTypesKey(bucketDate) {
+    return `error:day:${bucketDate}:types`;
+}
+
+function getErrorStatsDayTypeCountKey(bucketDate, errorType) {
+    return `error:day:${bucketDate}:type:${encodeURIComponent(String(errorType || "unknown"))}:count`;
+}
+
+async function getErrorStatsRedisSummary(befly, periodType, periodValue) {
+    return {
+        count: toNumber(await befly.redis.getString(getErrorStatsPeriodCountKey(periodType, periodValue)))
+    };
+}
+
+async function getErrorStatsRedisTrend(befly, bucketDate) {
+    const bucketSet = await befly.redis.smembers(getErrorStatsDayBucketsKey(bucketDate));
+    const trend = [];
+
+    for (const bucketTime of bucketSet) {
+        const itemBucketTime = toNumber(bucketTime);
+
+        trend.push({
+            bucketTime: itemBucketTime,
+            bucketDate: bucketDate,
+            count: toNumber(await befly.redis.getString(getErrorStatsDayBucketCountKey(bucketDate, itemBucketTime)))
+        });
+    }
+
+    trend.sort((a, b) => a.bucketTime - b.bucketTime);
+    return trend;
+}
+
+async function getErrorStatsRedisDays(befly, recentDateList) {
+    const days = [];
+
+    for (const item of recentDateList) {
+        days.push({
+            bucketDate: item,
+            count: toNumber(await befly.redis.getString(getErrorStatsPeriodCountKey("day", item)))
+        });
+    }
+
+    return days;
+}
+
+async function getErrorStatsRedisTopTypes(befly, bucketDate) {
+    const typeSet = await befly.redis.smembers(getErrorStatsDayTypesKey(bucketDate));
+    const topTypes = [];
+
+    for (const errorType of typeSet) {
+        const name = String(errorType || "unknown");
+        const count = toNumber(await befly.redis.getString(getErrorStatsDayTypeCountKey(bucketDate, name)));
+
+        if (count <= 0) {
+            continue;
+        }
+
+        topTypes.push({
+            errorType: name,
+            count: count
+        });
+    }
+
+    topTypes.sort((a, b) => {
+        if (b.count !== a.count) {
+            return b.count - a.count;
+        }
+
+        return String(a.errorType).localeCompare(String(b.errorType), "zh-CN");
+    });
+
+    return topTypes.slice(0, 5);
+}
+
+async function getErrorStatsFromRedis(befly, bucketDate, weekStartDate, monthStartDate, recentDateList) {
+    const todaySummary = await getErrorStatsRedisSummary(befly, "day", bucketDate);
+    const weekSummary = await getErrorStatsRedisSummary(befly, "week", weekStartDate);
+    const monthSummary = await getErrorStatsRedisSummary(befly, "month", monthStartDate);
+    const trend = await getErrorStatsRedisTrend(befly, bucketDate);
+    const days = await getErrorStatsRedisDays(befly, recentDateList);
+    const topTypes = await getErrorStatsRedisTopTypes(befly, bucketDate);
+
+    if (todaySummary.count <= 0 && weekSummary.count <= 0 && monthSummary.count <= 0 && trend.length === 0 && topTypes.length === 0) {
+        return null;
+    }
+
+    return {
+        today: {
+            bucketDate: bucketDate,
+            count: todaySummary.count
+        },
+        week: {
+            startDate: weekStartDate,
+            endDate: bucketDate,
+            count: weekSummary.count
+        },
+        month: {
+            startDate: monthStartDate,
+            endDate: bucketDate,
+            count: monthSummary.count
+        },
+        trend: trend,
+        days: days,
+        topTypes: topTypes
+    };
+}
+
+async function incrErrorStatsFallbackCount(befly, bucketDate) {
+    if (!befly.redis) {
+        return;
+    }
+
+    const dailyKey = getErrorStatsFallbackDailyCountKey(bucketDate);
+
+    if (typeof befly.redis.incrWithExpire === "function") {
+        await befly.redis.incrWithExpire(ERROR_STATS_FALLBACK_COUNT_KEY, ERROR_STATS_FALLBACK_TTL_SECONDS);
+        await befly.redis.incrWithExpire(dailyKey, ERROR_STATS_FALLBACK_TTL_SECONDS);
+        return;
+    }
+
+    await befly.redis.incr(ERROR_STATS_FALLBACK_COUNT_KEY);
+    await befly.redis.expire(ERROR_STATS_FALLBACK_COUNT_KEY, ERROR_STATS_FALLBACK_TTL_SECONDS);
+    await befly.redis.incr(dailyKey);
+    await befly.redis.expire(dailyKey, ERROR_STATS_FALLBACK_TTL_SECONDS);
+}
+
 async function getErrorStatsSummary(befly, startDate, endDate) {
     const result = await befly.mysql.execute("SELECT SUM(hit_count) as count FROM befly_error_report WHERE state = 1 AND bucket_date BETWEEN ? AND ?", [startDate, endDate]);
     const detail = result.data?.[0] || {};
@@ -56,6 +201,9 @@ export default {
         const tableReady = tableExistsResult.data === true;
         const now = Date.now();
         const bucketDate = getDateYmdNumber(now);
+        const weekStartDate = getErrorStatsWeekStartDate(now);
+        const monthStartDate = getErrorStatsMonthStartDate(now);
+        const recentDateList = getErrorStatsRecentDateList(now);
 
         if (!tableReady) {
             return befly.tool.Yes("获取成功", {
@@ -64,12 +212,12 @@ export default {
                     count: 0
                 },
                 week: {
-                    startDate: getErrorStatsWeekStartDate(now),
+                    startDate: weekStartDate,
                     endDate: bucketDate,
                     count: 0
                 },
                 month: {
-                    startDate: getErrorStatsMonthStartDate(now),
+                    startDate: monthStartDate,
                     endDate: bucketDate,
                     count: 0
                 },
@@ -79,9 +227,16 @@ export default {
             });
         }
 
-        const weekStartDate = getErrorStatsWeekStartDate(now);
-        const monthStartDate = getErrorStatsMonthStartDate(now);
-        const recentDateList = getErrorStatsRecentDateList(now);
+        if (befly.redis) {
+            const redisResult = await getErrorStatsFromRedis(befly, bucketDate, weekStartDate, monthStartDate, recentDateList);
+
+            if (redisResult) {
+                return befly.tool.Yes("获取成功", redisResult);
+            }
+
+            await incrErrorStatsFallbackCount(befly, bucketDate);
+        }
+
         const todaySummary = await getErrorStatsSummary(befly, bucketDate, bucketDate);
         const weekSummary = await getErrorStatsSummary(befly, weekStartDate, bucketDate);
         const monthSummary = await getErrorStatsSummary(befly, monthStartDate, bucketDate);

package/apis/tongJi/fallbackReset.js

@@ -0,0 +1,69 @@
+import { getDateYmdNumber } from "#root/utils/datetime.js";
+
+const INFO_STATS_FALLBACK_COUNT_KEY = "stats:fallback:infoStats:count";
+const ERROR_STATS_FALLBACK_COUNT_KEY = "stats:fallback:errorStats:count";
+
+function getInfoStatsFallbackDailyCountKey(reportDate) {
+    return `stats:fallback:infoStats:count:day:${reportDate}`;
+}
+
+function getErrorStatsFallbackDailyCountKey(reportDate) {
+    return `stats:fallback:errorStats:count:day:${reportDate}`;
+}
+
+function toNumber(value) {
+    const num = Number(value);
+
+    if (!Number.isFinite(num)) {
+        return 0;
+    }
+
+    return num;
+}
+
+export default {
+    name: "重置统计回退计数",
+    method: "POST",
+    body: "none",
+    auth: true,
+    fields: {},
+    required: [],
+    handler: async (befly) => {
+        if (!befly.redis) {
+            return befly.tool.No("Redis 不可用，无法重置", {
+                reset: false
+            });
+        }
+
+        const reportDate = getDateYmdNumber(Date.now());
+        const infoDailyKey = getInfoStatsFallbackDailyCountKey(reportDate);
+        const errorDailyKey = getErrorStatsFallbackDailyCountKey(reportDate);
+        const resetKeys = [INFO_STATS_FALLBACK_COUNT_KEY, ERROR_STATS_FALLBACK_COUNT_KEY, infoDailyKey, errorDailyKey];
+        const before = {
+            infoStats: toNumber(await befly.redis.getString(INFO_STATS_FALLBACK_COUNT_KEY)),
+            errorStats: toNumber(await befly.redis.getString(ERROR_STATS_FALLBACK_COUNT_KEY)),
+            today: {
+                infoStats: toNumber(await befly.redis.getString(infoDailyKey)),
+                errorStats: toNumber(await befly.redis.getString(errorDailyKey))
+            }
+        };
+
+        const removedCount = await befly.redis.delBatch(resetKeys);
+
+        return befly.tool.Yes("重置成功", {
+            reset: true,
+            reportDate: reportDate,
+            removedCount: toNumber(removedCount),
+            keys: resetKeys,
+            before: before,
+            after: {
+                infoStats: 0,
+                errorStats: 0,
+                today: {
+                    infoStats: 0,
+                    errorStats: 0
+                }
+            }
+        });
+    }
+};

package/apis/tongJi/infoReport.js

@@ -1,8 +1,27 @@
 import { UAParser } from "ua-parser-js";
 
-import { getDateYmdNumber } from "#root/utils/datetime.js";
+import { addDays, getDateYmdNumber } from "#root/utils/datetime.js";
 import { isValidPositiveInt } from "#root/utils/is.js";
 
+const INFO_STATS_REDIS_TTL_SECONDS = 45 * 24 * 60 * 60;
+const INFO_STATS_FIELDS = [
+    { key: "sources", valueKey: "source" },
+    { key: "productNames", valueKey: "productName" },
+    { key: "productCodes", valueKey: "productCode" },
+    { key: "productVersions", valueKey: "productVersion" },
+    { key: "pagePaths", valueKey: "pagePath" },
+    { key: "pageNames", valueKey: "pageName" },
+    { key: "deviceTypes", valueKey: "deviceType" },
+    { key: "browsers", valueKey: "browserName" },
+    { key: "browserVersions", valueKey: "browserVersion" },
+    { key: "osList", valueKey: "osName" },
+    { key: "osVersions", valueKey: "osVersion" },
+    { key: "deviceVendors", valueKey: "deviceVendor" },
+    { key: "deviceModels", valueKey: "deviceModel" },
+    { key: "engines", valueKey: "engineName" },
+    { key: "cpuArchitectures", valueKey: "cpuArchitecture" }
+];
+
 function getInfoStatsMember(ctx) {
     if (isValidPositiveInt(ctx.userId)) {
         return `user:${ctx.userId}`;
@@ -11,6 +30,67 @@ function getInfoStatsMember(ctx) {
     return `ip:${ctx.ip || "unknown"}`;
 }
 
+function getInfoStatsWeekStartDate(timestamp = Date.now()) {
+    const date = Reflect.construct(Date, [timestamp]);
+    const day = date.getDay();
+    const offset = day === 0 ? -6 : 1 - day;
+
+    return getDateYmdNumber(addDays(timestamp, offset));
+}
+
+function getInfoStatsMonthStartDate(timestamp = Date.now()) {
+    const date = Reflect.construct(Date, [timestamp]);
+
+    return getDateYmdNumber(Reflect.construct(Date, [date.getFullYear(), date.getMonth(), 1, 0, 0, 0, 0]).getTime());
+}
+
+function getInfoStatsNamesKey(periodType, periodValue, fieldKey) {
+    return `info:${periodType}:${periodValue}:${fieldKey}:names`;
+}
+
+function getInfoStatsCountKey(periodType, periodValue, fieldKey, value) {
+    return `info:${periodType}:${periodValue}:${fieldKey}:count:${encodeURIComponent(String(value || ""))}`;
+}
+
+function getInfoStatsReportTimeKey(periodType, periodValue) {
+    return `info:${periodType}:${periodValue}:reportTime`;
+}
+
+async function updateInfoStatsPeriodRedis(befly, periodType, periodValue, statData, now) {
+    const reportTimeKey = getInfoStatsReportTimeKey(periodType, periodValue);
+    await befly.redis.setString(reportTimeKey, String(now), INFO_STATS_REDIS_TTL_SECONDS);
+    await befly.redis.expire(reportTimeKey, INFO_STATS_REDIS_TTL_SECONDS);
+
+    for (const item of INFO_STATS_FIELDS) {
+        const rawValue = String(statData[item.valueKey] || "").trim();
+
+        if (!rawValue) {
+            continue;
+        }
+
+        const namesKey = getInfoStatsNamesKey(periodType, periodValue, item.key);
+        const countKey = getInfoStatsCountKey(periodType, periodValue, item.key, rawValue);
+
+        await befly.redis.sadd(namesKey, [rawValue]);
+        await befly.redis.incr(countKey);
+        await befly.redis.expire(namesKey, INFO_STATS_REDIS_TTL_SECONDS);
+        await befly.redis.expire(countKey, INFO_STATS_REDIS_TTL_SECONDS);
+    }
+}
+
+async function updateInfoStatsRedis(befly, now, reportDate, statData) {
+    if (!befly.redis) {
+        return;
+    }
+
+    const weekStartDate = getInfoStatsWeekStartDate(now);
+    const monthStartDate = getInfoStatsMonthStartDate(now);
+
+    await updateInfoStatsPeriodRedis(befly, "day", reportDate, statData, now);
+    await updateInfoStatsPeriodRedis(befly, "week", weekStartDate, statData, now);
+    await updateInfoStatsPeriodRedis(befly, "month", monthStartDate, statData, now);
+}
+
 export default {
     name: "上报访问信息统计",
     method: "POST",
@@ -68,32 +148,52 @@ export default {
             });
         }
 
+        const statData = {
+            source: body.source || "",
+            productName: body.productName || "",
+            productCode: body.productCode || "",
+            productVersion: body.productVersion || "",
+            pagePath: body.pagePath || "",
+            pageName: body.pageName || "",
+            deviceType: uaResult.device.type || "desktop",
+            browserName: uaResult.browser.name || "",
+            browserVersion: uaResult.browser.version || "",
+            osName: uaResult.os.name || "",
+            osVersion: uaResult.os.version || "",
+            deviceVendor: uaResult.device.vendor || "",
+            deviceModel: uaResult.device.model || "",
+            engineName: uaResult.engine.name || "",
+            cpuArchitecture: uaResult.cpu.architecture || ""
+        };
+
         await befly.mysql.insData({
             table: "beflyInfoReport",
             data: {
                 reportTime: now,
                 reportDate: reportDate,
                 memberKey: member,
-                source: body.source || "",
-                productName: body.productName || "",
-                productCode: body.productCode || "",
-                productVersion: body.productVersion || "",
-                pagePath: body.pagePath || "",
-                pageName: body.pageName || "",
+                source: statData.source,
+                productName: statData.productName,
+                productCode: statData.productCode,
+                productVersion: statData.productVersion,
+                pagePath: statData.pagePath,
+                pageName: statData.pageName,
                 detail: body.detail || "",
                 userAgent: userAgent,
-                deviceType: uaResult.device.type || "desktop",
-                browserName: uaResult.browser.name || "",
-                browserVersion: uaResult.browser.version || "",
-                osName: uaResult.os.name || "",
-                osVersion: uaResult.os.version || "",
-                deviceVendor: uaResult.device.vendor || "",
-                deviceModel: uaResult.device.model || "",
-                engineName: uaResult.engine.name || "",
-                cpuArchitecture: uaResult.cpu.architecture || ""
+                deviceType: statData.deviceType,
+                browserName: statData.browserName,
+                browserVersion: statData.browserVersion,
+                osName: statData.osName,
+                osVersion: statData.osVersion,
+                deviceVendor: statData.deviceVendor,
+                deviceModel: statData.deviceModel,
+                engineName: statData.engineName,
+                cpuArchitecture: statData.cpuArchitecture
             }
         });
 
+        await updateInfoStatsRedis(befly, now, reportDate, statData);
+
         return befly.tool.Yes("上报成功", {
             reportTime: now,
             reportDate: reportDate,