@abtnode/analytics 1.16.13-beta-2eb54cbc

package/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2018-2020 ArcBlock
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

package/README.md

@@ -0,0 +1,3 @@
+# Blocklet Insights
+
+Utility to help get blocklet insights

package/goaccess.conf

@@ -0,0 +1,4 @@
+time-format %T
+date-format %d/%b/%Y
+log_format %h - %^ [%d:%t %^] %^ "%v" "%r" %s %b "%R" "%u" "%^" %T
+date_spec hr

package/index.js

@@ -0,0 +1,130 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getJsonResultPath = exports.getHtmlResultPath = exports.processLogByDate = exports.hasLogByDate = exports.processLogForGroup = exports.splitLogByGroup = exports.decompressLog = exports.getGoAccessBinary = void 0;
+/* eslint-disable @typescript-eslint/comma-dangle */
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+const tar_1 = __importDefault(require("tar"));
+const shelljs_1 = __importDefault(require("shelljs"));
+const debug_1 = __importDefault(require("debug"));
+const child_process_1 = require("child_process");
+const run_script_1 = __importDefault(require("@abtnode/util/lib/run-script"));
+const GOACCESS_CONFIG = path_1.default.join(__dirname, 'goaccess.conf');
+const ANALYTICS_DIR = '.analytics';
+const debug = (0, debug_1.default)('@abtnode/analytics');
+const runAsync = (command, options = {}) => new Promise((resolve, reject) => {
+    const startedAt = Date.now();
+    (0, child_process_1.exec)(command, { windowsHide: true, ...options }, (err, stdout, stderr) => {
+        debug('executed command', { command, time: Date.now() - startedAt, stderr });
+        if (err) {
+            reject(err);
+        }
+        else {
+            resolve(stdout);
+        }
+    });
+});
+function getGoAccessBinary() {
+    const { stdout } = shelljs_1.default.which('goaccess') || { stdout: '' };
+    return stdout.trim();
+}
+exports.getGoAccessBinary = getGoAccessBinary;
+async function decompressLog(source, tmpDir) {
+    fs_extra_1.default.ensureDirSync(tmpDir);
+    const dest = path_1.default.join(tmpDir, path_1.default.basename(source, '.gz'));
+    if (fs_extra_1.default.existsSync(dest)) {
+        fs_extra_1.default.removeSync(dest);
+    }
+    await tar_1.default.x({ file: source, C: tmpDir });
+    return dest;
+}
+exports.decompressLog = decompressLog;
+function splitLogByGroup(filePath, dataDir, date, groups) {
+    return Promise.all(groups.map(async (group) => {
+        if (group.type === 'server') {
+            group.log = filePath;
+            return group;
+        }
+        const groupDir = path_1.default.join(dataDir, group.did, ANALYTICS_DIR);
+        fs_extra_1.default.ensureDirSync(groupDir);
+        const dest = path_1.default.join(groupDir, `${date}.log`);
+        if (fs_extra_1.default.existsSync(dest)) {
+            fs_extra_1.default.removeSync(dest);
+        }
+        const command = `grep -E "${group.hosts.join('|')}" ${filePath} > ${dest}`;
+        try {
+            await runAsync(command);
+            group.log = dest;
+        }
+        catch (err) {
+            console.error('Failed to split log', { command, group, err });
+        }
+        return group;
+    }));
+}
+exports.splitLogByGroup = splitLogByGroup;
+async function processLogForGroup(dataDir, date, group) {
+    try {
+        const groupDir = path_1.default.join(dataDir, group.did, ANALYTICS_DIR);
+        fs_extra_1.default.ensureDirSync(groupDir);
+        if (!group.log) {
+            return group;
+        }
+        if (!fs_extra_1.default.existsSync(group.log)) {
+            return group;
+        }
+        if (fs_extra_1.default.statSync(group.log).size === 0) {
+            fs_extra_1.default.removeSync(group.log);
+            return group;
+        }
+        // FIXME: @wangshijun does `LANG` work on any location
+        const env = { ...process.env, LANG: 'en_US.UTF-8' };
+        const command = `goaccess --no-query-string --no-html-last-updated --html-prefs='{"theme":"bright","perPage":10}' --log-file ${group.log} --config-file ${GOACCESS_CONFIG}`;
+        await Promise.all([
+            (0, run_script_1.default)(`${command} --output ${date}.json`, 'goaccess', { cwd: groupDir, env, silent: true }),
+            (0, run_script_1.default)(`${command} --output ${date}.html`, 'goaccess', { cwd: groupDir, env, silent: true }),
+        ]);
+        fs_extra_1.default.removeSync(group.log);
+        const result = fs_extra_1.default.readJSONSync(path_1.default.join(groupDir, `${date}.json`));
+        group.result = result.general;
+    }
+    catch {
+        console.error('Failed to analyze log for group', { group, date });
+    }
+    return group;
+}
+exports.processLogForGroup = processLogForGroup;
+function hasLogByDate(logDir, date) {
+    const compressed = path_1.default.join(logDir, `access-${date}.log.gz`);
+    return fs_extra_1.default.existsSync(compressed);
+}
+exports.hasLogByDate = hasLogByDate;
+async function processLogByDate(logDir, tmpDir, dataDir, date, groups) {
+    const binary = getGoAccessBinary();
+    if (!binary) {
+        throw new Error('goaccess binary not found');
+    }
+    const compressed = path_1.default.join(logDir, `access-${date}.log.gz`);
+    if (fs_extra_1.default.existsSync(compressed) === false) {
+        return groups;
+    }
+    const decompressed = await decompressLog(compressed, tmpDir);
+    debug('decompressed', decompressed);
+    const splitted = await splitLogByGroup(decompressed, dataDir, date, groups);
+    debug('splitted', splitted);
+    const results = await Promise.all(splitted.map((group) => processLogForGroup(dataDir, date, group)));
+    fs_extra_1.default.removeSync(decompressed);
+    return results;
+}
+exports.processLogByDate = processLogByDate;
+function getHtmlResultPath(dataDir, date, did) {
+    return path_1.default.join(dataDir, did, ANALYTICS_DIR, `${date}.html`);
+}
+exports.getHtmlResultPath = getHtmlResultPath;
+function getJsonResultPath(dataDir, date, did) {
+    return path_1.default.join(dataDir, did, ANALYTICS_DIR, `${date}.json`);
+}
+exports.getJsonResultPath = getJsonResultPath;

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@abtnode/analytics",
+  "publishConfig": {
+    "access": "public"
+  },
+  "version": "1.16.13-beta-2eb54cbc",
+  "description": "Lib to help blocklet developers to get insights of their blocklet",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "goaccess.conf"
+  ],
+  "scripts": {
+    "lint": "eslint tests index.ts",
+    "lint:fix": "npm run lint -- --fix",
+    "test": "NODE_ENV=test node tools/jest.js",
+    "coverage": "npm run test -- --coverage",
+    "clean": "rm -f index.d.ts index.js",
+    "build": "npm run clean && tsc"
+  },
+  "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@abtnode/util": "1.16.13-beta-2eb54cbc",
+    "debug": "^4.3.4",
+    "fs-extra": "^10.1.0",
+    "shelljs": "^0.8.5",
+    "tar": "^6.1.11"
+  },
+  "devDependencies": {
+    "@abtnode/types": "1.16.13-beta-2eb54cbc",
+    "@arcblock/eslint-config-ts": "^0.2.4",
+    "@types/jest": "^29.2.0",
+    "@types/node": "^18.11.0",
+    "@typescript-eslint/eslint-plugin": "^5.40.1",
+    "@typescript-eslint/parser": "^5.40.1",
+    "eslint": "^8.25.0",
+    "jest": "^27.5.1",
+    "prettier": "^2.7.1",
+    "ts-jest": "^27.1.5",
+    "typescript": "^5.0.4"
+  },
+  "gitHead": "7adf96c2eb31762d8eb14b8121fce9865d6a458c"
+}