@tmsfe/tms-core 0.0.69 → 0.0.70

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tmsfe/tms-core",
-  "version": "0.0.69",
+  "version": "0.0.70",
   "description": "tms运行时框架",
   "repository": {
     "type": "git",
@@ -17,9 +17,6 @@
       "esm"
     ]
   },
-  "dependencies": {
-    "md5": "^2.3.0"
-  },
   "devDependencies": {
     "@babel/core": "^7.15.0",
     "@babel/plugin-proposal-class-properties": "^7.14.5",

package/src/index.js

@@ -6,7 +6,7 @@ import syncApi from './syncfnmanager';
 import nav from './navigator';
 import { getLogManager, getRealtimeLogManager } from './log';
 import { setEnvInfo, getEnvInfo, setAuthInfo, setAppPagePaths, isAppPageExist, getHomePage } from './env';
-import md5 from 'md5';
+import md5 from './md5';
 import { callCloudFunc } from './cloudService';
 import EventDispatcher from './eventDispatcher';
 import { serialize } from './objUtils';

package/src/md5.js

@@ -0,0 +1,190 @@
+/**
+ * @copyright 2017-present, Tencent, Inc. All rights reserved.
+ * @author Davis.Lu <davislu@tencent.com>
+ *
+ * @file crypto tools.
+ *
+**/
+
+/**
+ * @public
+ * @description  基于md5算法对源字符串进行hash，生成hash字符串
+ * @param {String} str 源字符串
+ * @returns {String} 源字符串的md5 hash值
+ */
+const md5 = function (str) {
+  /**
+   * 将unicode编码成utf-8
+   * @private
+   * @param {string} encoedStr unicode字符
+   * @returns {string} utf8格式的字符串
+   */
+  const encodeUtf8 = (encoedStr) => {
+    const string = encoedStr.replace(/\r\n/g, '\n');
+    /**
+     * @private
+     * @param {string} c unicode字符
+     * @returns {string} 字符串
+     */
+    const charCode = c => String.fromCharCode(c);
+    const utftextArr = [];
+    for (let n = 0; n < string.length; n += 1) {
+      let c = string.charCodeAt(n);
+      if (c < 128) {
+        utftextArr.push(charCode(c));
+      } else if (c < 2048) {
+        utftextArr.push(charCode((c >> 6) | 192), charCode((c & 63) | 128));
+      } else if (c < 55296 || c >= 57344) {
+        utftextArr.push(
+          charCode((c >> 12) | 224),
+          charCode(((c >> 6) & 63) | 128),
+          charCode((c & 63) | 128),
+        );
+      } else {
+        c = 65536 + (((c & 1023) << 10) | (string.charCodeAt(n += 1) & 1023));
+        utftextArr.push(
+          charCode((c >> 18) | 240),
+          charCode(((c >> 12) & 63) | 128),
+          charCode(((c >> 6) & 63) | 128),
+          charCode((c & 63) | 128),
+        );
+      }
+    }
+    return utftextArr.join('');
+  };
+
+  /**
+   * @private
+   * @param {string} string 字符串
+   * @returns {array} 字符串分组
+   */
+  const convertToWordArray = (string) => {
+    const msgLen = string.length;
+    const lNumberOfWords = ((((msgLen + 8) - ((msgLen + 8) % 64)) / 64) + 1) * 16;
+    const lWordArray = Array(lNumberOfWords - 1);
+    let lByteCount = 0;
+    while (lByteCount <= msgLen) {
+      const wordCount = (lByteCount - (lByteCount % 4)) / 4;
+      const lBytePosition = (lByteCount % 4) * 8;
+      const byteWord = lByteCount === msgLen ? 0x80 : string.charCodeAt(lByteCount);
+      lWordArray[wordCount] |= (byteWord << lBytePosition);
+      lByteCount += 1;
+    }
+    lWordArray[lNumberOfWords - 2] = msgLen << 3;
+    lWordArray[lNumberOfWords - 1] = msgLen >>> 29;
+    return lWordArray;
+  };
+
+  /**
+   * @private
+   * @param {string} lValue 字符串
+   * @param {number} iShiftBits 移动位数
+   * @returns {string} 字符串
+   */
+  const rotateLeft = (
+    lValue,
+    iShiftBits,
+  ) => (lValue << iShiftBits) | (lValue >>> (32 - iShiftBits));
+
+  /**
+   * @private
+   * @param {string} lX 字符串
+   * @param {string} lY 字符串
+   * @returns {string} 字符串
+   */
+  const addUnsigned = (lX, lY) => {
+    const lX8 = (lX & 0x80000000);
+    const lY8 = (lY & 0x80000000);
+    const lX4 = (lX & 0x40000000);
+    const lY4 = (lY & 0x40000000);
+    const lResult = (lX & 0x3FFFFFFF) + (lY & 0x3FFFFFFF);
+    if (lX4 & lY4) return (lResult ^ 0x80000000 ^ lX8 ^ lY8);
+    if (!(lX4 | lY4)) return (lResult ^ lX8 ^ lY8);
+    return (lResult & 0x40000000)
+      ? (lResult ^ 0xC0000000 ^ lX8 ^ lY8)
+      : (lResult ^ 0x40000000 ^ lX8 ^ lY8);
+  };
+
+  /**
+   * @private
+   * @param {object} recycleData 对象
+   * @returns {string} 字符串
+   */
+  const addRecycling = (recycleData) => {
+    const { FN, a, b, c, d, x, s, ac } = recycleData;
+    const aa = addUnsigned(a, addUnsigned(addUnsigned(FN(b, c, d), x), ac));
+    return addUnsigned(rotateLeft(aa, s), b);
+  };
+
+  /**
+   * @private
+   * @param {string} lValue 字符串
+   * @returns {string} 字符串
+   */
+  const wordToHex = (lValue) => {
+    let WordToHexValue = '';
+    for (let lCount = 0; lCount <= 3; lCount += 1) {
+      const lByte = (lValue >>> (lCount * 8)) & 255;
+      const WordToHexValueTemp = `0${lByte.toString(16)}`;
+      WordToHexValue += WordToHexValueTemp.substr(WordToHexValueTemp.length - 2, 2);
+    }
+    return WordToHexValue;
+  };
+  let [a, b, c, d] = [0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476];
+  const sArr = [[7, 12, 17, 22], [5, 9, 14, 20], [4, 11, 16, 23], [6, 10, 15, 21]];
+  const kiArr = ('16b05af49e38d27c58be147ad0369cf207e5c3a18f6d4b29').split('').map(n => parseInt(n, 16));
+  const hxArr = [
+    0xD76AA478, 0xE8C7B756, 0x242070DB, 0xC1BDCEEE, 0xF57C0FAF, 0x4787C62A, 0xA8304613, 0xFD469501,
+    0x698098D8, 0x8B44F7AF, 0xFFFF5BB1, 0x895CD7BE, 0x6B901122, 0xFD987193, 0xA679438E, 0x49B40821,
+    0xF61E2562, 0xC040B340, 0x265E5A51, 0xE9B6C7AA, 0xD62F105D, 0x2441453, 0xD8A1E681, 0xE7D3FBC8,
+    0x21E1CDE6, 0xC33707D6, 0xF4D50D87, 0x455A14ED, 0xA9E3E905, 0xFCEFA3F8, 0x676F02D9, 0x8D2A4C8A,
+    0xFFFA3942, 0x8771F681, 0x6D9D6122, 0xFDE5380C, 0xA4BEEA44, 0x4BDECFA9, 0xF6BB4B60, 0xBEBFBC70,
+    0x289B7EC6, 0xEAA127FA, 0xD4EF3085, 0x4881D05, 0xD9D4D039, 0xE6DB99E5, 0x1FA27CF8, 0xC4AC5665,
+    0xF4292244, 0x432AFF97, 0xAB9423A7, 0xFC93A039, 0x655B59C3, 0x8F0CCC92, 0xFFEFF47D, 0x85845DD1,
+    0x6FA87E4F, 0xFE2CE6E0, 0xA3014314, 0x4E0811A1, 0xF7537E82, 0xBD3AF235, 0x2AD7D2BB, 0xEB86D391,
+  ];
+  // eslint-disable-next-line require-jsdoc
+  const cyc = (i, r = 0) => (i + r) % 4;
+  // 4组处理位操作函数
+  // eslint-disable-next-line require-jsdoc
+  const md5F = (x, y, z) => (x & y) | ((~x) & z);
+  // eslint-disable-next-line require-jsdoc
+  const md5G = (x, y, z) => (x & z) | (y & (~z));
+  // eslint-disable-next-line require-jsdoc
+  const md5H = (x, y, z) => (x ^ y ^ z);
+  // eslint-disable-next-line require-jsdoc
+  const md5I = (x, y, z) => (y ^ (x | (~z)));
+  const string = encodeUtf8(str);
+  const x = convertToWordArray(string);
+  for (let k = 0; k < x.length; k += 16) {
+    const AA = a;
+    const BB = b;
+    const CC = c;
+    const DD = d;
+    const arr = [a, d, c, b];
+    hxArr.forEach((hx, m) => {
+      const i = m % 16;
+      const g = m / 16 << 0;
+      const ki = m < 16 ? m : kiArr[m - 16];
+      const FN = [md5F, md5G, md5H, md5I][g];
+
+      arr[cyc(i)] = addRecycling({
+        FN,
+        a: arr[cyc(i)],
+        b: arr[cyc(i, 3)],
+        c: arr[cyc(i, 2)],
+        d: arr[cyc(i, 1)],
+        x: x[k + ki],
+        s: sArr[g][i % 4],
+        ac: hx,
+      });
+    });
+    a = addUnsigned(arr[0], AA);
+    b = addUnsigned(arr[3], BB);
+    c = addUnsigned(arr[2], CC);
+    d = addUnsigned(arr[1], DD);
+  }
+  return (wordToHex(a) + wordToHex(b) + wordToHex(c) + wordToHex(d)).toLowerCase();
+};
+
+export default md5;

package/src/report/helper.ts

@@ -77,8 +77,13 @@ function convert2String(value: any): string | null {
  */
 function convert2StringArray(arr: DataItem[]) {
   for (let i = 0; i < arr.length; i++) {
-    // eslint-disable-next-line no-param-reassign
+    /* eslint-disable */
     arr[i] = convert2String(arr[i]);
+    if (arr[i]) {
+      // 后端是根据 | 来分割字段的，这里只好过滤掉
+      arr[i] = arr[i].replace(/\|/g, '[');
+    }
+    /* eslint-enable */
   }
 }
 

package/src/request.js

@@ -8,7 +8,7 @@
  * 考虑到对不同运行环境的支持，强依赖运行环境的依赖，比如 wx.request，应通过注入的形式提供。
  * 框架判断在不同的运行环境，切换调用不同运行环境提供的方法。
  */
-import md5 from 'md5';
+import md5 from './md5';
 import { getLogManager } from './log';
 import { getEnvInfo, getAuthInfo } from './env';
 import { safeJsonParse } from './objUtils';

package/test/data.js

@@ -1 +0,0 @@
-{"__FILENAME__":"/var/log/tke-log-agent/ns-prj6wdcj-1134791-production/ns-prj6wdcj-1134791-production-stsp-sinanserver-f-ab66d07e-025c-44f6-9bff-fe6a44db1dd7/sinanserver-f-1/ab66d07e-025c-44f6-9bff-fe6a44db1dd7/v_applog/4b5dcbe6f3fc0d2e0173491dc16a7461/Wecar.SinanServer_access.log","__PKGID__":"CA5498C625624F5-8B57","__PKG_LOGID__":10354794,"__SOURCE__":"9.144.184.20","__TAG__":{"container_id":"505ccdbca43b91595f5f328820fb949e1c39c0dcdbecacea5587139a0e0ce260","container_name":"main-container","image_name":"csighub.tencentyun.com/th-deploy/th_wecar_sinanserver:feat-albertluo-sinan77a-20220101_154653-c873ef","namespace":"ns-prj6wdcj-1134791-production","pod_label_clusterId":"cls-991r4bpx","pod_label_controller-revision-hash":"sinanserver-f-66999dd78c","pod_label_env":"production","pod_label_k8s-app":"sinanserver-f","pod_label_moduleFourId":"1134791","pod_label_projectName":"prj6wdcj","pod_label_qcloud-app":"sinanserver-f","pod_label_region":"ap-tianjin","pod_label_statefulsetplus-kubernetes-io/pod-name":"sinanserver-f-1","pod_label_workload-kind":"statefulsetplus","pod_name":"sinanserver-f-1","pod_uid":"e135e0e8-b655-11ec-a3e8-525400b30d18"},"__TIMESTAMP__":1649848303067,"clientAddr":"111.206.145.51:55349","deviceFingerprint":"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 wechatdevtools/1.05.2111300 MicroMessenger/8.0.5 Language/zh_CN webview/","downstreamTimeCost":"58","file":"main.go:Run:158","isTxScanner":"false","level":"INFO","method":"POST","requestContent":"\"{\\\"appVersion\\\":\\\"2022.16.0\\\",\\\"nonce\\\":\\\"g0es1cn9ln\\\",\\\"seqId\\\":\\\"1649848302200869298\\\",\\\"sign\\\":\\\"3242dba92a77bef52c7110446f3ce156\\\",\\\"timestamp\\\":1649848302200,\\\"token\\\":\\\"22d6a3884c7494fd754d8851eb29d83f\\\",\\\"types\\\":[{\\\"bussClassify\\\":3,\\\"spIds\\\":[],\\\"status\\\":[3,5,6,7,8,11,31,32,34]}],\\\"userId\\\":\\\"306914\\\",\\\"version\\\":\\\"1.0\\\",\\\"wxAppId\\\":\\\"wx65cc950f42e8fff1\\\"}\"","responseCode":"0","responseContent":"\"{\\\"errCode\\\":0,\\\"errMsg\\\":\\\"success\\\",\\\"resData\\\":{\\\"count\\\":[{\\\"bussClassify\\\":3,\\\"count\\\":51}]}}\"}","seqId":"1649848302200869298","statusCode":"200","time":"2022-04-13 19:11:42.686","timeCost":"138","upstreamAddr":"9.144.241.102:5334","urlPath":"/user/order/countv2","userId":"306914"}

package/test/test.js

@@ -1,154 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-const readline = require('readline');
-// const { md5 } = require('../src/md5');
-const openSourceMd5 = require('md5');
-
-const scriptStartDir = process.cwd();
-const srcDataPathArg = process.argv[2];
-const srcDataAbsPath = path.resolve(scriptStartDir, srcDataPathArg);
-const startLine = parseInt(process.argv[3], 10);
-const maxLine = parseInt(process.argv[4], 10);
-
-const readLineByLine = (filePath, onReadLine, startLine, endLine) => { // eslint-disable-line
-  const promise = new Promise((resolve, reject) => {
-    if (!fs.existsSync(filePath)) {
-      reject(`${filePath} not exists`);
-      return;
-    }
-    if (endLine <= 0) resolve();
-
-    const rl = readline.createInterface({
-      input: fs.createReadStream(filePath),
-      crlfDelay: Infinity,
-    });
-    let line = 0;
-    rl.on('line', (data) => {
-      line += 1;
-      if (line > endLine) return rl.close();
-      if (line < startLine) return;
-      onReadLine(data);
-    });
-    rl.on('close', resolve);
-    rl.on('error', reject);
-  });
-  return promise;
-};
-
-const formatLog = (str) => {
-  const log = JSON.parse(str);
-  const request = JSON.parse(JSON.parse(log.requestContent));
-  const errCode = log.responseContent.replace(/.*(errCode[^\d]*)(\d*)(.*)/g, (m, g1, g2) => {
-    if (m) return g2;
-    const response = JSON.parse(JSON.parse(`${log.responseContent.substring(0, log.responseContent.length - 2)}"`));
-    return response.errCode;
-  });
-  return { request, errCode: parseInt(errCode, 10), urlPath: log.urlPath };
-};
-
-/**
- * 用于序列化需要签名的参数
- * @private
- * @param {object} param 需要序列化的参数
- * @returns {string} 序列化之后的参数字符串
- */
-const seriesParam = (param) => {
-  const keys = Object.keys(param)
-    .sort();
-  const series = keys.map((key) => {
-    const val = param[key];
-    return `${key}${typeof val === 'object' ? JSON.stringify(val) : val}`;
-  });
-  return series.join('');
-};
-
-const sign = (param = {}) => {
-  const data = Object.assign({}, param);
-  delete data.sign;
-  const str = seriesParam(data);
-  // return { sign: md5(str), openSourceSign: openSourceMd5(str), str };
-  return { sign: '', openSourceSign: openSourceMd5(str), str };
-};
-
-(async () => {
-  const count = {
-    total: 0,
-    valid: 0,
-    invalid: {
-      total: 0,
-      list: [],
-    },
-    match: {
-      total: 0,
-      urlPaths: new Set(),
-      list: [],
-    },
-    same: {
-      total: 0,
-      urlPaths: new Set(),
-      list: [],
-    },
-    mismatch: {
-      ignore: {
-        total: 0,
-        patterns: [
-          '^/aggre/saas/.*', '^/aggre/datacenter/.*', '^/aggre/rent/.*',
-          '^/dd/api/.*',
-          '^/fmbapi/business/.*',
-          '^/ruqimobility/.*',
-          '^/ruqireport/.*',
-          '^/taiweb/.*',
-          '^/v2/bus/.*',
-        ],
-        urlPaths: new Set(),
-        list: [],
-      },
-      problem: {
-        total: 0,
-        list: [],
-        urlPaths: new Set(),
-      },
-    },
-  };
-  const testLog = (logStr) => {
-    let log;
-    count.total += 1;
-    console.log(count.total + startLine);
-    try {
-      log = formatLog(logStr);
-      count.valid += 1;
-    } catch (e) {
-      log = null;
-      count.invalid.total += 1;
-      count.invalid.list.push(logStr);
-    }
-    if (!log) return;
-
-    const { sign: signature, openSourceSign, str } = sign(log.request);
-    if (openSourceSign === log.request.sign) {
-      count.match.total += 1;
-      count.match.urlPaths.add(log.urlPath);
-    } else if (signature === openSourceSign) {
-      count.same.total += 1;
-      count.same.urlPaths.add(log.urlPath);
-      count.same.list.push({ ...log, str });
-    } else {
-      const ignore = count.mismatch.ignore.patterns.some(pattern => new RegExp(pattern).test(log.urlPath));
-      const store = count.mismatch[ignore ? 'ignore' : 'problem'];
-      store.total += 1;
-      if (!store.urlPaths.has(log.urlPath)) {
-        store.urlPaths.add(log.urlPath);
-        store.list.push({ ...log, str });
-      }
-      // if (!ignore) console.warn(log.request.seqId, str, log.request.sign, signature);
-    }
-  };
-  await readLineByLine(srcDataAbsPath, testLog, isNaN(startLine) ? 0 : startLine, isNaN(maxLine) ? 10 : maxLine);
-  console.log('total', count.total, 'valid', count.valid, 'invalid', count.invalid.total);
-  console.log('match', count.match.total, 'same', count.same.total, 'mismatch ignore', count.mismatch.ignore.total, 'mismatch problem', count.mismatch.problem.total);
-  // console.log('match urls', count.match.urlPaths);
-  console.log('same logs', count.same.list);
-  // console.log('known mismatch ignore urls', count.mismatch.ignore.urlPaths);
-  // console.log('unknown mismatch urls', count.mismatch.problem.urlPaths);
-  // console.log('unknown mismatch urls', count.mismatch.problem.list);
-})();