@cicctencent/tars2node-cli 0.0.1

package/lib/parser/parser.js

@@ -0,0 +1,383 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const peg = require('pegjs');
+const assert = require('assert');
+const globby = require('globby');
+const util = require('util');
+
+const PEG_SOURCE = './../../jce.pegjs';
+const parser = buildParser();
+
+class AST {
+    /**
+     * constructor
+     * @param {string} pathname
+     */
+    constructor(pathname) {
+        assert(pathname.length > 0, 'Need at least one entry file to parse.');
+
+        this.pathname = pathname;
+        /** @type {Object.<string, *>} */
+        this.parsed = {};
+        this.parser = parser;
+        /** @type {Object.<string, *>} */
+        this.refMap = {};
+        /** @type {Object.<string, *>} */
+        this.fileRefMap = {};
+        // 倒排索引
+        this.invertedMap = {};
+        /** @type {string[]} */
+        this.entry = [];
+        /** 入口文件路径 */
+        this.entryFilepath = '';
+        /** 记录文件是否已经被加载 */
+        this.refFilePath = new Map();
+    }
+
+    // 原始编译方式
+    buildAST() {
+        const result = this.parseTars();
+
+        this.setRefMap(result.modules, result.pathname, true);
+        this.handleRefRecursively(result.refs, this.pathname);
+        return this.refMap;
+    }
+
+    // 按文件模块解析AST
+    buildFileAST() {
+        const result = this.parseTars();
+        this.setFileRefMap(result.modules, result.pathname);
+        this.handleFileRefRecursively(result.refs, this.pathname);
+        // fs.writeFileSync('fileRefMap.json', JSON.stringify(this.fileRefMap, null, 2),{encoding: 'utf8'});
+        // fs.writeFileSync('invertedMap.json', JSON.stringify(this.invertedMap, null, 2),{encoding: 'utf8'});
+        this.changeModuleName()
+        // fs.writeFileSync('refMap.json', JSON.stringify(this.refMap, null, 2),{encoding: 'utf8'});
+        return this.refMap;
+    }
+
+    // 解决递归问题，合并依赖文件。
+    buildASTRecursiveMerge() {
+        const result = this.parseTars();
+        this.setRefMapRecuriveMerge(result.modules, result.pathname, true);
+        this.handleRefRecursivelyMerge(result.refs, this.pathname, true);
+        // TODO:: 归一化 this.changeModuleName()
+        this.changeModuleNameOrigin();
+        // 合并文件,并去除依赖
+        this.mergeModuleFile();
+        return this.refMap;
+    }
+
+    // 合并 include 文件 
+    mergeModuleFile() {
+        const entryModelName = path.basename(this.pathname, '.tars');
+        Object.keys(this.refMap).forEach(key => {
+            const moduleName = path.basename(this.refMap[key].pathname, '.tars');
+            // 入口文件作为合并基准
+            if (moduleName == entryModelName) {
+            } else {
+                this.refMap[entryModelName].value = this.refMap[entryModelName].value.concat(this.refMap[key].value);
+                delete this.refMap[key];
+            }
+            // this.refMap[entryModelName].value.map(m => {
+            //     if (m.type === 'interface') {
+            //         // TODO:: 处理方法
+            //         // m.methods.forEach(({ params, output }) => {
+            //         //     this._checkIsTypeNeedImport(output, moduleName);
+            //         //     params && params.forEach(({ type }) => { // 测试函数允许没有参数
+            //         //         this._checkIsTypeNeedImport(type, moduleName);
+            //         //     });
+            //         // });
+            //     } else if (m.type === 'struct') {
+            //         // 处理结构体
+            //         m.members.forEach(({ type }) => {
+            //             this.renameImport(type, entryModelName);
+            //         });
+            //     }
+            // })
+        });
+    }
+    /**
+     * 修改 module name
+     * */
+    changeModuleNameOrigin() {
+        // 记录映射关系
+        let moduleMap = new Map();
+        Object.keys(this.refMap).forEach(key => {
+            const moduleName = path.basename(this.refMap[key].pathname, '.tars');
+            moduleMap.set(key, moduleName);
+            this.refMap[moduleName] = this.refMap[key];
+            // 去除改变之前的多余的module
+            delete this.refMap[key];
+        });
+        // 改变包的引用名称
+        Object.keys(this.refMap).forEach(key => {
+            this.refMap[key].value &&
+                this.refMap[key].value.forEach(item => {
+                    item.members &&
+                        item.members.forEach(member => {
+                            // 需要改变引用包名称
+                            if (util.isObject(member.type) && member.type.nested && member.type.nested.length > 0) {
+                                member.type.nested = member.type.nested.map(module => {
+                                    return moduleMap.get(module);
+                                });
+                            }
+                        });
+                });
+        });
+    }
+
+    /**
+     * @param {*[]} modules
+     * @param {string} pathname
+     */
+    setFileRefMap(modules, pathname) {
+        const fileRefMap = this.fileRefMap[pathname] = {};
+        modules.forEach(({ id, type, value }) => {
+            assert(type === 'module', `File ${id} is not a tars module.`);
+            (value || []).forEach(item => {
+                // 这里不考虑从哪里引用了，因为法确定
+                this.invertedMap[item.id] = {
+                    pathname,
+                    moduleId: id
+                };
+            });
+            fileRefMap[id] = value;
+        });
+    }
+
+    /** 递归处理引用
+     * @param {{type: string, target: string}[]} refList
+     * @param {string} pathname
+     */
+    handleFileRefRecursively(refList, pathname) {
+        const parsedRefs = this.parseRefs(refList, pathname);
+
+        parsedRefs.forEach(({ refs, pathname: _pathname, modules }) => {
+            this.setFileRefMap(modules, _pathname);
+            this.handleFileRefRecursively(refs, _pathname);
+        });
+    }
+
+    /** 递归处理引用
+     * @param {{type: string, target: string}[]} refList
+     * @param {string} pathname
+     */
+    handleRefRecursively(refList, pathname) {
+        const parsedRefs = this.parseRefs(refList, pathname);
+
+        parsedRefs.forEach(({ refs, pathname: _pathname, modules }) => {
+            this.setRefMap(modules, _pathname);
+            this.handleFileRefRecursively(refs, _pathname);
+        });
+    }
+
+    /**
+     * 修改 module name
+     * */
+    changeModuleName() {
+        // 记录映射关系
+        let moduleMap = {};
+        Object.keys(this.fileRefMap).forEach(key => {
+            const moduleName = path.basename(key).replace(/\.\w+$/, '');
+            const module = this.fileRefMap[key];
+            // path到name
+            moduleMap[key] = moduleName;
+            // todo： 目前一个文件只允许导出1个module
+            this.refMap[moduleName] = {
+                pathname: key,
+                value: module[Object.keys(module)[0]],
+            };
+        });
+
+        const changeNest = type => {
+            if (typeof type !== 'object') {
+                return;
+            }
+            if (typeof type.key === 'object') {
+                changeNest(type.key);
+            }
+            if (typeof type.value === 'object') {
+                changeNest(type.value);
+            }
+
+            if (typeof type.target === 'object') {
+                changeNest(type.target);
+            }
+
+            if (typeof type.target !== 'string') {
+                return;
+            }
+            const conf = this.invertedMap[type.target];
+            // 需要改变引用包名称
+            if (conf) {
+                const moduleName = moduleMap[conf.pathname];
+                if (moduleName) {
+                    type.nested = [moduleName];
+                }
+            }
+        };
+
+        // 改变包的引用名称
+        Object.keys(this.refMap).forEach(key => {
+            const refObj = this.refMap[key];
+            refObj.value.forEach(item => {
+                (item.members || []).forEach(member => {
+                    changeNest(member.type);
+                });
+            });
+        });
+    }
+
+    /**
+   * @param {*[]} modules
+   * @param {string} pathname
+   * @param {boolean} [isEntry = false]
+   */
+    setRefMap(modules, pathname, isEntry = false) {
+        modules.forEach(({
+            id,
+            type,
+            value,
+        }) => {
+            assert(type === 'module', `File ${id} is not a tars module.`);
+            if (!this.refMap[id]) {
+                if (isEntry) {
+                    this.entry.push(id);
+                }
+
+                this.refMap[id] = {};
+                this.refMap[id].pathname = pathname;
+                this.refMap[id].value = value;
+            } else {
+                this.refMap[id].value = this.refMap[id].value.concat(value);
+            }
+        });
+    }
+
+
+    /**
+     * @param {*[]} modules
+     * @param {string} pathname
+     * @param {boolean} [isEntry = false]
+     */
+    setRefMapRecuriveMerge(modules, pathname, isEntry = false) {
+        return modules.map(({ id, type, value }) => {
+            assert(type === 'module', `File ${id} is not a tars module.`);
+            if (!this.refMap[id]) {
+                if (isEntry) {
+                    this.entry.push(id);
+                    this.entryFilepath = pathname;
+                }
+
+                this.refMap[id] = {};
+                this.refMap[id].pathname = pathname;
+                this.refMap[id].value = value;
+            } else {
+                // 判断文件路径是否已经被使用。由于用户定义的协议的 module name 不跟随文件名
+                if (this.refFilePath.get(`${pathname}${id}`)) {
+                    // console.error('文件已被加载>>>', this.refFilePath.get(`${pathname}${id}`), pathname);
+                    // Object.keys(this.refMap).forEach(key => {
+                    //     console.error(`key>>>> ${key}`, this.refMap[key].value.length, pathname);
+                    // })
+                    return false;
+                }
+                this.refMap[id].value = this.refMap[id].value.concat(value);
+            }
+            // 记录加载的文件
+            this.refFilePath.set(`${pathname}${id}`, id);
+            return true;
+        }).filter(item => item).length > 0
+    }
+
+    /**
+     * @param {string} [pathname]
+     */
+    parseTars(pathname) {
+        pathname = pathname || this.pathname;
+        assert(pathname.length > 0, 'Need at least one .tars file to parse.');
+        let result;
+        try {
+            result = parser.parse(fs.readFileSync(pathname, 'utf8'));
+        } catch (e) {
+            console.error('\x1b[31m', `💥 Parse ${pathname} fail.`);
+            if (e.location && e.location.start && e.location.end) {
+                console.error('\x1b[31m', `💥 Error line: start at ${e.location.start.line} end at ${e.location.end.line}`);
+                console.error('\x1b[31m', `💥 Error column: start at ${e.location.start.column} end at ${e.location.end.column}`);
+            }
+            console.error('\x1b[31m', `💥 ${e.stack}`);
+            throw new Error('An error occured, compiling stoped');
+        }
+        const refs = /** @type {{type: string, target: string}[]} */ (result).filter(({ type }) => type === 'ref');
+        const modules = /** @type {{type: string, target: string}[]} */ (result).filter(({ type }) => type === 'module');
+        assert(modules.length > 0, 'A file must include at least one module.');
+
+        const parsed = {
+            refs,
+            modules,
+            pathname,
+        };
+        this.parsed[pathname] = parsed;
+        return parsed;
+    }
+
+    /**
+     * @param {{type: string, target: string}[]} refs
+     * @param {string} pathname
+     */
+    parseRefs(refs, pathname) {
+        pathname = pathname || this.pathname;
+        const { dir } = path.parse(pathname);
+        return refs.map(ref => {
+            let targetPath = path.resolve(dir, ref.target);
+            let result = this.parsed[targetPath];
+            if (!result) {
+                if (!fs.existsSync(targetPath)) {
+                    const { base } = path.parse(targetPath);
+                    const cwd = path.resolve(dir, '../../'); // 从上2级目录开始找
+                    const filePaths = globby.sync(`**/${base}`, {
+                        cwd,
+                    });
+                    if (filePaths.length) {
+                        targetPath = path.resolve(cwd, filePaths[0]);
+                    } else {
+                        throw new Error(`找不到${pathname}依赖的${base}文件！`);
+                    }
+                }
+                result = this.parseTars(targetPath);
+            }
+            return result;
+        });
+    }
+
+    /**
+     * 递归处理引用
+     * @param {{type: string, target: string}[]} refList
+     * @param {string} pathname
+     */
+    handleRefRecursivelyMerge(refList, pathname, init = false) {
+        // 入口文件只需要执行一次。不需要重复导入
+        if (this.entryFilepath == pathname && !init) {
+            return;
+        }
+        const parsedRefs = this.parseRefs(refList, pathname);
+        parsedRefs.forEach(({ refs, pathname: _pathname, modules }) => {
+            const hasNew = this.setRefMapRecuriveMerge(modules, _pathname);
+            // 判断是否有新增文件，有新增文件，需要递归查找引用文件
+            if (!hasNew) {
+                // console.error("Recursively >>> error", _pathname);
+                return;
+            }
+            this.handleRefRecursivelyMerge(refs, _pathname);
+        });
+    }
+}
+
+function buildParser() {
+    const content = fs.readFileSync(path.resolve(__dirname, PEG_SOURCE), 'utf8');
+    const source = content.toString();
+    return peg.generate(source);
+}
+
+module.exports = AST;

package/main.d.ts

@@ -0,0 +1,29 @@
+export interface Config {
+  withReturn?: boolean;
+  interface?: boolean;
+  gather?: boolean;
+  long?: 'string'|'number'|'bigint';
+}
+
+export declare function run(
+  entry: string,
+  type: 'ts' | 'es5',
+  dir?: string,
+  mode?: 'service' | 'client' | 'all',
+  config?: Config
+): Promise<Array<void>>;
+
+export declare function process(
+  entry: string,
+  type: 'ts' | 'es5',
+  dir?: string,
+  mode?: 'service' | 'client' | 'all',
+  config?: Config
+): Promise<object>;
+
+export interface AST {
+  entry: string[];
+  buildAST(): object;
+}
+
+export declare function ast(entry: string): AST;

package/main.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const path = require('path');
+
+const Generator = require('./lib/generator/ts/generator');
+const TsParser = require('./lib/parser/parser');
+
+function runner(entry, type, dir, mode = 'service', config = {}) {
+  dir = dir || path.parse(entry).dir;
+
+  const parser = new TsParser(entry);
+  let ast
+    if (config.useModuleName === false) {
+        ast = parser.buildAST()
+    } else if (config.fixRecurAndMerge) {
+        ast = parser.buildASTRecursiveMerge()
+    } else { 
+        ast = parser.buildFileAST()
+    }
+  return new Generator(ast, entry, {
+    mode,
+    type,
+    output: dir,
+    long: 'string',
+    ...config,
+  });
+}
+
+module.exports.ast = entry => new TsParser(entry).buildAST();
+module.exports.memo = (...args) => runner(...args).memo();
+module.exports.run = (...args) => runner(...args).build();
+module.exports.process = (...args) => {
+  const generator = runner(...args);
+  return generator.build().then(() => generator.getTree());
+};

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@cicctencent/tars2node-cli",
+  "version": "0.0.1",
+  "description": "",
+  "main": "main.js",
+  "types": "main.d.ts",
+  "access": "public",
+  "files": [
+    "lib",
+    "bin",
+    "main.d.ts",
+    "jce.pegjs"
+  ],
+  "scripts": {
+    "test": "mocha --recursive \"./test/**/*.test.js\" -w --watch-extensions js",
+    "test:lint": "eslint main.js bin/ lib/"
+  },
+  "author": "liquidliang",
+  "original_author": "kealxiao",
+  "license": "MIT",
+  "bin": {
+    "tars": "./bin/tars.js"
+  },
+  "dependencies": {
+    "chokidar": "^2.1.8",
+    "commander": "^2.20.3",
+    "fs-extra": "^8.1.0",
+    "globby": "^11.1.0",
+    "pegjs": "^0.10.0",
+    "typescript": "^3.9.10"
+  },
+  "devDependencies": {
+    "@tars/rpc": "latest",
+    "@tars/stream": "latest",
+    "@types/fs-extra": "^8.1.5",
+    "@types/node": "^10.17.60",
+    "@types/pegjs": "^0.10.6",
+    "chai": "^4.5.0",
+    "eslint": "^5.16.0",
+    "eslint-config-airbnb-base": "^13.2.0",
+    "eslint-plugin-import": "^2.32.0",
+    "mocha": "^5.2.0"
+  }
+}