@bratel/dgit 0.0.13

package/src/cmd/action.ts

@@ -0,0 +1,128 @@
+import { Command } from 'commander';
+import ora, { Ora } from 'ora';
+import ProgressBar from 'progress';
+import dgit from '../dgit';
+import {
+    ParseGithubHttpsLink, TextEllipsis, isHttpsLink,
+} from './utils';
+import { CommandInfo } from './type';
+
+import { DownloadPrompt, PasswordPrompt } from './prompt';
+
+const MAX_TEXT_ELLIPSIS = 30;
+
+const DownloadAction = async (
+    githubLink: string | undefined,
+    cmd: Command & CommandInfo,
+): Promise<any> => {
+    let {
+        ref = '',
+        dest = '',
+        owner = '',
+        repoName = '',
+        relativePath = '',
+        password,
+    } = cmd;
+
+    const {
+        exclude = '',
+        include = '',
+        log = false,
+        logPrefix = '[dgit-logger]',
+    } = cmd;
+
+    const {
+        parallelLimit = '', username, token,
+    } = cmd;
+    const { proxy = '' } = cmd;
+
+    if (githubLink && isHttpsLink(githubLink)) {
+        const parseResult = ParseGithubHttpsLink(githubLink);
+        ref = parseResult.ref;
+        owner = parseResult.owner;
+        repoName = parseResult.repoName;
+        relativePath = parseResult.relativePath;
+    }
+
+    if (username && !password) {
+        const pwdAnswer = await PasswordPrompt();
+        password = pwdAnswer.password;
+    }
+
+    const answer = await DownloadPrompt({
+        ref,
+        dest,
+        owner,
+        repoName,
+        relativePath,
+    });
+
+    ref = answer.ref;
+    dest = answer.dest;
+    owner = answer.owner;
+    repoName = answer.repoName;
+    relativePath = answer.relativePath;
+
+    const excludeList = exclude.split(',').filter(Boolean);
+    const includeList = include.split(',').filter(Boolean);
+
+    const spinner: Ora = ora(' loading remote repo tree...');
+    let bar: ProgressBar;
+
+    try {
+        await dgit(
+            {
+                ref,
+                owner,
+                repoName,
+                relativePath,
+                username,
+                password,
+                token,
+                proxy,
+            },
+            dest,
+            {
+                log,
+                logPrefix,
+                parallelLimit: Number(parallelLimit.trim()),
+                exclude      : excludeList,
+                include      : includeList,
+            },
+            {
+                beforeLoadTree () {
+                    spinner.start();
+                },
+                afterLoadTree () {
+                    spinner.succeed(' load remote repo tree succeed! ');
+                },
+                onResolved (status) {
+                    if (log) return;
+                    const green = '\u001b[42m \u001b[0m';
+                    const red = '\u001b[41m \u001b[0m';
+                    const index = 0;
+                    bar = new ProgressBar(
+                        ' DOWNLOAD |:bar| :current/:total :percent elapsed: :elapseds eta: :eta :file, done.',
+                        {
+                            total     : status.totalCount,
+                            width     : 50,
+                            complete  : green,
+                            incomplete: red,
+                        },
+                    );
+                    bar.update(index);
+                },
+                onProgress (_, node) {
+                    if (log) return;
+                    bar.tick({ file: TextEllipsis(node.path, MAX_TEXT_ELLIPSIS) });
+                },
+            },
+        );
+        spinner.succeed(' download all files succeed!');
+    } catch (error) {
+        console.error(error);
+        spinner.fail(' download files failed!');
+    }
+};
+
+export default DownloadAction;

package/src/cmd/main.ts

@@ -0,0 +1,70 @@
+import program from 'commander';
+import chalk from 'chalk';
+
+import { PackageInfo } from './type';
+import { GetPackageInfo } from './utils';
+import DownloadAction from './action';
+
+const EXIT_CODE = 1;
+
+const Exit = (): void => {
+    process.exit(EXIT_CODE);
+};
+
+const UnknownCommand = (cmdName: string): void => {
+    console.log(`${ chalk.red('Unknown command') } ${ chalk.yellow(cmdName) }.`);
+};
+
+const packageInfo: PackageInfo = GetPackageInfo();
+
+program.version(packageInfo.version);
+
+program
+    .command('download [githubLink]')
+    .option('--owner <ownerName>', 'git repo author.')
+    .option('--repo-name <repoName>', 'git repo name.')
+    .option('--ref <refName>', 'git repo branch, commit hash or tagname.')
+    .option(
+        '--relative-path <relativePath>',
+        'specified repo relative path to download.',
+    )
+    .option('-d, --dest <destPath>', 'specified dest path.')
+    .option(
+        '-l, --parallel-limit, <number>',
+        'specified download max parallel limit.',
+    )
+    .option('-u, --username, <username>', 'specified git account username.')
+    .option('-p --password, <password>', 'specified git account password.')
+    .option(
+        '-t --token, <token>',
+        'specified git account personal access token.',
+    )
+    .option(
+        '-e --exclude, <relativePath,...,relativePath>',
+        'indicates which file paths need to be excluded in the current directory.',
+    )
+    .option(
+        '-i --include, <relativePath,...,relativePath>',
+        'indicates which files need to be included in the exclusion file list.',
+    )
+    .option('--log', 'output dgit internal log details.')
+    .option('--log-prefix, <log>', 'dgit internal log prefix.')
+    .option('--proxy, <proxyHttp>', 'dgit proxy download url.')
+    .alias('d')
+    .description('download the file with the specified path of the remote repo.')
+    .action(DownloadAction);
+
+program.on('command:*', (cmdObj = []) => {
+    const [ cmd ] = cmdObj;
+    if (cmd) {
+        program.outputHelp();
+        UnknownCommand(cmd);
+        Exit();
+    }
+});
+
+if (process.argv.slice(2).length <= 0) {
+    program.help();
+}
+
+program.parse(process.argv);

package/src/cmd/prompt.ts

@@ -0,0 +1,98 @@
+import inquirer, { Question } from 'inquirer';
+import { DownloadPromptInfo, PasswordPromptInfo } from './type';
+
+export const CreatePrompt = (questions: Array<Question>): Promise<any> => inquirer.prompt(questions);
+
+export const DownloadPrompt = async (
+    currentInfo: DownloadPromptInfo,
+): Promise<DownloadPromptInfo> => {
+    if (
+        currentInfo.owner &&
+        currentInfo.repoName &&
+        currentInfo.ref &&
+        currentInfo.relativePath &&
+        currentInfo.dest
+    ) return currentInfo;
+
+    const questions = [
+        {
+            type: 'input',
+            name: 'owner',
+            when () {
+                return !currentInfo.owner;
+            },
+            validate (input: string) {
+                return input && input.length > 0;
+            },
+            message: 'input github ownername.',
+        },
+        {
+            type: 'input',
+            name: 'repoName',
+            when () {
+                return !currentInfo.repoName;
+            },
+            validate (input: string) {
+                return input && input.length > 0;
+            },
+            message: 'input github repoName.',
+        },
+        {
+            type: 'input',
+            name: 'ref',
+            when () {
+                return !currentInfo.ref;
+            },
+            validate (input: string) {
+                return input && input.length > 0;
+            },
+            'default': 'master',
+            message  : 'input github branch or commit hash or tagname.',
+        },
+        {
+            type: 'input',
+            name: 'relativePath',
+            when () {
+                return !currentInfo.relativePath;
+            },
+            validate (input: string) {
+                return input && input.length > 0;
+            },
+            'default': '.',
+            message  : 'input github relative path.',
+        },
+        {
+            type: 'input',
+            name: 'dest',
+            when () {
+                return !currentInfo.dest;
+            },
+            validate (input: string) {
+                return input && input.length > 0;
+            },
+            'default': '.',
+            message  : 'input template output dest path.',
+        },
+    ];
+
+    const answer = await CreatePrompt(questions);
+    return {
+        owner       : answer.owner || currentInfo.owner,
+        dest        : answer.dest || currentInfo.dest,
+        repoName    : answer.repoName || currentInfo.repoName,
+        relativePath: answer.relativePath || currentInfo.relativePath,
+        ref         : answer.ref || currentInfo.ref,
+    };
+};
+
+export const PasswordPrompt = (): Promise<PasswordPromptInfo> => {
+    const question = {
+        type: 'password',
+        name: 'password',
+        validate (input: string) {
+            return input && input.length > 0;
+        },
+        message: 'input github account password.',
+    };
+    return CreatePrompt([ question ]);
+};

package/src/cmd/type.ts

@@ -0,0 +1,41 @@
+export interface PackageInfo {
+    version: string;
+    name: string;
+}
+
+export interface CommandInfo {
+    dest?: string;
+    owner?: string;
+    repoName?: string;
+    ref?: string;
+    relativePath?: string;
+    parallelLimit?: string;
+    username?: string;
+    password?: string;
+    token?: string;
+    exclude?: string;
+    include?: string;
+    log?: boolean;
+    logPrefix?: string;
+    proxy?: string;
+}
+
+export interface DownloadPromptInfo {
+    dest: string;
+    owner: string;
+    repoName: string;
+    ref: string;
+    relativePath: string;
+}
+
+export interface GithubLinkInfo {
+    owner: string;
+    repoName: string;
+    ref: string;
+    relativePath: string;
+    type: string;
+}
+
+export interface PasswordPromptInfo {
+    password: string;
+}

package/src/cmd/utils.ts

@@ -0,0 +1,88 @@
+import fs from 'fs';
+import path from 'path';
+import { PackageInfo, GithubLinkInfo } from './type';
+
+export const GetPackageInfo = (): PackageInfo => {
+    const buffer = fs.readFileSync(path.resolve(__dirname, '../../package.json'));
+    return JSON.parse(buffer.toString());
+};
+
+export const GITHUB_ADDRESS = 'https://github.com/';
+export const isHttpsLink = (link: string) => link.trim().startsWith(GITHUB_ADDRESS);
+
+export const ParseGithubHttpsLink = (httpsLink: string): GithubLinkInfo => {
+    let nextLink = httpsLink.trim().slice(GITHUB_ADDRESS.length);
+    let index = nextLink.indexOf('/');
+    if (index === -1) throw new Error('invalid github address.');
+    const owner = nextLink.slice(0, index);
+    nextLink = nextLink.slice(owner.length + 1);
+    index = nextLink.indexOf('/');
+    let repoName: string;
+    if (index === -1) {
+        repoName = nextLink.slice(0);
+        if (!repoName) throw new Error('invalid github address.');
+        return {
+            owner,
+            repoName,
+            ref         : 'master',
+            relativePath: '',
+            type        : 'tree',
+        };
+    }
+    repoName = nextLink.slice(0, index);
+    nextLink = nextLink.slice(repoName.length + 1);
+    index = nextLink.indexOf('/');
+    let ref = 'master';
+    let relativePath = '';
+    let type = 'tree';
+    if (index === -1) {
+        if (repoName.endsWith('.git')) {
+            const lastIndex = -4;
+            repoName = repoName.slice(0, lastIndex);
+        }
+    } else {
+        type = nextLink.slice(0, index);
+        nextLink = nextLink.slice(type.length + 1);
+        index = nextLink.indexOf('/');
+        if (index === -1) {
+            ref = nextLink.slice(0) || 'master';
+        } else {
+            ref = nextLink.slice(0, index);
+            relativePath = nextLink.slice(ref.length + 1);
+        }
+    }
+
+    return {
+        owner,
+        repoName,
+        ref,
+        relativePath,
+        type,
+    };
+};
+
+export const TextEllipsis = (text: string, maxLen: number): string => (text.length >= maxLen ? `${ text.slice(0, maxLen) }...` : text);
+
+export const MakeDirs = (dirs: string): void => {
+    const mkdirs = (dir: string, callback?: ()=> void) => {
+        if (fs.existsSync(dir)) {
+            callback && callback();
+            return;
+        }
+
+        mkdirs(path.dirname(dir), () => {
+            fs.mkdirSync(dir);
+            callback && callback();
+        });
+    };
+
+    if (fs.existsSync(dirs)) return;
+    mkdirs(dirs);
+};
+
+export const AddExtraRandomQs = (origin: string): string => {
+    if (origin.indexOf('?') !== -1) {
+        return `${ origin }&_t=${ Math.random() }`;
+    }
+    return `${ origin }?_t=${ Math.random() }`;
+};

package/src/dgit.ts

@@ -0,0 +1,256 @@
+import fs from 'fs';
+import path from 'path';
+import async from 'async';
+import repo from './repo';
+import { createLogger } from './log';
+import { requestGetPromise, requestOnStream } from './request';
+import {
+    DgitGlobalOption,
+    RepoOptionType,
+    RepoTreeNode,
+    DgitLifeCycle,
+    DgitLoadGitTree,
+} from './type';
+import {
+    ParseGithubHttpsLink, isHttpsLink, MakeDirs,
+} from './cmd/utils';
+
+const UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36';
+const DEFAULT_PARALLEL_LIMIT = 10;
+const MAX_PARALLEL_LIMIT = 100;
+const JSON_STRINGIFY_PADDING = 2;
+
+const dgit = async (
+    repoOption: RepoOptionType,
+    dPath: string,
+    dgitOptions?: DgitGlobalOption,
+    hooks?: DgitLifeCycle & DgitLoadGitTree,
+): Promise<void> => {
+    const {
+        username, password, token, githubLink, proxy = '',
+    } = repoOption;
+
+    let {
+        owner, repoName, ref = 'master', relativePath = '.',
+    } = repoOption;
+
+    if (githubLink && isHttpsLink(githubLink)) {
+        const parseResult = ParseGithubHttpsLink(githubLink);
+        owner = parseResult.owner;
+        repoName = parseResult.repoName;
+        ref = parseResult.ref;
+        relativePath = parseResult.relativePath;
+    }
+
+    if (!owner || !repoName) {
+        throw new Error('invalid repo option.');
+    }
+
+    const logger = createLogger(dgitOptions);
+
+    const { exclude = [], include = []} = dgitOptions || {};
+
+    let { parallelLimit = DEFAULT_PARALLEL_LIMIT } = dgitOptions || {};
+    if (!parallelLimit || parallelLimit <= 0) {
+        logger('parallelLimit value is invalid.');
+        parallelLimit = DEFAULT_PARALLEL_LIMIT;
+    }
+
+    parallelLimit > MAX_PARALLEL_LIMIT && (parallelLimit = MAX_PARALLEL_LIMIT);
+
+    const {
+        onSuccess,
+        onError,
+        onProgress,
+        onFinish,
+        onRetry,
+        onResolved,
+        beforeLoadTree,
+        afterLoadTree,
+    } = hooks || {};
+
+    let onSuccessResolve: (data?: any)=> void = () => {};
+    let onErrorReject: (err?: any)=> void = () => {};
+
+    const prom: Promise<void> = new Promise((resolve, reject) => {
+        onSuccessResolve = resolve;
+        onErrorReject = reject;
+    });
+
+    const { getRepoTreeUrl, getDownloadUrl } = repo(owner, repoName, ref, proxy);
+    const url = getRepoTreeUrl();
+
+    const headers = {
+        'User-Agent' : UserAgent,
+        Authorization: token ? `token ${ token }` : undefined,
+    };
+
+    const auth = username && password ?
+        {
+            user           : username,
+            pass           : password,
+            sendImmediately: true,
+        } :
+        undefined;
+
+    const options = {
+        url, headers, auth,
+    };
+
+    const destPath = path.isAbsolute(dPath) ? dPath : path.resolve(process.cwd(), dPath);
+
+    logger(' request repo tree options.');
+    logger(JSON.stringify(options, null, JSON_STRINGIFY_PADDING));
+
+    try {
+        logger(' loading remote repo tree...');
+        beforeLoadTree && beforeLoadTree();
+        const body = await requestGetPromise(options, dgitOptions || {}, {
+            onRetry () {
+                logger(` request ${ url } failed. Retrying...`);
+                onRetry && onRetry();
+            },
+        });
+
+        logger(' loading remote repo tree succeed.');
+        afterLoadTree && afterLoadTree();
+        const result = JSON.parse(body);
+
+        if (!result.tree || result.tree.length <= 0) {
+            throw new Error('404 repo not found!');
+        }
+
+        const treeNodeList: RepoTreeNode[] = result.tree;
+        const includeTreeNodeList = treeNodeList.filter(node => {
+            const nPath = path.resolve(__dirname, node.path);
+            const rPath = path.resolve(__dirname, relativePath);
+            if (!nPath.startsWith(rPath) || node.type !== 'blob') {
+                return false;
+            }
+            if (
+                exclude.some(v => nPath.startsWith(path.resolve(rPath, v))) &&
+                include.every(v => !nPath.startsWith(path.resolve(rPath, v)))
+            ) {
+                return false;
+            }
+            return true;
+        });
+
+        if (includeTreeNodeList.length <= 0) {
+            throw new Error(`404 repo ${ relativePath } not found!`);
+        }
+
+        const totalStatus = includeTreeNodeList.reduce(
+            (prev, cur) => {
+                if (cur.type === 'blob') {
+                    prev.size += cur.size;
+                    prev.count++;
+                }
+                return prev;
+            },
+            { size: 0, count: 0 },
+        );
+
+        let currentSize = 0;
+        let currentCount = 0;
+
+        onResolved &&
+            onResolved({
+                currentSize,
+                currentCount,
+                totalSize : totalStatus.size,
+                totalCount: totalStatus.count,
+            });
+
+        logger(' include files resolved.');
+        logger(
+            '',
+            JSON.stringify({
+                currentSize,
+                currentCount,
+                totalSize : totalStatus.size,
+                totalCount: totalStatus.count,
+            }),
+        );
+
+        async.eachLimit(
+            includeTreeNodeList,
+            parallelLimit,
+            (node, callback) => {
+                const downloadUrl = getDownloadUrl(node.path);
+
+                const rPath = path.resolve(destPath, relativePath);
+                const tPath = path.resolve(destPath, node.path);
+                const root = path.resolve(destPath, '.');
+
+                let targetPath: string;
+                if (rPath === tPath) {
+                    targetPath = path.resolve(destPath, path.basename(tPath));
+                } else {
+                    targetPath = tPath.replace(rPath, root);
+                }
+
+                logger('', node.path, relativePath, targetPath);
+
+                if (!fs.existsSync(path.dirname(targetPath))) {
+                    MakeDirs(path.dirname(targetPath));
+                }
+
+                const ws = fs.createWriteStream(targetPath);
+
+                logger(` downloading from ${ downloadUrl }...`);
+
+                requestOnStream(downloadUrl, ws, dgitOptions || {}, {
+                    onSuccess () {
+                        currentCount++;
+                        currentSize += node.size;
+
+                        logger(` write file ${ node.path } succeed. 
+                            size: [${ currentSize }/${ totalStatus.size }], 
+                            count: [${ currentCount }/${ totalStatus.count }]`);
+
+                        onProgress &&
+                            onProgress(
+                                {
+                                    totalCount: totalStatus.count,
+                                    totalSize : totalStatus.size,
+                                    currentSize,
+                                    currentCount,
+                                },
+                                node,
+                            );
+
+                        callback();
+                    },
+                    onError (err) {
+                        logger('', err);
+                        callback(new Error(` request ${ downloadUrl } failed.`));
+                    },
+                    onRetry () {
+                        logger(` request ${ downloadUrl } failed. Retrying...`);
+                        onRetry && onRetry();
+                    },
+                });
+            },
+            err => {
+                if (err) {
+                    onError && onError(err);
+                    onFinish && onFinish();
+                    onErrorReject(err);
+                } else {
+                    onSuccess && onSuccess();
+                    onFinish && onFinish();
+                    onSuccessResolve();
+                }
+            },
+        );
+    } catch (error) {
+        onError && onError(error);
+        onFinish && onFinish();
+        onErrorReject(error);
+    }
+
+    return prom;
+};
+
+export default dgit;

package/src/log.ts

@@ -0,0 +1,12 @@
+import { DgitGlobalOption } from './type';
+
+const DEFAULT_PREFIX = '[dgit-logger]';
+
+export const createLogger = (option?: DgitGlobalOption) => (...message: any[]) => {
+    if (option && option.log) {
+        const prefix = option ?
+            option.logPrefix || DEFAULT_PREFIX :
+            DEFAULT_PREFIX;
+        console.log(prefix, ...message, '\n');
+    }
+};

package/src/repo.ts

@@ -0,0 +1,6 @@
+const repoUtils = (owner: string, repoName: string, ref: string, proxy: string) => ({
+    getRepoTreeUrl: () => `https://api.github.com/repos/${ owner }/${ repoName }/git/trees/${ ref }?recursive=1`,
+    getDownloadUrl: (path: string) => `${ proxy ? `${ proxy }/` : '' }https://raw.githubusercontent.com/${ owner }/${ repoName }/${ ref }/${ path }`,
+});
+
+export default repoUtils;