node-adlt 0.16.1

package/README.md

@@ -0,0 +1,30 @@
+# node-adlt
+
+This is an npm module for using [adlt](https://github.com/mbehr1/adlt) in a Node project. It's used by VS Code Extension [dlt-logs](https://github.com/mbehr1/dlt-logs).
+
+## How it works
+
+- adlt is built in [adlt](https://github.com/mbehr1/adlt) and published to releases for each tag in that repo.
+- In this module's postinstall task, it determines which platform it is being installed on and downloads the correct binary from adlt for the platform.
+- The path to the adlt binary is exported as `adltPath`.
+- This modules version reflects 1:1 the adlt version.
+
+### Usage example
+
+```js
+const { adltPath } = require('node-adlt');
+
+// child_process.spawn(adltPath, ...)
+```
+
+### Dev note
+
+Runtime dependencies are not allowed in this project. This code runs on postinstall, and any dependencies would only be needed for postinstall, but they would have to be declared as `dependencies`, not `devDependencies`. Then if they were not cleaned up manually, they would end up being included in any project that uses this. I allow `https-proxy-agent` as an exception because we already ship that in VS Code, and `proxy-from-env` because it's very small and much easier to use it than reimplement it.
+
+### GitHub API Limit note
+
+You can produce an API key, set the GITHUB_TOKEN environment var to it, and vscode-ripgrep will use it when downloading from GitHub. This increases your API limit.
+
+### License
+
+This code/project is licensed under MIT license (as its mainly a copy+paste from microsoft/vscode-ripgrep repo/examples). But the adlt binary installed is licensed under CC-BY-NC-SA-4.0 so I reflect that here to avoid confusion on installing it in a node project.

package/jsconfig.json

@@ -0,0 +1,8 @@
+{
+    "compilerOptions": {
+        "resolveJsonModule": true,
+        "lib": [
+            "esnext"
+        ]
+    }
+}

package/lib/download.js

@@ -0,0 +1,328 @@
+// @ts-check
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const os = require('os');
+const https = require('https');
+const util = require('util');
+const url = require('url');
+const URL = url.URL;
+const child_process = require('child_process');
+const proxy_from_env = require('proxy-from-env');
+
+const packageVersion = require('../package.json').version;
+const tmpDir = path.join(os.tmpdir(), `node-adlt-cache-${packageVersion}`);
+
+const fsUnlink = util.promisify(fs.unlink);
+const fsExists = util.promisify(fs.exists);
+const fsMkdir = util.promisify(fs.mkdir);
+
+const isWindows = os.platform() === 'win32';
+
+const REPO = 'mbehr1/adlt';
+
+function isGithubUrl(_url) {
+    return url.parse(_url).hostname === 'api.github.com';
+}
+
+function downloadWin(url, dest, opts) {
+    return new Promise((resolve, reject) => {
+        let userAgent;
+        if (opts.headers['user-agent']) {
+            userAgent = opts.headers['user-agent'];
+            delete opts.headers['user-agent'];
+        }
+        const headerValues = Object.keys(opts.headers)
+            .map(key => `\\"${key}\\"=\\"${opts.headers[key]}\\"`)
+            .join('; ');
+        const headers = `@{${headerValues}}`;
+        console.log('Downloading with Invoke-WebRequest');
+        dest = sanitizePathForPowershell(dest);
+        let iwrCmd = `[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; Invoke-WebRequest -URI ${url} -UseBasicParsing -OutFile ${dest} -Headers ${headers}`;
+        if (userAgent) {
+            iwrCmd += ' -UserAgent ' + userAgent;
+        }
+        if (opts.proxy) {
+            iwrCmd += ' -Proxy ' + opts.proxy;
+
+            try {
+                const { username, password } = new URL(opts.proxy);
+                if (username && password) {
+                    const decodedPassword = decodeURIComponent(password);
+                    iwrCmd += ` -ProxyCredential (New-Object PSCredential ('${username}', (ConvertTo-SecureString '${decodedPassword}' -AsPlainText -Force)))`;
+                }
+            } catch (err) {
+                reject(err);
+            }
+        }
+
+        iwrCmd = `powershell "${iwrCmd}"`;
+
+        child_process.exec(iwrCmd, err => {
+            if (err) {
+                reject(err);
+                return;
+            }
+            resolve();
+        });
+    });
+}
+
+function download(_url, dest, opts) {
+
+    const proxy = proxy_from_env.getProxyForUrl(url.parse(_url));
+    if (proxy !== '') {
+        var HttpsProxyAgent = require('https-proxy-agent');
+        opts = {
+            ...opts,
+            "agent": HttpsProxyAgent(proxy),
+            proxy
+        };
+    }
+
+    if (isWindows) {
+        // This alternative strategy shouldn't be necessary but sometimes on Windows the file does not get closed,
+        // so unzipping it fails, and I don't know why.
+        return downloadWin(_url, dest, opts);
+    }
+
+    if (opts.headers && opts.headers.authorization && !isGithubUrl(_url)) {
+        delete opts.headers.authorization;
+    }
+
+    return new Promise((resolve, reject) => {
+        console.log(`Download options: ${JSON.stringify(opts)}`);
+        const outFile = fs.createWriteStream(dest);
+        const mergedOpts = {
+            ...url.parse(_url),
+            ...opts
+        };
+        https.get(mergedOpts, response => {
+            console.log('statusCode: ' + response.statusCode);
+            if (response.statusCode === 302) {
+                console.log('Following redirect to: ' + response.headers.location);
+                return download(response.headers.location, dest, opts)
+                    .then(resolve, reject);
+            } else if (response.statusCode !== 200) {
+                reject(new Error('Download failed with ' + response.statusCode));
+                return;
+            }
+
+            response.pipe(outFile);
+            outFile.on('finish', () => {
+                resolve();
+            });
+        }).on('error', async err => {
+            await fsUnlink(dest);
+            reject(err);
+        });
+    });
+}
+
+function get(_url, opts) {
+    console.log(`GET ${_url}`);
+
+    const proxy = proxy_from_env.getProxyForUrl(url.parse(_url));
+    if (proxy !== '') {
+        var HttpsProxyAgent = require('https-proxy-agent');
+        opts = {
+            ...opts,
+            "agent": HttpsProxyAgent(proxy)
+        };
+    }
+
+    return new Promise((resolve, reject) => {
+        let result = '';
+        opts = {
+            ...url.parse(_url),
+            ...opts
+        };
+        https.get(opts, response => {
+            if (response.statusCode !== 200) {
+                reject(new Error('Request failed: ' + response.statusCode));
+            }
+
+            response.on('data', d => {
+                result += d.toString();
+            });
+
+            response.on('end', () => {
+                resolve(result);
+            });
+
+            response.on('error', e => {
+                reject(e);
+            });
+        });
+    });
+}
+
+function getApiUrl(repo, tag) {
+    return `https://api.github.com/repos/${repo}/releases/tags/${tag}`;
+}
+
+/**
+ * @param {{ force: boolean; token: string; version: string; }} opts
+ * @param {string} assetName
+ * @param {string} downloadFolder
+ */
+async function getAssetFromGithubApi(opts, assetName, downloadFolder) {
+    const assetDownloadPath = path.join(downloadFolder, assetName);
+
+    // We can just use the cached binary
+    if (!opts.force && await fsExists(assetDownloadPath)) {
+        console.log('Using cached download: ' + assetDownloadPath);
+        return assetDownloadPath;
+    }
+
+    const downloadOpts = {
+        headers: {
+            'user-agent': 'node-adlt'
+        }
+    };
+
+    if (opts.token) {
+        downloadOpts.headers.authorization = `token ${opts.token}`;
+    }
+
+    console.log(`Finding release for ${opts.version}`);
+    const release = await get(getApiUrl(REPO, opts.version), downloadOpts);
+    let jsonRelease;
+    try {
+        jsonRelease = JSON.parse(release);
+    } catch (e) {
+        throw new Error('Malformed API response: ' + e.stack);
+    }
+
+    if (!jsonRelease.assets) {
+        throw new Error('Bad API response: ' + JSON.stringify(release));
+    }
+
+    const asset = jsonRelease.assets.find(a => a.name === assetName);
+    if (!asset) {
+        throw new Error('Asset not found with name: ' + assetName);
+    }
+
+    console.log(`Downloading from ${asset.url}`);
+    console.log(`Downloading to ${assetDownloadPath}`);
+
+    downloadOpts.headers.accept = 'application/octet-stream';
+    await download(asset.url, assetDownloadPath, downloadOpts);
+}
+
+function unzipWindows(zipPath, destinationDir) {
+    return new Promise((resolve, reject) => {
+        zipPath = sanitizePathForPowershell(zipPath);
+        destinationDir = sanitizePathForPowershell(destinationDir);
+        const expandCmd = 'powershell -ExecutionPolicy Bypass -Command Expand-Archive ' + ['-Path', zipPath, '-DestinationPath', destinationDir, '-Force'].join(' ');
+        child_process.exec(expandCmd, (err, _stdout, stderr) => {
+            if (err) {
+                reject(err);
+                return;
+            }
+
+            if (stderr) {
+                console.log(stderr);
+                reject(new Error(stderr));
+                return;
+            }
+
+            console.log('Expand-Archive completed');
+            resolve();
+        });
+    });
+}
+
+// Handle whitespace in filepath as powershell split's path with whitespaces
+function sanitizePathForPowershell(path) {
+    path = path.replace(/ /g, '` '); // replace whitespace with "` " as solution provided here https://stackoverflow.com/a/18537344/7374562
+    return path;
+}
+
+function untar(zipPath, destinationDir) {
+    return new Promise((resolve, reject) => {
+        const unzipProc = child_process.spawn('tar', ['xvf', zipPath, '-C', destinationDir], { stdio: 'inherit' });
+        unzipProc.on('error', err => {
+            reject(err);
+        });
+        unzipProc.on('close', code => {
+            console.log(`tar xvf exited with ${code}`);
+            if (code !== 0) {
+                reject(new Error(`tar xvf exited with ${code}`));
+                return;
+            }
+
+            resolve();
+        });
+    });
+}
+
+async function unzipAdlt(zipPath, destinationDir) {
+    if (isWindows) {
+        await unzipWindows(zipPath, destinationDir);
+    } else {
+        await untar(zipPath, destinationDir);
+    }
+
+    // the binary is extracted into a folder like adlt-vx.y.z-target/
+    // determine that folder name:
+    const dirEntries = fs.readdirSync(destinationDir);
+    let adltDir = dirEntries.find((dirEntry) => dirEntry.startsWith("adlt-v"));
+
+    const expectedName = path.join(destinationDir, adltDir, 'adlt');
+    if (await fsExists(expectedName)) {
+        return expectedName;
+    }
+
+    if (await fsExists(expectedName + '.exe')) {
+        return expectedName + '.exe';
+    }
+
+    throw new Error(`Expecting adlt or adlt.exe unzipped into ${destinationDir}, didn't find one.`);
+}
+
+module.exports = async opts => {
+    if (!opts.version) {
+        return Promise.reject(new Error('Missing version'));
+    }
+
+    if (!opts.target) {
+        return Promise.reject(new Error('Missing target'));
+    }
+
+    const extension = isWindows ? '.zip' : '.tar.gz';
+    const assetName = ['adlt', opts.version, opts.target].join('-') + extension;
+
+    if (!await fsExists(tmpDir)) {
+        await fsMkdir(tmpDir);
+    }
+
+    const assetDownloadPath = path.join(tmpDir, assetName);
+    try {
+        await getAssetFromGithubApi(opts, assetName, tmpDir)
+    } catch (e) {
+        console.log('Deleting invalid download cache');
+        try {
+            await fsUnlink(assetDownloadPath);
+        } catch (e) { }
+
+        throw e;
+    }
+
+    console.log(`Unzipping to ${opts.destDir}`);
+    try {
+        const destinationPath = await unzipAdlt(assetDownloadPath, opts.destDir);
+        if (!isWindows) {
+            await util.promisify(fs.chmod)(destinationPath, '755');
+        }
+    } catch (e) {
+        console.log('Deleting invalid download');
+
+        try {
+            await fsUnlink(assetDownloadPath);
+        } catch (e) { }
+
+        throw e;
+    }
+};

package/lib/index.d.ts

@@ -0,0 +1 @@
+export declare const adltPath: string;

package/lib/index.js

@@ -0,0 +1,8 @@
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const dirEntries = fs.readdirSync(path.join(__dirname, "../bin"));
+let adltDir = dirEntries.find((dirEntry) => dirEntry.startsWith("adlt-v"));
+
+module.exports.adltPath = path.join(__dirname, adltDir, `adlt${process.platform === 'win32' ? '.exe' : ''}`);

package/lib/postinstall.js

@@ -0,0 +1,91 @@
+// @ts-check
+'use strict';
+
+const os = require('os');
+const fs = require('fs');
+const path = require('path');
+const util = require('util');
+const child_process = require('child_process');
+
+const download = require('./download');
+
+const fsExists = util.promisify(fs.exists);
+const mkdir = util.promisify(fs.mkdir);
+const exec = util.promisify(child_process.exec);
+
+const forceInstall = process.argv.includes('--force');
+if (forceInstall) {
+    console.log('--force, ignoring caches');
+}
+
+// we do use this package version as well for the binary
+const { version } = require('./../package.json');
+const BIN_PATH = path.join(__dirname, '../bin');
+
+process.on('unhandledRejection', (reason, promise) => {
+    console.log('Unhandled rejection: ', promise, 'reason:', reason);
+});
+
+async function isMusl() {
+    let stderr;
+    try {
+        stderr = (await exec('ldd --version')).stderr;
+    } catch (err) {
+        stderr = err.stderr;
+    }
+    if (stderr.indexOf('musl') > -1) {
+        return true;
+    }
+    return false;
+}
+
+async function getTarget() {
+    const arch = process.env.npm_config_arch || os.arch();
+
+    switch (os.platform()) {
+        case 'darwin':
+            return arch === 'arm64' ? 'aarch64-apple-darwin' :
+                'x86_64-apple-darwin';
+        case 'win32':
+            return arch === 'x64' ? 'x86_64-pc-windows-msvc' :
+                arch === 'arm' ? 'aarch64-pc-windows-msvc' :
+                    'i686-pc-windows-msvc';
+        case 'linux':
+            return arch === 'x64' ? 'x86_64-unknown-linux-musl' :
+                arch === 'arm' ? 'arm-unknown-linux-gnueabihf' :
+                    arch === 'armv7l' ? 'arm-unknown-linux-gnueabihf' :
+                        arch === 'arm64' ? await isMusl() ? 'aarch64-unknown-linux-musl' : 'aarch64-unknown-linux-gnu' :
+                            arch === 'ppc64' ? 'powerpc64le-unknown-linux-gnu' :
+                                arch === 's390x' ? 's390x-unknown-linux-gnu' :
+                                    'i686-unknown-linux-musl'
+        default: throw new Error('Unknown platform: ' + os.platform());
+    }
+}
+
+async function main() {
+    const binExists = await fsExists(BIN_PATH);
+    if (!forceInstall && binExists) {
+        console.log('bin/ folder already exists, exiting');
+        process.exit(0);
+    }
+
+    if (!binExists) {
+        await mkdir(BIN_PATH);
+    }
+
+    const opts = {
+        version: `v${version}`,
+        token: process.env['GITHUB_TOKEN'],
+        target: await getTarget(),
+        destDir: BIN_PATH,
+        force: forceInstall
+    };
+    try {
+        await download(opts);
+    } catch (err) {
+        console.error(`Downloading adlt failed: ${err.stack}`);
+        process.exit(1);
+    }
+}
+
+main();

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "node-adlt",
+  "version": "0.16.1",
+  "description": "A npm module for using adlt binary in a Node project. Used e.g. by VS Code extension dlt-logs.",
+  "main": "lib/index.js",
+  "typings": "lib/index.d.ts",
+  "scripts": {
+    "postinstall": "node ./lib/postinstall.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mbehr1/node-adlt"
+  },
+  "keywords": [
+    "dlt",
+    "binaries",
+    "log+trace"
+  ],
+  "author": "Matthias Behr",
+  "license": "CC-BY-NC-SA-4.0",
+  "bugs": {
+    "url": "https://github.com/mbehr1/node-adlt/issues"
+  },
+  "homepage": "https://github.com/mbehr1/node-adlt#readme",
+  "dependencies": {
+    "https-proxy-agent": "^5.0.0",
+    "proxy-from-env": "^1.1.0"
+  },
+  "overrides": {
+    "agent-base": "^6.0.2"
+  },
+  "devDependencies": {
+    "@types/node": "^10.12.0"
+  }
+}

package/test/adltPathNotEmpty.js

@@ -0,0 +1,4 @@
+const assert = require('assert');
+
+const { adltPath } = require('../lib/index.js');
+assert(!adltPath.len > 0);