linguist-js 2.1.2 → 2.2.1

package/dist/cli.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const VERSION = require('../package.json').version;
 const commander_1 = require("commander");
 const index_1 = __importDefault(require("./index"));
+const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
 const colouredMsg = ([r, g, b], msg) => `\u001B[${38};2;${r};${g};${b}m${msg}${'\u001b[0m'}`;
 const hexToRgb = (hex) => [parseInt(hex.slice(1, 3), 16), parseInt(hex.slice(3, 5), 16), parseInt(hex.slice(5, 7), 16)];
 commander_1.program
@@ -21,6 +22,7 @@ commander_1.program
     .option('-q|--quick [bool]', 'Skip complex language analysis (alias for -{A|I|H|S}=false)', false)
     .option('-V|--keepVendored [bool]', 'Prevent skipping over vendored/generated files', false)
     .option('-B|--keepBinary [bool]', 'Prevent skipping over binary files', false)
+    .option('-r|--relativePaths [bool]', 'Convert absolute file paths to relative', false)
     .option('-A|--checkAttributes [bool]', 'Force the checking of gitattributes files', true)
     .option('-I|--checkIgnored [bool]', 'Force the checking of gitignore files', true)
     .option('-H|--checkHeuristics [bool]', 'Apply heuristics to ambiguous languages', true)
@@ -49,17 +51,23 @@ if (args.analyze)
     (async () => {
         // Fetch language data
         const root = args.analyze === true ? '.' : args.analyze;
-        const { files, languages, unknown } = await (0, index_1.default)(root, args);
+        const data = await (0, index_1.default)(root, args);
+        const { files, languages, unknown } = data;
+        // Get file count
+        let totalFiles = (0, walk_tree_1.default)(root).files.length;
         // Print output
         if (!args.json) {
             const sortedEntries = Object.entries(languages.results).sort((a, b) => a[1].bytes < b[1].bytes ? +1 : -1);
             const totalBytes = languages.bytes;
-            console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${files.count} files with linguist-js`);
+            console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${totalFiles} files (${totalFiles - files.count} ignored) with linguist-js`);
             console.log(`\n Language analysis results:`);
-            let i = 0;
+            let count = 0;
+            if (sortedEntries.length === 0)
+                console.log(`  None`);
+            // List parsed results
             for (const [lang, { bytes, color }] of sortedEntries) {
                 const fmtd = {
-                    index: (++i).toString().padStart(2, ' '),
+                    index: (++count).toString().padStart(2, ' '),
                     lang: lang.padEnd(24, ' '),
                     percent: (bytes / (totalBytes || 1) * 100).toFixed(2).padStart(5, ' '),
                     bytes: bytes.toLocaleString().padStart(10, ' '),
@@ -68,35 +76,33 @@ if (args.analyze)
                 console.log(`  ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B`);
             }
             console.log(` Total: ${totalBytes.toLocaleString()} B`);
+            // List unknown files/extensions
             if (unknown.bytes > 0) {
                 console.log(`\n Unknown files and extensions:`);
                 for (const [name, bytes] of Object.entries(unknown.filenames)) {
                     console.log(`  '${name}': ${bytes.toLocaleString()} B`);
                 }
                 for (const [ext, bytes] of Object.entries(unknown.extensions)) {
-                    console.log(`  '${ext}': ${bytes.toLocaleString()} B`);
+                    console.log(`  '*${ext}': ${bytes.toLocaleString()} B`);
                 }
                 console.log(` Total: ${unknown.bytes.toLocaleString()} B`);
             }
         }
-        else {
-            const data = { files, languages, unknown };
-            if (args.tree) {
-                const treeParts = args.tree.split('.');
-                let nestedData = data;
-                for (const part of treeParts) {
-                    if (!nestedData[part])
-                        throw Error(`TraversalError: Key '${part}' cannot be found on output object.`);
-                    nestedData = nestedData[part];
-                }
-                console.log(nestedData);
-            }
-            else {
-                console.log(JSON.stringify(data, null, 2).replace(/{\s+"type".+?}/sg, obj => obj.replace(/\n\s+/g, ' ')));
+        else if (args.tree) {
+            const treeParts = args.tree.split('.');
+            let nestedData = data;
+            for (const part of treeParts) {
+                if (!nestedData[part])
+                    throw Error(`TraversalError: Key '${part}' cannot be found on output object.`);
+                nestedData = nestedData[part];
             }
+            console.log(nestedData);
+        }
+        else {
+            console.dir(data, { depth: null });
         }
     })();
 else {
-    console.log(`Welcome to linguist-js, the JavaScript port of GitHub's language analyzer.`);
+    console.log(`Welcome to linguist-js, a JavaScript port of GitHub's language analyzer.`);
     console.log(`Type 'linguist --help' for a list of commands.`);
 }

package/dist/helpers/convert-glob.js

@@ -0,0 +1,11 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const glob_to_regexp_1 = __importDefault(require("glob-to-regexp"));
+function parseGitGlob(path) {
+    const globPath = `**/${path}`.replace(/\\/g, '/').replace(/\[:(space|digit):\]/g, (_, val) => `\\${val[0]}`);
+    return (0, glob_to_regexp_1.default)(globPath, { globstar: true, extended: true });
+}
+exports.default = parseGitGlob;

package/dist/helpers/convert-pcre.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 /** Convert a PCRE regex into JS. */
 function pcre(regex) {
     let finalRegex = regex;
-    let finalFlags = new Set();
+    const finalFlags = new Set();
     // Convert inline flag declarations
     const inlineMatches = regex.matchAll(/\?([a-z]):/g);
     const startMatches = regex.matchAll(/\(\?([a-z]+)\)/g);

package/dist/helpers/read-file.js

@@ -4,12 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
-/** Read part of a file on disc. */
+/**
+ * Read part of a file on disc.
+ * @throws 'EPERM' if the file is not readable.
+ */
 async function readFile(filename, onlyFirstLine = false) {
     const chunkSize = 100;
     const stream = fs_1.default.createReadStream(filename, { highWaterMark: chunkSize });
     let content = '';
-    for await (const data of stream) {
+    for await (const data of stream) { // may throw
         content += data.toString();
         if (onlyFirstLine && content.includes('\n')) {
             return content.split(/\r?\n/)[0];

package/dist/index.js

@@ -12,7 +12,7 @@ const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
 const load_data_1 = __importDefault(require("./helpers/load-data"));
 const read_file_1 = __importDefault(require("./helpers/read-file"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
-const convertToRegex = (path) => (0, glob_to_regexp_1.default)('**/' + path, { globstar: true, extended: true });
+const convert_glob_1 = __importDefault(require("./helpers/convert-glob"));
 async function analyse(input, opts = {}) {
     var _a, _b, _c, _d, _e, _f, _g, _h, _j;
     var _k, _l, _m;
@@ -31,7 +31,7 @@ async function analyse(input, opts = {}) {
     };
     const ignoredFiles = [
         /\/\.git\//,
-        opts.keepVendored ? [] : vendorData.map(path => (0, convert_pcre_1.default)(path)),
+        opts.keepVendored ? [] : vendorData.map(path => RegExp(path)),
         (_b = (_a = opts.ignoredFiles) === null || _a === void 0 ? void 0 : _a.map(path => (0, glob_to_regexp_1.default)('*' + path + '*', { extended: true }))) !== null && _b !== void 0 ? _b : [],
     ].flat();
     let { files, folders } = (0, walk_tree_1.default)(input !== null && input !== void 0 ? input : '.', ignoredFiles);
@@ -48,27 +48,38 @@ async function analyse(input, opts = {}) {
     }
     // Load gitattributes
     const customIgnored = [];
+    const customBinary = [];
+    const customText = [];
     if (!opts.quick) {
         for (const folder of folders) {
             // Skip if folder is marked in gitattributes
-            if (customIgnored.some(path => (0, convert_pcre_1.default)(path).test(folder)))
+            if (customIgnored.some(path => (0, convert_glob_1.default)(path).test(folder)))
                 continue;
             // Parse gitignores
             const ignoresFile = path_1.default.join(folder, '.gitignore');
             if (opts.checkIgnored && fs_1.default.existsSync(ignoresFile)) {
                 const ignoresData = await (0, read_file_1.default)(ignoresFile);
                 const ignoresList = ignoresData.split(/\r?\n/).filter(line => line.trim() && !line.startsWith('#'));
-                const ignoredPaths = ignoresList.map(path => convertToRegex(path).source);
-                customIgnored.push(...ignoredPaths.map(file => file.replace(folder, '')));
+                const ignoredPaths = ignoresList.map(path => (0, convert_glob_1.default)(path).source);
+                customIgnored.push(...ignoredPaths);
             }
             // Parse gitattributes
             const attributesFile = path_1.default.join(folder, '.gitattributes');
             if (opts.checkAttributes && fs_1.default.existsSync(attributesFile)) {
                 const attributesData = await (0, read_file_1.default)(attributesFile);
+                const relPathToRegex = (path) => (0, convert_glob_1.default)(path).source.substr(1).replace(folder, '');
+                // Explicit text/binary associations
+                const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)(?!=auto)/gm);
+                for (const [_line, path, type] of contentTypeMatches) {
+                    if (['text', '-binary'].includes(type))
+                        customText.push(relPathToRegex(path));
+                    if (['-text', 'binary'].includes(type))
+                        customBinary.push(relPathToRegex(path));
+                }
                 // Custom vendor options
                 const vendorMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-(vendored|generated|documentation)(?!=false)/gm);
                 for (const [_line, path] of vendorMatches) {
-                    customIgnored.push(convertToRegex(path).source.substr(1).replace(folder, ''));
+                    customIgnored.push(relPathToRegex(path));
                 }
                 // Custom file associations
                 const customLangMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-language=(\S+)/gm);
@@ -79,7 +90,7 @@ async function analyse(input, opts = {}) {
                         if (overrideLang)
                             forcedLang = overrideLang[0];
                     }
-                    const fullPath = folder + convertToRegex(path).source.substr(1);
+                    const fullPath = folder + (0, convert_glob_1.default)(path).source.substr(1);
                     overrides[fullPath] = forcedLang;
                 }
             }
@@ -88,7 +99,7 @@ async function analyse(input, opts = {}) {
     // Check vendored files
     if (!opts.keepVendored) {
         // Filter out any files that match a vendor file path
-        const matcher = (match) => (0, convert_pcre_1.default)(match.replace(/\/$/, '/.+$').replace(/^\.\//, ''));
+        const matcher = (match) => RegExp(match.replace(/\/$/, '/.+$').replace(/^\.\//, ''));
         files = files.filter(file => !customIgnored.some(pattern => matcher(pattern).test(file)));
     }
     // Load all files and parse languages
@@ -106,16 +117,18 @@ async function analyse(input, opts = {}) {
     for (const file of files) {
         if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
             continue;
+        const firstLine = await (0, read_file_1.default)(file, true).catch(() => null);
+        // Skip if file is unreadable
+        if (firstLine === null)
+            continue;
         // Check shebang line for explicit classification
-        if (!opts.quick && opts.checkShebang) {
-            const firstLine = await (0, read_file_1.default)(file, true);
-            if (firstLine.startsWith('#!')) {
-                const matches = Object.entries(langData).filter(([, data]) => { var _a; return (_a = data.interpreters) === null || _a === void 0 ? void 0 : _a.some(interpreter => firstLine.match('\\b' + interpreter + '\\b')); });
-                if (matches.length) {
-                    const forcedLang = matches[0][0];
-                    addResult(file, forcedLang);
-                    continue;
-                }
+        if (!opts.quick && opts.checkShebang && firstLine.startsWith('#!')) {
+            // Find matching interpreters
+            const matches = Object.entries(langData).filter(([, data]) => { var _a; return (_a = data.interpreters) === null || _a === void 0 ? void 0 : _a.some(interpreter => firstLine.match('\\b' + interpreter + '\\b')); });
+            if (matches.length) {
+                // Add explicitly-identified language
+                const forcedLang = matches[0][0];
+                addResult(file, forcedLang);
             }
         }
         // Check override for manual language classification
@@ -151,8 +164,13 @@ async function analyse(input, opts = {}) {
     // Narrow down file associations to the best fit
     for (const file in fileAssociations) {
         // Skip binary files
-        if (!opts.keepBinary && (binary_extensions_1.default.some(ext => file.endsWith('.' + ext)) || await (0, isbinaryfile_1.isBinaryFile)(file))) {
-            continue;
+        if (!opts.keepBinary) {
+            const isCustomText = customText.some(path => RegExp(path).test(file));
+            const isCustomBinary = customBinary.some(path => RegExp(path).test(file));
+            const isBinaryExt = binary_extensions_1.default.some(ext => file.endsWith('.' + ext));
+            if (!isCustomText && (isCustomBinary || isBinaryExt || await (0, isbinaryfile_1.isBinaryFile)(file))) {
+                continue;
+            }
         }
         // Parse heuristics if applicable
         if (opts.checkHeuristics)
@@ -180,7 +198,9 @@ async function analyse(input, opts = {}) {
                     if (heuristic.named_pattern)
                         normalise(heuristicsData.named_patterns[heuristic.named_pattern]);
                     // Check file contents and apply heuristic patterns
-                    const fileContent = await (0, read_file_1.default)(file);
+                    const fileContent = await (0, read_file_1.default)(file).catch(() => null);
+                    if (fileContent === null)
+                        continue;
                     if (!patterns.length || patterns.some(pattern => (0, convert_pcre_1.default)(pattern).test(fileContent))) {
                         results.files.results[file] = heuristic.language;
                         break;
@@ -208,6 +228,17 @@ async function analyse(input, opts = {}) {
             }
         }
     }
+    // Convert paths to relative
+    if (opts.relativePaths) {
+        const newMap = {};
+        for (const [file, lang] of Object.entries(results.files.results)) {
+            let relPath = path_1.default.relative(process.cwd(), file).replace(/\\/g, '/');
+            if (!relPath.startsWith('../'))
+                relPath = './' + relPath;
+            newMap[relPath] = lang;
+        }
+        results.files.results = newMap;
+    }
     // Load language bytes size
     for (const [file, lang] of Object.entries(results.files.results)) {
         if (lang && !langData[lang])

package/dist/types.d.ts

@@ -10,6 +10,7 @@ export interface Options {
     categories?: Category[];
     keepVendored?: boolean;
     keepBinary?: boolean;
+    relativePaths?: boolean;
     childLanguages?: boolean;
     quick?: boolean;
     checkIgnored?: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "linguist-js",
-	"version": "2.1.2",
+	"version": "2.2.1",
 	"description": "Analyse languages used in a folder. Powered by GitHub Linguist, although it doesn't need to be installed.",
 	"main": "dist/index.js",
 	"bin": {
@@ -10,7 +10,6 @@
 		"node": ">=12"
 	},
 	"scripts": {
-		"prepublish": "tsc",
 		"prepare": "npm test && npm run perf",
 		"perf": "tsc && node test/perf",
 		"test": "tsc && node test/test"
@@ -38,9 +37,9 @@
 	"homepage": "https://github.com/Nixinova/Linguist#readme",
 	"dependencies": {
 		"binary-extensions": "^2.2.0",
-		"commander": "^8.2.0",
-		"cross-fetch": "^3.1.4",
-		"glob-to-regexp": "^0.4.1",
+		"commander": "^9.0.0",
+		"cross-fetch": "^3.1.5",
+		"glob-to-regexp": "~0.4.1",
 		"isbinaryfile": "^4.0.8",
 		"js-yaml": "^4.1.0",
 		"node-cache": "^5.1.2"
@@ -49,7 +48,7 @@
 		"@types/glob-to-regexp": "ts4.4",
 		"@types/js-yaml": "ts4.4",
 		"@types/node": "ts4.4",
-		"deep-object-diff": "^1.1.0",
-		"typescript": "~4.4.3"
+		"deep-object-diff": "^1.1.7",
+		"typescript": "~4.5.5"
 	}
 }

package/readme.md

@@ -10,6 +10,7 @@ Powered by [github-linguist](https://github.com/github/linguist), although it do
 
 ## Install
 
+[Node.js](https://nodejs.org) must be installed to be able to use this.
 Linguist is available [on npm](https://npmjs.com/package/linguist-js) as `linguist-js`.
 
 Install locally using `npm install linguist-js` and import it into your code like so:
@@ -114,6 +115,8 @@ const { files, languages, unknown } = linguist(folder, options);
       Whether to keep vendored files (dependencies, etc) (defaults to `false`).
     - `keepBinary` (boolean):
       Whether binary files should be included in the output (defaults to `false`).
+    - `relativePaths` (boolean):
+      Change the absolute file paths in the output to be relative to the current working directory (defaults to `false`).
     - `checkAttributes` (boolean):
       Force the checking of `.gitattributes` files (defaults to `true` unless `quick` is set).
     - `checkIgnored` (boolean):
@@ -132,35 +135,37 @@ linguist --help
 
 - `--analyze`:
   Analyse the language of all files found in a folder.
-  - `<folders...>` (optional):
+  - `<folders...>`:
     The folders to analyse (defaults to `./`).
-  - `--ignoredFiles <paths...>` (optional):
+  - `--ignoredFiles <paths...>`:
     A list of space-delimited file path globs to ignore.
-  - `--ignoredLanguages` (optional):
+  - `--ignoredLanguages`:
     A list of languages to ignore.
-  - `--categories <categories...>` (optional):
+  - `--categories <categories...>`:
     A list of space-delimited categories that should be displayed in the output.
-  - `--childLanguages` (optional):
+  - `--childLanguages`:
     Whether to display sub-languages instead of their parents, when possible.
-  - `--json` (optional):
+  - `--json`:
     Display the outputted language data as JSON.
-  - `--tree <traversal>` (optional):
+  - `--tree <traversal>`:
     A dot-delimited traversal to the nested object that should be logged to the console instead of the entire output.
     Requires `--json` to be specified.
-  - `--quick` (optional):
+  - `--quick`:
     Whether to skip the checking of `.gitattributes` and `.gitignore` files for manual language classifications.
     Alias for `--checkAttributes=false --checkIgnored=false --checkHeuristics=false --checkShebang=false`.
-  - `--keepVendored` (optional):
+  - `--keepVendored`:
     Whether to include vendored files (auto-generated files, dependencies folder, etc).
-  - `--keepBinary` (optional):
+  - `--keepBinary`:
     Whether binary files should be excluded from the output.
-  - `--checkAttributes` (optional):
+  - `--relativePaths`:
+    Change the absolute file paths in the output to be relative to the current working directory.
+  - `--checkAttributes`:
     Force the checking of `.gitatributes` files (use alongside `--quick` to overwrite).
-  - `--checkIgnored` (optional):
+  - `--checkIgnored`:
     Force the checking of `.gitignore` files (use alongside `--quick` to overwrite).
-  - `--checkHeuristics` (optional):
+  - `--checkHeuristics`:
     Apply heuristics to ambiguous languages (use alongside `--quick` to overwrite).
-  - `--checkShebang` (optional):
+  - `--checkShebang`:
     Check shebang (`#!`) lines for explicit classification (use alongside `--quick` to overwrite).
 - `--help`:
   Display a help message.