linguist-js 2.1.4 → 2.3.1

package/dist/cli.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const VERSION = require('../package.json').version;
 const commander_1 = require("commander");
 const index_1 = __importDefault(require("./index"));
+const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
 const colouredMsg = ([r, g, b], msg) => `\u001B[${38};2;${r};${g};${b}m${msg}${'\u001b[0m'}`;
 const hexToRgb = (hex) => [parseInt(hex.slice(1, 3), 16), parseInt(hex.slice(3, 5), 16), parseInt(hex.slice(5, 7), 16)];
 commander_1.program
@@ -21,6 +22,7 @@ commander_1.program
     .option('-q|--quick [bool]', 'Skip complex language analysis (alias for -{A|I|H|S}=false)', false)
     .option('-V|--keepVendored [bool]', 'Prevent skipping over vendored/generated files', false)
     .option('-B|--keepBinary [bool]', 'Prevent skipping over binary files', false)
+    .option('-r|--relativePaths [bool]', 'Convert absolute file paths to relative', false)
     .option('-A|--checkAttributes [bool]', 'Force the checking of gitattributes files', true)
     .option('-I|--checkIgnored [bool]', 'Force the checking of gitignore files', true)
     .option('-H|--checkHeuristics [bool]', 'Apply heuristics to ambiguous languages', true)
@@ -51,16 +53,21 @@ if (args.analyze)
         const root = args.analyze === true ? '.' : args.analyze;
         const data = await (0, index_1.default)(root, args);
         const { files, languages, unknown } = data;
+        // Get file count
+        const totalFiles = (0, walk_tree_1.default)(root).files.length;
         // Print output
         if (!args.json) {
             const sortedEntries = Object.entries(languages.results).sort((a, b) => a[1].bytes < b[1].bytes ? +1 : -1);
             const totalBytes = languages.bytes;
-            console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${files.count} files with linguist-js`);
+            console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${totalFiles} files (${totalFiles - files.count} ignored) with linguist-js`);
             console.log(`\n Language analysis results:`);
-            let i = 0;
+            let count = 0;
+            if (sortedEntries.length === 0)
+                console.log(`  None`);
+            // List parsed results
             for (const [lang, { bytes, color }] of sortedEntries) {
                 const fmtd = {
-                    index: (++i).toString().padStart(2, ' '),
+                    index: (++count).toString().padStart(2, ' '),
                     lang: lang.padEnd(24, ' '),
                     percent: (bytes / (totalBytes || 1) * 100).toFixed(2).padStart(5, ' '),
                     bytes: bytes.toLocaleString().padStart(10, ' '),
@@ -69,13 +76,14 @@ if (args.analyze)
                 console.log(`  ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B`);
             }
             console.log(` Total: ${totalBytes.toLocaleString()} B`);
+            // List unknown files/extensions
             if (unknown.bytes > 0) {
                 console.log(`\n Unknown files and extensions:`);
                 for (const [name, bytes] of Object.entries(unknown.filenames)) {
                     console.log(`  '${name}': ${bytes.toLocaleString()} B`);
                 }
                 for (const [ext, bytes] of Object.entries(unknown.extensions)) {
-                    console.log(`  '${ext}': ${bytes.toLocaleString()} B`);
+                    console.log(`  '*${ext}': ${bytes.toLocaleString()} B`);
                 }
                 console.log(` Total: ${unknown.bytes.toLocaleString()} B`);
             }
@@ -95,6 +103,6 @@ if (args.analyze)
         }
     })();
 else {
-    console.log(`Welcome to linguist-js, the JavaScript port of GitHub's language analyzer.`);
+    console.log(`Welcome to linguist-js, a JavaScript port of GitHub's language analyzer.`);
     console.log(`Type 'linguist --help' for a list of commands.`);
 }

package/dist/helpers/convert-glob.js

@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const glob_to_regexp_1 = __importDefault(require("glob-to-regexp"));
 function parseGitGlob(path) {
-    const globPath = `**/${path}`.replace(/\[:(space|digit):\]/, (_, val) => `\\${val[0]}`);
+    const globPath = `**/${path}`.replace(/\\/g, '/').replace(/\[:(space|digit):\]/g, (_, val) => `\\${val[0]}`);
     return (0, glob_to_regexp_1.default)(globPath, { globstar: true, extended: true });
 }
 exports.default = parseGitGlob;

package/dist/helpers/convert-pcre.js

@@ -3,23 +3,26 @@ Object.defineProperty(exports, "__esModule", { value: true });
 /** Convert a PCRE regex into JS. */
 function pcre(regex) {
     let finalRegex = regex;
+    const replace = (search, replace) => finalRegex = finalRegex.replace(search, replace);
     const finalFlags = new Set();
     // Convert inline flag declarations
     const inlineMatches = regex.matchAll(/\?([a-z]):/g);
     const startMatches = regex.matchAll(/\(\?([a-z]+)\)/g);
     for (const [match, flags] of [...inlineMatches, ...startMatches]) {
-        finalRegex = finalRegex.replace(match, '');
+        replace(match, '');
         [...flags].forEach(flag => finalFlags.add(flag));
     }
-    // Remove invalid modifiers
-    finalRegex = finalRegex.replace(/([*+]){2}/g, '$1');
+    // Remove invalid syntax
+    replace(/([*+]){2}/g, '$1');
+    replace(/\(\?>/g, '(?:');
     // Remove start/end-of-file markers
     if (/\\[AZ]/.test(finalRegex)) {
-        finalRegex = finalRegex.replace(/\\A/g, '^').replace(/\\Z/g, '$');
+        replace(/\\A/g, '^').replace(/\\Z/g, '$');
         finalFlags.delete('m');
     }
     else
         finalFlags.add('m');
+    // Return final regex
     return RegExp(finalRegex, [...finalFlags].join(''));
 }
 exports.default = pcre;

package/dist/helpers/read-file.js

@@ -4,12 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
-/** Read part of a file on disc. */
+/**
+ * Read part of a file on disc.
+ * @throws 'EPERM' if the file is not readable.
+ */
 async function readFile(filename, onlyFirstLine = false) {
     const chunkSize = 100;
     const stream = fs_1.default.createReadStream(filename, { highWaterMark: chunkSize });
     let content = '';
-    for await (const data of stream) {
+    for await (const data of stream) { // may throw
         content += data.toString();
         if (onlyFirstLine && content.includes('\n')) {
             return content.split(/\r?\n/)[0];

package/dist/index.js

@@ -14,8 +14,9 @@ const read_file_1 = __importDefault(require("./helpers/read-file"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
 const convert_glob_1 = __importDefault(require("./helpers/convert-glob"));
 async function analyse(input, opts = {}) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j;
-    var _k, _l, _m;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r;
+    var _s, _t, _u;
+    const useRawContent = opts.fileContent !== undefined;
     const langData = await (0, load_data_1.default)('languages.yml').then(js_yaml_1.default.load);
     const vendorData = await (0, load_data_1.default)('vendor.yml').then(js_yaml_1.default.load);
     const heuristicsData = await (0, load_data_1.default)('heuristics.yml').then(js_yaml_1.default.load);
@@ -34,7 +35,16 @@ async function analyse(input, opts = {}) {
         opts.keepVendored ? [] : vendorData.map(path => RegExp(path)),
         (_b = (_a = opts.ignoredFiles) === null || _a === void 0 ? void 0 : _a.map(path => (0, glob_to_regexp_1.default)('*' + path + '*', { extended: true }))) !== null && _b !== void 0 ? _b : [],
     ].flat();
-    let { files, folders } = (0, walk_tree_1.default)(input !== null && input !== void 0 ? input : '.', ignoredFiles);
+    let files, folders;
+    if (opts.fileContent) {
+        opts.fileContent = Array.isArray(opts.fileContent) ? opts.fileContent : [opts.fileContent];
+        files = [`${input}`];
+        folders = [''];
+    }
+    else {
+        const data = (0, walk_tree_1.default)(input !== null && input !== void 0 ? input : '.', ignoredFiles);
+        ({ files, folders } = data);
+    }
     // Apply aliases
     opts = { checkIgnored: !opts.quick, checkAttributes: !opts.quick, checkHeuristics: !opts.quick, checkShebang: !opts.quick, ...opts };
     // Ignore specific languages
@@ -50,7 +60,7 @@ async function analyse(input, opts = {}) {
     const customIgnored = [];
     const customBinary = [];
     const customText = [];
-    if (!opts.quick) {
+    if (!useRawContent && !opts.quick) {
         for (const folder of folders) {
             // Skip if folder is marked in gitattributes
             if (customIgnored.some(path => (0, convert_glob_1.default)(path).test(folder)))
@@ -69,7 +79,7 @@ async function analyse(input, opts = {}) {
                 const attributesData = await (0, read_file_1.default)(attributesFile);
                 const relPathToRegex = (path) => (0, convert_glob_1.default)(path).source.substr(1).replace(folder, '');
                 // Explicit text/binary associations
-                const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)/gm);
+                const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)(?!=auto)/gm);
                 for (const [_line, path, type] of contentTypeMatches) {
                     if (['text', '-binary'].includes(type))
                         customText.push(relPathToRegex(path));
@@ -97,7 +107,7 @@ async function analyse(input, opts = {}) {
         }
     }
     // Check vendored files
-    if (!opts.keepVendored) {
+    if (!useRawContent && !opts.keepVendored) {
         // Filter out any files that match a vendor file path
         const matcher = (match) => RegExp(match.replace(/\/$/, '/.+$').replace(/^\.\//, ''));
         files = files.filter(file => !customIgnored.some(pattern => matcher(pattern).test(file)));
@@ -110,27 +120,35 @@ async function analyse(input, opts = {}) {
         }
         const parent = !opts.childLanguages && result && langData[result].group || false;
         fileAssociations[file].push(parent || result);
-        extensions[file] = path_1.default.extname(file);
+        extensions[file] = path_1.default.extname(file).toLowerCase();
     };
     const overridesArray = Object.entries(overrides);
     // List all languages that could be associated with a given file
     for (const file of files) {
-        if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
+        let firstLine;
+        if (opts.fileContent) {
+            firstLine = (_f = (_e = (_d = opts.fileContent) === null || _d === void 0 ? void 0 : _d[files.indexOf(file)]) === null || _e === void 0 ? void 0 : _e.split('\n')[0]) !== null && _f !== void 0 ? _f : null;
+        }
+        else {
+            if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
+                continue;
+            firstLine = await (0, read_file_1.default)(file, true).catch(() => null);
+        }
+        // Skip if file is unreadable
+        if (firstLine === null)
             continue;
         // Check shebang line for explicit classification
-        if (!opts.quick && opts.checkShebang) {
-            const firstLine = await (0, read_file_1.default)(file, true);
-            if (firstLine.startsWith('#!')) {
-                const matches = Object.entries(langData).filter(([, data]) => { var _a; return (_a = data.interpreters) === null || _a === void 0 ? void 0 : _a.some(interpreter => firstLine.match('\\b' + interpreter + '\\b')); });
-                if (matches.length) {
-                    const forcedLang = matches[0][0];
-                    addResult(file, forcedLang);
-                    continue;
-                }
+        if (!opts.quick && opts.checkShebang && firstLine.startsWith('#!')) {
+            // Find matching interpreters
+            const matches = Object.entries(langData).filter(([, data]) => { var _a; return (_a = data.interpreters) === null || _a === void 0 ? void 0 : _a.some(interpreter => firstLine.match('\\b' + interpreter + '\\b')); });
+            if (matches.length) {
+                // Add explicitly-identified language
+                const forcedLang = matches[0][0];
+                addResult(file, forcedLang);
             }
         }
         // Check override for manual language classification
-        if (!opts.quick && opts.checkAttributes) {
+        if (!useRawContent && !opts.quick && opts.checkAttributes) {
             const match = overridesArray.find(item => RegExp(item[0]).test(file));
             if (match) {
                 const forcedLang = match[1];
@@ -142,7 +160,7 @@ async function analyse(input, opts = {}) {
         let skipExts = false;
         for (const lang in langData) {
             // Check if filename is a match
-            const matchesName = (_d = langData[lang].filenames) === null || _d === void 0 ? void 0 : _d.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
+            const matchesName = (_g = langData[lang].filenames) === null || _g === void 0 ? void 0 : _g.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
             if (matchesName) {
                 addResult(file, lang);
                 skipExts = true;
@@ -151,7 +169,7 @@ async function analyse(input, opts = {}) {
         if (!skipExts)
             for (const lang in langData) {
                 // Check if extension is a match
-                const matchesExt = (_e = langData[lang].extensions) === null || _e === void 0 ? void 0 : _e.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
+                const matchesExt = (_h = langData[lang].extensions) === null || _h === void 0 ? void 0 : _h.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
                 if (matchesExt)
                     addResult(file, lang);
             }
@@ -162,7 +180,7 @@ async function analyse(input, opts = {}) {
     // Narrow down file associations to the best fit
     for (const file in fileAssociations) {
         // Skip binary files
-        if (!opts.keepBinary) {
+        if (!useRawContent && !opts.keepBinary) {
             const isCustomText = customText.some(path => RegExp(path).test(file));
             const isCustomBinary = customBinary.some(path => RegExp(path).test(file));
             const isBinaryExt = binary_extensions_1.default.some(ext => file.endsWith('.' + ext));
@@ -184,8 +202,9 @@ async function analyse(input, opts = {}) {
                         heuristic.language = heuristic.language[0];
                     }
                     // Make sure the results includes this language
+                    const languageGroup = (_j = langData[heuristic.language]) === null || _j === void 0 ? void 0 : _j.group;
                     const matchesLang = fileAssociations[file].includes(heuristic.language);
-                    const matchesParent = langData[heuristic.language].group && fileAssociations[file].includes(langData[heuristic.language].group);
+                    const matchesParent = languageGroup && fileAssociations[file].includes(languageGroup);
                     if (!matchesLang && !matchesParent)
                         continue;
                     // Normalise heuristic data
@@ -196,7 +215,9 @@ async function analyse(input, opts = {}) {
                     if (heuristic.named_pattern)
                         normalise(heuristicsData.named_patterns[heuristic.named_pattern]);
                     // Check file contents and apply heuristic patterns
-                    const fileContent = await (0, read_file_1.default)(file);
+                    const fileContent = await (0, read_file_1.default)(file).catch(() => null);
+                    if (fileContent === null)
+                        continue;
                     if (!patterns.length || patterns.some(pattern => (0, convert_pcre_1.default)(pattern).test(fileContent))) {
                         results.files.results[file] = heuristic.language;
                         break;
@@ -204,10 +225,10 @@ async function analyse(input, opts = {}) {
                 }
             }
         // If no heuristics, assign a language
-        (_f = (_k = results.files.results)[file]) !== null && _f !== void 0 ? _f : (_k[file] = fileAssociations[file][0]);
+        (_k = (_s = results.files.results)[file]) !== null && _k !== void 0 ? _k : (_s[file] = fileAssociations[file][0]);
     }
     // Skip specified categories
-    if ((_g = opts.categories) === null || _g === void 0 ? void 0 : _g.length) {
+    if ((_l = opts.categories) === null || _l === void 0 ? void 0 : _l.length) {
         const categories = ['data', 'markup', 'programming', 'prose'];
         const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
         for (const [file, lang] of Object.entries(results.files.results)) {
@@ -224,25 +245,36 @@ async function analyse(input, opts = {}) {
             }
         }
     }
+    // Convert paths to relative
+    if (!useRawContent && opts.relativePaths) {
+        const newMap = {};
+        for (const [file, lang] of Object.entries(results.files.results)) {
+            let relPath = path_1.default.relative(process.cwd(), file).replace(/\\/g, '/');
+            if (!relPath.startsWith('../'))
+                relPath = './' + relPath;
+            newMap[relPath] = lang;
+        }
+        results.files.results = newMap;
+    }
     // Load language bytes size
     for (const [file, lang] of Object.entries(results.files.results)) {
         if (lang && !langData[lang])
             continue;
-        const fileSize = fs_1.default.statSync(file).size;
+        const fileSize = (_p = (_o = (_m = opts.fileContent) === null || _m === void 0 ? void 0 : _m[files.indexOf(file)]) === null || _o === void 0 ? void 0 : _o.length) !== null && _p !== void 0 ? _p : fs_1.default.statSync(file).size;
         results.files.bytes += fileSize;
         // If no language found, add extension in other section
         if (!lang) {
             const ext = path_1.default.extname(file);
             const unknownType = ext === '' ? 'filenames' : 'extensions';
             const name = ext === '' ? path_1.default.basename(file) : ext;
-            (_h = (_l = results.unknown[unknownType])[name]) !== null && _h !== void 0 ? _h : (_l[name] = 0);
+            (_q = (_t = results.unknown[unknownType])[name]) !== null && _q !== void 0 ? _q : (_t[name] = 0);
             results.unknown[unknownType][name] += fileSize;
             results.unknown.bytes += fileSize;
             continue;
         }
         // Add language and bytes data to corresponding section
         const { type } = langData[lang];
-        (_j = (_m = results.languages.results)[lang]) !== null && _j !== void 0 ? _j : (_m[lang] = { type, bytes: 0, color: langData[lang].color });
+        (_r = (_u = results.languages.results)[lang]) !== null && _r !== void 0 ? _r : (_u[lang] = { type, bytes: 0, color: langData[lang].color });
         if (opts.childLanguages)
             results.languages.results[lang].parent = langData[lang].group;
         results.languages.results[lang].bytes += fileSize;

package/dist/types.d.ts

@@ -5,11 +5,13 @@ export declare type FilePath = string;
 export declare type Bytes = Integer;
 export declare type Integer = number;
 export interface Options {
+    fileContent?: string | string[];
     ignoredFiles?: string[];
     ignoredLanguages?: Language[];
     categories?: Category[];
     keepVendored?: boolean;
     keepBinary?: boolean;
+    relativePaths?: boolean;
     childLanguages?: boolean;
     quick?: boolean;
     checkIgnored?: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "linguist-js",
-	"version": "2.1.4",
+	"version": "2.3.1",
 	"description": "Analyse languages used in a folder. Powered by GitHub Linguist, although it doesn't need to be installed.",
 	"main": "dist/index.js",
 	"bin": {
@@ -37,18 +37,18 @@
 	"homepage": "https://github.com/Nixinova/Linguist#readme",
 	"dependencies": {
 		"binary-extensions": "^2.2.0",
-		"commander": "^8.2.0",
-		"cross-fetch": "^3.1.4",
+		"commander": "^9.0.0",
+		"cross-fetch": "^3.1.5",
 		"glob-to-regexp": "~0.4.1",
 		"isbinaryfile": "^4.0.8",
 		"js-yaml": "^4.1.0",
 		"node-cache": "^5.1.2"
 	},
 	"devDependencies": {
-		"@types/glob-to-regexp": "ts4.4",
-		"@types/js-yaml": "ts4.4",
-		"@types/node": "ts4.4",
-		"deep-object-diff": "^1.1.0",
-		"typescript": "~4.4.3"
+		"@types/glob-to-regexp": "ts4.6",
+		"@types/js-yaml": "ts4.6",
+		"@types/node": "ts4.6",
+		"deep-object-diff": "^1.1.7",
+		"typescript": "~4.6.1-rc"
 	}
 }

package/readme.md

@@ -77,6 +77,7 @@ Running Linguist on this folder will return the following JSON:
 ### Notes
 
 - File paths in the output use only forward slashes as delimiters, even on Windows.
+- This tool does not work when offline.
 - Do not rely on any language classification output from Linguist being unchanged between runs.
   Language data is fetched each run from the latest classifications of [`github-linguist`](https://github.com/github/linguist).
   This data is subject to change at any time and may change the results of a run even when using the same version of Linguist.
@@ -99,6 +100,8 @@ const { files, languages, unknown } = linguist(folder, options);
     Analyse multiple folders using the syntax `"{folder1,folder2,...}"`.
   - `opts` (optional; object):
     An object containing analyser options.
+    - `fileContent` (string or string array):
+      Provides the file content associated with the file name(s) given as `entry` to analyse instead of reading from a folder on disk.
     - `ignoredFiles` (string array):
       A list of file path globs to explicitly ignore.
     - `ignoredLanguages` (string array):
@@ -115,6 +118,8 @@ const { files, languages, unknown } = linguist(folder, options);
       Whether to keep vendored files (dependencies, etc) (defaults to `false`).
     - `keepBinary` (boolean):
       Whether binary files should be included in the output (defaults to `false`).
+    - `relativePaths` (boolean):
+      Change the absolute file paths in the output to be relative to the current working directory (defaults to `false`).
     - `checkAttributes` (boolean):
       Force the checking of `.gitattributes` files (defaults to `true` unless `quick` is set).
     - `checkIgnored` (boolean):
@@ -155,6 +160,8 @@ linguist --help
     Whether to include vendored files (auto-generated files, dependencies folder, etc).
   - `--keepBinary`:
     Whether binary files should be excluded from the output.
+  - `--relativePaths`:
+    Change the absolute file paths in the output to be relative to the current working directory.
   - `--checkAttributes`:
     Force the checking of `.gitatributes` files (use alongside `--quick` to overwrite).
   - `--checkIgnored`: