linguist-js 2.2.0 → 2.3.2

package/dist/cli.js

@@ -6,7 +6,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const VERSION = require('../package.json').version;
 const commander_1 = require("commander");
 const index_1 = __importDefault(require("./index"));
-const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
 const colouredMsg = ([r, g, b], msg) => `\u001B[${38};2;${r};${g};${b}m${msg}${'\u001b[0m'}`;
 const hexToRgb = (hex) => [parseInt(hex.slice(1, 3), 16), parseInt(hex.slice(3, 5), 16), parseInt(hex.slice(5, 7), 16)];
 commander_1.program
@@ -53,17 +52,16 @@ if (args.analyze)
         const root = args.analyze === true ? '.' : args.analyze;
         const data = await (0, index_1.default)(root, args);
         const { files, languages, unknown } = data;
-        // Get file count
-        let totalFiles = (0, walk_tree_1.default)(root).files.length;
         // Print output
         if (!args.json) {
             const sortedEntries = Object.entries(languages.results).sort((a, b) => a[1].bytes < b[1].bytes ? +1 : -1);
             const totalBytes = languages.bytes;
-            console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${totalFiles} files (${totalFiles - files.count} ignored) with linguist-js`);
+            console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${files.count} files with linguist-js`);
             console.log(`\n Language analysis results:`);
             let count = 0;
             if (sortedEntries.length === 0)
                 console.log(`  None`);
+            // List parsed results
             for (const [lang, { bytes, color }] of sortedEntries) {
                 const fmtd = {
                     index: (++count).toString().padStart(2, ' '),
@@ -75,6 +73,7 @@ if (args.analyze)
                 console.log(`  ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B`);
             }
             console.log(` Total: ${totalBytes.toLocaleString()} B`);
+            // List unknown files/extensions
             if (unknown.bytes > 0) {
                 console.log(`\n Unknown files and extensions:`);
                 for (const [name, bytes] of Object.entries(unknown.filenames)) {
@@ -101,6 +100,6 @@ if (args.analyze)
         }
     })();
 else {
-    console.log(`Welcome to linguist-js, the JavaScript port of GitHub's language analyzer.`);
+    console.log(`Welcome to linguist-js, a JavaScript port of GitHub's language analyzer.`);
     console.log(`Type 'linguist --help' for a list of commands.`);
 }

package/dist/helpers/convert-pcre.js

@@ -3,23 +3,26 @@ Object.defineProperty(exports, "__esModule", { value: true });
 /** Convert a PCRE regex into JS. */
 function pcre(regex) {
     let finalRegex = regex;
+    const replace = (search, replace) => finalRegex = finalRegex.replace(search, replace);
     const finalFlags = new Set();
     // Convert inline flag declarations
     const inlineMatches = regex.matchAll(/\?([a-z]):/g);
     const startMatches = regex.matchAll(/\(\?([a-z]+)\)/g);
     for (const [match, flags] of [...inlineMatches, ...startMatches]) {
-        finalRegex = finalRegex.replace(match, '');
+        replace(match, '');
         [...flags].forEach(flag => finalFlags.add(flag));
     }
-    // Remove invalid modifiers
-    finalRegex = finalRegex.replace(/([*+]){2}/g, '$1');
+    // Remove invalid syntax
+    replace(/([*+]){2}/g, '$1');
+    replace(/\(\?>/g, '(?:');
     // Remove start/end-of-file markers
     if (/\\[AZ]/.test(finalRegex)) {
-        finalRegex = finalRegex.replace(/\\A/g, '^').replace(/\\Z/g, '$');
+        replace(/\\A/g, '^').replace(/\\Z/g, '$');
         finalFlags.delete('m');
     }
     else
         finalFlags.add('m');
+    // Return final regex
     return RegExp(finalRegex, [...finalFlags].join(''));
 }
 exports.default = pcre;

package/dist/helpers/read-file.js

@@ -4,12 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
-/** Read part of a file on disc. */
+/**
+ * Read part of a file on disc.
+ * @throws 'EPERM' if the file is not readable.
+ */
 async function readFile(filename, onlyFirstLine = false) {
     const chunkSize = 100;
     const stream = fs_1.default.createReadStream(filename, { highWaterMark: chunkSize });
     let content = '';
-    for await (const data of stream) {
+    for await (const data of stream) { // may throw
         content += data.toString();
         if (onlyFirstLine && content.includes('\n')) {
             return content.split(/\r?\n/)[0];

package/dist/helpers/walk-tree.js

@@ -8,28 +8,50 @@ const path_1 = __importDefault(require("path"));
 const allFiles = new Set();
 const allFolders = new Set();
 /** Generate list of files in a directory. */
-function walk(folder, ignored = []) {
-    if (Array.isArray(folder)) {
-        for (const path of folder)
-            walk(path, ignored);
-    }
-    else {
-        const files = fs_1.default.readdirSync(folder);
+function walk(root, folders, ignored = []) {
+    // Switch out paths that expect being in root
+    ignored = ignored.map(match => match.replace(/^\^/, '^\\./'));
+    // Walk tree of a folder
+    if (folders.length === 1) {
+        const folder = folders[0];
+        // Get list of files and folders inside this folder
+        const files = fs_1.default.readdirSync(folder).map(file => {
+            // Create path relative to root
+            const base = path_1.default.resolve(folder, file).replace(/\\/g, '/').replace(root, '.');
+            // Add trailing slash to mark directories
+            const isDir = fs_1.default.lstatSync(path_1.default.resolve(root, base)).isDirectory();
+            return isDir ? `${base}/` : base;
+        });
+        // Loop through files and folders
         for (const file of files) {
-            const path = path_1.default.resolve(folder, file).replace(/\\/g, '/');
-            if (!fs_1.default.existsSync(path) || ignored.some(pattern => pattern.test(path)))
+            // Create absolute path for disc operations
+            const path = path_1.default.resolve(root, file).replace(/\\/g, '/');
+            // Skip if nonexistant or ignored
+            if (!fs_1.default.existsSync(path) || ignored.some(pattern => RegExp(pattern).test(file)))
                 continue;
-            allFolders.add(folder.replace(/\\/g, '/'));
-            if (fs_1.default.lstatSync(path).isDirectory()) {
+            // Add absolute folder path to list
+            allFolders.add(path_1.default.resolve(folder).replace(/\\/g, '/'));
+            // Check if this is a folder or file
+            if (file.endsWith('/')) {
+                // Recurse into subfolders
                 allFolders.add(path);
-                walk(path, ignored);
-                continue;
+                walk(root, [path], ignored);
+            }
+            else {
+                // Add relative file path to list
+                allFiles.add(path);
             }
-            allFiles.add(path);
         }
     }
+    // Recurse into all folders
+    else {
+        for (const path of folders) {
+            walk(root, [path], ignored);
+        }
+    }
+    // Return absolute files and folders lists
     return {
-        files: [...allFiles],
+        files: [...allFiles].map(file => file.replace(/^\./, root)),
         folders: [...allFolders],
     };
 }

package/dist/index.js

@@ -6,6 +6,7 @@ const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const js_yaml_1 = __importDefault(require("js-yaml"));
 const glob_to_regexp_1 = __importDefault(require("glob-to-regexp"));
+const common_path_prefix_1 = __importDefault(require("common-path-prefix"));
 const binary_extensions_1 = __importDefault(require("binary-extensions"));
 const isbinaryfile_1 = require("isbinaryfile");
 const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
@@ -14,13 +15,19 @@ const read_file_1 = __importDefault(require("./helpers/read-file"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
 const convert_glob_1 = __importDefault(require("./helpers/convert-glob"));
 async function analyse(input, opts = {}) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j;
-    var _k, _l, _m;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
+    var _r, _s, _t;
+    const useRawContent = opts.fileContent !== undefined;
+    input = [input !== null && input !== void 0 ? input : []].flat();
+    opts.fileContent = [(_a = opts.fileContent) !== null && _a !== void 0 ? _a : []].flat();
+    // Load data from github-linguist web repo
     const langData = await (0, load_data_1.default)('languages.yml').then(js_yaml_1.default.load);
     const vendorData = await (0, load_data_1.default)('vendor.yml').then(js_yaml_1.default.load);
+    const docData = await (0, load_data_1.default)('documentation.yml').then(js_yaml_1.default.load);
     const heuristicsData = await (0, load_data_1.default)('heuristics.yml').then(js_yaml_1.default.load);
     const generatedData = await (0, load_data_1.default)('generated.rb').then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/\))/gm)) !== null && _a !== void 0 ? _a : []; });
-    vendorData.push(...generatedData);
+    const vendorPaths = [...vendorData, ...docData, ...generatedData];
+    // Setup main variables
     const fileAssociations = {};
     const extensions = {};
     const overrides = {};
@@ -29,16 +36,34 @@ async function analyse(input, opts = {}) {
         languages: { count: 0, bytes: 0, results: {} },
         unknown: { count: 0, bytes: 0, extensions: {}, filenames: {} },
     };
-    const ignoredFiles = [
-        /\/\.git\//,
-        opts.keepVendored ? [] : vendorData.map(path => RegExp(path)),
-        (_b = (_a = opts.ignoredFiles) === null || _a === void 0 ? void 0 : _a.map(path => (0, glob_to_regexp_1.default)('*' + path + '*', { extended: true }))) !== null && _b !== void 0 ? _b : [],
-    ].flat();
-    let { files, folders } = (0, walk_tree_1.default)(input !== null && input !== void 0 ? input : '.', ignoredFiles);
+    // Prepare list of ignored files
+    const ignoredFiles = ['(^|\\/)\\.git\\/'];
+    if (!opts.keepVendored) {
+        ignoredFiles.push(...vendorPaths);
+    }
+    if (opts.ignoredFiles) {
+        ignoredFiles.push(...opts.ignoredFiles.map(path => (0, glob_to_regexp_1.default)('*' + path + '*', { extended: true }).source));
+    }
+    // Load file paths and folders
+    let files, folders;
+    if (useRawContent) {
+        // Uses raw file content
+        files = input;
+        folders = [''];
+    }
+    else {
+        // Uses directory on disc
+        // Set a common root path so that vendor paths do not incorrectly match parent folders
+        const resolvedInput = input.map(path => path_1.default.resolve(path).replace(/\\/g, '/'));
+        const commonRoot = (input.length > 1 ? (0, common_path_prefix_1.default)(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
+        const data = (0, walk_tree_1.default)(commonRoot, input, ignoredFiles);
+        files = data.files;
+        folders = data.folders;
+    }
     // Apply aliases
     opts = { checkIgnored: !opts.quick, checkAttributes: !opts.quick, checkHeuristics: !opts.quick, checkShebang: !opts.quick, ...opts };
     // Ignore specific languages
-    for (const lang of (_c = opts.ignoredLanguages) !== null && _c !== void 0 ? _c : []) {
+    for (const lang of (_b = opts.ignoredLanguages) !== null && _b !== void 0 ? _b : []) {
         for (const key in langData) {
             if (lang.toLowerCase() === key.toLowerCase()) {
                 delete langData[key];
@@ -50,7 +75,7 @@ async function analyse(input, opts = {}) {
     const customIgnored = [];
     const customBinary = [];
     const customText = [];
-    if (!opts.quick) {
+    if (!useRawContent && opts.checkAttributes) {
         for (const folder of folders) {
             // Skip if folder is marked in gitattributes
             if (customIgnored.some(path => (0, convert_glob_1.default)(path).test(folder)))
@@ -69,7 +94,7 @@ async function analyse(input, opts = {}) {
                 const attributesData = await (0, read_file_1.default)(attributesFile);
                 const relPathToRegex = (path) => (0, convert_glob_1.default)(path).source.substr(1).replace(folder, '');
                 // Explicit text/binary associations
-                const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)/gm);
+                const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)(?!=auto)/gm);
                 for (const [_line, path, type] of contentTypeMatches) {
                     if (['text', '-binary'].includes(type))
                         customText.push(relPathToRegex(path));
@@ -97,7 +122,7 @@ async function analyse(input, opts = {}) {
         }
     }
     // Check vendored files
-    if (!opts.keepVendored) {
+    if (!useRawContent && !opts.keepVendored) {
         // Filter out any files that match a vendor file path
         const matcher = (match) => RegExp(match.replace(/\/$/, '/.+$').replace(/^\.\//, ''));
         files = files.filter(file => !customIgnored.some(pattern => matcher(pattern).test(file)));
@@ -110,27 +135,35 @@ async function analyse(input, opts = {}) {
         }
         const parent = !opts.childLanguages && result && langData[result].group || false;
         fileAssociations[file].push(parent || result);
-        extensions[file] = path_1.default.extname(file);
+        extensions[file] = path_1.default.extname(file).toLowerCase();
     };
     const overridesArray = Object.entries(overrides);
     // List all languages that could be associated with a given file
     for (const file of files) {
-        if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
+        let firstLine;
+        if (useRawContent) {
+            firstLine = (_e = (_d = (_c = opts.fileContent) === null || _c === void 0 ? void 0 : _c[files.indexOf(file)]) === null || _d === void 0 ? void 0 : _d.split('\n')[0]) !== null && _e !== void 0 ? _e : null;
+        }
+        else {
+            if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
+                continue;
+            firstLine = await (0, read_file_1.default)(file, true).catch(() => null);
+        }
+        // Skip if file is unreadable
+        if (firstLine === null)
             continue;
         // Check shebang line for explicit classification
-        if (!opts.quick && opts.checkShebang) {
-            const firstLine = await (0, read_file_1.default)(file, true);
-            if (firstLine.startsWith('#!')) {
-                const matches = Object.entries(langData).filter(([, data]) => { var _a; return (_a = data.interpreters) === null || _a === void 0 ? void 0 : _a.some(interpreter => firstLine.match('\\b' + interpreter + '\\b')); });
-                if (matches.length) {
-                    const forcedLang = matches[0][0];
-                    addResult(file, forcedLang);
-                    continue;
-                }
+        if (!opts.quick && opts.checkShebang && firstLine.startsWith('#!')) {
+            // Find matching interpreters
+            const matches = Object.entries(langData).filter(([, data]) => { var _a; return (_a = data.interpreters) === null || _a === void 0 ? void 0 : _a.some(interpreter => firstLine.match('\\b' + interpreter + '\\b')); });
+            if (matches.length) {
+                // Add explicitly-identified language
+                const forcedLang = matches[0][0];
+                addResult(file, forcedLang);
             }
         }
         // Check override for manual language classification
-        if (!opts.quick && opts.checkAttributes) {
+        if (!useRawContent && !opts.quick && opts.checkAttributes) {
             const match = overridesArray.find(item => RegExp(item[0]).test(file));
             if (match) {
                 const forcedLang = match[1];
@@ -142,7 +175,7 @@ async function analyse(input, opts = {}) {
         let skipExts = false;
         for (const lang in langData) {
             // Check if filename is a match
-            const matchesName = (_d = langData[lang].filenames) === null || _d === void 0 ? void 0 : _d.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
+            const matchesName = (_f = langData[lang].filenames) === null || _f === void 0 ? void 0 : _f.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
             if (matchesName) {
                 addResult(file, lang);
                 skipExts = true;
@@ -151,7 +184,7 @@ async function analyse(input, opts = {}) {
         if (!skipExts)
             for (const lang in langData) {
                 // Check if extension is a match
-                const matchesExt = (_e = langData[lang].extensions) === null || _e === void 0 ? void 0 : _e.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
+                const matchesExt = (_g = langData[lang].extensions) === null || _g === void 0 ? void 0 : _g.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
                 if (matchesExt)
                     addResult(file, lang);
             }
@@ -162,7 +195,7 @@ async function analyse(input, opts = {}) {
     // Narrow down file associations to the best fit
     for (const file in fileAssociations) {
         // Skip binary files
-        if (!opts.keepBinary) {
+        if (!useRawContent && !opts.keepBinary) {
             const isCustomText = customText.some(path => RegExp(path).test(file));
             const isCustomBinary = customBinary.some(path => RegExp(path).test(file));
             const isBinaryExt = binary_extensions_1.default.some(ext => file.endsWith('.' + ext));
@@ -184,19 +217,22 @@ async function analyse(input, opts = {}) {
                         heuristic.language = heuristic.language[0];
                     }
                     // Make sure the results includes this language
+                    const languageGroup = (_h = langData[heuristic.language]) === null || _h === void 0 ? void 0 : _h.group;
                     const matchesLang = fileAssociations[file].includes(heuristic.language);
-                    const matchesParent = langData[heuristic.language].group && fileAssociations[file].includes(langData[heuristic.language].group);
+                    const matchesParent = languageGroup && fileAssociations[file].includes(languageGroup);
                     if (!matchesLang && !matchesParent)
                         continue;
                     // Normalise heuristic data
                     const patterns = [];
-                    const normalise = (contents) => patterns.push(...(Array.isArray(contents) ? contents : [contents]));
+                    const normalise = (contents) => patterns.push(...[contents].flat());
                     if (heuristic.pattern)
                         normalise(heuristic.pattern);
                     if (heuristic.named_pattern)
                         normalise(heuristicsData.named_patterns[heuristic.named_pattern]);
                     // Check file contents and apply heuristic patterns
-                    const fileContent = await (0, read_file_1.default)(file);
+                    const fileContent = await (0, read_file_1.default)(file).catch(() => null);
+                    if (fileContent === null)
+                        continue;
                     if (!patterns.length || patterns.some(pattern => (0, convert_pcre_1.default)(pattern).test(fileContent))) {
                         results.files.results[file] = heuristic.language;
                         break;
@@ -204,10 +240,10 @@ async function analyse(input, opts = {}) {
                 }
             }
         // If no heuristics, assign a language
-        (_f = (_k = results.files.results)[file]) !== null && _f !== void 0 ? _f : (_k[file] = fileAssociations[file][0]);
+        (_j = (_r = results.files.results)[file]) !== null && _j !== void 0 ? _j : (_r[file] = fileAssociations[file][0]);
     }
     // Skip specified categories
-    if ((_g = opts.categories) === null || _g === void 0 ? void 0 : _g.length) {
+    if ((_k = opts.categories) === null || _k === void 0 ? void 0 : _k.length) {
         const categories = ['data', 'markup', 'programming', 'prose'];
         const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
         for (const [file, lang] of Object.entries(results.files.results)) {
@@ -225,7 +261,7 @@ async function analyse(input, opts = {}) {
         }
     }
     // Convert paths to relative
-    if (opts.relativePaths) {
+    if (!useRawContent && opts.relativePaths) {
         const newMap = {};
         for (const [file, lang] of Object.entries(results.files.results)) {
             let relPath = path_1.default.relative(process.cwd(), file).replace(/\\/g, '/');
@@ -239,21 +275,21 @@ async function analyse(input, opts = {}) {
     for (const [file, lang] of Object.entries(results.files.results)) {
         if (lang && !langData[lang])
             continue;
-        const fileSize = fs_1.default.statSync(file).size;
+        const fileSize = (_o = (_m = (_l = opts.fileContent) === null || _l === void 0 ? void 0 : _l[files.indexOf(file)]) === null || _m === void 0 ? void 0 : _m.length) !== null && _o !== void 0 ? _o : fs_1.default.statSync(file).size;
         results.files.bytes += fileSize;
         // If no language found, add extension in other section
         if (!lang) {
             const ext = path_1.default.extname(file);
             const unknownType = ext === '' ? 'filenames' : 'extensions';
             const name = ext === '' ? path_1.default.basename(file) : ext;
-            (_h = (_l = results.unknown[unknownType])[name]) !== null && _h !== void 0 ? _h : (_l[name] = 0);
+            (_p = (_s = results.unknown[unknownType])[name]) !== null && _p !== void 0 ? _p : (_s[name] = 0);
             results.unknown[unknownType][name] += fileSize;
             results.unknown.bytes += fileSize;
             continue;
         }
         // Add language and bytes data to corresponding section
         const { type } = langData[lang];
-        (_j = (_m = results.languages.results)[lang]) !== null && _j !== void 0 ? _j : (_m[lang] = { type, bytes: 0, color: langData[lang].color });
+        (_q = (_t = results.languages.results)[lang]) !== null && _q !== void 0 ? _q : (_t[lang] = { type, bytes: 0, color: langData[lang].color });
         if (opts.childLanguages)
             results.languages.results[lang].parent = langData[lang].group;
         results.languages.results[lang].bytes += fileSize;

package/dist/types.d.ts

@@ -5,6 +5,7 @@ export declare type FilePath = string;
 export declare type Bytes = Integer;
 export declare type Integer = number;
 export interface Options {
+    fileContent?: string | string[];
     ignoredFiles?: string[];
     ignoredLanguages?: Language[];
     categories?: Category[];

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "linguist-js",
-	"version": "2.2.0",
+	"version": "2.3.2",
 	"description": "Analyse languages used in a folder. Powered by GitHub Linguist, although it doesn't need to be installed.",
 	"main": "dist/index.js",
 	"bin": {
@@ -37,18 +37,19 @@
 	"homepage": "https://github.com/Nixinova/Linguist#readme",
 	"dependencies": {
 		"binary-extensions": "^2.2.0",
-		"commander": "^8.3.0",
-		"cross-fetch": "^3.1.4",
+		"commander": "^9.0.0",
+		"common-path-prefix": "^3.0.0",
+		"cross-fetch": "^3.1.5",
 		"glob-to-regexp": "~0.4.1",
 		"isbinaryfile": "^4.0.8",
 		"js-yaml": "^4.1.0",
 		"node-cache": "^5.1.2"
 	},
 	"devDependencies": {
-		"@types/glob-to-regexp": "ts4.4",
-		"@types/js-yaml": "ts4.4",
-		"@types/node": "ts4.4",
-		"deep-object-diff": "^1.1.0",
-		"typescript": "~4.5.4"
+		"@types/glob-to-regexp": "ts4.6",
+		"@types/js-yaml": "ts4.6",
+		"@types/node": "ts4.6",
+		"deep-object-diff": "^1.1.7",
+		"typescript": "~4.6.1-rc"
 	}
 }

package/readme.md

@@ -77,6 +77,7 @@ Running Linguist on this folder will return the following JSON:
 ### Notes
 
 - File paths in the output use only forward slashes as delimiters, even on Windows.
+- This tool does not work when offline.
 - Do not rely on any language classification output from Linguist being unchanged between runs.
   Language data is fetched each run from the latest classifications of [`github-linguist`](https://github.com/github/linguist).
   This data is subject to change at any time and may change the results of a run even when using the same version of Linguist.
@@ -99,6 +100,8 @@ const { files, languages, unknown } = linguist(folder, options);
     Analyse multiple folders using the syntax `"{folder1,folder2,...}"`.
   - `opts` (optional; object):
     An object containing analyser options.
+    - `fileContent` (string or string array):
+      Provides the file content associated with the file name(s) given as `entry` to analyse instead of reading from a folder on disk.
     - `ignoredFiles` (string array):
       A list of file path globs to explicitly ignore.
     - `ignoredLanguages` (string array):