linguist-js 2.3.1 → 2.3.2

package/dist/cli.js

@@ -6,7 +6,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const VERSION = require('../package.json').version;
 const commander_1 = require("commander");
 const index_1 = __importDefault(require("./index"));
-const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
 const colouredMsg = ([r, g, b], msg) => `\u001B[${38};2;${r};${g};${b}m${msg}${'\u001b[0m'}`;
 const hexToRgb = (hex) => [parseInt(hex.slice(1, 3), 16), parseInt(hex.slice(3, 5), 16), parseInt(hex.slice(5, 7), 16)];
 commander_1.program
@@ -53,13 +52,11 @@ if (args.analyze)
         const root = args.analyze === true ? '.' : args.analyze;
         const data = await (0, index_1.default)(root, args);
         const { files, languages, unknown } = data;
-        // Get file count
-        const totalFiles = (0, walk_tree_1.default)(root).files.length;
         // Print output
         if (!args.json) {
             const sortedEntries = Object.entries(languages.results).sort((a, b) => a[1].bytes < b[1].bytes ? +1 : -1);
             const totalBytes = languages.bytes;
-            console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${totalFiles} files (${totalFiles - files.count} ignored) with linguist-js`);
+            console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${files.count} files with linguist-js`);
             console.log(`\n Language analysis results:`);
             let count = 0;
             if (sortedEntries.length === 0)

package/dist/helpers/walk-tree.js

@@ -8,28 +8,50 @@ const path_1 = __importDefault(require("path"));
 const allFiles = new Set();
 const allFolders = new Set();
 /** Generate list of files in a directory. */
-function walk(folder, ignored = []) {
-    if (Array.isArray(folder)) {
-        for (const path of folder)
-            walk(path, ignored);
-    }
-    else {
-        const files = fs_1.default.readdirSync(folder);
+function walk(root, folders, ignored = []) {
+    // Switch out paths that expect being in root
+    ignored = ignored.map(match => match.replace(/^\^/, '^\\./'));
+    // Walk tree of a folder
+    if (folders.length === 1) {
+        const folder = folders[0];
+        // Get list of files and folders inside this folder
+        const files = fs_1.default.readdirSync(folder).map(file => {
+            // Create path relative to root
+            const base = path_1.default.resolve(folder, file).replace(/\\/g, '/').replace(root, '.');
+            // Add trailing slash to mark directories
+            const isDir = fs_1.default.lstatSync(path_1.default.resolve(root, base)).isDirectory();
+            return isDir ? `${base}/` : base;
+        });
+        // Loop through files and folders
         for (const file of files) {
-            const path = path_1.default.resolve(folder, file).replace(/\\/g, '/');
-            if (!fs_1.default.existsSync(path) || ignored.some(pattern => pattern.test(path)))
+            // Create absolute path for disc operations
+            const path = path_1.default.resolve(root, file).replace(/\\/g, '/');
+            // Skip if nonexistant or ignored
+            if (!fs_1.default.existsSync(path) || ignored.some(pattern => RegExp(pattern).test(file)))
                 continue;
-            allFolders.add(folder.replace(/\\/g, '/'));
-            if (fs_1.default.lstatSync(path).isDirectory()) {
+            // Add absolute folder path to list
+            allFolders.add(path_1.default.resolve(folder).replace(/\\/g, '/'));
+            // Check if this is a folder or file
+            if (file.endsWith('/')) {
+                // Recurse into subfolders
                 allFolders.add(path);
-                walk(path, ignored);
-                continue;
+                walk(root, [path], ignored);
+            }
+            else {
+                // Add relative file path to list
+                allFiles.add(path);
             }
-            allFiles.add(path);
         }
     }
+    // Recurse into all folders
+    else {
+        for (const path of folders) {
+            walk(root, [path], ignored);
+        }
+    }
+    // Return absolute files and folders lists
     return {
-        files: [...allFiles],
+        files: [...allFiles].map(file => file.replace(/^\./, root)),
         folders: [...allFolders],
     };
 }

package/dist/index.js

@@ -6,6 +6,7 @@ const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const js_yaml_1 = __importDefault(require("js-yaml"));
 const glob_to_regexp_1 = __importDefault(require("glob-to-regexp"));
+const common_path_prefix_1 = __importDefault(require("common-path-prefix"));
 const binary_extensions_1 = __importDefault(require("binary-extensions"));
 const isbinaryfile_1 = require("isbinaryfile");
 const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
@@ -14,14 +15,19 @@ const read_file_1 = __importDefault(require("./helpers/read-file"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
 const convert_glob_1 = __importDefault(require("./helpers/convert-glob"));
 async function analyse(input, opts = {}) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r;
-    var _s, _t, _u;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
+    var _r, _s, _t;
     const useRawContent = opts.fileContent !== undefined;
+    input = [input !== null && input !== void 0 ? input : []].flat();
+    opts.fileContent = [(_a = opts.fileContent) !== null && _a !== void 0 ? _a : []].flat();
+    // Load data from github-linguist web repo
     const langData = await (0, load_data_1.default)('languages.yml').then(js_yaml_1.default.load);
     const vendorData = await (0, load_data_1.default)('vendor.yml').then(js_yaml_1.default.load);
+    const docData = await (0, load_data_1.default)('documentation.yml').then(js_yaml_1.default.load);
     const heuristicsData = await (0, load_data_1.default)('heuristics.yml').then(js_yaml_1.default.load);
     const generatedData = await (0, load_data_1.default)('generated.rb').then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/\))/gm)) !== null && _a !== void 0 ? _a : []; });
-    vendorData.push(...generatedData);
+    const vendorPaths = [...vendorData, ...docData, ...generatedData];
+    // Setup main variables
     const fileAssociations = {};
     const extensions = {};
     const overrides = {};
@@ -30,25 +36,34 @@ async function analyse(input, opts = {}) {
         languages: { count: 0, bytes: 0, results: {} },
         unknown: { count: 0, bytes: 0, extensions: {}, filenames: {} },
     };
-    const ignoredFiles = [
-        /\/\.git\//,
-        opts.keepVendored ? [] : vendorData.map(path => RegExp(path)),
-        (_b = (_a = opts.ignoredFiles) === null || _a === void 0 ? void 0 : _a.map(path => (0, glob_to_regexp_1.default)('*' + path + '*', { extended: true }))) !== null && _b !== void 0 ? _b : [],
-    ].flat();
+    // Prepare list of ignored files
+    const ignoredFiles = ['(^|\\/)\\.git\\/'];
+    if (!opts.keepVendored) {
+        ignoredFiles.push(...vendorPaths);
+    }
+    if (opts.ignoredFiles) {
+        ignoredFiles.push(...opts.ignoredFiles.map(path => (0, glob_to_regexp_1.default)('*' + path + '*', { extended: true }).source));
+    }
+    // Load file paths and folders
     let files, folders;
-    if (opts.fileContent) {
-        opts.fileContent = Array.isArray(opts.fileContent) ? opts.fileContent : [opts.fileContent];
-        files = [`${input}`];
+    if (useRawContent) {
+        // Uses raw file content
+        files = input;
         folders = [''];
     }
     else {
-        const data = (0, walk_tree_1.default)(input !== null && input !== void 0 ? input : '.', ignoredFiles);
-        ({ files, folders } = data);
+        // Uses directory on disc
+        // Set a common root path so that vendor paths do not incorrectly match parent folders
+        const resolvedInput = input.map(path => path_1.default.resolve(path).replace(/\\/g, '/'));
+        const commonRoot = (input.length > 1 ? (0, common_path_prefix_1.default)(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
+        const data = (0, walk_tree_1.default)(commonRoot, input, ignoredFiles);
+        files = data.files;
+        folders = data.folders;
     }
     // Apply aliases
     opts = { checkIgnored: !opts.quick, checkAttributes: !opts.quick, checkHeuristics: !opts.quick, checkShebang: !opts.quick, ...opts };
     // Ignore specific languages
-    for (const lang of (_c = opts.ignoredLanguages) !== null && _c !== void 0 ? _c : []) {
+    for (const lang of (_b = opts.ignoredLanguages) !== null && _b !== void 0 ? _b : []) {
         for (const key in langData) {
             if (lang.toLowerCase() === key.toLowerCase()) {
                 delete langData[key];
@@ -60,7 +75,7 @@ async function analyse(input, opts = {}) {
     const customIgnored = [];
     const customBinary = [];
     const customText = [];
-    if (!useRawContent && !opts.quick) {
+    if (!useRawContent && opts.checkAttributes) {
         for (const folder of folders) {
             // Skip if folder is marked in gitattributes
             if (customIgnored.some(path => (0, convert_glob_1.default)(path).test(folder)))
@@ -126,8 +141,8 @@ async function analyse(input, opts = {}) {
     // List all languages that could be associated with a given file
     for (const file of files) {
         let firstLine;
-        if (opts.fileContent) {
-            firstLine = (_f = (_e = (_d = opts.fileContent) === null || _d === void 0 ? void 0 : _d[files.indexOf(file)]) === null || _e === void 0 ? void 0 : _e.split('\n')[0]) !== null && _f !== void 0 ? _f : null;
+        if (useRawContent) {
+            firstLine = (_e = (_d = (_c = opts.fileContent) === null || _c === void 0 ? void 0 : _c[files.indexOf(file)]) === null || _d === void 0 ? void 0 : _d.split('\n')[0]) !== null && _e !== void 0 ? _e : null;
         }
         else {
             if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
@@ -160,7 +175,7 @@ async function analyse(input, opts = {}) {
         let skipExts = false;
         for (const lang in langData) {
             // Check if filename is a match
-            const matchesName = (_g = langData[lang].filenames) === null || _g === void 0 ? void 0 : _g.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
+            const matchesName = (_f = langData[lang].filenames) === null || _f === void 0 ? void 0 : _f.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
             if (matchesName) {
                 addResult(file, lang);
                 skipExts = true;
@@ -169,7 +184,7 @@ async function analyse(input, opts = {}) {
         if (!skipExts)
             for (const lang in langData) {
                 // Check if extension is a match
-                const matchesExt = (_h = langData[lang].extensions) === null || _h === void 0 ? void 0 : _h.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
+                const matchesExt = (_g = langData[lang].extensions) === null || _g === void 0 ? void 0 : _g.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
                 if (matchesExt)
                     addResult(file, lang);
             }
@@ -202,14 +217,14 @@ async function analyse(input, opts = {}) {
                         heuristic.language = heuristic.language[0];
                     }
                     // Make sure the results includes this language
-                    const languageGroup = (_j = langData[heuristic.language]) === null || _j === void 0 ? void 0 : _j.group;
+                    const languageGroup = (_h = langData[heuristic.language]) === null || _h === void 0 ? void 0 : _h.group;
                     const matchesLang = fileAssociations[file].includes(heuristic.language);
                     const matchesParent = languageGroup && fileAssociations[file].includes(languageGroup);
                     if (!matchesLang && !matchesParent)
                         continue;
                     // Normalise heuristic data
                     const patterns = [];
-                    const normalise = (contents) => patterns.push(...(Array.isArray(contents) ? contents : [contents]));
+                    const normalise = (contents) => patterns.push(...[contents].flat());
                     if (heuristic.pattern)
                         normalise(heuristic.pattern);
                     if (heuristic.named_pattern)
@@ -225,10 +240,10 @@ async function analyse(input, opts = {}) {
                 }
             }
         // If no heuristics, assign a language
-        (_k = (_s = results.files.results)[file]) !== null && _k !== void 0 ? _k : (_s[file] = fileAssociations[file][0]);
+        (_j = (_r = results.files.results)[file]) !== null && _j !== void 0 ? _j : (_r[file] = fileAssociations[file][0]);
     }
     // Skip specified categories
-    if ((_l = opts.categories) === null || _l === void 0 ? void 0 : _l.length) {
+    if ((_k = opts.categories) === null || _k === void 0 ? void 0 : _k.length) {
         const categories = ['data', 'markup', 'programming', 'prose'];
         const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
         for (const [file, lang] of Object.entries(results.files.results)) {
@@ -260,21 +275,21 @@ async function analyse(input, opts = {}) {
     for (const [file, lang] of Object.entries(results.files.results)) {
         if (lang && !langData[lang])
             continue;
-        const fileSize = (_p = (_o = (_m = opts.fileContent) === null || _m === void 0 ? void 0 : _m[files.indexOf(file)]) === null || _o === void 0 ? void 0 : _o.length) !== null && _p !== void 0 ? _p : fs_1.default.statSync(file).size;
+        const fileSize = (_o = (_m = (_l = opts.fileContent) === null || _l === void 0 ? void 0 : _l[files.indexOf(file)]) === null || _m === void 0 ? void 0 : _m.length) !== null && _o !== void 0 ? _o : fs_1.default.statSync(file).size;
         results.files.bytes += fileSize;
         // If no language found, add extension in other section
         if (!lang) {
             const ext = path_1.default.extname(file);
             const unknownType = ext === '' ? 'filenames' : 'extensions';
             const name = ext === '' ? path_1.default.basename(file) : ext;
-            (_q = (_t = results.unknown[unknownType])[name]) !== null && _q !== void 0 ? _q : (_t[name] = 0);
+            (_p = (_s = results.unknown[unknownType])[name]) !== null && _p !== void 0 ? _p : (_s[name] = 0);
             results.unknown[unknownType][name] += fileSize;
             results.unknown.bytes += fileSize;
             continue;
         }
         // Add language and bytes data to corresponding section
         const { type } = langData[lang];
-        (_r = (_u = results.languages.results)[lang]) !== null && _r !== void 0 ? _r : (_u[lang] = { type, bytes: 0, color: langData[lang].color });
+        (_q = (_t = results.languages.results)[lang]) !== null && _q !== void 0 ? _q : (_t[lang] = { type, bytes: 0, color: langData[lang].color });
         if (opts.childLanguages)
             results.languages.results[lang].parent = langData[lang].group;
         results.languages.results[lang].bytes += fileSize;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "linguist-js",
-	"version": "2.3.1",
+	"version": "2.3.2",
 	"description": "Analyse languages used in a folder. Powered by GitHub Linguist, although it doesn't need to be installed.",
 	"main": "dist/index.js",
 	"bin": {
@@ -38,6 +38,7 @@
 	"dependencies": {
 		"binary-extensions": "^2.2.0",
 		"commander": "^9.0.0",
+		"common-path-prefix": "^3.0.0",
 		"cross-fetch": "^3.1.5",
 		"glob-to-regexp": "~0.4.1",
 		"isbinaryfile": "^4.0.8",