linguist-js 2.4.0 → 2.4.1

package/dist/helpers/walk-tree.js

@@ -8,9 +8,7 @@ const path_1 = __importDefault(require("path"));
 const allFiles = new Set();
 const allFolders = new Set();
 /** Generate list of files in a directory. */
-function walk(root, folders, ignored = []) {
-    // Switch out paths that expect being in root
-    ignored = ignored.map(match => match.replace(/^\^/, '^\\./'));
+function walk(root, folders, gitignores, regexIgnores) {
     // Walk tree of a folder
     if (folders.length === 1) {
         const folder = folders[0];
@@ -27,7 +25,10 @@ function walk(root, folders, ignored = []) {
             // Create absolute path for disc operations
             const path = path_1.default.resolve(root, file).replace(/\\/g, '/');
             // Skip if nonexistant or ignored
-            if (!fs_1.default.existsSync(path) || ignored.some(pattern => RegExp(pattern).test(file)))
+            const nonExistant = !fs_1.default.existsSync(path);
+            const isGitIgnored = gitignores.test(file.replace('./', '')).ignored;
+            const isRegexIgnored = regexIgnores.find(match => file.replace('./', '').match(match));
+            if (nonExistant || isGitIgnored || isRegexIgnored)
                 continue;
             // Add absolute folder path to list
             allFolders.add(path_1.default.resolve(folder).replace(/\\/g, '/'));
@@ -35,7 +36,7 @@ function walk(root, folders, ignored = []) {
             if (file.endsWith('/')) {
                 // Recurse into subfolders
                 allFolders.add(path);
-                walk(root, [path], ignored);
+                walk(root, [path], gitignores, regexIgnores);
             }
             else {
                 // Add relative file path to list
@@ -46,7 +47,7 @@ function walk(root, folders, ignored = []) {
     // Recurse into all folders
     else {
         for (const path of folders) {
-            walk(root, [path], ignored);
+            walk(root, [path], gitignores, regexIgnores);
         }
     }
     // Return absolute files and folders lists

package/dist/index.js

@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const js_yaml_1 = __importDefault(require("js-yaml"));
-const glob_to_regexp_1 = __importDefault(require("glob-to-regexp"));
+const ignore_1 = __importDefault(require("ignore"));
 const common_path_prefix_1 = __importDefault(require("common-path-prefix"));
 const binary_extensions_1 = __importDefault(require("binary-extensions"));
 const isbinaryfile_1 = require("isbinaryfile");
@@ -13,7 +13,6 @@ const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
 const load_data_1 = __importDefault(require("./helpers/load-data"));
 const read_file_1 = __importDefault(require("./helpers/read-file"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
-const convert_glob_1 = __importDefault(require("./helpers/convert-glob"));
 async function analyse(input, opts = {}) {
     var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
     var _t, _u, _v;
@@ -25,7 +24,7 @@ async function analyse(input, opts = {}) {
     const vendorData = await (0, load_data_1.default)('vendor.yml').then(js_yaml_1.default.load);
     const docData = await (0, load_data_1.default)('documentation.yml').then(js_yaml_1.default.load);
     const heuristicsData = await (0, load_data_1.default)('heuristics.yml').then(js_yaml_1.default.load);
-    const generatedData = await (0, load_data_1.default)('generated.rb').then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/\))/gm)) !== null && _a !== void 0 ? _a : []; });
+    const generatedData = await (0, load_data_1.default)('generated.rb').then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/)/gm)) !== null && _a !== void 0 ? _a : []; });
     const vendorPaths = [...vendorData, ...docData, ...generatedData];
     // Setup main variables
     const fileAssociations = {};
@@ -38,13 +37,18 @@ async function analyse(input, opts = {}) {
         unknown: { count: 0, bytes: 0, extensions: {}, filenames: {} },
     };
     // Prepare list of ignored files
-    const ignoredFiles = ['(^|\\/)\\.git\\/'];
-    if (!opts.keepVendored) {
-        ignoredFiles.push(...vendorPaths);
-    }
-    if (opts.ignoredFiles) {
-        ignoredFiles.push(...opts.ignoredFiles.map(path => (0, glob_to_regexp_1.default)('*' + path + '*', { extended: true }).source));
-    }
+    const gitignores = (0, ignore_1.default)();
+    const regexIgnores = [];
+    gitignores.add('/.git');
+    if (!opts.keepVendored)
+        regexIgnores.push(...vendorPaths.map(path => RegExp(path, 'i')));
+    if (opts.ignoredFiles)
+        gitignores.add(opts.ignoredFiles);
+    // Set a common root path so that vendor paths do not incorrectly match parent folders
+    const resolvedInput = input.map(path => path_1.default.resolve(path).replace(/\\/g, '/'));
+    const commonRoot = (input.length > 1 ? (0, common_path_prefix_1.default)(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
+    const relPath = (file) => path_1.default.relative(commonRoot, file).replace(/\\/g, '/');
+    const unRelPath = (file) => path_1.default.resolve(commonRoot, file).replace(/\\/g, '/');
     // Load file paths and folders
     let files, folders;
     if (useRawContent) {
@@ -54,10 +58,7 @@ async function analyse(input, opts = {}) {
     }
     else {
         // Uses directory on disc
-        // Set a common root path so that vendor paths do not incorrectly match parent folders
-        const resolvedInput = input.map(path => path_1.default.resolve(path).replace(/\\/g, '/'));
-        const commonRoot = (input.length > 1 ? (0, common_path_prefix_1.default)(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
-        const data = (0, walk_tree_1.default)(commonRoot, input, ignoredFiles);
+        const data = (0, walk_tree_1.default)(commonRoot, input, gitignores, regexIgnores);
         files = data.files;
         folders = data.folders;
     }
@@ -79,40 +80,36 @@ async function analyse(input, opts = {}) {
             }
         }
     }
-    // Load gitattributes
-    const customIgnored = [];
-    const customBinary = [];
-    const customText = [];
+    // Load gitignores and gitattributes
+    const customBinary = (0, ignore_1.default)();
+    const customText = (0, ignore_1.default)();
     if (!useRawContent && opts.checkAttributes) {
         for (const folder of folders) {
             // Skip if folder is marked in gitattributes
-            if (customIgnored.some(path => (0, convert_glob_1.default)(path).test(folder)))
+            if (relPath(folder) && gitignores.test(relPath(folder)).ignored)
                 continue;
             // Parse gitignores
             const ignoresFile = path_1.default.join(folder, '.gitignore');
             if (opts.checkIgnored && fs_1.default.existsSync(ignoresFile)) {
                 const ignoresData = await (0, read_file_1.default)(ignoresFile);
-                const ignoresList = ignoresData.split(/\r?\n/).filter(line => line.trim() && !line.startsWith('#'));
-                const ignoredPaths = ignoresList.map(path => (0, convert_glob_1.default)(path).source);
-                customIgnored.push(...ignoredPaths);
+                gitignores.add(ignoresData);
             }
             // Parse gitattributes
             const attributesFile = path_1.default.join(folder, '.gitattributes');
             if (opts.checkAttributes && fs_1.default.existsSync(attributesFile)) {
                 const attributesData = await (0, read_file_1.default)(attributesFile);
-                const relPathToRegex = (path) => (0, convert_glob_1.default)(path).source.substr(1).replace(folder, '');
                 // Explicit text/binary associations
                 const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)(?!=auto)/gm);
                 for (const [_line, path, type] of contentTypeMatches) {
                     if (['text', '-binary'].includes(type))
-                        customText.push(relPathToRegex(path));
+                        customText.add(path);
                     if (['-text', 'binary'].includes(type))
-                        customBinary.push(relPathToRegex(path));
+                        customBinary.add(path);
                 }
                 // Custom vendor options
                 const vendorMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-(vendored|generated|documentation)(?!=false)/gm);
                 for (const [_line, path] of vendorMatches) {
-                    customIgnored.push(relPathToRegex(path));
+                    gitignores.add(path);
                 }
                 // Custom file associations
                 const customLangMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-language=(\S+)/gm);
@@ -123,17 +120,22 @@ async function analyse(input, opts = {}) {
                         if (overrideLang)
                             forcedLang = overrideLang[0];
                     }
-                    const fullPath = folder + (0, convert_glob_1.default)(path).source.substr(1);
+                    const fullPath = relPath(folder) + '/' + path;
                     overrides[fullPath] = forcedLang;
                 }
             }
         }
     }
     // Check vendored files
-    if (!useRawContent && !opts.keepVendored) {
+    if (!opts.keepVendored) {
         // Filter out any files that match a vendor file path
-        const matcher = (match) => RegExp(match.replace(/\/$/, '/.+$').replace(/^\.\//, ''));
-        files = files.filter(file => !customIgnored.some(pattern => matcher(pattern).test(file)));
+        if (useRawContent) {
+            files = gitignores.filter(files);
+            files = files.filter(file => !regexIgnores.find(match => match.test(file)));
+        }
+        else {
+            files = gitignores.filter(files.map(relPath)).map(unRelPath);
+        }
     }
     // Load all files and parse languages
     const addResult = (file, result) => {
@@ -166,7 +168,7 @@ async function analyse(input, opts = {}) {
         if (!opts.quick && (hasShebang || hasModeline)) {
             const matches = [];
             for (const [lang, data] of Object.entries(langData)) {
-                const langMatcher = (lang) => `\\b${lang.toLowerCase().replace(/\W/g, '\\$&')}\\b`;
+                const langMatcher = (lang) => `\\b${lang.toLowerCase().replace(/\W/g, '\\$&')}(?![\\w#+*]|-\*-)`;
                 // Check for interpreter match
                 const matchesInterpretor = (_f = data.interpreters) === null || _f === void 0 ? void 0 : _f.some(interpreter => firstLine.match(`\\b${interpreter}\\b`));
                 // Check modeline declaration
@@ -188,7 +190,8 @@ async function analyse(input, opts = {}) {
         }
         // Check override for manual language classification
         if (!useRawContent && !opts.quick && opts.checkAttributes) {
-            const match = overridesArray.find(item => RegExp(item[0]).test(file));
+            const isOverridden = (path) => (0, ignore_1.default)().add(path).test(relPath(file)).ignored;
+            const match = overridesArray.find(item => isOverridden(item[0]));
             if (match) {
                 const forcedLang = match[1];
                 addResult(file, forcedLang);
@@ -226,8 +229,8 @@ async function analyse(input, opts = {}) {
         }
         // Skip binary files
         if (!useRawContent && !opts.keepBinary) {
-            const isCustomText = customText.some(path => RegExp(path).test(file));
-            const isCustomBinary = customBinary.some(path => RegExp(path).test(file));
+            const isCustomText = customText.test(relPath(file)).ignored;
+            const isCustomBinary = customBinary.test(relPath(file)).ignored;
             const isBinaryExt = binary_extensions_1.default.some(ext => file.endsWith('.' + ext));
             if (!isCustomText && (isCustomBinary || isBinaryExt || await (0, isbinaryfile_1.isBinaryFile)(file))) {
                 continue;

package/license.md

@@ -1,6 +1,6 @@
 # ISC License
 
-Copyright © 2021 Nixinova
+Copyright © Nixinova 2021–2022
 
 Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "linguist-js",
-	"version": "2.4.0",
+	"version": "2.4.1",
 	"description": "Analyse languages used in a folder. Powered by GitHub Linguist, although it doesn't need to be installed.",
 	"main": "dist/index.js",
 	"bin": {
@@ -40,7 +40,7 @@
 		"commander": "^9.0.0",
 		"common-path-prefix": "^3.0.0",
 		"cross-fetch": "^3.1.5",
-		"glob-to-regexp": "~0.4.1",
+		"ignore": "^5.2.0",
 		"isbinaryfile": "^4.0.8",
 		"js-yaml": "^4.1.0",
 		"node-cache": "^5.1.2"

package/readme.md

@@ -97,7 +97,6 @@ const { files, languages, unknown } = linguist(folder, options);
   Analyse the language of all files found in a folder.
   - `entry` (optional; string or string array):
     The folder(s) to analyse (defaults to `./`).
-    Analyse multiple folders using the syntax `"{folder1,folder2,...}"`.
   - `opts` (optional; object):
     An object containing analyser options.
     - `fileContent` (string or string array):
@@ -160,7 +159,7 @@ linguist --help
     Requires `--json` to be specified.
   - `--quick`:
     Whether to skip the checking of `.gitattributes` and `.gitignore` files for manual language classifications.
-    Alias for `--checkAttributes=false --checkIgnored=false --checkHeuristics=false --checkExplicit=false --checkModeline=false`.
+    Alias for `--checkAttributes=false --checkIgnored=false --checkHeuristics=false --checkShebang=false --checkModeline=false`.
   - `--keepVendored`:
     Whether to include vendored files (auto-generated files, dependencies folder, etc).
   - `--keepBinary`:

package/dist/helpers/convert-glob.js

@@ -1,11 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const glob_to_regexp_1 = __importDefault(require("glob-to-regexp"));
-function parseGitGlob(path) {
-    const globPath = `**/${path}`.replace(/\\/g, '/').replace(/\[:(space|digit):\]/g, (_, val) => `\\${val[0]}`);
-    return (0, glob_to_regexp_1.default)(globPath, { globstar: true, extended: true });
-}
-exports.default = parseGitGlob;