linguist-js 2.4.0 → 2.5.0

package/dist/cli.js

@@ -19,6 +19,7 @@ commander_1.program
     .option('-j|--json [bool]', 'Display the output as JSON', false)
     .option('-t|--tree <traversal>', 'Which part of the output JSON to display (dot-delimited)')
     .option('-q|--quick [bool]', 'Skip complex language analysis (alias for -{A|I|H|S}=false)', false)
+    .option('-o|--offline [bool]', 'Use packaged data files instead of fetching latest from GitHub', false)
     .option('-V|--keepVendored [bool]', 'Prevent skipping over vendored/generated files', false)
     .option('-B|--keepBinary [bool]', 'Prevent skipping over binary files', false)
     .option('-r|--relativePaths [bool]', 'Convert absolute file paths to relative', false)

package/dist/helpers/load-data.js

@@ -3,11 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
 const cross_fetch_1 = __importDefault(require("cross-fetch"));
 const node_cache_1 = __importDefault(require("node-cache"));
 const cache = new node_cache_1.default({});
-/** Load a data file from github-linguist. */
-async function loadFile(file) {
+async function loadWebFile(file) {
     // Return cache if it exists
     const cachedContent = cache.get(file);
     if (cachedContent)
@@ -18,4 +19,12 @@ async function loadFile(file) {
     cache.set(file, fileContent);
     return fileContent;
 }
+async function loadLocalFile(file) {
+    const filePath = path_1.default.resolve(__dirname, '../../ext', file);
+    return promises_1.default.readFile(filePath).then(buffer => buffer.toString());
+}
+/** Load a data file from github-linguist. */
+async function loadFile(file, offline = false) {
+    return offline ? loadLocalFile(file) : loadWebFile(file);
+}
 exports.default = loadFile;

package/dist/helpers/walk-tree.js

@@ -5,12 +5,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
-const allFiles = new Set();
-const allFolders = new Set();
+let allFiles;
+let allFolders;
 /** Generate list of files in a directory. */
-function walk(root, folders, ignored = []) {
-    // Switch out paths that expect being in root
-    ignored = ignored.map(match => match.replace(/^\^/, '^\\./'));
+function walk(init, root, folders, gitignores, regexIgnores) {
+    // Initialise files and folders lists
+    if (init) {
+        allFiles = new Set();
+        allFolders = new Set();
+    }
     // Walk tree of a folder
     if (folders.length === 1) {
         const folder = folders[0];
@@ -27,7 +30,10 @@ function walk(root, folders, ignored = []) {
             // Create absolute path for disc operations
             const path = path_1.default.resolve(root, file).replace(/\\/g, '/');
             // Skip if nonexistant or ignored
-            if (!fs_1.default.existsSync(path) || ignored.some(pattern => RegExp(pattern).test(file)))
+            const nonExistant = !fs_1.default.existsSync(path);
+            const isGitIgnored = gitignores.test(file.replace('./', '')).ignored;
+            const isRegexIgnored = regexIgnores.find(match => file.replace('./', '').match(match));
+            if (nonExistant || isGitIgnored || isRegexIgnored)
                 continue;
             // Add absolute folder path to list
             allFolders.add(path_1.default.resolve(folder).replace(/\\/g, '/'));
@@ -35,7 +41,7 @@ function walk(root, folders, ignored = []) {
             if (file.endsWith('/')) {
                 // Recurse into subfolders
                 allFolders.add(path);
-                walk(root, [path], ignored);
+                walk(false, root, [path], gitignores, regexIgnores);
             }
             else {
                 // Add relative file path to list
@@ -46,7 +52,7 @@ function walk(root, folders, ignored = []) {
     // Recurse into all folders
     else {
         for (const path of folders) {
-            walk(root, [path], ignored);
+            walk(false, root, [path], gitignores, regexIgnores);
         }
     }
     // Return absolute files and folders lists

package/dist/index.js

@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const js_yaml_1 = __importDefault(require("js-yaml"));
-const glob_to_regexp_1 = __importDefault(require("glob-to-regexp"));
+const ignore_1 = __importDefault(require("ignore"));
 const common_path_prefix_1 = __importDefault(require("common-path-prefix"));
 const binary_extensions_1 = __importDefault(require("binary-extensions"));
 const isbinaryfile_1 = require("isbinaryfile");
@@ -13,7 +13,6 @@ const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
 const load_data_1 = __importDefault(require("./helpers/load-data"));
 const read_file_1 = __importDefault(require("./helpers/read-file"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
-const convert_glob_1 = __importDefault(require("./helpers/convert-glob"));
 async function analyse(input, opts = {}) {
     var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
     var _t, _u, _v;
@@ -21,11 +20,11 @@ async function analyse(input, opts = {}) {
     input = [input !== null && input !== void 0 ? input : []].flat();
     opts.fileContent = [(_a = opts.fileContent) !== null && _a !== void 0 ? _a : []].flat();
     // Load data from github-linguist web repo
-    const langData = await (0, load_data_1.default)('languages.yml').then(js_yaml_1.default.load);
-    const vendorData = await (0, load_data_1.default)('vendor.yml').then(js_yaml_1.default.load);
-    const docData = await (0, load_data_1.default)('documentation.yml').then(js_yaml_1.default.load);
-    const heuristicsData = await (0, load_data_1.default)('heuristics.yml').then(js_yaml_1.default.load);
-    const generatedData = await (0, load_data_1.default)('generated.rb').then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/\))/gm)) !== null && _a !== void 0 ? _a : []; });
+    const langData = await (0, load_data_1.default)('languages.yml', opts.offline).then(js_yaml_1.default.load);
+    const vendorData = await (0, load_data_1.default)('vendor.yml', opts.offline).then(js_yaml_1.default.load);
+    const docData = await (0, load_data_1.default)('documentation.yml', opts.offline).then(js_yaml_1.default.load);
+    const heuristicsData = await (0, load_data_1.default)('heuristics.yml', opts.offline).then(js_yaml_1.default.load);
+    const generatedData = await (0, load_data_1.default)('generated.rb', opts.offline).then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/)/gm)) !== null && _a !== void 0 ? _a : []; });
     const vendorPaths = [...vendorData, ...docData, ...generatedData];
     // Setup main variables
     const fileAssociations = {};
@@ -38,13 +37,18 @@ async function analyse(input, opts = {}) {
         unknown: { count: 0, bytes: 0, extensions: {}, filenames: {} },
     };
     // Prepare list of ignored files
-    const ignoredFiles = ['(^|\\/)\\.git\\/'];
-    if (!opts.keepVendored) {
-        ignoredFiles.push(...vendorPaths);
-    }
-    if (opts.ignoredFiles) {
-        ignoredFiles.push(...opts.ignoredFiles.map(path => (0, glob_to_regexp_1.default)('*' + path + '*', { extended: true }).source));
-    }
+    const gitignores = (0, ignore_1.default)();
+    const regexIgnores = [];
+    gitignores.add('/.git');
+    if (!opts.keepVendored)
+        regexIgnores.push(...vendorPaths.map(path => RegExp(path, 'i')));
+    if (opts.ignoredFiles)
+        gitignores.add(opts.ignoredFiles);
+    // Set a common root path so that vendor paths do not incorrectly match parent folders
+    const resolvedInput = input.map(path => path_1.default.resolve(path).replace(/\\/g, '/'));
+    const commonRoot = (input.length > 1 ? (0, common_path_prefix_1.default)(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
+    const relPath = (file) => path_1.default.relative(commonRoot, file).replace(/\\/g, '/');
+    const unRelPath = (file) => path_1.default.resolve(commonRoot, file).replace(/\\/g, '/');
     // Load file paths and folders
     let files, folders;
     if (useRawContent) {
@@ -54,10 +58,7 @@ async function analyse(input, opts = {}) {
     }
     else {
         // Uses directory on disc
-        // Set a common root path so that vendor paths do not incorrectly match parent folders
-        const resolvedInput = input.map(path => path_1.default.resolve(path).replace(/\\/g, '/'));
-        const commonRoot = (input.length > 1 ? (0, common_path_prefix_1.default)(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
-        const data = (0, walk_tree_1.default)(commonRoot, input, ignoredFiles);
+        const data = (0, walk_tree_1.default)(true, commonRoot, input, gitignores, regexIgnores);
         files = data.files;
         folders = data.folders;
     }
@@ -79,40 +80,39 @@ async function analyse(input, opts = {}) {
             }
         }
     }
-    // Load gitattributes
-    const customIgnored = [];
-    const customBinary = [];
-    const customText = [];
+    // Load gitignores and gitattributes
+    const customBinary = (0, ignore_1.default)();
+    const customText = (0, ignore_1.default)();
     if (!useRawContent && opts.checkAttributes) {
         for (const folder of folders) {
             // Skip if folder is marked in gitattributes
-            if (customIgnored.some(path => (0, convert_glob_1.default)(path).test(folder)))
+            if (relPath(folder) && gitignores.ignores(relPath(folder))) {
                 continue;
+            }
             // Parse gitignores
             const ignoresFile = path_1.default.join(folder, '.gitignore');
             if (opts.checkIgnored && fs_1.default.existsSync(ignoresFile)) {
                 const ignoresData = await (0, read_file_1.default)(ignoresFile);
-                const ignoresList = ignoresData.split(/\r?\n/).filter(line => line.trim() && !line.startsWith('#'));
-                const ignoredPaths = ignoresList.map(path => (0, convert_glob_1.default)(path).source);
-                customIgnored.push(...ignoredPaths);
+                gitignores.add(ignoresData);
             }
             // Parse gitattributes
             const attributesFile = path_1.default.join(folder, '.gitattributes');
             if (opts.checkAttributes && fs_1.default.existsSync(attributesFile)) {
                 const attributesData = await (0, read_file_1.default)(attributesFile);
-                const relPathToRegex = (path) => (0, convert_glob_1.default)(path).source.substr(1).replace(folder, '');
                 // Explicit text/binary associations
                 const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)(?!=auto)/gm);
                 for (const [_line, path, type] of contentTypeMatches) {
-                    if (['text', '-binary'].includes(type))
-                        customText.push(relPathToRegex(path));
-                    if (['-text', 'binary'].includes(type))
-                        customBinary.push(relPathToRegex(path));
+                    if (['text', '-binary'].includes(type)) {
+                        customText.add(path);
+                    }
+                    if (['-text', 'binary'].includes(type)) {
+                        customBinary.add(path);
+                    }
                 }
                 // Custom vendor options
                 const vendorMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-(vendored|generated|documentation)(?!=false)/gm);
                 for (const [_line, path] of vendorMatches) {
-                    customIgnored.push(relPathToRegex(path));
+                    gitignores.add(path);
                 }
                 // Custom file associations
                 const customLangMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-language=(\S+)/gm);
@@ -120,20 +120,26 @@ async function analyse(input, opts = {}) {
                     // If specified language is an alias, associate it with its full name
                     if (!langData[forcedLang]) {
                         const overrideLang = Object.entries(langData).find(entry => { var _a; return (_a = entry[1].aliases) === null || _a === void 0 ? void 0 : _a.includes(forcedLang.toLowerCase()); });
-                        if (overrideLang)
+                        if (overrideLang) {
                             forcedLang = overrideLang[0];
+                        }
                     }
-                    const fullPath = folder + (0, convert_glob_1.default)(path).source.substr(1);
+                    const fullPath = relPath(folder) + '/' + path;
                     overrides[fullPath] = forcedLang;
                 }
             }
         }
     }
     // Check vendored files
-    if (!useRawContent && !opts.keepVendored) {
+    if (!opts.keepVendored) {
         // Filter out any files that match a vendor file path
-        const matcher = (match) => RegExp(match.replace(/\/$/, '/.+$').replace(/^\.\//, ''));
-        files = files.filter(file => !customIgnored.some(pattern => matcher(pattern).test(file)));
+        if (useRawContent) {
+            files = gitignores.filter(files);
+            files = files.filter(file => !regexIgnores.find(match => match.test(file)));
+        }
+        else {
+            files = gitignores.filter(files.map(relPath)).map(unRelPath);
+        }
     }
     // Load all files and parse languages
     const addResult = (file, result) => {
@@ -152,11 +158,11 @@ async function analyse(input, opts = {}) {
         if (useRawContent) {
             firstLine = (_e = (_d = (_c = opts.fileContent) === null || _c === void 0 ? void 0 : _c[files.indexOf(file)]) === null || _d === void 0 ? void 0 : _d.split('\n')[0]) !== null && _e !== void 0 ? _e : null;
         }
-        else {
-            if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
-                continue;
+        else if (fs_1.default.existsSync(file) && !fs_1.default.lstatSync(file).isDirectory()) {
             firstLine = await (0, read_file_1.default)(file, true).catch(() => null);
         }
+        else
+            continue;
         // Skip if file is unreadable
         if (firstLine === null)
             continue;
@@ -166,17 +172,18 @@ async function analyse(input, opts = {}) {
         if (!opts.quick && (hasShebang || hasModeline)) {
             const matches = [];
             for (const [lang, data] of Object.entries(langData)) {
-                const langMatcher = (lang) => `\\b${lang.toLowerCase().replace(/\W/g, '\\$&')}\\b`;
+                const langMatcher = (lang) => `\\b${lang.toLowerCase().replace(/\W/g, '\\$&')}(?![\\w#+*]|-\*-)`;
                 // Check for interpreter match
                 const matchesInterpretor = (_f = data.interpreters) === null || _f === void 0 ? void 0 : _f.some(interpreter => firstLine.match(`\\b${interpreter}\\b`));
                 // Check modeline declaration
                 const matchesLang = firstLine.toLowerCase().match(langMatcher(lang));
                 const matchesAlias = (_g = data.aliases) === null || _g === void 0 ? void 0 : _g.some(lang => firstLine.toLowerCase().match(langMatcher(lang)));
                 // Add language
-                if (opts.checkShebang && matchesInterpretor)
-                    matches.push(lang);
-                if (opts.checkModeline && (matchesLang || matchesAlias))
+                const interpretorCheck = opts.checkShebang && matchesInterpretor;
+                const modelineCheck = opts.checkModeline && (matchesLang || matchesAlias);
+                if (interpretorCheck || modelineCheck) {
                     matches.push(lang);
+                }
             }
             if (matches.length) {
                 // Add explicitly-identified language
@@ -188,7 +195,8 @@ async function analyse(input, opts = {}) {
         }
         // Check override for manual language classification
         if (!useRawContent && !opts.quick && opts.checkAttributes) {
-            const match = overridesArray.find(item => RegExp(item[0]).test(file));
+            const isOverridden = (path) => (0, ignore_1.default)().add(path).ignores(relPath(file));
+            const match = overridesArray.find(item => isOverridden(item[0]));
             if (match) {
                 const forcedLang = match[1];
                 addResult(file, forcedLang);
@@ -210,12 +218,14 @@ async function analyse(input, opts = {}) {
             for (const lang in langData) {
                 // Check if extension is a match
                 const matchesExt = (_j = langData[lang].extensions) === null || _j === void 0 ? void 0 : _j.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
-                if (matchesExt)
+                if (matchesExt) {
                     addResult(file, lang);
+                }
             }
         // Fallback to null if no language matches
-        if (!fileAssociations[file])
+        if (!fileAssociations[file]) {
             addResult(file, null);
+        }
     }
     // Narrow down file associations to the best fit
     for (const file in fileAssociations) {
@@ -226,8 +236,8 @@ async function analyse(input, opts = {}) {
         }
         // Skip binary files
         if (!useRawContent && !opts.keepBinary) {
-            const isCustomText = customText.some(path => RegExp(path).test(file));
-            const isCustomBinary = customBinary.some(path => RegExp(path).test(file));
+            const isCustomText = customText.ignores(relPath(file));
+            const isCustomBinary = customBinary.ignores(relPath(file));
             const isBinaryExt = binary_extensions_1.default.some(ext => file.endsWith('.' + ext));
             if (!isCustomText && (isCustomBinary || isBinaryExt || await (0, isbinaryfile_1.isBinaryFile)(file))) {
                 continue;
@@ -260,7 +270,7 @@ async function analyse(input, opts = {}) {
                     if (heuristic.named_pattern)
                         normalise(heuristicsData.named_patterns[heuristic.named_pattern]);
                     // Check file contents and apply heuristic patterns
-                    const fileContent = await (0, read_file_1.default)(file).catch(() => null);
+                    const fileContent = opts.fileContent ? opts.fileContent[files.indexOf(file)] : await (0, read_file_1.default)(file).catch(() => null);
                     if (fileContent === null)
                         continue;
                     if (!patterns.length || patterns.some(pattern => (0, convert_pcre_1.default)(pattern).test(fileContent))) {
@@ -277,16 +287,19 @@ async function analyse(input, opts = {}) {
         const categories = ['data', 'markup', 'programming', 'prose'];
         const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
         for (const [file, lang] of Object.entries(results.files.results)) {
-            if (!hiddenCategories.some(cat => { var _a; return lang && ((_a = langData[lang]) === null || _a === void 0 ? void 0 : _a.type) === cat; }))
+            if (!hiddenCategories.some(cat => { var _a; return lang && ((_a = langData[lang]) === null || _a === void 0 ? void 0 : _a.type) === cat; })) {
                 continue;
+            }
             delete results.files.results[file];
-            if (lang)
+            if (lang) {
                 delete results.languages.results[lang];
+            }
         }
         for (const category of hiddenCategories) {
             for (const [lang, { type }] of Object.entries(results.languages.results)) {
-                if (type === category)
+                if (type === category) {
                     delete results.languages.results[lang];
+                }
             }
         }
     }
@@ -295,8 +308,9 @@ async function analyse(input, opts = {}) {
         const newMap = {};
         for (const [file, lang] of Object.entries(results.files.results)) {
             let relPath = path_1.default.relative(process.cwd(), file).replace(/\\/g, '/');
-            if (!relPath.startsWith('../'))
+            if (!relPath.startsWith('../')) {
                 relPath = './' + relPath;
+            }
             newMap[relPath] = lang;
         }
         results.files.results = newMap;
@@ -320,8 +334,9 @@ async function analyse(input, opts = {}) {
         // Add language and bytes data to corresponding section
         const { type } = langData[lang];
         (_s = (_v = results.languages.results)[lang]) !== null && _s !== void 0 ? _s : (_v[lang] = { type, bytes: 0, color: langData[lang].color });
-        if (opts.childLanguages)
+        if (opts.childLanguages) {
             results.languages.results[lang].parent = langData[lang].group;
+        }
         results.languages.results[lang].bytes += fileSize;
         results.languages.bytes += fileSize;
     }

package/dist/types.d.ts

@@ -14,6 +14,7 @@ export interface Options {
     relativePaths?: boolean;
     childLanguages?: boolean;
     quick?: boolean;
+    offline?: boolean;
     checkIgnored?: boolean;
     checkAttributes?: boolean;
     checkHeuristics?: boolean;

package/ext/documentation.yml

@@ -0,0 +1,17 @@
+- ^[Dd]ocs?/
+- (^|/)[Dd]ocumentation/
+- (^|/)[Gg]roovydoc/
+- (^|/)[Jj]avadoc/
+- ^[Mm]an/
+- ^[Ee]xamples/
+- ^[Dd]emos?/
+- (^|/)inst/doc/- (^|/)CITATION(\.cff|(S)?(\.(bib|md))?)$
+- (^|/)CHANGE(S|LOG)?(\.|$)
+- (^|/)CONTRIBUTING(\.|$)
+- (^|/)COPYING(\.|$)
+- (^|/)INSTALL(\.|$)
+- (^|/)LICEN[CS]E(\.|$)
+- (^|/)[Ll]icen[cs]e(\.|$)
+- (^|/)README(\.|$)
+- (^|/)[Rr]eadme(\.|$)
+- ^[Ss]amples?/