linguist-js 2.6.0 → 2.7.0-pre

package/dist/index.js

@@ -1,389 +1,429 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-const fs_1 = __importDefault(require("fs"));
-const path_1 = __importDefault(require("path"));
-const js_yaml_1 = __importDefault(require("js-yaml"));
-const ignore_1 = __importDefault(require("ignore"));
-const common_path_prefix_1 = __importDefault(require("common-path-prefix"));
-const binary_extensions_1 = __importDefault(require("binary-extensions"));
-const isbinaryfile_1 = require("isbinaryfile");
-const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
-const load_data_1 = __importDefault(require("./helpers/load-data"));
-const read_file_1 = __importDefault(require("./helpers/read-file"));
-const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
-async function analyse(input, opts = {}) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
-    var _t, _u;
-    const useRawContent = opts.fileContent !== undefined;
-    input = [input !== null && input !== void 0 ? input : []].flat();
-    opts.fileContent = [(_a = opts.fileContent) !== null && _a !== void 0 ? _a : []].flat();
-    // Load data from github-linguist web repo
-    const langData = await (0, load_data_1.default)('languages.yml', opts.offline).then(js_yaml_1.default.load);
-    const vendorData = await (0, load_data_1.default)('vendor.yml', opts.offline).then(js_yaml_1.default.load);
-    const docData = await (0, load_data_1.default)('documentation.yml', opts.offline).then(js_yaml_1.default.load);
-    const heuristicsData = await (0, load_data_1.default)('heuristics.yml', opts.offline).then(js_yaml_1.default.load);
-    const generatedData = await (0, load_data_1.default)('generated.rb', opts.offline).then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/)/gm)) !== null && _a !== void 0 ? _a : []; });
-    const vendorPaths = [...vendorData, ...docData, ...generatedData];
-    // Setup main variables
-    const fileAssociations = {};
-    const extensions = {};
-    const overrides = {};
-    const results = {
-        files: { count: 0, bytes: 0, results: {}, alternatives: {} },
-        languages: { count: 0, bytes: 0, results: {} },
-        unknown: { count: 0, bytes: 0, extensions: {}, filenames: {} },
-    };
-    // Prepare list of ignored files
-    const gitignores = (0, ignore_1.default)();
-    const regexIgnores = [];
-    gitignores.add('/.git');
-    if (!opts.keepVendored)
-        regexIgnores.push(...vendorPaths.map(path => RegExp(path, 'i')));
-    if (opts.ignoredFiles)
-        gitignores.add(opts.ignoredFiles);
-    // Set a common root path so that vendor paths do not incorrectly match parent folders
-    const resolvedInput = input.map(path => path_1.default.resolve(path).replace(/\\/g, '/'));
-    const commonRoot = (input.length > 1 ? (0, common_path_prefix_1.default)(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
-    const relPath = (file) => path_1.default.relative(commonRoot, file).replace(/\\/g, '/');
-    const unRelPath = (file) => path_1.default.resolve(commonRoot, file).replace(/\\/g, '/');
-    // Load file paths and folders
-    let files, folders;
-    if (useRawContent) {
-        // Uses raw file content
-        files = input;
-        folders = [''];
-    }
-    else {
-        // Uses directory on disc
-        const data = (0, walk_tree_1.default)(true, commonRoot, input, gitignores, regexIgnores);
-        files = data.files;
-        folders = data.folders;
-    }
-    // Apply aliases
-    opts = {
-        checkIgnored: !opts.quick,
-        checkAttributes: !opts.quick,
-        checkHeuristics: !opts.quick,
-        checkShebang: !opts.quick,
-        checkModeline: !opts.quick,
-        ...opts
-    };
-    // Ignore specific languages
-    for (const lang of (_b = opts.ignoredLanguages) !== null && _b !== void 0 ? _b : []) {
-        for (const key in langData) {
-            if (lang.toLowerCase() === key.toLowerCase()) {
-                delete langData[key];
-                break;
-            }
-        }
-    }
-    // Load gitignores and gitattributes
-    const customBinary = (0, ignore_1.default)();
-    const customText = (0, ignore_1.default)();
-    if (!useRawContent && opts.checkAttributes) {
-        for (const folder of folders) {
-            // Skip if folder is marked in gitattributes
-            if (relPath(folder) && gitignores.ignores(relPath(folder))) {
-                continue;
-            }
-            // Parse gitignores
-            const ignoresFile = path_1.default.join(folder, '.gitignore');
-            if (opts.checkIgnored && fs_1.default.existsSync(ignoresFile)) {
-                const ignoresData = await (0, read_file_1.default)(ignoresFile);
-                gitignores.add(ignoresData);
-            }
-            // Parse gitattributes
-            const attributesFile = path_1.default.join(folder, '.gitattributes');
-            if (opts.checkAttributes && fs_1.default.existsSync(attributesFile)) {
-                const attributesData = await (0, read_file_1.default)(attributesFile);
-                // Explicit text/binary associations
-                const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)(?!=auto)/gm);
-                for (const [_line, path, type] of contentTypeMatches) {
-                    if (['text', '-binary'].includes(type)) {
-                        customText.add(path);
-                    }
-                    if (['-text', 'binary'].includes(type)) {
-                        customBinary.add(path);
-                    }
-                }
-                // Custom vendor options
-                const vendorMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-(vendored|generated|documentation)(?!=false)/gm);
-                for (const [_line, path] of vendorMatches) {
-                    gitignores.add(path);
-                }
-                // Custom file associations
-                const customLangMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-language=(\S+)/gm);
-                for (let [_line, path, forcedLang] of customLangMatches) {
-                    // If specified language is an alias, associate it with its full name
-                    if (!langData[forcedLang]) {
-                        const overrideLang = Object.entries(langData).find(entry => { var _a; return (_a = entry[1].aliases) === null || _a === void 0 ? void 0 : _a.includes(forcedLang.toLowerCase()); });
-                        if (overrideLang) {
-                            forcedLang = overrideLang[0];
-                        }
-                    }
-                    const fullPath = path_1.default.join(relPath(folder), path);
-                    overrides[fullPath] = forcedLang;
-                }
-            }
-        }
-    }
-    // Check vendored files
-    if (!opts.keepVendored) {
-        // Filter out any files that match a vendor file path
-        if (useRawContent) {
-            files = gitignores.filter(files);
-            files = files.filter(file => !regexIgnores.find(match => match.test(file)));
-        }
-        else {
-            files = gitignores.filter(files.map(relPath)).map(unRelPath);
-        }
-    }
-    // Load all files and parse languages
-    const addResult = (file, result) => {
-        if (!fileAssociations[file]) {
-            fileAssociations[file] = [];
-            extensions[file] = '';
-        }
-        // Set parent to result group if it is present
-        // Is nullish if either `opts.childLanguages` is set or if there is no group
-        const finalResult = !opts.childLanguages && result && langData[result].group || result;
-        if (!fileAssociations[file].includes(finalResult))
-            fileAssociations[file].push(finalResult);
-        extensions[file] = path_1.default.extname(file).toLowerCase();
-    };
-    const overridesArray = Object.entries(overrides);
-    // List all languages that could be associated with a given file
-    const definiteness = {};
-    const fromShebang = {};
-    for (const file of files) {
-        let firstLine;
-        if (useRawContent) {
-            firstLine = (_e = (_d = (_c = opts.fileContent) === null || _c === void 0 ? void 0 : _c[files.indexOf(file)]) === null || _d === void 0 ? void 0 : _d.split('\n')[0]) !== null && _e !== void 0 ? _e : null;
-        }
-        else if (fs_1.default.existsSync(file) && !fs_1.default.lstatSync(file).isDirectory()) {
-            firstLine = await (0, read_file_1.default)(file, true).catch(() => null);
-        }
-        else
-            continue;
-        // Skip if file is unreadable
-        if (firstLine === null)
-            continue;
-        // Check first line for explicit classification
-        const hasShebang = opts.checkShebang && /^#!/.test(firstLine);
-        const hasModeline = opts.checkModeline && /-\*-|(syntax|filetype|ft)\s*=/.test(firstLine);
-        if (!opts.quick && (hasShebang || hasModeline)) {
-            const matches = [];
-            for (const [lang, data] of Object.entries(langData)) {
-                const langMatcher = (lang) => `\\b${lang.toLowerCase().replace(/\W/g, '\\$&')}(?![\\w#+*]|-\*-)`;
-                // Check for interpreter match
-                if (opts.checkShebang && hasShebang) {
-                    const matchesInterpretor = (_f = data.interpreters) === null || _f === void 0 ? void 0 : _f.some(interpreter => firstLine.match(`\\b${interpreter}\\b`));
-                    if (matchesInterpretor)
-                        matches.push(lang);
-                }
-                // Check modeline declaration
-                if (opts.checkModeline && hasModeline) {
-                    const modelineText = firstLine.toLowerCase().replace(/^.*-\*-(.+)-\*-.*$/, '$1');
-                    const matchesLang = modelineText.match(langMatcher(lang));
-                    const matchesAlias = (_g = data.aliases) === null || _g === void 0 ? void 0 : _g.some(lang => modelineText.match(langMatcher(lang)));
-                    if (matchesLang || matchesAlias)
-                        matches.push(lang);
-                }
-            }
-            // Add identified language(s)
-            if (matches.length) {
-                for (const match of matches)
-                    addResult(file, match);
-                if (matches.length === 1)
-                    definiteness[file] = true;
-                fromShebang[file] = true;
-                continue;
-            }
-        }
-        // Check override for manual language classification
-        if (!useRawContent && !opts.quick && opts.checkAttributes) {
-            const isOverridden = (path) => (0, ignore_1.default)().add(path).ignores(relPath(file));
-            const match = overridesArray.find(item => isOverridden(item[0]));
-            if (match) {
-                const forcedLang = match[1];
-                addResult(file, forcedLang);
-                definiteness[file] = true;
-                continue;
-            }
-        }
-        // Search each language
-        let skipExts = false;
-        // Check if filename is a match
-        for (const lang in langData) {
-            const matchesName = (_h = langData[lang].filenames) === null || _h === void 0 ? void 0 : _h.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
-            if (matchesName) {
-                addResult(file, lang);
-                skipExts = true;
-            }
-        }
-        // Check if extension is a match
-        const possibleExts = [];
-        if (!skipExts)
-            for (const lang in langData) {
-                const extMatches = (_j = langData[lang].extensions) === null || _j === void 0 ? void 0 : _j.filter(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
-                if (extMatches === null || extMatches === void 0 ? void 0 : extMatches.length) {
-                    for (const ext of extMatches)
-                        possibleExts.push({ ext, lang });
-                }
-            }
-        // Apply more specific extension if available
-        const isComplexExt = (ext) => /\..+\./.test(ext);
-        const hasComplexExt = possibleExts.some(data => isComplexExt(data.ext));
-        for (const { ext, lang } of possibleExts) {
-            if (hasComplexExt && !isComplexExt(ext))
-                continue;
-            if (!hasComplexExt && isComplexExt(ext))
-                continue;
-            addResult(file, lang);
-        }
-        // Fallback to null if no language matches
-        if (!fileAssociations[file]) {
-            addResult(file, null);
-        }
-    }
-    // Narrow down file associations to the best fit
-    for (const file in fileAssociations) {
-        // Skip if file has explicit association
-        if (definiteness[file]) {
-            results.files.results[file] = fileAssociations[file][0];
-            continue;
-        }
-        // Skip binary files
-        if (!useRawContent && !opts.keepBinary) {
-            const isCustomText = customText.ignores(relPath(file));
-            const isCustomBinary = customBinary.ignores(relPath(file));
-            const isBinaryExt = binary_extensions_1.default.some(ext => file.endsWith('.' + ext));
-            if (!isCustomText && (isCustomBinary || isBinaryExt || await (0, isbinaryfile_1.isBinaryFile)(file))) {
-                continue;
-            }
-        }
-        // Parse heuristics if applicable
-        if (opts.checkHeuristics)
-            for (const heuristics of heuristicsData.disambiguations) {
-                // Make sure the extension matches the current file
-                if (!fromShebang[file] && !heuristics.extensions.includes(extensions[file]))
-                    continue;
-                // Load heuristic rules
-                for (const heuristic of heuristics.rules) {
-                    // Make sure the language is not an array
-                    if (Array.isArray(heuristic.language)) {
-                        heuristic.language = heuristic.language[0];
-                    }
-                    // Make sure the results includes this language
-                    const languageGroup = (_k = langData[heuristic.language]) === null || _k === void 0 ? void 0 : _k.group;
-                    const matchesLang = fileAssociations[file].includes(heuristic.language);
-                    const matchesParent = languageGroup && fileAssociations[file].includes(languageGroup);
-                    if (!matchesLang && !matchesParent)
-                        continue;
-                    // Normalise heuristic data
-                    const patterns = [];
-                    const normalise = (contents) => patterns.push(...[contents].flat());
-                    if (heuristic.pattern)
-                        normalise(heuristic.pattern);
-                    if (heuristic.named_pattern)
-                        normalise(heuristicsData.named_patterns[heuristic.named_pattern]);
-                    if (heuristic.and) {
-                        for (const data of heuristic.and) {
-                            if (data.pattern)
-                                normalise(data.pattern);
-                            if (data.named_pattern)
-                                normalise(heuristicsData.named_patterns[data.named_pattern]);
-                        }
-                    }
-                    // Check file contents and apply heuristic patterns
-                    const fileContent = ((_l = opts.fileContent) === null || _l === void 0 ? void 0 : _l.length) ? opts.fileContent[files.indexOf(file)] : await (0, read_file_1.default)(file).catch(() => null);
-                    // Skip if file read errors
-                    if (fileContent === null)
-                        continue;
-                    // Apply heuristics
-                    if (!patterns.length || patterns.some(pattern => (0, convert_pcre_1.default)(pattern).test(fileContent))) {
-                        results.files.results[file] = heuristic.language;
-                        break;
-                    }
-                }
-            }
-        // If no heuristics, assign a language
-        if (!results.files.results[file]) {
-            const possibleLangs = fileAssociations[file];
-            // Assign first language as a default option
-            const defaultLang = possibleLangs[0];
-            const alternativeLangs = possibleLangs.slice(1);
-            results.files.results[file] = defaultLang;
-            // List alternative languages if there are any
-            if (alternativeLangs.length > 0)
-                results.files.alternatives[file] = alternativeLangs;
-        }
-    }
-    // Skip specified categories
-    if ((_m = opts.categories) === null || _m === void 0 ? void 0 : _m.length) {
-        const categories = ['data', 'markup', 'programming', 'prose'];
-        const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
-        for (const [file, lang] of Object.entries(results.files.results)) {
-            if (!hiddenCategories.some(cat => { var _a; return lang && ((_a = langData[lang]) === null || _a === void 0 ? void 0 : _a.type) === cat; })) {
-                continue;
-            }
-            delete results.files.results[file];
-            if (lang) {
-                delete results.languages.results[lang];
-            }
-        }
-        for (const category of hiddenCategories) {
-            for (const [lang, { type }] of Object.entries(results.languages.results)) {
-                if (type === category) {
-                    delete results.languages.results[lang];
-                }
-            }
-        }
-    }
-    // Convert paths to relative
-    if (!useRawContent && opts.relativePaths) {
-        const newMap = {};
-        for (const [file, lang] of Object.entries(results.files.results)) {
-            let relPath = path_1.default.relative(process.cwd(), file).replace(/\\/g, '/');
-            if (!relPath.startsWith('../')) {
-                relPath = './' + relPath;
-            }
-            newMap[relPath] = lang;
-        }
-        results.files.results = newMap;
-    }
-    // Load language bytes size
-    for (const [file, lang] of Object.entries(results.files.results)) {
-        if (lang && !langData[lang])
-            continue;
-        const fileSize = (_q = (_p = (_o = opts.fileContent) === null || _o === void 0 ? void 0 : _o[files.indexOf(file)]) === null || _p === void 0 ? void 0 : _p.length) !== null && _q !== void 0 ? _q : fs_1.default.statSync(file).size;
-        results.files.bytes += fileSize;
-        // If no language found, add extension in other section
-        if (!lang) {
-            const ext = path_1.default.extname(file);
-            const unknownType = ext === '' ? 'filenames' : 'extensions';
-            const name = ext === '' ? path_1.default.basename(file) : ext;
-            (_r = (_t = results.unknown[unknownType])[name]) !== null && _r !== void 0 ? _r : (_t[name] = 0);
-            results.unknown[unknownType][name] += fileSize;
-            results.unknown.bytes += fileSize;
-            continue;
-        }
-        // Add language and bytes data to corresponding section
-        const { type } = langData[lang];
-        (_s = (_u = results.languages.results)[lang]) !== null && _s !== void 0 ? _s : (_u[lang] = { type, bytes: 0, color: langData[lang].color });
-        if (opts.childLanguages) {
-            results.languages.results[lang].parent = langData[lang].group;
-        }
-        results.languages.results[lang].bytes += fileSize;
-        results.languages.bytes += fileSize;
-    }
-    // Set counts
-    results.files.count = Object.keys(results.files.results).length;
-    results.languages.count = Object.keys(results.languages.results).length;
-    results.unknown.count = Object.keys({ ...results.unknown.extensions, ...results.unknown.filenames }).length;
-    // Return
-    return results;
-}
-module.exports = analyse;
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const ignore_1 = __importDefault(require("ignore"));
+const common_path_prefix_1 = __importDefault(require("common-path-prefix"));
+const binary_extensions_1 = __importDefault(require("binary-extensions"));
+const isbinaryfile_1 = require("isbinaryfile");
+const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
+const load_data_1 = __importStar(require("./helpers/load-data"));
+const read_file_1 = __importDefault(require("./helpers/read-file"));
+const parse_gitattributes_1 = __importDefault(require("./helpers/parse-gitattributes"));
+const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
+async function analyse(rawInput, opts = {}) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
+    var _r, _s;
+    const useRawContent = opts.fileContent !== undefined;
+    const input = [rawInput !== null && rawInput !== void 0 ? rawInput : []].flat();
+    const manualFileContent = [(_a = opts.fileContent) !== null && _a !== void 0 ? _a : []].flat();
+    // Normalise input option arguments
+    opts = {
+        checkIgnored: !opts.quick,
+        checkAttributes: !opts.quick,
+        checkHeuristics: !opts.quick,
+        checkShebang: !opts.quick,
+        checkModeline: !opts.quick,
+        ...opts,
+    };
+    // Load data from github-linguist web repo
+    const langData = await (0, load_data_1.default)('languages.yml', opts.offline).then(js_yaml_1.default.load);
+    const vendorData = await (0, load_data_1.default)('vendor.yml', opts.offline).then(js_yaml_1.default.load);
+    const docData = await (0, load_data_1.default)('documentation.yml', opts.offline).then(js_yaml_1.default.load);
+    const heuristicsData = await (0, load_data_1.default)('heuristics.yml', opts.offline).then(js_yaml_1.default.load);
+    const generatedData = await (0, load_data_1.default)('generated.rb', opts.offline).then(load_data_1.parseGeneratedDataFile);
+    const vendorPaths = [...vendorData, ...docData, ...generatedData];
+    // Setup main variables
+    const fileAssociations = {};
+    const extensions = {};
+    const globOverrides = {};
+    const results = {
+        files: { count: 0, bytes: 0, results: {}, alternatives: {} },
+        languages: { count: 0, bytes: 0, results: {} },
+        unknown: { count: 0, bytes: 0, extensions: {}, filenames: {} },
+    };
+    // Set a common root path so that vendor paths do not incorrectly match parent folders
+    const normPath = (file) => file.replace(/\\/g, '/');
+    const resolvedInput = input.map(path => normPath(path_1.default.resolve(path)));
+    const commonRoot = (input.length > 1 ? (0, common_path_prefix_1.default)(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
+    const localRoot = (folder) => folder.replace(commonRoot, '').replace(/^\//, '');
+    const relPath = (file) => useRawContent ? file : normPath(path_1.default.relative(commonRoot, file));
+    const unRelPath = (file) => useRawContent ? file : normPath(path_1.default.resolve(commonRoot, file));
+    // Other helper functions
+    const fileMatchesGlobs = (file, ...globs) => (0, ignore_1.default)().add(globs).ignores(relPath(file));
+    const filterOutIgnored = (files, ignored) => ignored.filter(files.map(relPath)).map(unRelPath);
+    //*PREPARE FILES AND DATA*//
+    // Prepare list of ignored files
+    const ignored = (0, ignore_1.default)();
+    ignored.add('.git/');
+    ignored.add((_b = opts.ignoredFiles) !== null && _b !== void 0 ? _b : []);
+    const regexIgnores = [];
+    if (!opts.keepVendored)
+        regexIgnores.push(...vendorPaths.map(path => RegExp(path, 'i')));
+    // Load file paths and folders
+    let files;
+    let folders;
+    if (useRawContent) {
+        // Uses raw file content
+        files = input;
+        folders = [''];
+    }
+    else {
+        // Uses directory on disc
+        const data = (0, walk_tree_1.default)({ init: true, commonRoot, folderRoots: resolvedInput, folders: resolvedInput, ignored });
+        files = data.files;
+        folders = data.folders;
+    }
+    // Load gitignore data and apply ignores rules
+    if (!useRawContent) {
+        // TODO switch to be like the gitattributes code
+        for (const folder of folders) {
+            // Parse gitignores
+            const ignoresFile = path_1.default.join(folder, '.gitignore');
+            if (opts.checkIgnored && fs_1.default.existsSync(ignoresFile)) {
+                const ignoresData = await (0, read_file_1.default)(ignoresFile);
+                const localIgnoresData = ignoresData.replace(/^[\/\\]/g, localRoot(folder) + '/');
+                ignored.add(localIgnoresData);
+                files = filterOutIgnored(files, ignored);
+            }
+        }
+    }
+    // Fetch and normalise gitattributes data of all subfolders and save to metadata
+    const manualAttributes = {}; // Maps file globs to gitattribute boolean flags
+    const getFlaggedGlobs = (attr, val) => {
+        return Object.entries(manualAttributes).filter(([, attrs]) => attrs[attr] === val).map(([glob,]) => glob);
+    };
+    if (!useRawContent && opts.checkAttributes) {
+        const nestedAttrFiles = files.filter(file => file.endsWith('.gitattributes'));
+        for (const attrFile of nestedAttrFiles) {
+            const relAttrFile = relPath(attrFile);
+            const relAttrFolder = path_1.default.dirname(relAttrFile);
+            const contents = await (0, read_file_1.default)(attrFile);
+            const parsed = (0, parse_gitattributes_1.default)(contents, relAttrFolder);
+            for (const { glob, attrs } of parsed) {
+                manualAttributes[glob] = attrs;
+            }
+        }
+    }
+    // Apply vendor file path matches and filter out vendored files
+    if (!opts.keepVendored) {
+        // Get data of files that have been manually marked with metadata
+        const vendorTrueGlobs = [...getFlaggedGlobs('vendored', true), ...getFlaggedGlobs('generated', true), ...getFlaggedGlobs('documentation', true)];
+        const vendorFalseGlobs = [...getFlaggedGlobs('vendored', false), ...getFlaggedGlobs('generated', false), ...getFlaggedGlobs('documentation', false)];
+        // Set up glob ignore object to use for expanding globs to match files
+        const vendorOverrides = (0, ignore_1.default)().add(vendorFalseGlobs);
+        // Remove all files marked as vendored by default
+        const excludedFiles = files.filter(file => vendorPaths.some(pathPtn => RegExp(pathPtn, 'i').test(relPath(file))));
+        files = files.filter(file => !excludedFiles.includes(file));
+        // Re-add removed files that are overridden manually in gitattributes
+        const overriddenExcludedFiles = excludedFiles.filter(file => vendorOverrides.ignores(relPath(file)));
+        files.push(...overriddenExcludedFiles);
+        // Remove files explicitly marked as vendored in gitattributes
+        // TODO change globs.includes(file) to parse the glob using ignore()
+        files = files.filter(file => !vendorTrueGlobs.includes(relPath(file)));
+    }
+    // Filter out binary files
+    if (!opts.keepBinary) {
+        // Filter out files that are binary by default
+        files = files.filter(file => !binary_extensions_1.default.some(ext => file.endsWith('.' + ext)));
+        // Filter out manually specified binary files
+        const binaryIgnored = (0, ignore_1.default)().add(getFlaggedGlobs('binary', true));
+        files = filterOutIgnored(files, binaryIgnored);
+        // Re-add files manually marked not as binary
+        const binaryUnignored = (0, ignore_1.default)().add(getFlaggedGlobs('binary', false));
+        // TODO parse the globs using ignore()
+        const unignoredList = filterOutIgnored(files, binaryUnignored);
+        files.push(...unignoredList);
+    }
+    // Ignore specific languages
+    for (const lang of (_c = opts.ignoredLanguages) !== null && _c !== void 0 ? _c : []) {
+        for (const key in langData) {
+            if (lang.toLowerCase() === key.toLowerCase()) {
+                delete langData[key];
+                break;
+            }
+        }
+    }
+    // Establish language overrides taken from gitattributes
+    const forcedLangs = Object.entries(manualAttributes).filter(([, attrs]) => attrs.language);
+    for (const [globPath, attrs] of forcedLangs) {
+        let forcedLang = attrs.language;
+        if (!forcedLang)
+            continue;
+        // If specified language is an alias, associate it with its full name
+        if (!langData[forcedLang]) {
+            const overrideLang = Object.entries(langData).find(entry => { var _a; return (_a = entry[1].aliases) === null || _a === void 0 ? void 0 : _a.includes(forcedLang.toLowerCase()); });
+            if (overrideLang) {
+                forcedLang = overrideLang[0];
+            }
+        }
+        globOverrides[globPath] = forcedLang;
+    }
+    //*PARSE LANGUAGES*//
+    const addResult = (file, result) => {
+        if (!fileAssociations[file]) {
+            fileAssociations[file] = [];
+            extensions[file] = '';
+        }
+        // Set parent to result group if it is present
+        // Is nullish if either `opts.childLanguages` is set or if there is no group
+        const finalResult = !opts.childLanguages && result && langData[result] && langData[result].group || result;
+        if (!fileAssociations[file].includes(finalResult)) {
+            fileAssociations[file].push(finalResult);
+        }
+        extensions[file] = path_1.default.extname(file).toLowerCase();
+    };
+    const definiteness = {};
+    const fromShebang = {};
+    fileLoop: for (const file of files) {
+        // Check manual override
+        for (const globMatch in globOverrides) {
+            if (!fileMatchesGlobs(file, globMatch))
+                continue;
+            // If the given file matches the glob, apply the override to the file
+            const forcedLang = globOverrides[globMatch];
+            addResult(file, forcedLang);
+            definiteness[file] = true;
+            continue fileLoop; // no need to check other heuristics, the classified language has been found
+        }
+        // Check first line for readability
+        let firstLine;
+        if (useRawContent) {
+            firstLine = (_e = (_d = manualFileContent[files.indexOf(file)]) === null || _d === void 0 ? void 0 : _d.split('\n')[0]) !== null && _e !== void 0 ? _e : null;
+        }
+        else if (fs_1.default.existsSync(file) && !fs_1.default.lstatSync(file).isDirectory()) {
+            firstLine = await (0, read_file_1.default)(file, true).catch(() => null);
+        }
+        else
+            continue;
+        // Skip if file is unreadable or blank
+        if (firstLine === null)
+            continue;
+        // Check first line for explicit classification
+        const hasShebang = opts.checkShebang && /^#!/.test(firstLine);
+        const hasModeline = opts.checkModeline && /-\*-|(syntax|filetype|ft)\s*=/.test(firstLine);
+        if (!opts.quick && (hasShebang || hasModeline)) {
+            const matches = [];
+            for (const [lang, data] of Object.entries(langData)) {
+                const langMatcher = (lang) => `\\b${lang.toLowerCase().replace(/\W/g, '\\$&')}(?![\\w#+*]|-\*-)`;
+                // Check for interpreter match
+                if (opts.checkShebang && hasShebang) {
+                    const matchesInterpretor = (_f = data.interpreters) === null || _f === void 0 ? void 0 : _f.some(interpreter => firstLine.match(`\\b${interpreter}\\b`));
+                    if (matchesInterpretor)
+                        matches.push(lang);
+                }
+                // Check modeline declaration
+                if (opts.checkModeline && hasModeline) {
+                    const modelineText = firstLine.toLowerCase().replace(/^.*-\*-(.+)-\*-.*$/, '$1');
+                    const matchesLang = modelineText.match(langMatcher(lang));
+                    const matchesAlias = (_g = data.aliases) === null || _g === void 0 ? void 0 : _g.some(lang => modelineText.match(langMatcher(lang)));
+                    if (matchesLang || matchesAlias)
+                        matches.push(lang);
+                }
+            }
+            // Add identified language(s)
+            if (matches.length) {
+                for (const match of matches)
+                    addResult(file, match);
+                if (matches.length === 1)
+                    definiteness[file] = true;
+                fromShebang[file] = true;
+                continue;
+            }
+        }
+        // Search each language
+        let skipExts = false;
+        // Check if filename is a match
+        for (const lang in langData) {
+            const matchesName = (_h = langData[lang].filenames) === null || _h === void 0 ? void 0 : _h.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
+            if (matchesName) {
+                addResult(file, lang);
+                skipExts = true;
+            }
+        }
+        // Check if extension is a match
+        const possibleExts = [];
+        if (!skipExts)
+            for (const lang in langData) {
+                const extMatches = (_j = langData[lang].extensions) === null || _j === void 0 ? void 0 : _j.filter(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
+                if (extMatches === null || extMatches === void 0 ? void 0 : extMatches.length) {
+                    for (const ext of extMatches)
+                        possibleExts.push({ ext, lang });
+                }
+            }
+        // Apply more specific extension if available
+        const isComplexExt = (ext) => /\..+\./.test(ext);
+        const hasComplexExt = possibleExts.some(data => isComplexExt(data.ext));
+        for (const { ext, lang } of possibleExts) {
+            if (hasComplexExt && !isComplexExt(ext))
+                continue;
+            if (!hasComplexExt && isComplexExt(ext))
+                continue;
+            addResult(file, lang);
+        }
+        // Fallback to null if no language matches
+        if (!fileAssociations[file]) {
+            addResult(file, null);
+        }
+    }
+    // Narrow down file associations to the best fit
+    for (const file in fileAssociations) {
+        // Skip if file has explicit association
+        if (definiteness[file]) {
+            results.files.results[file] = fileAssociations[file][0];
+            continue;
+        }
+        // Skip binary files
+        if (!useRawContent && !opts.keepBinary) {
+            if (await (0, isbinaryfile_1.isBinaryFile)(file))
+                continue;
+        }
+        // Parse heuristics if applicable
+        if (opts.checkHeuristics)
+            for (const heuristics of heuristicsData.disambiguations) {
+                // Make sure the extension matches the current file
+                if (!fromShebang[file] && !heuristics.extensions.includes(extensions[file]))
+                    continue;
+                // Load heuristic rules
+                for (const heuristic of heuristics.rules) {
+                    // Make sure the language is not an array
+                    if (Array.isArray(heuristic.language)) {
+                        heuristic.language = heuristic.language[0];
+                    }
+                    // Make sure the results includes this language
+                    const languageGroup = (_k = langData[heuristic.language]) === null || _k === void 0 ? void 0 : _k.group;
+                    const matchesLang = fileAssociations[file].includes(heuristic.language);
+                    const matchesParent = languageGroup && fileAssociations[file].includes(languageGroup);
+                    if (!matchesLang && !matchesParent)
+                        continue;
+                    // Normalise heuristic data
+                    const patterns = [];
+                    const normalise = (contents) => patterns.push(...[contents].flat());
+                    if (heuristic.pattern)
+                        normalise(heuristic.pattern);
+                    if (heuristic.named_pattern)
+                        normalise(heuristicsData.named_patterns[heuristic.named_pattern]);
+                    if (heuristic.and) {
+                        for (const data of heuristic.and) {
+                            if (data.pattern)
+                                normalise(data.pattern);
+                            if (data.named_pattern)
+                                normalise(heuristicsData.named_patterns[data.named_pattern]);
+                        }
+                    }
+                    // Check file contents and apply heuristic patterns
+                    const fileContent = opts.fileContent ? manualFileContent[files.indexOf(file)] : await (0, read_file_1.default)(file).catch(() => null);
+                    // Skip if file read errors
+                    if (fileContent === null)
+                        continue;
+                    // Apply heuristics
+                    if (!patterns.length || patterns.some(pattern => (0, convert_pcre_1.default)(pattern).test(fileContent))) {
+                        results.files.results[file] = heuristic.language;
+                        break;
+                    }
+                }
+            }
+        // If no heuristics, assign a language
+        if (!results.files.results[file]) {
+            const possibleLangs = fileAssociations[file];
+            // Assign first language as a default option
+            const defaultLang = possibleLangs[0];
+            const alternativeLangs = possibleLangs.slice(1);
+            results.files.results[file] = defaultLang;
+            // List alternative languages if there are any
+            if (alternativeLangs.length > 0)
+                results.files.alternatives[file] = alternativeLangs;
+        }
+    }
+    // Skip specified categories
+    if ((_l = opts.categories) === null || _l === void 0 ? void 0 : _l.length) {
+        const categories = ['data', 'markup', 'programming', 'prose'];
+        const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
+        for (const [file, lang] of Object.entries(results.files.results)) {
+            if (!hiddenCategories.some(cat => { var _a; return lang && ((_a = langData[lang]) === null || _a === void 0 ? void 0 : _a.type) === cat; })) {
+                continue;
+            }
+            delete results.files.results[file];
+            if (lang) {
+                delete results.languages.results[lang];
+            }
+        }
+        for (const category of hiddenCategories) {
+            for (const [lang, { type }] of Object.entries(results.languages.results)) {
+                if (type === category) {
+                    delete results.languages.results[lang];
+                }
+            }
+        }
+    }
+    // Convert paths to relative
+    if (!useRawContent && opts.relativePaths) {
+        const newMap = {};
+        for (const [file, lang] of Object.entries(results.files.results)) {
+            let relPath = path_1.default.relative(process.cwd(), file).replace(/\\/g, '/');
+            if (!relPath.startsWith('../')) {
+                relPath = './' + relPath;
+            }
+            newMap[relPath] = lang;
+        }
+        results.files.results = newMap;
+    }
+    // Load language bytes size
+    for (const [file, lang] of Object.entries(results.files.results)) {
+        if (lang && !langData[lang])
+            continue;
+        const fileSize = (_o = (_m = manualFileContent[files.indexOf(file)]) === null || _m === void 0 ? void 0 : _m.length) !== null && _o !== void 0 ? _o : fs_1.default.statSync(file).size;
+        results.files.bytes += fileSize;
+        // If no language found, add extension in other section
+        if (!lang) {
+            const ext = path_1.default.extname(file);
+            const unknownType = ext === '' ? 'filenames' : 'extensions';
+            const name = ext === '' ? path_1.default.basename(file) : ext;
+            (_p = (_r = results.unknown[unknownType])[name]) !== null && _p !== void 0 ? _p : (_r[name] = 0);
+            results.unknown[unknownType][name] += fileSize;
+            results.unknown.bytes += fileSize;
+            continue;
+        }
+        // Add language and bytes data to corresponding section
+        const { type } = langData[lang];
+        (_q = (_s = results.languages.results)[lang]) !== null && _q !== void 0 ? _q : (_s[lang] = { type, bytes: 0, color: langData[lang].color });
+        if (opts.childLanguages) {
+            results.languages.results[lang].parent = langData[lang].group;
+        }
+        results.languages.results[lang].bytes += fileSize;
+        results.languages.bytes += fileSize;
+    }
+    // Set counts
+    results.files.count = Object.keys(results.files.results).length;
+    results.languages.count = Object.keys(results.languages.results).length;
+    results.unknown.count = Object.keys({ ...results.unknown.extensions, ...results.unknown.filenames }).length;
+    // Return
+    return results;
+}
+module.exports = analyse;