linguist-js 2.6.1 → 2.7.0-pre

package/dist/cli.js

@@ -4,6 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const VERSION = require('../package.json').version;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
 const commander_1 = require("commander");
 const index_1 = __importDefault(require("./index"));
 const colouredMsg = ([r, g, b], msg) => `\u001B[${38};2;${r};${g};${b}m${msg}${'\u001b[0m'}`;
@@ -18,6 +20,7 @@ commander_1.program
     .option('-C|--childLanguages [bool]', 'Display child languages instead of their parents', false)
     .option('-j|--json [bool]', 'Display the output as JSON', false)
     .option('-t|--tree <traversal>', 'Which part of the output JSON to display (dot-delimited)')
+    .option('-F|--listFiles [bool]', 'Whether to list every matching file under the language results', false)
     .option('-q|--quick [bool]', 'Skip complex language analysis (alias for -{A|I|H|S}=false)', false)
     .option('-o|--offline [bool]', 'Use packaged data files instead of fetching latest from GitHub', false)
     .option('-V|--keepVendored [bool]', 'Prevent skipping over vendored/generated files', false)
@@ -59,21 +62,52 @@ if (args.analyze)
             const sortedEntries = Object.entries(languages.results).sort((a, b) => a[1].bytes < b[1].bytes ? +1 : -1);
             const totalBytes = languages.bytes;
             console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${files.count} files with linguist-js`);
-            console.log(`\n Language analysis results:`);
+            console.log(`\n Language analysis results: \n`);
             let count = 0;
             if (sortedEntries.length === 0)
                 console.log(`  None`);
+            // Collate files per language
+            const filesPerLanguage = {};
+            if (args.listFiles) {
+                for (const language of Object.keys(languages.results)) {
+                    filesPerLanguage[language] = [];
+                }
+                for (const [file, lang] of Object.entries(files.results)) {
+                    if (lang)
+                        filesPerLanguage[lang].push(file);
+                }
+            }
             // List parsed results
             for (const [lang, { bytes, color }] of sortedEntries) {
+                const percent = (bytes) => bytes / (totalBytes || 1) * 100;
                 const fmtd = {
                     index: (++count).toString().padStart(2, ' '),
                     lang: lang.padEnd(24, ' '),
-                    percent: (bytes / (totalBytes || 1) * 100).toFixed(2).padStart(5, ' '),
+                    percent: percent(bytes).toFixed(2).padStart(5, ' '),
                     bytes: bytes.toLocaleString().padStart(10, ' '),
                     icon: colouredMsg(hexToRgb(color !== null && color !== void 0 ? color : '#ededed'), '\u2588'),
                 };
                 console.log(`  ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B`);
+                // If using `listFiles` option, list all files tagged as this language
+                if (args.listFiles) {
+                    console.log(); // padding
+                    for (const file of filesPerLanguage[lang]) {
+                        let relFile = path_1.default.relative(path_1.default.resolve('.'), file).replace(/\\/g, '/');
+                        if (!relFile.startsWith('../'))
+                            relFile = './' + relFile;
+                        const bytes = (await fs_1.default.promises.stat(file)).size;
+                        const fmtd2 = {
+                            file: relFile.padEnd(42, ' '),
+                            percent: percent(bytes).toFixed(2).padStart(5, ' '),
+                            bytes: bytes.toLocaleString().padStart(10, ' '),
+                        };
+                        console.log(`        ${fmtd.icon} ${fmtd2.file}  ${fmtd2.percent}% ${fmtd2.bytes} B`);
+                    }
+                    console.log(); // padding
+                }
             }
+            if (!args.listFiles)
+                console.log(); // padding
             console.log(` Total: ${totalBytes.toLocaleString()} B`);
             // List unknown files/extensions
             if (unknown.bytes > 0) {

package/dist/helpers/parse-gitattributes.d.ts

@@ -0,0 +1,16 @@
+import * as T from '../types';
+export type FlagAttributes = {
+    'vendored': boolean | null;
+    'generated': boolean | null;
+    'documentation': boolean | null;
+    'binary': boolean | null;
+    'language': T.LanguageResult;
+};
+export type ParsedGitattributes = Array<{
+    glob: T.FileGlob;
+    attrs: FlagAttributes;
+}>;
+/**
+ * Parses a gitattributes file.
+ */
+export default function parseAttributes(content: string, folderRoot?: string): ParsedGitattributes;

package/dist/helpers/parse-gitattributes.js

@@ -0,0 +1,37 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path_1 = __importDefault(require("path"));
+/**
+ * Parses a gitattributes file.
+ */
+function parseAttributes(content, folderRoot = '.') {
+    var _a, _b;
+    const output = [];
+    for (const line of content.split('\n')) {
+        if (!line)
+            continue;
+        const parts = line.split(/\s+/g);
+        const fileGlob = parts[0];
+        const relFileGlob = path_1.default.join(folderRoot, fileGlob).replace(/\\/g, '/');
+        const attrParts = parts.slice(1);
+        const isTrue = (str) => !str.startsWith('-') && !str.endsWith('=false');
+        const isFalse = (str) => str.startsWith('-') || str.endsWith('=false');
+        const trueParts = (str) => attrParts.filter(part => part.includes(str) && isTrue(part));
+        const falseParts = (str) => attrParts.filter(part => part.includes(str) && isFalse(part));
+        const hasTrueParts = (str) => trueParts(str).length > 0;
+        const hasFalseParts = (str) => falseParts(str).length > 0;
+        const attrs = {
+            'generated': hasTrueParts('linguist-generated') ? true : hasFalseParts('linguist-generated') ? false : null,
+            'vendored': hasTrueParts('linguist-vendored') ? true : hasFalseParts('linguist-vendored') ? false : null,
+            'documentation': hasTrueParts('linguist-documentation') ? true : hasFalseParts('linguist-documentation') ? false : null,
+            'binary': hasTrueParts('binary') || hasFalseParts('text') ? true : hasFalseParts('binary') || hasTrueParts('text') ? false : null,
+            'language': (_b = (_a = trueParts('linguist-language').at(-1)) === null || _a === void 0 ? void 0 : _a.split('=')[1]) !== null && _b !== void 0 ? _b : null,
+        };
+        output.push({ glob: relFileGlob, attrs });
+    }
+    return output;
+}
+exports.default = parseAttributes;

package/dist/helpers/walk-tree.d.ts

@@ -8,8 +8,8 @@ interface WalkInput {
     folderRoots: string[];
     /** The absolute path of folders being checked */
     folders: string[];
-    gitignores: Ignore;
-    regexIgnores: RegExp[];
+    /** An instantiated Ignore object listing ignored files */
+    ignored: Ignore;
 }
 interface WalkOutput {
     files: string[];

package/dist/helpers/walk-tree.js

@@ -11,7 +11,7 @@ let allFolders;
 ;
 /** Generate list of files in a directory. */
 function walk(data) {
-    const { init, commonRoot, folderRoots, folders, gitignores, regexIgnores } = data;
+    const { init, commonRoot, folderRoots, folders, ignored } = data;
     // Initialise files and folders lists
     if (init) {
         allFiles = new Set();
@@ -34,11 +34,13 @@ function walk(data) {
             // Create absolute path for disc operations
             const path = path_1.default.resolve(commonRoot, file).replace(/\\/g, '/');
             const localPath = localRoot ? file.replace(`./${localRoot}/`, '') : file.replace('./', '');
-            // Skip if nonexistant or ignored
+            // Skip if nonexistant
             const nonExistant = !fs_1.default.existsSync(path);
-            const isGitIgnored = gitignores.test(localPath).ignored;
-            const isRegexIgnored = regexIgnores.find(match => localPath.match(match));
-            if (nonExistant || isGitIgnored || isRegexIgnored)
+            if (nonExistant)
+                continue;
+            // Skip if marked as ignored
+            const isIgnored = ignored.test(localPath).ignored;
+            if (isIgnored)
                 continue;
             // Add absolute folder path to list
             allFolders.add(path_1.default.resolve(folder).replace(/\\/g, '/'));
@@ -46,7 +48,7 @@ function walk(data) {
             if (file.endsWith('/')) {
                 // Recurse into subfolders
                 allFolders.add(path);
-                walk({ init: false, commonRoot: commonRoot, folderRoots, folders: [path], gitignores, regexIgnores });
+                walk({ init: false, commonRoot, folderRoots, folders: [path], ignored });
             }
             else {
                 // Add file path to list
@@ -57,7 +59,7 @@ function walk(data) {
     // Recurse into all folders
     else {
         for (const i in folders) {
-            walk({ init: false, commonRoot: commonRoot, folderRoots: [folderRoots[i]], folders: [folders[i]], gitignores, regexIgnores });
+            walk({ init: false, commonRoot, folderRoots: [folderRoots[i]], folders: [folders[i]], ignored });
         }
     }
     // Return absolute files and folders lists

package/dist/index.js

@@ -35,13 +35,23 @@ const isbinaryfile_1 = require("isbinaryfile");
 const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
 const load_data_1 = __importStar(require("./helpers/load-data"));
 const read_file_1 = __importDefault(require("./helpers/read-file"));
+const parse_gitattributes_1 = __importDefault(require("./helpers/parse-gitattributes"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
-async function analyse(input, opts = {}) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
-    var _t, _u;
+async function analyse(rawInput, opts = {}) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
+    var _r, _s;
     const useRawContent = opts.fileContent !== undefined;
-    input = [input !== null && input !== void 0 ? input : []].flat();
-    opts.fileContent = [(_a = opts.fileContent) !== null && _a !== void 0 ? _a : []].flat();
+    const input = [rawInput !== null && rawInput !== void 0 ? rawInput : []].flat();
+    const manualFileContent = [(_a = opts.fileContent) !== null && _a !== void 0 ? _a : []].flat();
+    // Normalise input option arguments
+    opts = {
+        checkIgnored: !opts.quick,
+        checkAttributes: !opts.quick,
+        checkHeuristics: !opts.quick,
+        checkShebang: !opts.quick,
+        checkModeline: !opts.quick,
+        ...opts,
+    };
     // Load data from github-linguist web repo
     const langData = await (0, load_data_1.default)('languages.yml', opts.offline).then(js_yaml_1.default.load);
     const vendorData = await (0, load_data_1.default)('vendor.yml', opts.offline).then(js_yaml_1.default.load);
@@ -52,30 +62,33 @@ async function analyse(input, opts = {}) {
     // Setup main variables
     const fileAssociations = {};
     const extensions = {};
-    const overrides = {};
+    const globOverrides = {};
     const results = {
         files: { count: 0, bytes: 0, results: {}, alternatives: {} },
         languages: { count: 0, bytes: 0, results: {} },
         unknown: { count: 0, bytes: 0, extensions: {}, filenames: {} },
     };
-    // Prepare list of ignored files
-    const gitignores = (0, ignore_1.default)();
-    const regexIgnores = [];
-    gitignores.add('.git');
-    if (!opts.keepVendored)
-        regexIgnores.push(...vendorPaths.map(path => RegExp(path, 'i')));
-    if (opts.ignoredFiles)
-        gitignores.add(opts.ignoredFiles);
     // Set a common root path so that vendor paths do not incorrectly match parent folders
     const normPath = (file) => file.replace(/\\/g, '/');
     const resolvedInput = input.map(path => normPath(path_1.default.resolve(path)));
     const commonRoot = (input.length > 1 ? (0, common_path_prefix_1.default)(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
     const localRoot = (folder) => folder.replace(commonRoot, '').replace(/^\//, '');
-    const relPath = (file) => normPath(path_1.default.relative(commonRoot, file));
-    const unRelPath = (file) => normPath(path_1.default.resolve(commonRoot, file));
-    const localPath = (file) => localRoot(unRelPath(file));
+    const relPath = (file) => useRawContent ? file : normPath(path_1.default.relative(commonRoot, file));
+    const unRelPath = (file) => useRawContent ? file : normPath(path_1.default.resolve(commonRoot, file));
+    // Other helper functions
+    const fileMatchesGlobs = (file, ...globs) => (0, ignore_1.default)().add(globs).ignores(relPath(file));
+    const filterOutIgnored = (files, ignored) => ignored.filter(files.map(relPath)).map(unRelPath);
+    //*PREPARE FILES AND DATA*//
+    // Prepare list of ignored files
+    const ignored = (0, ignore_1.default)();
+    ignored.add('.git/');
+    ignored.add((_b = opts.ignoredFiles) !== null && _b !== void 0 ? _b : []);
+    const regexIgnores = [];
+    if (!opts.keepVendored)
+        regexIgnores.push(...vendorPaths.map(path => RegExp(path, 'i')));
     // Load file paths and folders
-    let files, folders;
+    let files;
+    let folders;
     if (useRawContent) {
         // Uses raw file content
         files = input;
@@ -83,93 +96,96 @@ async function analyse(input, opts = {}) {
     }
     else {
         // Uses directory on disc
-        const data = (0, walk_tree_1.default)({ init: true, commonRoot, folderRoots: resolvedInput, folders: resolvedInput, gitignores, regexIgnores });
+        const data = (0, walk_tree_1.default)({ init: true, commonRoot, folderRoots: resolvedInput, folders: resolvedInput, ignored });
         files = data.files;
         folders = data.folders;
     }
-    // Apply aliases
-    opts = {
-        checkIgnored: !opts.quick,
-        checkAttributes: !opts.quick,
-        checkHeuristics: !opts.quick,
-        checkShebang: !opts.quick,
-        checkModeline: !opts.quick,
-        ...opts
-    };
-    // Ignore specific languages
-    for (const lang of (_b = opts.ignoredLanguages) !== null && _b !== void 0 ? _b : []) {
-        for (const key in langData) {
-            if (lang.toLowerCase() === key.toLowerCase()) {
-                delete langData[key];
-                break;
-            }
-        }
-    }
-    // Load gitignores and gitattributes
-    const customBinary = (0, ignore_1.default)();
-    const customText = (0, ignore_1.default)();
-    if (!useRawContent && opts.checkAttributes) {
+    // Load gitignore data and apply ignores rules
+    if (!useRawContent) {
+        // TODO switch to be like the gitattributes code
         for (const folder of folders) {
-            // TODO FIX: this is absolute when only 1 path given
-            const localFilePath = (path) => localRoot(folder) ? localRoot(folder) + '/' + localPath(path) : path;
-            // Skip if folder is marked in gitattributes
-            if (relPath(folder) && gitignores.ignores(relPath(folder))) {
-                continue;
-            }
             // Parse gitignores
             const ignoresFile = path_1.default.join(folder, '.gitignore');
             if (opts.checkIgnored && fs_1.default.existsSync(ignoresFile)) {
                 const ignoresData = await (0, read_file_1.default)(ignoresFile);
                 const localIgnoresData = ignoresData.replace(/^[\/\\]/g, localRoot(folder) + '/');
-                gitignores.add(localIgnoresData);
+                ignored.add(localIgnoresData);
+                files = filterOutIgnored(files, ignored);
             }
-            // Parse gitattributes
-            const attributesFile = path_1.default.join(folder, '.gitattributes');
-            if (opts.checkAttributes && fs_1.default.existsSync(attributesFile)) {
-                const attributesData = await (0, read_file_1.default)(attributesFile);
-                // Explicit text/binary associations
-                const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)(?!=auto)/gm);
-                for (const [_line, path, type] of contentTypeMatches) {
-                    if (['text', '-binary'].includes(type)) {
-                        customText.add(localFilePath(path));
-                    }
-                    if (['-text', 'binary'].includes(type)) {
-                        customBinary.add(localFilePath(path));
-                    }
-                }
-                // Custom vendor options
-                const vendorMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-(vendored|generated|documentation)(?!=false)/gm);
-                for (const [_line, path] of vendorMatches) {
-                    gitignores.add(localFilePath(path));
-                }
-                // Custom file associations
-                const customLangMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-language=(\S+)/gm);
-                for (let [_line, path, forcedLang] of customLangMatches) {
-                    // If specified language is an alias, associate it with its full name
-                    if (!langData[forcedLang]) {
-                        const overrideLang = Object.entries(langData).find(entry => { var _a; return (_a = entry[1].aliases) === null || _a === void 0 ? void 0 : _a.includes(forcedLang.toLowerCase()); });
-                        if (overrideLang) {
-                            forcedLang = overrideLang[0];
-                        }
-                    }
-                    const fullPath = path_1.default.join(relPath(folder), path);
-                    overrides[fullPath] = forcedLang;
-                }
+        }
+    }
+    // Fetch and normalise gitattributes data of all subfolders and save to metadata
+    const manualAttributes = {}; // Maps file globs to gitattribute boolean flags
+    const getFlaggedGlobs = (attr, val) => {
+        return Object.entries(manualAttributes).filter(([, attrs]) => attrs[attr] === val).map(([glob,]) => glob);
+    };
+    if (!useRawContent && opts.checkAttributes) {
+        const nestedAttrFiles = files.filter(file => file.endsWith('.gitattributes'));
+        for (const attrFile of nestedAttrFiles) {
+            const relAttrFile = relPath(attrFile);
+            const relAttrFolder = path_1.default.dirname(relAttrFile);
+            const contents = await (0, read_file_1.default)(attrFile);
+            const parsed = (0, parse_gitattributes_1.default)(contents, relAttrFolder);
+            for (const { glob, attrs } of parsed) {
+                manualAttributes[glob] = attrs;
             }
         }
     }
-    // Check vendored files
+    // Apply vendor file path matches and filter out vendored files
     if (!opts.keepVendored) {
-        // Filter out any files that match a vendor file path
-        if (useRawContent) {
-            files = gitignores.filter(files);
-            files = files.filter(file => !regexIgnores.find(match => match.test(file)));
+        // Get data of files that have been manually marked with metadata
+        const vendorTrueGlobs = [...getFlaggedGlobs('vendored', true), ...getFlaggedGlobs('generated', true), ...getFlaggedGlobs('documentation', true)];
+        const vendorFalseGlobs = [...getFlaggedGlobs('vendored', false), ...getFlaggedGlobs('generated', false), ...getFlaggedGlobs('documentation', false)];
+        // Set up glob ignore object to use for expanding globs to match files
+        const vendorOverrides = (0, ignore_1.default)().add(vendorFalseGlobs);
+        // Remove all files marked as vendored by default
+        const excludedFiles = files.filter(file => vendorPaths.some(pathPtn => RegExp(pathPtn, 'i').test(relPath(file))));
+        files = files.filter(file => !excludedFiles.includes(file));
+        // Re-add removed files that are overridden manually in gitattributes
+        const overriddenExcludedFiles = excludedFiles.filter(file => vendorOverrides.ignores(relPath(file)));
+        files.push(...overriddenExcludedFiles);
+        // Remove files explicitly marked as vendored in gitattributes
+        // TODO change globs.includes(file) to parse the glob using ignore()
+        files = files.filter(file => !vendorTrueGlobs.includes(relPath(file)));
+    }
+    // Filter out binary files
+    if (!opts.keepBinary) {
+        // Filter out files that are binary by default
+        files = files.filter(file => !binary_extensions_1.default.some(ext => file.endsWith('.' + ext)));
+        // Filter out manually specified binary files
+        const binaryIgnored = (0, ignore_1.default)().add(getFlaggedGlobs('binary', true));
+        files = filterOutIgnored(files, binaryIgnored);
+        // Re-add files manually marked not as binary
+        const binaryUnignored = (0, ignore_1.default)().add(getFlaggedGlobs('binary', false));
+        // TODO parse the globs using ignore()
+        const unignoredList = filterOutIgnored(files, binaryUnignored);
+        files.push(...unignoredList);
+    }
+    // Ignore specific languages
+    for (const lang of (_c = opts.ignoredLanguages) !== null && _c !== void 0 ? _c : []) {
+        for (const key in langData) {
+            if (lang.toLowerCase() === key.toLowerCase()) {
+                delete langData[key];
+                break;
+            }
         }
-        else {
-            files = gitignores.filter(files.map(localPath)).map(unRelPath);
+    }
+    // Establish language overrides taken from gitattributes
+    const forcedLangs = Object.entries(manualAttributes).filter(([, attrs]) => attrs.language);
+    for (const [globPath, attrs] of forcedLangs) {
+        let forcedLang = attrs.language;
+        if (!forcedLang)
+            continue;
+        // If specified language is an alias, associate it with its full name
+        if (!langData[forcedLang]) {
+            const overrideLang = Object.entries(langData).find(entry => { var _a; return (_a = entry[1].aliases) === null || _a === void 0 ? void 0 : _a.includes(forcedLang.toLowerCase()); });
+            if (overrideLang) {
+                forcedLang = overrideLang[0];
+            }
         }
+        globOverrides[globPath] = forcedLang;
     }
-    // Load all files and parse languages
+    //*PARSE LANGUAGES*//
     const addResult = (file, result) => {
         if (!fileAssociations[file]) {
             fileAssociations[file] = [];
@@ -177,26 +193,36 @@ async function analyse(input, opts = {}) {
         }
         // Set parent to result group if it is present
         // Is nullish if either `opts.childLanguages` is set or if there is no group
-        const finalResult = !opts.childLanguages && result && langData[result].group || result;
-        if (!fileAssociations[file].includes(finalResult))
+        const finalResult = !opts.childLanguages && result && langData[result] && langData[result].group || result;
+        if (!fileAssociations[file].includes(finalResult)) {
             fileAssociations[file].push(finalResult);
+        }
         extensions[file] = path_1.default.extname(file).toLowerCase();
     };
-    const overridesArray = Object.entries(overrides);
-    // List all languages that could be associated with a given file
     const definiteness = {};
     const fromShebang = {};
-    for (const file of files) {
+    fileLoop: for (const file of files) {
+        // Check manual override
+        for (const globMatch in globOverrides) {
+            if (!fileMatchesGlobs(file, globMatch))
+                continue;
+            // If the given file matches the glob, apply the override to the file
+            const forcedLang = globOverrides[globMatch];
+            addResult(file, forcedLang);
+            definiteness[file] = true;
+            continue fileLoop; // no need to check other heuristics, the classified language has been found
+        }
+        // Check first line for readability
         let firstLine;
         if (useRawContent) {
-            firstLine = (_e = (_d = (_c = opts.fileContent) === null || _c === void 0 ? void 0 : _c[files.indexOf(file)]) === null || _d === void 0 ? void 0 : _d.split('\n')[0]) !== null && _e !== void 0 ? _e : null;
+            firstLine = (_e = (_d = manualFileContent[files.indexOf(file)]) === null || _d === void 0 ? void 0 : _d.split('\n')[0]) !== null && _e !== void 0 ? _e : null;
         }
         else if (fs_1.default.existsSync(file) && !fs_1.default.lstatSync(file).isDirectory()) {
             firstLine = await (0, read_file_1.default)(file, true).catch(() => null);
         }
         else
             continue;
-        // Skip if file is unreadable
+        // Skip if file is unreadable or blank
         if (firstLine === null)
             continue;
         // Check first line for explicit classification
@@ -231,17 +257,6 @@ async function analyse(input, opts = {}) {
                 continue;
             }
         }
-        // Check override for manual language classification
-        if (!useRawContent && !opts.quick && opts.checkAttributes) {
-            const isOverridden = (path) => (0, ignore_1.default)().add(path).ignores(relPath(file));
-            const match = overridesArray.find(item => isOverridden(item[0]));
-            if (match) {
-                const forcedLang = match[1];
-                addResult(file, forcedLang);
-                definiteness[file] = true;
-                continue;
-            }
-        }
         // Search each language
         let skipExts = false;
         // Check if filename is a match
@@ -286,12 +301,8 @@ async function analyse(input, opts = {}) {
         }
         // Skip binary files
         if (!useRawContent && !opts.keepBinary) {
-            const isCustomText = customText.ignores(relPath(file));
-            const isCustomBinary = customBinary.ignores(relPath(file));
-            const isBinaryExt = binary_extensions_1.default.some(ext => file.endsWith('.' + ext));
-            if (!isCustomText && (isCustomBinary || isBinaryExt || await (0, isbinaryfile_1.isBinaryFile)(file))) {
+            if (await (0, isbinaryfile_1.isBinaryFile)(file))
                 continue;
-            }
         }
         // Parse heuristics if applicable
         if (opts.checkHeuristics)
@@ -327,7 +338,7 @@ async function analyse(input, opts = {}) {
                         }
                     }
                     // Check file contents and apply heuristic patterns
-                    const fileContent = ((_l = opts.fileContent) === null || _l === void 0 ? void 0 : _l.length) ? opts.fileContent[files.indexOf(file)] : await (0, read_file_1.default)(file).catch(() => null);
+                    const fileContent = opts.fileContent ? manualFileContent[files.indexOf(file)] : await (0, read_file_1.default)(file).catch(() => null);
                     // Skip if file read errors
                     if (fileContent === null)
                         continue;
@@ -351,7 +362,7 @@ async function analyse(input, opts = {}) {
         }
     }
     // Skip specified categories
-    if ((_m = opts.categories) === null || _m === void 0 ? void 0 : _m.length) {
+    if ((_l = opts.categories) === null || _l === void 0 ? void 0 : _l.length) {
         const categories = ['data', 'markup', 'programming', 'prose'];
         const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
         for (const [file, lang] of Object.entries(results.files.results)) {
@@ -387,21 +398,21 @@ async function analyse(input, opts = {}) {
     for (const [file, lang] of Object.entries(results.files.results)) {
         if (lang && !langData[lang])
             continue;
-        const fileSize = (_q = (_p = (_o = opts.fileContent) === null || _o === void 0 ? void 0 : _o[files.indexOf(file)]) === null || _p === void 0 ? void 0 : _p.length) !== null && _q !== void 0 ? _q : fs_1.default.statSync(file).size;
+        const fileSize = (_o = (_m = manualFileContent[files.indexOf(file)]) === null || _m === void 0 ? void 0 : _m.length) !== null && _o !== void 0 ? _o : fs_1.default.statSync(file).size;
         results.files.bytes += fileSize;
         // If no language found, add extension in other section
         if (!lang) {
             const ext = path_1.default.extname(file);
             const unknownType = ext === '' ? 'filenames' : 'extensions';
             const name = ext === '' ? path_1.default.basename(file) : ext;
-            (_r = (_t = results.unknown[unknownType])[name]) !== null && _r !== void 0 ? _r : (_t[name] = 0);
+            (_p = (_r = results.unknown[unknownType])[name]) !== null && _p !== void 0 ? _p : (_r[name] = 0);
             results.unknown[unknownType][name] += fileSize;
             results.unknown.bytes += fileSize;
             continue;
         }
         // Add language and bytes data to corresponding section
         const { type } = langData[lang];
-        (_s = (_u = results.languages.results)[lang]) !== null && _s !== void 0 ? _s : (_u[lang] = { type, bytes: 0, color: langData[lang].color });
+        (_q = (_s = results.languages.results)[lang]) !== null && _q !== void 0 ? _q : (_s[lang] = { type, bytes: 0, color: langData[lang].color });
         if (opts.childLanguages) {
             results.languages.results[lang].parent = langData[lang].group;
         }

package/dist/types.d.ts

@@ -4,6 +4,10 @@ export type Category = 'data' | 'markup' | 'programming' | 'prose';
 export type FilePath = string;
 export type Bytes = Integer;
 export type Integer = number;
+export type RelFile = string & {};
+export type AbsFile = string & {};
+export type AbsFolder = string & {};
+export type FileGlob = string & {};
 export interface Options {
     fileContent?: string | string[];
     ignoredFiles?: string[];

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "linguist-js",
-	"version": "2.6.1",
+	"version": "2.7.0-pre",
 	"description": "Analyse languages used in a folder. Powered by GitHub Linguist, although it doesn't need to be installed.",
 	"main": "dist/index.js",
 	"bin": {
@@ -42,13 +42,13 @@
 		"commander": "^9.5.0 <10",
 		"common-path-prefix": "^3.0.0",
 		"cross-fetch": "^3.1.8 <4",
-		"ignore": "^5.2.4",
+		"ignore": "^5.3.1",
 		"isbinaryfile": "^4.0.10 <5",
 		"js-yaml": "^4.1.0",
 		"node-cache": "^5.1.2"
 	},
 	"devDependencies": {
-		"@types/js-yaml": "^4.0.5",
+		"@types/js-yaml": "^4.0.9",
 		"@types/node": "ts5.0",
 		"deep-object-diff": "^1.1.9",
 		"typescript": "~5.0.4 <5.1"

package/readme.md

@@ -122,7 +122,7 @@ const { files, languages, unknown } = linguist(folder, options);
       Alias for `checkAttributes:false, checkIgnored:false, checkHeuristics:false, checkShebang:false, checkModeline:false`.
     - `offline` (boolean):
       Whether to use pre-packaged metadata files instead of fetching them from GitHub at runtime (defaults to `false`).
-	- `keepVendored` (boolean):
+  - `keepVendored` (boolean):
       Whether to keep vendored files (dependencies, etc) (defaults to `false`).
       Does nothing when `fileContent` is set.
     - `keepBinary` (boolean):
@@ -160,14 +160,20 @@ linguist --version
     A list of languages to exclude from the output.
   - `--categories <categories...>`:
     A list of language categories that should be displayed in the output.
-	Must be one or more of `data`, `prose`, `programming`, `markup`.
+    Must be one or more of `data`, `prose`, `programming`, `markup`.
   - `--childLanguages`:
     Display sub-languages instead of their parents, when possible.
   - `--json`:
+    Only affects the CLI output.
     Display the outputted language data as JSON.
   - `--tree <traversal>`:
+    Only affects the CLI output.
     A dot-delimited traversal to the nested object that should be logged to the console instead of the entire output.
     Requires `--json` to be specified.
+  - `--listFiles`:
+    Only affects the visual CLI output.
+    List each matching file and its size under each outputted language result.
+    Does nothing if `--json` is specified.
   - `--quick`:
     Skip the checking of `.gitattributes` and `.gitignore` files for manual language classifications.
     Alias for `--checkAttributes=false --checkIgnored=false --checkHeuristics=false --checkShebang=false --checkModeline=false`.