linguist-js 2.1.3 → 2.3.0

package/dist/cli.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const VERSION = require('../package.json').version;
 const commander_1 = require("commander");
 const index_1 = __importDefault(require("./index"));
+const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
 const colouredMsg = ([r, g, b], msg) => `\u001B[${38};2;${r};${g};${b}m${msg}${'\u001b[0m'}`;
 const hexToRgb = (hex) => [parseInt(hex.slice(1, 3), 16), parseInt(hex.slice(3, 5), 16), parseInt(hex.slice(5, 7), 16)];
 commander_1.program
@@ -21,6 +22,7 @@ commander_1.program
     .option('-q|--quick [bool]', 'Skip complex language analysis (alias for -{A|I|H|S}=false)', false)
     .option('-V|--keepVendored [bool]', 'Prevent skipping over vendored/generated files', false)
     .option('-B|--keepBinary [bool]', 'Prevent skipping over binary files', false)
+    .option('-r|--relativePaths [bool]', 'Convert absolute file paths to relative', false)
     .option('-A|--checkAttributes [bool]', 'Force the checking of gitattributes files', true)
     .option('-I|--checkIgnored [bool]', 'Force the checking of gitignore files', true)
     .option('-H|--checkHeuristics [bool]', 'Apply heuristics to ambiguous languages', true)
@@ -49,17 +51,23 @@ if (args.analyze)
     (async () => {
         // Fetch language data
         const root = args.analyze === true ? '.' : args.analyze;
-        const { files, languages, unknown } = await (0, index_1.default)(root, args);
+        const data = await (0, index_1.default)(root, args);
+        const { files, languages, unknown } = data;
+        // Get file count
+        const totalFiles = (0, walk_tree_1.default)(root).files.length;
         // Print output
         if (!args.json) {
             const sortedEntries = Object.entries(languages.results).sort((a, b) => a[1].bytes < b[1].bytes ? +1 : -1);
             const totalBytes = languages.bytes;
-            console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${files.count} files with linguist-js`);
+            console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${totalFiles} files (${totalFiles - files.count} ignored) with linguist-js`);
             console.log(`\n Language analysis results:`);
-            let i = 0;
+            let count = 0;
+            if (sortedEntries.length === 0)
+                console.log(`  None`);
+            // List parsed results
             for (const [lang, { bytes, color }] of sortedEntries) {
                 const fmtd = {
-                    index: (++i).toString().padStart(2, ' '),
+                    index: (++count).toString().padStart(2, ' '),
                     lang: lang.padEnd(24, ' '),
                     percent: (bytes / (totalBytes || 1) * 100).toFixed(2).padStart(5, ' '),
                     bytes: bytes.toLocaleString().padStart(10, ' '),
@@ -68,35 +76,33 @@ if (args.analyze)
                 console.log(`  ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B`);
             }
             console.log(` Total: ${totalBytes.toLocaleString()} B`);
+            // List unknown files/extensions
             if (unknown.bytes > 0) {
                 console.log(`\n Unknown files and extensions:`);
                 for (const [name, bytes] of Object.entries(unknown.filenames)) {
                     console.log(`  '${name}': ${bytes.toLocaleString()} B`);
                 }
                 for (const [ext, bytes] of Object.entries(unknown.extensions)) {
-                    console.log(`  '${ext}': ${bytes.toLocaleString()} B`);
+                    console.log(`  '*${ext}': ${bytes.toLocaleString()} B`);
                 }
                 console.log(` Total: ${unknown.bytes.toLocaleString()} B`);
             }
         }
-        else {
-            const data = { files, languages, unknown };
-            if (args.tree) {
-                const treeParts = args.tree.split('.');
-                let nestedData = data;
-                for (const part of treeParts) {
-                    if (!nestedData[part])
-                        throw Error(`TraversalError: Key '${part}' cannot be found on output object.`);
-                    nestedData = nestedData[part];
-                }
-                console.log(nestedData);
-            }
-            else {
-                console.log(JSON.stringify(data, null, 2).replace(/{\s+"type".+?}/sg, obj => obj.replace(/\n\s+/g, ' ')));
+        else if (args.tree) {
+            const treeParts = args.tree.split('.');
+            let nestedData = data;
+            for (const part of treeParts) {
+                if (!nestedData[part])
+                    throw Error(`TraversalError: Key '${part}' cannot be found on output object.`);
+                nestedData = nestedData[part];
             }
+            console.log(nestedData);
+        }
+        else {
+            console.dir(data, { depth: null });
         }
     })();
 else {
-    console.log(`Welcome to linguist-js, the JavaScript port of GitHub's language analyzer.`);
+    console.log(`Welcome to linguist-js, a JavaScript port of GitHub's language analyzer.`);
     console.log(`Type 'linguist --help' for a list of commands.`);
 }

package/dist/helpers/convert-glob.js

@@ -0,0 +1,11 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const glob_to_regexp_1 = __importDefault(require("glob-to-regexp"));
+function parseGitGlob(path) {
+    const globPath = `**/${path}`.replace(/\\/g, '/').replace(/\[:(space|digit):\]/g, (_, val) => `\\${val[0]}`);
+    return (0, glob_to_regexp_1.default)(globPath, { globstar: true, extended: true });
+}
+exports.default = parseGitGlob;

package/dist/helpers/read-file.js

@@ -4,12 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
-/** Read part of a file on disc. */
+/**
+ * Read part of a file on disc.
+ * @throws 'EPERM' if the file is not readable.
+ */
 async function readFile(filename, onlyFirstLine = false) {
     const chunkSize = 100;
     const stream = fs_1.default.createReadStream(filename, { highWaterMark: chunkSize });
     let content = '';
-    for await (const data of stream) {
+    for await (const data of stream) { // may throw
         content += data.toString();
         if (onlyFirstLine && content.includes('\n')) {
             return content.split(/\r?\n/)[0];

package/dist/index.js

@@ -12,10 +12,10 @@ const walk_tree_1 = __importDefault(require("./helpers/walk-tree"));
 const load_data_1 = __importDefault(require("./helpers/load-data"));
 const read_file_1 = __importDefault(require("./helpers/read-file"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
-const convertToRegex = (path) => (0, glob_to_regexp_1.default)('**/' + path, { globstar: true, extended: true });
+const convert_glob_1 = __importDefault(require("./helpers/convert-glob"));
 async function analyse(input, opts = {}) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j;
-    var _k, _l, _m;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
+    var _r, _s, _t;
     const langData = await (0, load_data_1.default)('languages.yml').then(js_yaml_1.default.load);
     const vendorData = await (0, load_data_1.default)('vendor.yml').then(js_yaml_1.default.load);
     const heuristicsData = await (0, load_data_1.default)('heuristics.yml').then(js_yaml_1.default.load);
@@ -34,7 +34,16 @@ async function analyse(input, opts = {}) {
         opts.keepVendored ? [] : vendorData.map(path => RegExp(path)),
         (_b = (_a = opts.ignoredFiles) === null || _a === void 0 ? void 0 : _a.map(path => (0, glob_to_regexp_1.default)('*' + path + '*', { extended: true }))) !== null && _b !== void 0 ? _b : [],
     ].flat();
-    let { files, folders } = (0, walk_tree_1.default)(input !== null && input !== void 0 ? input : '.', ignoredFiles);
+    let files, folders;
+    if (opts.fileContent) {
+        opts.fileContent = Array.isArray(opts.fileContent) ? opts.fileContent : [opts.fileContent];
+        files = [`${input}`];
+        folders = [''];
+    }
+    else {
+        const data = (0, walk_tree_1.default)(input !== null && input !== void 0 ? input : '.', ignoredFiles);
+        ({ files, folders } = data);
+    }
     // Apply aliases
     opts = { checkIgnored: !opts.quick, checkAttributes: !opts.quick, checkHeuristics: !opts.quick, checkShebang: !opts.quick, ...opts };
     // Ignore specific languages
@@ -50,26 +59,26 @@ async function analyse(input, opts = {}) {
     const customIgnored = [];
     const customBinary = [];
     const customText = [];
-    if (!opts.quick) {
+    if (!opts.fileContent && !opts.quick) {
         for (const folder of folders) {
             // Skip if folder is marked in gitattributes
-            if (customIgnored.some(path => convertToRegex(path).test(folder)))
+            if (customIgnored.some(path => (0, convert_glob_1.default)(path).test(folder)))
                 continue;
             // Parse gitignores
             const ignoresFile = path_1.default.join(folder, '.gitignore');
             if (opts.checkIgnored && fs_1.default.existsSync(ignoresFile)) {
                 const ignoresData = await (0, read_file_1.default)(ignoresFile);
                 const ignoresList = ignoresData.split(/\r?\n/).filter(line => line.trim() && !line.startsWith('#'));
-                const ignoredPaths = ignoresList.map(path => convertToRegex(path).source);
-                customIgnored.push(...ignoredPaths.map(file => file.replace(folder, '')));
+                const ignoredPaths = ignoresList.map(path => (0, convert_glob_1.default)(path).source);
+                customIgnored.push(...ignoredPaths);
             }
             // Parse gitattributes
             const attributesFile = path_1.default.join(folder, '.gitattributes');
             if (opts.checkAttributes && fs_1.default.existsSync(attributesFile)) {
                 const attributesData = await (0, read_file_1.default)(attributesFile);
-                const relPathToRegex = (path) => convertToRegex(path).source.substr(1).replace(folder, '');
+                const relPathToRegex = (path) => (0, convert_glob_1.default)(path).source.substr(1).replace(folder, '');
                 // Explicit text/binary associations
-                const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)/gm);
+                const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)(?!=auto)/gm);
                 for (const [_line, path, type] of contentTypeMatches) {
                     if (['text', '-binary'].includes(type))
                         customText.push(relPathToRegex(path));
@@ -90,14 +99,14 @@ async function analyse(input, opts = {}) {
                         if (overrideLang)
                             forcedLang = overrideLang[0];
                     }
-                    const fullPath = folder + convertToRegex(path).source.substr(1);
+                    const fullPath = folder + (0, convert_glob_1.default)(path).source.substr(1);
                     overrides[fullPath] = forcedLang;
                 }
             }
         }
     }
     // Check vendored files
-    if (!opts.keepVendored) {
+    if (!opts.fileContent && !opts.keepVendored) {
         // Filter out any files that match a vendor file path
         const matcher = (match) => RegExp(match.replace(/\/$/, '/.+$').replace(/^\.\//, ''));
         files = files.filter(file => !customIgnored.some(pattern => matcher(pattern).test(file)));
@@ -115,22 +124,30 @@ async function analyse(input, opts = {}) {
     const overridesArray = Object.entries(overrides);
     // List all languages that could be associated with a given file
     for (const file of files) {
-        if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
+        let firstLine;
+        if (opts.fileContent) {
+            firstLine = (_f = (_e = (_d = opts.fileContent) === null || _d === void 0 ? void 0 : _d[files.indexOf(file)]) === null || _e === void 0 ? void 0 : _e.split('\n')[0]) !== null && _f !== void 0 ? _f : null;
+        }
+        else {
+            if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
+                continue;
+            firstLine = await (0, read_file_1.default)(file, true).catch(() => null);
+        }
+        // Skip if file is unreadable
+        if (firstLine === null)
             continue;
         // Check shebang line for explicit classification
-        if (!opts.quick && opts.checkShebang) {
-            const firstLine = await (0, read_file_1.default)(file, true);
-            if (firstLine.startsWith('#!')) {
-                const matches = Object.entries(langData).filter(([, data]) => { var _a; return (_a = data.interpreters) === null || _a === void 0 ? void 0 : _a.some(interpreter => firstLine.match('\\b' + interpreter + '\\b')); });
-                if (matches.length) {
-                    const forcedLang = matches[0][0];
-                    addResult(file, forcedLang);
-                    continue;
-                }
+        if (!opts.quick && opts.checkShebang && firstLine.startsWith('#!')) {
+            // Find matching interpreters
+            const matches = Object.entries(langData).filter(([, data]) => { var _a; return (_a = data.interpreters) === null || _a === void 0 ? void 0 : _a.some(interpreter => firstLine.match('\\b' + interpreter + '\\b')); });
+            if (matches.length) {
+                // Add explicitly-identified language
+                const forcedLang = matches[0][0];
+                addResult(file, forcedLang);
             }
         }
         // Check override for manual language classification
-        if (!opts.quick && opts.checkAttributes) {
+        if (!opts.fileContent && !opts.quick && opts.checkAttributes) {
             const match = overridesArray.find(item => RegExp(item[0]).test(file));
             if (match) {
                 const forcedLang = match[1];
@@ -142,7 +159,7 @@ async function analyse(input, opts = {}) {
         let skipExts = false;
         for (const lang in langData) {
             // Check if filename is a match
-            const matchesName = (_d = langData[lang].filenames) === null || _d === void 0 ? void 0 : _d.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
+            const matchesName = (_g = langData[lang].filenames) === null || _g === void 0 ? void 0 : _g.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
             if (matchesName) {
                 addResult(file, lang);
                 skipExts = true;
@@ -151,7 +168,7 @@ async function analyse(input, opts = {}) {
         if (!skipExts)
             for (const lang in langData) {
                 // Check if extension is a match
-                const matchesExt = (_e = langData[lang].extensions) === null || _e === void 0 ? void 0 : _e.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
+                const matchesExt = (_h = langData[lang].extensions) === null || _h === void 0 ? void 0 : _h.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
                 if (matchesExt)
                     addResult(file, lang);
             }
@@ -162,7 +179,7 @@ async function analyse(input, opts = {}) {
     // Narrow down file associations to the best fit
     for (const file in fileAssociations) {
         // Skip binary files
-        if (!opts.keepBinary) {
+        if (!opts.fileContent && !opts.keepBinary) {
             const isCustomText = customText.some(path => RegExp(path).test(file));
             const isCustomBinary = customBinary.some(path => RegExp(path).test(file));
             const isBinaryExt = binary_extensions_1.default.some(ext => file.endsWith('.' + ext));
@@ -196,7 +213,9 @@ async function analyse(input, opts = {}) {
                     if (heuristic.named_pattern)
                         normalise(heuristicsData.named_patterns[heuristic.named_pattern]);
                     // Check file contents and apply heuristic patterns
-                    const fileContent = await (0, read_file_1.default)(file);
+                    const fileContent = await (0, read_file_1.default)(file).catch(() => null);
+                    if (fileContent === null)
+                        continue;
                     if (!patterns.length || patterns.some(pattern => (0, convert_pcre_1.default)(pattern).test(fileContent))) {
                         results.files.results[file] = heuristic.language;
                         break;
@@ -204,10 +223,10 @@ async function analyse(input, opts = {}) {
                 }
             }
         // If no heuristics, assign a language
-        (_f = (_k = results.files.results)[file]) !== null && _f !== void 0 ? _f : (_k[file] = fileAssociations[file][0]);
+        (_j = (_r = results.files.results)[file]) !== null && _j !== void 0 ? _j : (_r[file] = fileAssociations[file][0]);
     }
     // Skip specified categories
-    if ((_g = opts.categories) === null || _g === void 0 ? void 0 : _g.length) {
+    if ((_k = opts.categories) === null || _k === void 0 ? void 0 : _k.length) {
         const categories = ['data', 'markup', 'programming', 'prose'];
         const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
         for (const [file, lang] of Object.entries(results.files.results)) {
@@ -224,25 +243,36 @@ async function analyse(input, opts = {}) {
             }
         }
     }
+    // Convert paths to relative
+    if (!opts.fileContent && opts.relativePaths) {
+        const newMap = {};
+        for (const [file, lang] of Object.entries(results.files.results)) {
+            let relPath = path_1.default.relative(process.cwd(), file).replace(/\\/g, '/');
+            if (!relPath.startsWith('../'))
+                relPath = './' + relPath;
+            newMap[relPath] = lang;
+        }
+        results.files.results = newMap;
+    }
     // Load language bytes size
     for (const [file, lang] of Object.entries(results.files.results)) {
         if (lang && !langData[lang])
             continue;
-        const fileSize = fs_1.default.statSync(file).size;
+        const fileSize = (_o = (_m = (_l = opts.fileContent) === null || _l === void 0 ? void 0 : _l[files.indexOf(file)]) === null || _m === void 0 ? void 0 : _m.length) !== null && _o !== void 0 ? _o : fs_1.default.statSync(file).size;
         results.files.bytes += fileSize;
         // If no language found, add extension in other section
         if (!lang) {
             const ext = path_1.default.extname(file);
             const unknownType = ext === '' ? 'filenames' : 'extensions';
             const name = ext === '' ? path_1.default.basename(file) : ext;
-            (_h = (_l = results.unknown[unknownType])[name]) !== null && _h !== void 0 ? _h : (_l[name] = 0);
+            (_p = (_s = results.unknown[unknownType])[name]) !== null && _p !== void 0 ? _p : (_s[name] = 0);
             results.unknown[unknownType][name] += fileSize;
             results.unknown.bytes += fileSize;
             continue;
         }
         // Add language and bytes data to corresponding section
         const { type } = langData[lang];
-        (_j = (_m = results.languages.results)[lang]) !== null && _j !== void 0 ? _j : (_m[lang] = { type, bytes: 0, color: langData[lang].color });
+        (_q = (_t = results.languages.results)[lang]) !== null && _q !== void 0 ? _q : (_t[lang] = { type, bytes: 0, color: langData[lang].color });
         if (opts.childLanguages)
             results.languages.results[lang].parent = langData[lang].group;
         results.languages.results[lang].bytes += fileSize;

package/dist/types.d.ts

@@ -5,11 +5,13 @@ export declare type FilePath = string;
 export declare type Bytes = Integer;
 export declare type Integer = number;
 export interface Options {
+    fileContent?: string | string[];
     ignoredFiles?: string[];
     ignoredLanguages?: Language[];
     categories?: Category[];
     keepVendored?: boolean;
     keepBinary?: boolean;
+    relativePaths?: boolean;
     childLanguages?: boolean;
     quick?: boolean;
     checkIgnored?: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "linguist-js",
-	"version": "2.1.3",
+	"version": "2.3.0",
 	"description": "Analyse languages used in a folder. Powered by GitHub Linguist, although it doesn't need to be installed.",
 	"main": "dist/index.js",
 	"bin": {
@@ -37,18 +37,18 @@
 	"homepage": "https://github.com/Nixinova/Linguist#readme",
 	"dependencies": {
 		"binary-extensions": "^2.2.0",
-		"commander": "^8.2.0",
-		"cross-fetch": "^3.1.4",
+		"commander": "^9.0.0",
+		"cross-fetch": "^3.1.5",
 		"glob-to-regexp": "~0.4.1",
 		"isbinaryfile": "^4.0.8",
 		"js-yaml": "^4.1.0",
 		"node-cache": "^5.1.2"
 	},
 	"devDependencies": {
-		"@types/glob-to-regexp": "ts4.4",
-		"@types/js-yaml": "ts4.4",
-		"@types/node": "ts4.4",
-		"deep-object-diff": "^1.1.0",
-		"typescript": "~4.4.3"
+		"@types/glob-to-regexp": "ts4.6",
+		"@types/js-yaml": "ts4.6",
+		"@types/node": "ts4.6",
+		"deep-object-diff": "^1.1.7",
+		"typescript": "~4.6.1-rc"
 	}
 }

package/readme.md

@@ -77,6 +77,7 @@ Running Linguist on this folder will return the following JSON:
 ### Notes
 
 - File paths in the output use only forward slashes as delimiters, even on Windows.
+- This tool does not work when offline.
 - Do not rely on any language classification output from Linguist being unchanged between runs.
   Language data is fetched each run from the latest classifications of [`github-linguist`](https://github.com/github/linguist).
   This data is subject to change at any time and may change the results of a run even when using the same version of Linguist.
@@ -99,6 +100,8 @@ const { files, languages, unknown } = linguist(folder, options);
     Analyse multiple folders using the syntax `"{folder1,folder2,...}"`.
   - `opts` (optional; object):
     An object containing analyser options.
+    - `fileContent` (string or string array):
+      Provides the file content associated with the file name(s) given as `entry` to analyse instead of reading from a folder on disk.
     - `ignoredFiles` (string array):
       A list of file path globs to explicitly ignore.
     - `ignoredLanguages` (string array):
@@ -115,6 +118,8 @@ const { files, languages, unknown } = linguist(folder, options);
       Whether to keep vendored files (dependencies, etc) (defaults to `false`).
     - `keepBinary` (boolean):
       Whether binary files should be included in the output (defaults to `false`).
+    - `relativePaths` (boolean):
+      Change the absolute file paths in the output to be relative to the current working directory (defaults to `false`).
     - `checkAttributes` (boolean):
       Force the checking of `.gitattributes` files (defaults to `true` unless `quick` is set).
     - `checkIgnored` (boolean):
@@ -155,6 +160,8 @@ linguist --help
     Whether to include vendored files (auto-generated files, dependencies folder, etc).
   - `--keepBinary`:
     Whether binary files should be excluded from the output.
+  - `--relativePaths`:
+    Change the absolute file paths in the output to be relative to the current working directory.
   - `--checkAttributes`:
     Force the checking of `.gitatributes` files (use alongside `--quick` to overwrite).
   - `--checkIgnored`:

package/dist/test.js

@@ -1,3 +0,0 @@
-const classificator = require('classificator');
-const b = classificator();
-b.toJson();