linguist-js 2.4.1 → 2.5.1

package/dist/cli.js

@@ -19,6 +19,7 @@ commander_1.program
     .option('-j|--json [bool]', 'Display the output as JSON', false)
     .option('-t|--tree <traversal>', 'Which part of the output JSON to display (dot-delimited)')
     .option('-q|--quick [bool]', 'Skip complex language analysis (alias for -{A|I|H|S}=false)', false)
+    .option('-o|--offline [bool]', 'Use packaged data files instead of fetching latest from GitHub', false)
     .option('-V|--keepVendored [bool]', 'Prevent skipping over vendored/generated files', false)
     .option('-B|--keepBinary [bool]', 'Prevent skipping over binary files', false)
     .option('-r|--relativePaths [bool]', 'Convert absolute file paths to relative', false)

package/dist/helpers/load-data.js

@@ -3,11 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
 const cross_fetch_1 = __importDefault(require("cross-fetch"));
 const node_cache_1 = __importDefault(require("node-cache"));
 const cache = new node_cache_1.default({});
-/** Load a data file from github-linguist. */
-async function loadFile(file) {
+async function loadWebFile(file) {
     // Return cache if it exists
     const cachedContent = cache.get(file);
     if (cachedContent)
@@ -18,4 +19,12 @@ async function loadFile(file) {
     cache.set(file, fileContent);
     return fileContent;
 }
+async function loadLocalFile(file) {
+    const filePath = path_1.default.resolve(__dirname, '../../ext', file);
+    return fs_1.default.promises.readFile(filePath).then(buffer => buffer.toString());
+}
+/** Load a data file from github-linguist. */
+async function loadFile(file, offline = false) {
+    return offline ? loadLocalFile(file) : loadWebFile(file);
+}
 exports.default = loadFile;

package/dist/helpers/walk-tree.js

@@ -5,10 +5,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
-const allFiles = new Set();
-const allFolders = new Set();
+let allFiles;
+let allFolders;
 /** Generate list of files in a directory. */
-function walk(root, folders, gitignores, regexIgnores) {
+function walk(init, root, folders, gitignores, regexIgnores) {
+    // Initialise files and folders lists
+    if (init) {
+        allFiles = new Set();
+        allFolders = new Set();
+    }
     // Walk tree of a folder
     if (folders.length === 1) {
         const folder = folders[0];
@@ -36,7 +41,7 @@ function walk(root, folders, gitignores, regexIgnores) {
             if (file.endsWith('/')) {
                 // Recurse into subfolders
                 allFolders.add(path);
-                walk(root, [path], gitignores, regexIgnores);
+                walk(false, root, [path], gitignores, regexIgnores);
             }
             else {
                 // Add relative file path to list
@@ -47,7 +52,7 @@ function walk(root, folders, gitignores, regexIgnores) {
     // Recurse into all folders
     else {
         for (const path of folders) {
-            walk(root, [path], gitignores, regexIgnores);
+            walk(false, root, [path], gitignores, regexIgnores);
         }
     }
     // Return absolute files and folders lists

package/dist/index.js

@@ -14,17 +14,17 @@ const load_data_1 = __importDefault(require("./helpers/load-data"));
 const read_file_1 = __importDefault(require("./helpers/read-file"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
 async function analyse(input, opts = {}) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
-    var _t, _u, _v;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t;
+    var _u, _v, _w;
     const useRawContent = opts.fileContent !== undefined;
     input = [input !== null && input !== void 0 ? input : []].flat();
     opts.fileContent = [(_a = opts.fileContent) !== null && _a !== void 0 ? _a : []].flat();
     // Load data from github-linguist web repo
-    const langData = await (0, load_data_1.default)('languages.yml').then(js_yaml_1.default.load);
-    const vendorData = await (0, load_data_1.default)('vendor.yml').then(js_yaml_1.default.load);
-    const docData = await (0, load_data_1.default)('documentation.yml').then(js_yaml_1.default.load);
-    const heuristicsData = await (0, load_data_1.default)('heuristics.yml').then(js_yaml_1.default.load);
-    const generatedData = await (0, load_data_1.default)('generated.rb').then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/)/gm)) !== null && _a !== void 0 ? _a : []; });
+    const langData = await (0, load_data_1.default)('languages.yml', opts.offline).then(js_yaml_1.default.load);
+    const vendorData = await (0, load_data_1.default)('vendor.yml', opts.offline).then(js_yaml_1.default.load);
+    const docData = await (0, load_data_1.default)('documentation.yml', opts.offline).then(js_yaml_1.default.load);
+    const heuristicsData = await (0, load_data_1.default)('heuristics.yml', opts.offline).then(js_yaml_1.default.load);
+    const generatedData = await (0, load_data_1.default)('generated.rb', opts.offline).then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/)/gm)) !== null && _a !== void 0 ? _a : []; });
     const vendorPaths = [...vendorData, ...docData, ...generatedData];
     // Setup main variables
     const fileAssociations = {};
@@ -58,7 +58,7 @@ async function analyse(input, opts = {}) {
     }
     else {
         // Uses directory on disc
-        const data = (0, walk_tree_1.default)(commonRoot, input, gitignores, regexIgnores);
+        const data = (0, walk_tree_1.default)(true, commonRoot, input, gitignores, regexIgnores);
         files = data.files;
         folders = data.folders;
     }
@@ -86,8 +86,9 @@ async function analyse(input, opts = {}) {
     if (!useRawContent && opts.checkAttributes) {
         for (const folder of folders) {
             // Skip if folder is marked in gitattributes
-            if (relPath(folder) && gitignores.test(relPath(folder)).ignored)
+            if (relPath(folder) && gitignores.ignores(relPath(folder))) {
                 continue;
+            }
             // Parse gitignores
             const ignoresFile = path_1.default.join(folder, '.gitignore');
             if (opts.checkIgnored && fs_1.default.existsSync(ignoresFile)) {
@@ -101,10 +102,12 @@ async function analyse(input, opts = {}) {
                 // Explicit text/binary associations
                 const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)(?!=auto)/gm);
                 for (const [_line, path, type] of contentTypeMatches) {
-                    if (['text', '-binary'].includes(type))
+                    if (['text', '-binary'].includes(type)) {
                         customText.add(path);
-                    if (['-text', 'binary'].includes(type))
+                    }
+                    if (['-text', 'binary'].includes(type)) {
                         customBinary.add(path);
+                    }
                 }
                 // Custom vendor options
                 const vendorMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-(vendored|generated|documentation)(?!=false)/gm);
@@ -117,8 +120,9 @@ async function analyse(input, opts = {}) {
                     // If specified language is an alias, associate it with its full name
                     if (!langData[forcedLang]) {
                         const overrideLang = Object.entries(langData).find(entry => { var _a; return (_a = entry[1].aliases) === null || _a === void 0 ? void 0 : _a.includes(forcedLang.toLowerCase()); });
-                        if (overrideLang)
+                        if (overrideLang) {
                             forcedLang = overrideLang[0];
+                        }
                     }
                     const fullPath = relPath(folder) + '/' + path;
                     overrides[fullPath] = forcedLang;
@@ -154,11 +158,11 @@ async function analyse(input, opts = {}) {
         if (useRawContent) {
             firstLine = (_e = (_d = (_c = opts.fileContent) === null || _c === void 0 ? void 0 : _c[files.indexOf(file)]) === null || _d === void 0 ? void 0 : _d.split('\n')[0]) !== null && _e !== void 0 ? _e : null;
         }
-        else {
-            if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
-                continue;
+        else if (fs_1.default.existsSync(file) && !fs_1.default.lstatSync(file).isDirectory()) {
             firstLine = await (0, read_file_1.default)(file, true).catch(() => null);
         }
+        else
+            continue;
         // Skip if file is unreadable
         if (firstLine === null)
             continue;
@@ -175,10 +179,11 @@ async function analyse(input, opts = {}) {
                 const matchesLang = firstLine.toLowerCase().match(langMatcher(lang));
                 const matchesAlias = (_g = data.aliases) === null || _g === void 0 ? void 0 : _g.some(lang => firstLine.toLowerCase().match(langMatcher(lang)));
                 // Add language
-                if (opts.checkShebang && matchesInterpretor)
-                    matches.push(lang);
-                if (opts.checkModeline && (matchesLang || matchesAlias))
+                const interpretorCheck = opts.checkShebang && matchesInterpretor;
+                const modelineCheck = opts.checkModeline && (matchesLang || matchesAlias);
+                if (interpretorCheck || modelineCheck) {
                     matches.push(lang);
+                }
             }
             if (matches.length) {
                 // Add explicitly-identified language
@@ -190,7 +195,7 @@ async function analyse(input, opts = {}) {
         }
         // Check override for manual language classification
         if (!useRawContent && !opts.quick && opts.checkAttributes) {
-            const isOverridden = (path) => (0, ignore_1.default)().add(path).test(relPath(file)).ignored;
+            const isOverridden = (path) => (0, ignore_1.default)().add(path).ignores(relPath(file));
             const match = overridesArray.find(item => isOverridden(item[0]));
             if (match) {
                 const forcedLang = match[1];
@@ -213,12 +218,14 @@ async function analyse(input, opts = {}) {
             for (const lang in langData) {
                 // Check if extension is a match
                 const matchesExt = (_j = langData[lang].extensions) === null || _j === void 0 ? void 0 : _j.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
-                if (matchesExt)
+                if (matchesExt) {
                     addResult(file, lang);
+                }
             }
         // Fallback to null if no language matches
-        if (!fileAssociations[file])
+        if (!fileAssociations[file]) {
             addResult(file, null);
+        }
     }
     // Narrow down file associations to the best fit
     for (const file in fileAssociations) {
@@ -229,8 +236,8 @@ async function analyse(input, opts = {}) {
         }
         // Skip binary files
         if (!useRawContent && !opts.keepBinary) {
-            const isCustomText = customText.test(relPath(file)).ignored;
-            const isCustomBinary = customBinary.test(relPath(file)).ignored;
+            const isCustomText = customText.ignores(relPath(file));
+            const isCustomBinary = customBinary.ignores(relPath(file));
             const isBinaryExt = binary_extensions_1.default.some(ext => file.endsWith('.' + ext));
             if (!isCustomText && (isCustomBinary || isBinaryExt || await (0, isbinaryfile_1.isBinaryFile)(file))) {
                 continue;
@@ -262,8 +269,16 @@ async function analyse(input, opts = {}) {
                         normalise(heuristic.pattern);
                     if (heuristic.named_pattern)
                         normalise(heuristicsData.named_patterns[heuristic.named_pattern]);
+                    if (heuristic.and) {
+                        for (const data of heuristic.and) {
+                            if (data.pattern)
+                                normalise(data.pattern);
+                            if (data.named_pattern)
+                                normalise(heuristicsData.named_patterns[data.named_pattern]);
+                        }
+                    }
                     // Check file contents and apply heuristic patterns
-                    const fileContent = await (0, read_file_1.default)(file).catch(() => null);
+                    const fileContent = ((_l = opts.fileContent) === null || _l === void 0 ? void 0 : _l.length) ? opts.fileContent[files.indexOf(file)] : await (0, read_file_1.default)(file).catch(() => null);
                     if (fileContent === null)
                         continue;
                     if (!patterns.length || patterns.some(pattern => (0, convert_pcre_1.default)(pattern).test(fileContent))) {
@@ -273,23 +288,26 @@ async function analyse(input, opts = {}) {
                 }
             }
         // If no heuristics, assign a language
-        (_l = (_t = results.files.results)[file]) !== null && _l !== void 0 ? _l : (_t[file] = fileAssociations[file][0]);
+        (_m = (_u = results.files.results)[file]) !== null && _m !== void 0 ? _m : (_u[file] = fileAssociations[file][0]);
     }
     // Skip specified categories
-    if ((_m = opts.categories) === null || _m === void 0 ? void 0 : _m.length) {
+    if ((_o = opts.categories) === null || _o === void 0 ? void 0 : _o.length) {
         const categories = ['data', 'markup', 'programming', 'prose'];
         const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
         for (const [file, lang] of Object.entries(results.files.results)) {
-            if (!hiddenCategories.some(cat => { var _a; return lang && ((_a = langData[lang]) === null || _a === void 0 ? void 0 : _a.type) === cat; }))
+            if (!hiddenCategories.some(cat => { var _a; return lang && ((_a = langData[lang]) === null || _a === void 0 ? void 0 : _a.type) === cat; })) {
                 continue;
+            }
             delete results.files.results[file];
-            if (lang)
+            if (lang) {
                 delete results.languages.results[lang];
+            }
         }
         for (const category of hiddenCategories) {
             for (const [lang, { type }] of Object.entries(results.languages.results)) {
-                if (type === category)
+                if (type === category) {
                     delete results.languages.results[lang];
+                }
             }
         }
     }
@@ -298,8 +316,9 @@ async function analyse(input, opts = {}) {
         const newMap = {};
         for (const [file, lang] of Object.entries(results.files.results)) {
             let relPath = path_1.default.relative(process.cwd(), file).replace(/\\/g, '/');
-            if (!relPath.startsWith('../'))
+            if (!relPath.startsWith('../')) {
                 relPath = './' + relPath;
+            }
             newMap[relPath] = lang;
         }
         results.files.results = newMap;
@@ -308,23 +327,24 @@ async function analyse(input, opts = {}) {
     for (const [file, lang] of Object.entries(results.files.results)) {
         if (lang && !langData[lang])
             continue;
-        const fileSize = (_q = (_p = (_o = opts.fileContent) === null || _o === void 0 ? void 0 : _o[files.indexOf(file)]) === null || _p === void 0 ? void 0 : _p.length) !== null && _q !== void 0 ? _q : fs_1.default.statSync(file).size;
+        const fileSize = (_r = (_q = (_p = opts.fileContent) === null || _p === void 0 ? void 0 : _p[files.indexOf(file)]) === null || _q === void 0 ? void 0 : _q.length) !== null && _r !== void 0 ? _r : fs_1.default.statSync(file).size;
         results.files.bytes += fileSize;
         // If no language found, add extension in other section
         if (!lang) {
             const ext = path_1.default.extname(file);
             const unknownType = ext === '' ? 'filenames' : 'extensions';
             const name = ext === '' ? path_1.default.basename(file) : ext;
-            (_r = (_u = results.unknown[unknownType])[name]) !== null && _r !== void 0 ? _r : (_u[name] = 0);
+            (_s = (_v = results.unknown[unknownType])[name]) !== null && _s !== void 0 ? _s : (_v[name] = 0);
             results.unknown[unknownType][name] += fileSize;
             results.unknown.bytes += fileSize;
             continue;
         }
         // Add language and bytes data to corresponding section
         const { type } = langData[lang];
-        (_s = (_v = results.languages.results)[lang]) !== null && _s !== void 0 ? _s : (_v[lang] = { type, bytes: 0, color: langData[lang].color });
-        if (opts.childLanguages)
+        (_t = (_w = results.languages.results)[lang]) !== null && _t !== void 0 ? _t : (_w[lang] = { type, bytes: 0, color: langData[lang].color });
+        if (opts.childLanguages) {
             results.languages.results[lang].parent = langData[lang].group;
+        }
         results.languages.results[lang].bytes += fileSize;
         results.languages.bytes += fileSize;
     }

package/dist/types.d.ts

@@ -14,6 +14,7 @@ export interface Options {
     relativePaths?: boolean;
     childLanguages?: boolean;
     quick?: boolean;
+    offline?: boolean;
     checkIgnored?: boolean;
     checkAttributes?: boolean;
     checkHeuristics?: boolean;

package/ext/documentation.yml

@@ -0,0 +1,17 @@
+- ^[Dd]ocs?/
+- (^|/)[Dd]ocumentation/
+- (^|/)[Gg]roovydoc/
+- (^|/)[Jj]avadoc/
+- ^[Mm]an/
+- ^[Ee]xamples/
+- ^[Dd]emos?/
+- (^|/)inst/doc/- (^|/)CITATION(\.cff|(S)?(\.(bib|md))?)$
+- (^|/)CHANGE(S|LOG)?(\.|$)
+- (^|/)CONTRIBUTING(\.|$)
+- (^|/)COPYING(\.|$)
+- (^|/)INSTALL(\.|$)
+- (^|/)LICEN[CS]E(\.|$)
+- (^|/)[Ll]icen[cs]e(\.|$)
+- (^|/)README(\.|$)
+- (^|/)[Rr]eadme(\.|$)
+- ^[Ss]amples?/