linguist-js 2.4.2 → 2.5.2

package/dist/cli.js

@@ -19,6 +19,7 @@ commander_1.program
     .option('-j|--json [bool]', 'Display the output as JSON', false)
     .option('-t|--tree <traversal>', 'Which part of the output JSON to display (dot-delimited)')
     .option('-q|--quick [bool]', 'Skip complex language analysis (alias for -{A|I|H|S}=false)', false)
+    .option('-o|--offline [bool]', 'Use packaged data files instead of fetching latest from GitHub', false)
     .option('-V|--keepVendored [bool]', 'Prevent skipping over vendored/generated files', false)
     .option('-B|--keepBinary [bool]', 'Prevent skipping over binary files', false)
     .option('-r|--relativePaths [bool]', 'Convert absolute file paths to relative', false)

package/dist/helpers/load-data.js

@@ -3,11 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
 const cross_fetch_1 = __importDefault(require("cross-fetch"));
 const node_cache_1 = __importDefault(require("node-cache"));
 const cache = new node_cache_1.default({});
-/** Load a data file from github-linguist. */
-async function loadFile(file) {
+async function loadWebFile(file) {
     // Return cache if it exists
     const cachedContent = cache.get(file);
     if (cachedContent)
@@ -18,4 +19,12 @@ async function loadFile(file) {
     cache.set(file, fileContent);
     return fileContent;
 }
+async function loadLocalFile(file) {
+    const filePath = path_1.default.resolve(__dirname, '../../ext', file);
+    return fs_1.default.promises.readFile(filePath).then(buffer => buffer.toString());
+}
+/** Load a data file from github-linguist. */
+async function loadFile(file, offline = false) {
+    return offline ? loadLocalFile(file) : loadWebFile(file);
+}
 exports.default = loadFile;

package/dist/index.js

@@ -14,21 +14,20 @@ const load_data_1 = __importDefault(require("./helpers/load-data"));
 const read_file_1 = __importDefault(require("./helpers/read-file"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
 async function analyse(input, opts = {}) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
-    var _t, _u, _v;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t;
+    var _u, _v, _w;
     const useRawContent = opts.fileContent !== undefined;
     input = [input !== null && input !== void 0 ? input : []].flat();
     opts.fileContent = [(_a = opts.fileContent) !== null && _a !== void 0 ? _a : []].flat();
     // Load data from github-linguist web repo
-    const langData = await (0, load_data_1.default)('languages.yml').then(js_yaml_1.default.load);
-    const vendorData = await (0, load_data_1.default)('vendor.yml').then(js_yaml_1.default.load);
-    const docData = await (0, load_data_1.default)('documentation.yml').then(js_yaml_1.default.load);
-    const heuristicsData = await (0, load_data_1.default)('heuristics.yml').then(js_yaml_1.default.load);
-    const generatedData = await (0, load_data_1.default)('generated.rb').then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/)/gm)) !== null && _a !== void 0 ? _a : []; });
+    const langData = await (0, load_data_1.default)('languages.yml', opts.offline).then(js_yaml_1.default.load);
+    const vendorData = await (0, load_data_1.default)('vendor.yml', opts.offline).then(js_yaml_1.default.load);
+    const docData = await (0, load_data_1.default)('documentation.yml', opts.offline).then(js_yaml_1.default.load);
+    const heuristicsData = await (0, load_data_1.default)('heuristics.yml', opts.offline).then(js_yaml_1.default.load);
+    const generatedData = await (0, load_data_1.default)('generated.rb', opts.offline).then(text => { var _a; return (_a = text.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/)/gm)) !== null && _a !== void 0 ? _a : []; });
     const vendorPaths = [...vendorData, ...docData, ...generatedData];
     // Setup main variables
     const fileAssociations = {};
-    const definiteness = {};
     const extensions = {};
     const overrides = {};
     const results = {
@@ -86,8 +85,9 @@ async function analyse(input, opts = {}) {
     if (!useRawContent && opts.checkAttributes) {
         for (const folder of folders) {
             // Skip if folder is marked in gitattributes
-            if (relPath(folder) && gitignores.ignores(relPath(folder)))
+            if (relPath(folder) && gitignores.ignores(relPath(folder))) {
                 continue;
+            }
             // Parse gitignores
             const ignoresFile = path_1.default.join(folder, '.gitignore');
             if (opts.checkIgnored && fs_1.default.existsSync(ignoresFile)) {
@@ -101,10 +101,12 @@ async function analyse(input, opts = {}) {
                 // Explicit text/binary associations
                 const contentTypeMatches = attributesData.matchAll(/^(\S+).*?(-?binary|-?text)(?!=auto)/gm);
                 for (const [_line, path, type] of contentTypeMatches) {
-                    if (['text', '-binary'].includes(type))
+                    if (['text', '-binary'].includes(type)) {
                         customText.add(path);
-                    if (['-text', 'binary'].includes(type))
+                    }
+                    if (['-text', 'binary'].includes(type)) {
                         customBinary.add(path);
+                    }
                 }
                 // Custom vendor options
                 const vendorMatches = attributesData.matchAll(/^(\S+).*[^-]linguist-(vendored|generated|documentation)(?!=false)/gm);
@@ -117,8 +119,9 @@ async function analyse(input, opts = {}) {
                     // If specified language is an alias, associate it with its full name
                     if (!langData[forcedLang]) {
                         const overrideLang = Object.entries(langData).find(entry => { var _a; return (_a = entry[1].aliases) === null || _a === void 0 ? void 0 : _a.includes(forcedLang.toLowerCase()); });
-                        if (overrideLang)
+                        if (overrideLang) {
                             forcedLang = overrideLang[0];
+                        }
                     }
                     const fullPath = relPath(folder) + '/' + path;
                     overrides[fullPath] = forcedLang;
@@ -149,16 +152,18 @@ async function analyse(input, opts = {}) {
     };
     const overridesArray = Object.entries(overrides);
     // List all languages that could be associated with a given file
+    const definiteness = {};
+    const fromShebang = {};
     for (const file of files) {
         let firstLine;
         if (useRawContent) {
             firstLine = (_e = (_d = (_c = opts.fileContent) === null || _c === void 0 ? void 0 : _c[files.indexOf(file)]) === null || _d === void 0 ? void 0 : _d.split('\n')[0]) !== null && _e !== void 0 ? _e : null;
         }
-        else {
-            if (!fs_1.default.existsSync(file) || fs_1.default.lstatSync(file).isDirectory())
-                continue;
+        else if (fs_1.default.existsSync(file) && !fs_1.default.lstatSync(file).isDirectory()) {
             firstLine = await (0, read_file_1.default)(file, true).catch(() => null);
         }
+        else
+            continue;
         // Skip if file is unreadable
         if (firstLine === null)
             continue;
@@ -170,21 +175,27 @@ async function analyse(input, opts = {}) {
             for (const [lang, data] of Object.entries(langData)) {
                 const langMatcher = (lang) => `\\b${lang.toLowerCase().replace(/\W/g, '\\$&')}(?![\\w#+*]|-\*-)`;
                 // Check for interpreter match
-                const matchesInterpretor = (_f = data.interpreters) === null || _f === void 0 ? void 0 : _f.some(interpreter => firstLine.match(`\\b${interpreter}\\b`));
+                if (opts.checkShebang && hasShebang) {
+                    const matchesInterpretor = (_f = data.interpreters) === null || _f === void 0 ? void 0 : _f.some(interpreter => firstLine.match(`\\b${interpreter}\\b`));
+                    if (matchesInterpretor)
+                        matches.push(lang);
+                }
                 // Check modeline declaration
-                const matchesLang = firstLine.toLowerCase().match(langMatcher(lang));
-                const matchesAlias = (_g = data.aliases) === null || _g === void 0 ? void 0 : _g.some(lang => firstLine.toLowerCase().match(langMatcher(lang)));
-                // Add language
-                if (opts.checkShebang && matchesInterpretor)
-                    matches.push(lang);
-                if (opts.checkModeline && (matchesLang || matchesAlias))
-                    matches.push(lang);
+                if (opts.checkModeline && hasModeline) {
+                    const modelineText = firstLine.toLowerCase().replace(/^.*-\*-(.+)-\*-.*$/, '$1');
+                    const matchesLang = modelineText.match(langMatcher(lang));
+                    const matchesAlias = (_g = data.aliases) === null || _g === void 0 ? void 0 : _g.some(lang => modelineText.match(langMatcher(lang)));
+                    if (matchesLang || matchesAlias)
+                        matches.push(lang);
+                }
             }
+            // Add identified language(s)
             if (matches.length) {
-                // Add explicitly-identified language
-                const forcedLang = matches[0];
-                addResult(file, forcedLang);
-                definiteness[file] = true;
+                for (const match of matches)
+                    addResult(file, match);
+                if (matches.length === 1)
+                    definiteness[file] = true;
+                fromShebang[file] = true;
                 continue;
             }
         }
@@ -201,24 +212,38 @@ async function analyse(input, opts = {}) {
         }
         // Search each language
         let skipExts = false;
+        // Check if filename is a match
         for (const lang in langData) {
-            // Check if filename is a match
             const matchesName = (_h = langData[lang].filenames) === null || _h === void 0 ? void 0 : _h.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
             if (matchesName) {
                 addResult(file, lang);
                 skipExts = true;
             }
         }
+        // Check if extension is a match
+        const possibleExts = [];
         if (!skipExts)
             for (const lang in langData) {
-                // Check if extension is a match
-                const matchesExt = (_j = langData[lang].extensions) === null || _j === void 0 ? void 0 : _j.some(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
-                if (matchesExt)
-                    addResult(file, lang);
+                const extMatches = (_j = langData[lang].extensions) === null || _j === void 0 ? void 0 : _j.filter(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
+                if (extMatches === null || extMatches === void 0 ? void 0 : extMatches.length) {
+                    for (const ext of extMatches)
+                        possibleExts.push({ ext, lang });
+                }
             }
+        // Apply more specific extension if available
+        const isComplexExt = (ext) => /\..+\./.test(ext);
+        const hasComplexExt = possibleExts.some(data => isComplexExt(data.ext));
+        for (const { ext, lang } of possibleExts) {
+            if (hasComplexExt && !isComplexExt(ext))
+                continue;
+            if (!hasComplexExt && isComplexExt(ext))
+                continue;
+            addResult(file, lang);
+        }
         // Fallback to null if no language matches
-        if (!fileAssociations[file])
+        if (!fileAssociations[file]) {
             addResult(file, null);
+        }
     }
     // Narrow down file associations to the best fit
     for (const file in fileAssociations) {
@@ -240,9 +265,8 @@ async function analyse(input, opts = {}) {
         if (opts.checkHeuristics)
             for (const heuristics of heuristicsData.disambiguations) {
                 // Make sure the extension matches the current file
-                if (!heuristics.extensions.includes(extensions[file])) {
+                if (!fromShebang[file] && !heuristics.extensions.includes(extensions[file]))
                     continue;
-                }
                 // Load heuristic rules
                 for (const heuristic of heuristics.rules) {
                     // Make sure the language is not an array
@@ -262,8 +286,16 @@ async function analyse(input, opts = {}) {
                         normalise(heuristic.pattern);
                     if (heuristic.named_pattern)
                         normalise(heuristicsData.named_patterns[heuristic.named_pattern]);
+                    if (heuristic.and) {
+                        for (const data of heuristic.and) {
+                            if (data.pattern)
+                                normalise(data.pattern);
+                            if (data.named_pattern)
+                                normalise(heuristicsData.named_patterns[data.named_pattern]);
+                        }
+                    }
                     // Check file contents and apply heuristic patterns
-                    const fileContent = opts.fileContent ? opts.fileContent[files.indexOf(file)] : await (0, read_file_1.default)(file).catch(() => null);
+                    const fileContent = ((_l = opts.fileContent) === null || _l === void 0 ? void 0 : _l.length) ? opts.fileContent[files.indexOf(file)] : await (0, read_file_1.default)(file).catch(() => null);
                     if (fileContent === null)
                         continue;
                     if (!patterns.length || patterns.some(pattern => (0, convert_pcre_1.default)(pattern).test(fileContent))) {
@@ -273,23 +305,26 @@ async function analyse(input, opts = {}) {
                 }
             }
         // If no heuristics, assign a language
-        (_l = (_t = results.files.results)[file]) !== null && _l !== void 0 ? _l : (_t[file] = fileAssociations[file][0]);
+        (_m = (_u = results.files.results)[file]) !== null && _m !== void 0 ? _m : (_u[file] = fileAssociations[file][0]);
     }
     // Skip specified categories
-    if ((_m = opts.categories) === null || _m === void 0 ? void 0 : _m.length) {
+    if ((_o = opts.categories) === null || _o === void 0 ? void 0 : _o.length) {
         const categories = ['data', 'markup', 'programming', 'prose'];
         const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
         for (const [file, lang] of Object.entries(results.files.results)) {
-            if (!hiddenCategories.some(cat => { var _a; return lang && ((_a = langData[lang]) === null || _a === void 0 ? void 0 : _a.type) === cat; }))
+            if (!hiddenCategories.some(cat => { var _a; return lang && ((_a = langData[lang]) === null || _a === void 0 ? void 0 : _a.type) === cat; })) {
                 continue;
+            }
             delete results.files.results[file];
-            if (lang)
+            if (lang) {
                 delete results.languages.results[lang];
+            }
         }
         for (const category of hiddenCategories) {
             for (const [lang, { type }] of Object.entries(results.languages.results)) {
-                if (type === category)
+                if (type === category) {
                     delete results.languages.results[lang];
+                }
             }
         }
     }
@@ -298,8 +333,9 @@ async function analyse(input, opts = {}) {
         const newMap = {};
         for (const [file, lang] of Object.entries(results.files.results)) {
             let relPath = path_1.default.relative(process.cwd(), file).replace(/\\/g, '/');
-            if (!relPath.startsWith('../'))
+            if (!relPath.startsWith('../')) {
                 relPath = './' + relPath;
+            }
             newMap[relPath] = lang;
         }
         results.files.results = newMap;
@@ -308,23 +344,24 @@ async function analyse(input, opts = {}) {
     for (const [file, lang] of Object.entries(results.files.results)) {
         if (lang && !langData[lang])
             continue;
-        const fileSize = (_q = (_p = (_o = opts.fileContent) === null || _o === void 0 ? void 0 : _o[files.indexOf(file)]) === null || _p === void 0 ? void 0 : _p.length) !== null && _q !== void 0 ? _q : fs_1.default.statSync(file).size;
+        const fileSize = (_r = (_q = (_p = opts.fileContent) === null || _p === void 0 ? void 0 : _p[files.indexOf(file)]) === null || _q === void 0 ? void 0 : _q.length) !== null && _r !== void 0 ? _r : fs_1.default.statSync(file).size;
         results.files.bytes += fileSize;
         // If no language found, add extension in other section
         if (!lang) {
             const ext = path_1.default.extname(file);
             const unknownType = ext === '' ? 'filenames' : 'extensions';
             const name = ext === '' ? path_1.default.basename(file) : ext;
-            (_r = (_u = results.unknown[unknownType])[name]) !== null && _r !== void 0 ? _r : (_u[name] = 0);
+            (_s = (_v = results.unknown[unknownType])[name]) !== null && _s !== void 0 ? _s : (_v[name] = 0);
             results.unknown[unknownType][name] += fileSize;
             results.unknown.bytes += fileSize;
             continue;
         }
         // Add language and bytes data to corresponding section
         const { type } = langData[lang];
-        (_s = (_v = results.languages.results)[lang]) !== null && _s !== void 0 ? _s : (_v[lang] = { type, bytes: 0, color: langData[lang].color });
-        if (opts.childLanguages)
+        (_t = (_w = results.languages.results)[lang]) !== null && _t !== void 0 ? _t : (_w[lang] = { type, bytes: 0, color: langData[lang].color });
+        if (opts.childLanguages) {
             results.languages.results[lang].parent = langData[lang].group;
+        }
         results.languages.results[lang].bytes += fileSize;
         results.languages.bytes += fileSize;
     }

package/dist/types.d.ts

@@ -14,6 +14,7 @@ export interface Options {
     relativePaths?: boolean;
     childLanguages?: boolean;
     quick?: boolean;
+    offline?: boolean;
     checkIgnored?: boolean;
     checkAttributes?: boolean;
     checkHeuristics?: boolean;

package/ext/documentation.yml

@@ -0,0 +1,17 @@
+- ^[Dd]ocs?/
+- (^|/)[Dd]ocumentation/
+- (^|/)[Gg]roovydoc/
+- (^|/)[Jj]avadoc/
+- ^[Mm]an/
+- ^[Ee]xamples/
+- ^[Dd]emos?/
+- (^|/)inst/doc/- (^|/)CITATION(\.cff|(S)?(\.(bib|md))?)$
+- (^|/)CHANGE(S|LOG)?(\.|$)
+- (^|/)CONTRIBUTING(\.|$)
+- (^|/)COPYING(\.|$)
+- (^|/)INSTALL(\.|$)
+- (^|/)LICEN[CS]E(\.|$)
+- (^|/)[Ll]icen[cs]e(\.|$)
+- (^|/)README(\.|$)
+- (^|/)[Rr]eadme(\.|$)
+- ^[Ss]amples?/