linguist-js 2.7.0-pre → 2.7.0

package/dist/helpers/parse-gitattributes.js

@@ -10,7 +10,8 @@ const path_1 = __importDefault(require("path"));
 function parseAttributes(content, folderRoot = '.') {
     var _a, _b;
     const output = [];
-    for (const line of content.split('\n')) {
+    for (const rawLine of content.split('\n')) {
+        const line = rawLine.replace(/#.*/, '').trim();
         if (!line)
             continue;
         const parts = line.split(/\s+/g);

package/dist/index.js

@@ -37,11 +37,11 @@ const load_data_1 = __importStar(require("./helpers/load-data"));
 const read_file_1 = __importDefault(require("./helpers/read-file"));
 const parse_gitattributes_1 = __importDefault(require("./helpers/parse-gitattributes"));
 const convert_pcre_1 = __importDefault(require("./helpers/convert-pcre"));
-async function analyse(rawInput, opts = {}) {
+async function analyse(rawPaths, opts = {}) {
     var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
     var _r, _s;
     const useRawContent = opts.fileContent !== undefined;
-    const input = [rawInput !== null && rawInput !== void 0 ? rawInput : []].flat();
+    const input = [rawPaths !== null && rawPaths !== void 0 ? rawPaths : []].flat();
     const manualFileContent = [(_a = opts.fileContent) !== null && _a !== void 0 ? _a : []].flat();
     // Normalise input option arguments
     opts = {
@@ -101,17 +101,21 @@ async function analyse(rawInput, opts = {}) {
         folders = data.folders;
     }
     // Load gitignore data and apply ignores rules
-    if (!useRawContent) {
-        // TODO switch to be like the gitattributes code
-        for (const folder of folders) {
+    if (!useRawContent && opts.checkIgnored) {
+        const nestedIgnoreFiles = files.filter(file => file.endsWith('.gitignore'));
+        for (const ignoresFile of nestedIgnoreFiles) {
+            const relIgnoresFile = relPath(ignoresFile);
+            const relIgnoresFolder = path_1.default.dirname(relIgnoresFile);
             // Parse gitignores
-            const ignoresFile = path_1.default.join(folder, '.gitignore');
-            if (opts.checkIgnored && fs_1.default.existsSync(ignoresFile)) {
-                const ignoresData = await (0, read_file_1.default)(ignoresFile);
-                const localIgnoresData = ignoresData.replace(/^[\/\\]/g, localRoot(folder) + '/');
-                ignored.add(localIgnoresData);
-                files = filterOutIgnored(files, ignored);
-            }
+            const ignoresDataRaw = await (0, read_file_1.default)(ignoresFile);
+            const ignoresData = ignoresDataRaw.replace(/#.+|\s+$/gm, '');
+            const absoluteIgnoresData = ignoresData
+                // '.file' -> 'root/*/.file'
+                .replace(/^(?=[^\s\/\\])/gm, localRoot(relIgnoresFolder) + '/*/')
+                // '/folder' -> 'root/folder'
+                .replace(/^[\/\\]/gm, localRoot(relIgnoresFolder) + '/');
+            ignored.add(absoluteIgnoresData);
+            files = filterOutIgnored(files, ignored);
         }
     }
     // Fetch and normalise gitattributes data of all subfolders and save to metadata
@@ -137,16 +141,16 @@ async function analyse(rawInput, opts = {}) {
         const vendorTrueGlobs = [...getFlaggedGlobs('vendored', true), ...getFlaggedGlobs('generated', true), ...getFlaggedGlobs('documentation', true)];
         const vendorFalseGlobs = [...getFlaggedGlobs('vendored', false), ...getFlaggedGlobs('generated', false), ...getFlaggedGlobs('documentation', false)];
         // Set up glob ignore object to use for expanding globs to match files
-        const vendorOverrides = (0, ignore_1.default)().add(vendorFalseGlobs);
+        const vendorTrueIgnore = (0, ignore_1.default)().add(vendorTrueGlobs);
+        const vendorFalseIgnore = (0, ignore_1.default)().add(vendorFalseGlobs);
         // Remove all files marked as vendored by default
         const excludedFiles = files.filter(file => vendorPaths.some(pathPtn => RegExp(pathPtn, 'i').test(relPath(file))));
         files = files.filter(file => !excludedFiles.includes(file));
         // Re-add removed files that are overridden manually in gitattributes
-        const overriddenExcludedFiles = excludedFiles.filter(file => vendorOverrides.ignores(relPath(file)));
+        const overriddenExcludedFiles = excludedFiles.filter(file => vendorFalseIgnore.ignores(relPath(file)));
         files.push(...overriddenExcludedFiles);
         // Remove files explicitly marked as vendored in gitattributes
-        // TODO change globs.includes(file) to parse the glob using ignore()
-        files = files.filter(file => !vendorTrueGlobs.includes(relPath(file)));
+        files = files.filter(file => !vendorTrueIgnore.ignores(relPath(file)));
     }
     // Filter out binary files
     if (!opts.keepBinary) {
@@ -157,7 +161,6 @@ async function analyse(rawInput, opts = {}) {
         files = filterOutIgnored(files, binaryIgnored);
         // Re-add files manually marked not as binary
         const binaryUnignored = (0, ignore_1.default)().add(getFlaggedGlobs('binary', false));
-        // TODO parse the globs using ignore()
         const unignoredList = filterOutIgnored(files, binaryUnignored);
         files.push(...unignoredList);
     }

package/ext/generated.rb

@@ -22,3 +22,4 @@
 - ppport\.h$
 - __generated__\/
 - _tlb\.pas$
+- (?:^|\/)htmlcov\/

package/ext/heuristics.yml

@@ -45,7 +45,7 @@ disambiguations:
   - language: Public Key
     pattern: '^(----[- ]BEGIN|ssh-(rsa|dss)) '
   - language: AsciiDoc
-    pattern: '^[=-]+(\s|\n)|\{\{[A-Za-z]'
+    pattern: '^[=-]+\s|\{\{[A-Za-z]'
   - language: AGS Script
     pattern: '^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])'
 - extensions: ['.asm']
@@ -101,6 +101,8 @@ disambiguations:
     - (?i:^\s*(end\ssub)$)
     - (?i:^\s*(?=^function\s)(?:function\s*\w+\(.*?\)\s*as\s*\w*)|(?::\s*function\(.*?\)\s*as\s*\w*)$)
     - (?i:^\s*(end\sfunction)$)
+  - language: Bluespec BH
+    pattern: '^package\s+[A-Za-z_][A-Za-z0-9_'']*(?:\s*\(|\s+where)'
 - extensions: ['.builds']
   rules:
   - language: XML
@@ -132,7 +134,7 @@ disambiguations:
 - extensions: ['.cmp']
   rules:
   - language: Gerber Image
-    pattern: '^[DGMT][0-9]{2}\*\r?\n'
+    pattern: '^[DGMT][0-9]{2}\*(?:\r?\n|\r)'
 - extensions: ['.cs']
   rules:
   - language: Smalltalk
@@ -242,7 +244,7 @@ disambiguations:
 - extensions: ['.ftl']
   rules:
   - language: FreeMarker
-    pattern: '^(?:<|[a-zA-Z-][a-zA-Z0-9_-]+[ \t]+\w)|\$\{\w+[^\n]*?\}|^[ \t]*(?:<#--.*?-->|<#([a-z]+)(?=\s|>)[^>]*>.*?</#\1>|\[#--.*?--\]|\[#([a-z]+)(?=\s|\])[^\]]*\].*?\[#\2\])'
+    pattern: '^(?:<|[a-zA-Z-][a-zA-Z0-9_-]+[ \t]+\w)|\$\{\w+[^\r\n]*?\}|^[ \t]*(?:<#--.*?-->|<#([a-z]+)(?=\s|>)[^>]*>.*?</#\1>|\[#--.*?--\]|\[#([a-z]+)(?=\s|\])[^\]]*\].*?\[#\2\])'
   - language: Fluent
     pattern: '^-?[a-zA-Z][a-zA-Z0-9_-]* *=|\{\$-?[a-zA-Z][-\w]*(?:\.[a-zA-Z][-\w]*)?\}'
 - extensions: ['.g']
@@ -250,7 +252,7 @@ disambiguations:
   - language: GAP
     pattern: '\s*(Declare|BindGlobal|KeyDependentOperation|Install(Method|GlobalFunction)|SetPackageInfo)'
   - language: G-code
-    pattern: '^[MG][0-9]+\n'
+    pattern: '^[MG][0-9]+(?:\r?\n|\r)'
 - extensions: ['.gd']
   rules:
   - language: GAP
@@ -350,6 +352,12 @@ disambiguations:
     pattern: '^\.[A-Za-z]{2}(\s|$)'
   - language: PicoLisp
     pattern: '^\((de|class|rel|code|data|must)\s'
+- extensions: ['.lean']
+  rules:
+  - language: Lean
+    pattern: '^import [a-z]'
+  - language: Lean 4
+    pattern: '^import [A-Z]'
 - extensions: ['.ls']
   rules:
   - language: LoomScript
@@ -395,7 +403,7 @@ disambiguations:
   - language: Win32 Message File
     pattern: '(?i)^[ \t]*(?>\/\*\s*)?MessageId=|^\.$'
   - language: M4
-    pattern: '^dnl|^divert\((?:-?\d+)?\)|^\w+\(`[^\n]*?''[),]'
+    pattern: '^dnl|^divert\((?:-?\d+)?\)|^\w+\(`[^\r\n]*?''[),]'
   - language: Monkey C
     pattern: '\b(?:using|module|function|class|var)\s+\w'
 - extensions: ['.md']
@@ -440,7 +448,7 @@ disambiguations:
   - language: XML
     pattern: '^\s*<\?xml\s+version'
   - language: Gerber Image
-    pattern: '^[DGMT][0-9]{2}\*\r?\n'
+    pattern: '^[DGMT][0-9]{2}\*(?:\r?\n|\r)'
   - language: Text
     pattern: 'THE_TITLE'
 - extensions: ['.nl']
@@ -504,7 +512,7 @@ disambiguations:
 - extensions: ['.pod']
   rules:
   - language: Pod 6
-    pattern: '^[\s&&[^\n]]*=(comment|begin pod|begin para|item\d+)'
+    pattern: '^[\s&&[^\r\n]]*=(comment|begin pod|begin para|item\d+)'
   - language: Pod
 - extensions: ['.pp']
   rules:
@@ -543,7 +551,7 @@ disambiguations:
 - extensions: ['.q']
   rules:
   - language: q
-    pattern: '((?i:[A-Z.][\w.]*:\{)|(^|\n)\\(cd?|d|l|p|ts?) )'
+    pattern: '((?i:[A-Z.][\w.]*:\{)|^\\(cd?|d|l|p|ts?) )'
   - language: HiveQL
     pattern: '(?i:SELECT\s+[\w*,]+\s+FROM|(CREATE|ALTER|DROP)\s(DATABASE|SCHEMA|TABLE))'
 - extensions: ['.qs']
@@ -556,6 +564,8 @@ disambiguations:
   rules:
   - language: Rebol
     pattern: '(?i:\bRebol\b)'
+  - language: Rez
+    pattern: '(#include\s+["<](Types\.r|Carbon\/Carbon\.r)[">])|((resource|data|type)\s+''[A-Za-z0-9]{4}''\s+((\(.*\)\s+){0,1}){)'
   - language: R
     pattern: '<-|^\s*#'
 - extensions: ['.re']
@@ -615,7 +625,7 @@ disambiguations:
   - language: Solidity
     pattern: '\bpragma\s+solidity\b|\b(?:abstract\s+)?contract\s+(?!\d)[a-zA-Z0-9$_]+(?:\s+is\s+(?:[a-zA-Z0-9$_][^\{]*?)?)?\s*\{'
   - language: Gerber Image
-    pattern: '^[DGMT][0-9]{2}\*\r?\n'
+    pattern: '^[DGMT][0-9]{2}\*(?:\r?\n|\r)'
 - extensions: ['.sql']
   rules:
   - language: PLpgSQL
@@ -645,7 +655,7 @@ disambiguations:
 - extensions: ['.stl']
   rules:
   - language: STL
-    pattern: '\A\s*solid(?=$|\s)(?:.|[\r\n])*?^endsolid(?:$|\s)'
+    pattern: '\A\s*solid(?:$|\s)[\s\S]*^endsolid(?:$|\s)'
 - extensions: ['.sw']
   rules:
   - language: Sway
@@ -699,10 +709,15 @@ disambiguations:
   - language: Adblock Filter List
     pattern: '\A\[(?<version>(?:[Aa]d[Bb]lock(?:[ \t][Pp]lus)?|u[Bb]lock(?:[ \t][Oo]rigin)?|[Aa]d[Gg]uard)(?:[ \t] \d+(?:\.\d+)*+)?)(?:[ \t]?;[ \t]?\g<version>)*+\]'
   - language: Text
+- extensions: ['.typ']
+  rules:
+  - language: Typst
+    pattern: '^#(import|show|let|set)'
+  - language: XML
 - extensions: ['.url']
   rules:
   - language: INI
-    pattern: '^\[InternetShortcut\](?:\r?\n|\r)(?>[^\s\[][^\n]*(?:\r?\n|\r))*URL='
+    pattern: '^\[InternetShortcut\](?:\r?\n|\r)(?>[^\s\[][^\r\n]*(?:\r?\n|\r))*URL='
 - extensions: ['.v']
   rules:
   - language: Coq

package/ext/languages.yml

@@ -606,9 +606,28 @@ Bluespec:
   color: "#12223c"
   extensions:
   - ".bsv"
+  aliases:
+  - bluespec bsv
+  - bsv
   tm_scope: source.bsv
   ace_mode: verilog
+  codemirror_mode: verilog
+  codemirror_mime_type: text/x-systemverilog
   language_id: 36
+Bluespec BH:
+  type: programming
+  group: Bluespec
+  color: "#12223c"
+  extensions:
+  - ".bs"
+  aliases:
+  - bh
+  - bluespec classic
+  tm_scope: source.haskell
+  ace_mode: haskell
+  codemirror_mode: haskell
+  codemirror_mime_type: text/x-haskell
+  language_id: 641580358
 Boo:
   type: programming
   color: "#d4bec1"
@@ -1607,7 +1626,7 @@ ECL:
 ECLiPSe:
   type: programming
   color: "#001d9d"
-  group: prolog
+  group: Prolog
   extensions:
   - ".ecl"
   tm_scope: source.prolog.eclipse
@@ -1689,6 +1708,17 @@ Ecmarkup:
   aliases:
   - ecmarkdown
   language_id: 844766630
+EdgeQL:
+  type: programming
+  color: "#31A7FF"
+  aliases:
+  - esdl
+  extensions:
+  - ".edgeql"
+  - ".esdl"
+  ace_mode: text
+  tm_scope: source.edgeql
+  language_id: 925235833
 EditorConfig:
   type: data
   color: "#fff1f2"
@@ -2144,6 +2174,7 @@ GLSL:
   - ".tese"
   - ".vert"
   - ".vrx"
+  - ".vs"
   - ".vsh"
   - ".vshader"
   tm_scope: source.glsl
@@ -2205,20 +2236,20 @@ Gemini:
   wrap: true
   tm_scope: source.gemini
   language_id: 310828396
-Genero:
+Genero 4gl:
   type: programming
   color: "#63408e"
   extensions:
   - ".4gl"
-  tm_scope: source.genero
+  tm_scope: source.genero-4gl
   ace_mode: text
   language_id: 986054050
-Genero Forms:
+Genero per:
   type: markup
   color: "#d8df39"
   extensions:
   - ".per"
-  tm_scope: source.genero-forms
+  tm_scope: source.genero-per
   ace_mode: text
   language_id: 902995658
 Genie:
@@ -2316,7 +2347,6 @@ Gherkin:
 Git Attributes:
   type: data
   color: "#F44D27"
-  group: INI
   aliases:
   - gitattributes
   filenames:
@@ -2361,6 +2391,15 @@ Gleam:
   - ".gleam"
   tm_scope: source.gleam
   language_id: 1054258749
+Glimmer JS:
+  type: programming
+  extensions:
+  - ".gjs"
+  ace_mode: javascript
+  color: "#F5835F"
+  tm_scope: source.gjs
+  group: JavaScript
+  language_id: 5523150
 Glyph:
   type: programming
   color: "#c1ac7f"
@@ -2489,6 +2528,15 @@ Gradle:
   tm_scope: source.groovy.gradle
   ace_mode: text
   language_id: 136
+Gradle Kotlin DSL:
+  group: Gradle
+  type: data
+  color: "#02303a"
+  extensions:
+  - ".gradle.kts"
+  ace_mode: text
+  tm_scope: source.kotlin
+  language_id: 432600901
 Grammatical Framework:
   type: programming
   aliases:
@@ -2603,8 +2651,8 @@ HOCON:
   extensions:
   - ".hocon"
   filenames:
-  - .scalafix.conf
-  - .scalafmt.conf
+  - ".scalafix.conf"
+  - ".scalafmt.conf"
   tm_scope: source.hocon
   ace_mode: text
   language_id: 679725279
@@ -2812,6 +2860,8 @@ Hosts File:
   filenames:
   - HOSTS
   - hosts
+  aliases:
+  - hosts
   tm_scope: source.hosts
   ace_mode: text
   language_id: 231021894
@@ -2910,7 +2960,6 @@ Idris:
 Ignore List:
   type: data
   color: "#000000"
-  group: INI
   aliases:
   - ignore
   - gitignore
@@ -3104,6 +3153,7 @@ JSON:
   - ".watchmanconfig"
   - Pipfile.lock
   - composer.lock
+  - deno.lock
   - flake.lock
   - mcmod.info
   language_id: 174
@@ -3120,6 +3170,7 @@ JSON with Comments:
   extensions:
   - ".jsonc"
   - ".code-snippets"
+  - ".code-workspace"
   - ".sublime-build"
   - ".sublime-commands"
   - ".sublime-completions"
@@ -3623,6 +3674,14 @@ Lean:
   tm_scope: source.lean
   ace_mode: text
   language_id: 197
+Lean 4:
+  type: programming
+  group: Lean
+  extensions:
+  - ".lean"
+  tm_scope: source.lean4
+  ace_mode: text
+  language_id: 455147478
 Less:
   type: markup
   color: "#1d365d"
@@ -4374,7 +4433,7 @@ Nasal:
   extensions:
   - ".nas"
   tm_scope: source.nasal
-  ace_mode: text
+  ace_mode: nasal
   language_id: 178322513
 Nearley:
   type: programming
@@ -4823,6 +4882,8 @@ Option List:
   - ackrc
   filenames:
   - ".ackrc"
+  - ".rspec"
+  - ".yardopts"
   - ackrc
   - mocha.opts
   tm_scope: source.opts
@@ -5101,6 +5162,8 @@ Pic:
   extensions:
   - ".pic"
   - ".chem"
+  aliases:
+  - pikchr
   ace_mode: text
   codemirror_mode: troff
   codemirror_mime_type: text/troff
@@ -5257,6 +5320,14 @@ PowerShell:
   interpreters:
   - pwsh
   language_id: 293
+Praat:
+  type: programming
+  color: "#c8506d"
+  tm_scope: source.praat
+  ace_mode: praat
+  extensions:
+  - ".praat"
+  language_id: 106029007
 Prisma:
   type: data
   color: "#0c344b"
@@ -5851,6 +5922,14 @@ RenderScript:
   tm_scope: none
   ace_mode: text
   language_id: 323
+Rez:
+  type: programming
+  extensions:
+  - ".r"
+  tm_scope: source.rez
+  ace_mode: text
+  color: "#FFDAB3"
+  language_id: 498022874
 Rich Text Format:
   type: markup
   extensions:
@@ -6740,7 +6819,7 @@ Svelte:
   language_id: 928734530
 Sway:
   type: programming
-  color: "#dea584"
+  color: "#00F58C"
   extensions:
   - ".sw"
   tm_scope: source.sway
@@ -6748,6 +6827,14 @@ Sway:
   codemirror_mode: rust
   codemirror_mime_type: text/x-rustsrc
   language_id: 271471144
+Sweave:
+  type: prose
+  color: "#198ce7"
+  extensions:
+  - ".rnw"
+  tm_scope: text.tex.latex.sweave
+  ace_mode: tex
+  language_id: 558779190
 Swift:
   type: programming
   color: "#F05138"
@@ -6784,7 +6871,7 @@ TI Program:
 TL-Verilog:
   type: programming
   extensions:
-    - ".tlv"
+  - ".tlv"
   tm_scope: source.tlverilog
   ace_mode: verilog
   color: "#C40023"
@@ -6939,6 +7026,17 @@ Terra:
   interpreters:
   - lua
   language_id: 371
+Terraform Template:
+  type: markup
+  extensions:
+  - ".tftpl"
+  color: "#7b42bb"
+  tm_scope: source.hcl.terraform
+  ace_mode: ruby
+  codemirror_mode: ruby
+  codemirror_mime_type: text/x-ruby
+  group: HCL
+  language_id: 856832701
 Texinfo:
   type: prose
   wrap: true
@@ -7023,6 +7121,14 @@ Thrift:
   - ".thrift"
   ace_mode: text
   language_id: 374
+Toit:
+  type: programming
+  color: "#c2c9fb"
+  extensions:
+  - ".toit"
+  tm_scope: source.toit
+  ace_mode: text
+  language_id: 356554395
 Turing:
   type: programming
   color: "#cf142b"
@@ -7077,6 +7183,16 @@ TypeScript:
   codemirror_mode: javascript
   codemirror_mime_type: application/typescript
   language_id: 378
+Typst:
+  type: programming
+  color: "#239dad"
+  aliases:
+  - typ
+  extensions:
+  - ".typ"
+  tm_scope: source.typst
+  ace_mode: text
+  language_id: 704730682
 Unified Parallel C:
   type: programming
   color: "#4e3617"
@@ -7687,6 +7803,7 @@ XML:
   - ".tml"
   - ".ts"
   - ".tsx"
+  - ".typ"
   - ".ui"
   - ".urdf"
   - ".ux"

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "linguist-js",
-	"version": "2.7.0-pre",
+	"version": "2.7.0",
 	"description": "Analyse languages used in a folder. Powered by GitHub Linguist, although it doesn't need to be installed.",
 	"main": "dist/index.js",
 	"bin": {

package/readme.md

@@ -95,9 +95,17 @@ Running LinguistJS on this folder will return the following JSON:
 
 ```js
 const linguist = require('linguist-js');
-let folder = './src';
-let options = { keepVendored: false, quick: false };
-const { files, languages, unknown } = linguist(folder, options);
+
+// Analyse folder on disc
+const folder = './src';
+const options = { keepVendored: false, quick: false };
+const { files, languages, unknown } = await linguist(folder, options);
+
+// Analyse file content from raw input
+const fileNames = ['file1.ts', 'file2.ts', 'ignoreme.js'];
+const fileContent = ['#!/usr/bin/env node', 'console.log("Example");', '"ignored"'];
+const options = { ignoredFiles: ['ignore*'] };
+const { files, languages, unknown } = await linguist(fileNames, { fileContent, ...options });
 ```
 
 - `linguist(entry?, opts?)` (default export):
@@ -122,7 +130,7 @@ const { files, languages, unknown } = linguist(folder, options);
       Alias for `checkAttributes:false, checkIgnored:false, checkHeuristics:false, checkShebang:false, checkModeline:false`.
     - `offline` (boolean):
       Whether to use pre-packaged metadata files instead of fetching them from GitHub at runtime (defaults to `false`).
-  - `keepVendored` (boolean):
+    - `keepVendored` (boolean):
       Whether to keep vendored files (dependencies, etc) (defaults to `false`).
       Does nothing when `fileContent` is set.
     - `keepBinary` (boolean):