github-linguist 7.11.1 → 7.14.0

data/grammars/text.xml.xsl.json

@@ -1,88 +1 @@
-{
-  "name": "XSL",
-  "scopeName": "text.xml.xsl",
-  "patterns": [
-    {
-      "name": "meta.tag.xml.template",
-      "begin": "(\u003c)(xsl)((:))(template)",
-      "end": "(\u003e)",
-      "patterns": [
-        {
-          "match": " (?:([-_a-zA-Z0-9]+)((:)))?([a-zA-Z-]+)",
-          "captures": {
-            "1": {
-              "name": "entity.other.attribute-name.namespace.xml"
-            },
-            "2": {
-              "name": "entity.other.attribute-name.xml"
-            },
-            "3": {
-              "name": "punctuation.separator.namespace.xml"
-            },
-            "4": {
-              "name": "entity.other.attribute-name.localname.xml"
-            }
-          }
-        },
-        {
-          "include": "#doublequotedString"
-        },
-        {
-          "include": "#singlequotedString"
-        }
-      ],
-      "captures": {
-        "1": {
-          "name": "punctuation.definition.tag.xml"
-        },
-        "2": {
-          "name": "entity.name.tag.namespace.xml"
-        },
-        "3": {
-          "name": "entity.name.tag.xml"
-        },
-        "4": {
-          "name": "punctuation.separator.namespace.xml"
-        },
-        "5": {
-          "name": "entity.name.tag.localname.xml"
-        }
-      }
-    },
-    {
-      "include": "text.xml"
-    }
-  ],
-  "repository": {
-    "doublequotedString": {
-      "name": "string.quoted.double.xml",
-      "begin": "\"",
-      "end": "\"",
-      "beginCaptures": {
-        "0": {
-          "name": "punctuation.definition.string.begin.xml"
-        }
-      },
-      "endCaptures": {
-        "0": {
-          "name": "punctuation.definition.string.end.xml"
-        }
-      }
-    },
-    "singlequotedString": {
-      "name": "string.quoted.single.xml",
-      "begin": "'",
-      "end": "'",
-      "beginCaptures": {
-        "0": {
-          "name": "punctuation.definition.string.begin.xml"
-        }
-      },
-      "endCaptures": {
-        "0": {
-          "name": "punctuation.definition.string.end.xml"
-        }
-      }
-    }
-  }
-}
+{"name":"XSL","scopeName":"text.xml.xsl","patterns":[{"name":"meta.tag.xml.template","begin":"(\u003c)(xsl)((:))(template)","end":"(\u003e)","patterns":[{"match":" (?:([-_a-zA-Z0-9]+)((:)))?([a-zA-Z-]+)","captures":{"1":{"name":"entity.other.attribute-name.namespace.xml"},"2":{"name":"entity.other.attribute-name.xml"},"3":{"name":"punctuation.separator.namespace.xml"},"4":{"name":"entity.other.attribute-name.localname.xml"}}},{"include":"#doublequotedString"},{"include":"#singlequotedString"}],"captures":{"1":{"name":"punctuation.definition.tag.xml"},"2":{"name":"entity.name.tag.namespace.xml"},"3":{"name":"entity.name.tag.xml"},"4":{"name":"punctuation.separator.namespace.xml"},"5":{"name":"entity.name.tag.localname.xml"}}},{"include":"text.xml"}],"repository":{"doublequotedString":{"name":"string.quoted.double.xml","begin":"\"","end":"\"","beginCaptures":{"0":{"name":"punctuation.definition.string.begin.xml"}},"endCaptures":{"0":{"name":"punctuation.definition.string.end.xml"}}},"singlequotedString":{"name":"string.quoted.single.xml","begin":"'","end":"'","beginCaptures":{"0":{"name":"punctuation.definition.string.begin.xml"}},"endCaptures":{"0":{"name":"punctuation.definition.string.end.xml"}}}}}

data/grammars/text.zone_file.json

@@ -1,67 +1 @@
-{
-  "name": "Bind Zone Files",
-  "scopeName": "text.zone_file",
-  "patterns": [
-    {
-      "name": "comment.line.semicolon.zone_file",
-      "match": ";.*"
-    },
-    {
-      "name": "keyword.directive.zone_file",
-      "match": "@"
-    },
-    {
-      "name": "keyword.directive.zone_file",
-      "match": "\\$(ORIGIN|origin|TTL|ttl|INCLUDE|include)\\s*([^;]*)(;.*)?",
-      "captures": {
-        "2": {
-          "name": "variable.other.directive.zone_file"
-        },
-        "3": {
-          "name": "comment.line.semicolon.zone_file"
-        }
-      }
-    },
-    {
-      "name": "variable.other.timeunit.zone_file",
-      "match": "\\d+(H|h|D|d|W|w|M|m|Y|y)"
-    },
-    {
-      "name": "string.quoted.single.address.zone_file",
-      "begin": "([A-Za-z0-9_.-]*)\\s+(?:([0-9A-Za-z]*)\\s+)?([I|i][N|n]\\s+[A-Za-z0-9]+)\\s+(.*)\\(",
-      "end": "\\)",
-      "patterns": [
-        {
-          "name": "comment.line.semicolon.zone_file",
-          "match": ";.*"
-        }
-      ],
-      "beginCaptures": {
-        "2": {
-          "name": "variable.other.timeunit.zone_file"
-        },
-        "3": {
-          "name": "keyword.resourcetype.zone_file"
-        },
-        "4": {
-          "name": "string.quoted.single.resource.address.zone_file"
-        }
-      }
-    },
-    {
-      "name": "string.quoted.single.address.zone_file",
-      "match": "([A-Za-z0-9_.-]*)\\s+(?:([0-9A-Za-z]*)\\s+)?([I|i][N|n]\\s+[A-Za-z0-9]+)\\s+(.*)",
-      "captures": {
-        "2": {
-          "name": "variable.other.timeunit.zone_file"
-        },
-        "3": {
-          "name": "keyword.resourcetype.zone_file"
-        },
-        "4": {
-          "name": "string.quoted.single.resource.address.zone_file"
-        }
-      }
-    }
-  ]
-}
+{"name":"Bind Zone Files","scopeName":"text.zone_file","patterns":[{"name":"comment.line.semicolon.zone_file","match":";.*"},{"name":"keyword.directive.zone_file","match":"@"},{"name":"keyword.directive.zone_file","match":"\\$(ORIGIN|origin|TTL|ttl|INCLUDE|include)\\s*([^;]*)(;.*)?","captures":{"2":{"name":"variable.other.directive.zone_file"},"3":{"name":"comment.line.semicolon.zone_file"}}},{"name":"variable.other.timeunit.zone_file","match":"\\d+(H|h|D|d|W|w|M|m|Y|y)"},{"name":"string.quoted.single.address.zone_file","begin":"([A-Za-z0-9_.-]*)\\s+(?:([0-9A-Za-z]*)\\s+)?([I|i][N|n]\\s+[A-Za-z0-9]+)\\s+(.*)\\(","end":"\\)","patterns":[{"name":"comment.line.semicolon.zone_file","match":";.*"}],"beginCaptures":{"2":{"name":"variable.other.timeunit.zone_file"},"3":{"name":"keyword.resourcetype.zone_file"},"4":{"name":"string.quoted.single.resource.address.zone_file"}}},{"name":"string.quoted.single.address.zone_file","match":"([A-Za-z0-9_.-]*)\\s+(?:([0-9A-Za-z]*)\\s+)?([I|i][N|n]\\s+[A-Za-z0-9]+)\\s+(.*)","captures":{"2":{"name":"variable.other.timeunit.zone_file"},"3":{"name":"keyword.resourcetype.zone_file"},"4":{"name":"string.quoted.single.resource.address.zone_file"}}}]}

data/grammars/version

@@ -1 +1 @@
-7.11.1
+7.14.0

data/lib/linguist/VERSION

@@ -1 +1 @@
-7.11.1
+7.14.0

data/lib/linguist/classifier.rb

@@ -39,7 +39,11 @@ module Linguist
     # Set LINGUIST_DEBUG=1 or =2 to see probabilities per-token or
     # per-language.  See also #dump_all_tokens, below.
     def self.train!(db, language, data)
-      tokens = Tokenizer.tokenize(data)
+      tokens = data
+      tokens = Tokenizer.tokenize(tokens) if tokens.is_a?(String)
+
+      counts = Hash.new(0)
+      tokens.each { |tok| counts[tok] += 1 }
 
       db['tokens_total'] ||= 0
       db['languages_total'] ||= 0
@@ -47,13 +51,13 @@ module Linguist
       db['language_tokens'] ||= {}
       db['languages'] ||= {}
 
-      tokens.each do |token|
+      counts.each do |token, count|
         db['tokens'][language] ||= {}
         db['tokens'][language][token] ||= 0
-        db['tokens'][language][token] += 1
+        db['tokens'][language][token] += count
         db['language_tokens'][language] ||= 0
-        db['language_tokens'][language] += 1
-        db['tokens_total'] += 1
+        db['language_tokens'][language] += count
+        db['tokens_total'] += count
       end
       db['languages'][language] ||= 0
       db['languages'][language] += 1

data/lib/linguist/generated.rb

@@ -83,6 +83,7 @@ module Linguist
       generated_apache_thrift? ||
       generated_jni_header? ||
       vcr_cassette? ||
+      generated_antlr? ||
       generated_module? ||
       generated_unity3d_meta? ||
       generated_racc? ||
@@ -442,6 +443,15 @@ module Linguist
       return lines[-2].include?("recorded_with: VCR")
     end
 
+    # Is this a generated ANTLR file?
+    #
+    # Returns true or false
+    def generated_antlr?
+      return false unless extname == '.g'
+      return false unless lines.count > 2
+      return lines[1].include?("generated by Xtest")
+    end
+
     # Internal: Is this a compiled C/C++ file from Cython?
     #
     # Cython-compiled C/C++ files typically contain:

data/lib/linguist/generic.yml

@@ -0,0 +1,18 @@
+# This is a list of file extensions considered too common to safely associate
+# with only the formats Linguist recognises. In addition to the stipulated
+# extension requirement, candidate matches must also satisfy a heuristic
+# (or some other strategy) in order to be identified as any one language.
+
+---
+extensions:
+- ".1"
+- ".2"
+- ".3"
+- ".4"
+- ".5"
+- ".6"
+- ".7"
+- ".8"
+- ".9"
+- ".cmp"
+- ".sol"

data/lib/linguist/heuristics.yml

@@ -28,22 +28,30 @@
 #
 ---
 disambiguations:
-- extensions: ['.1', '.1in', '.1m', '.1x', '.2', '.3', '.3in', '.3m', '.3p', '.3pm', '.3qt', '.3x', '.4', '.5', '.6', '.7', '.8', '.9', '.man', '.mdoc']
+- extensions: ['.1', '.2', '.3', '.4', '.5', '.6', '.7', '.8', '.9']
   rules:
-  # Document which uses mdoc(7) macros
   - language: Roff Manpage
     and:
-    - pattern: '^[.''][ \t]*Dd +(?:[^"\s]+|"[^"]+")'
-    - pattern: '^[.''][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)'
-    - pattern: '^[.''][ \t]*Sh +(?:[^"\s]|"[^"]+")'
-
-  # Document which uses man(7) macros
+    - named_pattern: mdoc-date
+    - named_pattern: mdoc-title
+    - named_pattern: mdoc-heading
   - language: Roff Manpage
     and:
-    - pattern: '^[.''][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)'
-    - pattern: '^[.''][ \t]*SH +(?:[^"\s]+|"[^"\s]+)'
-
-  # Open-ended fallback to unrendered Roff source
+    - named_pattern: man-title
+    - named_pattern: man-heading
+  - language: Roff
+    pattern: '^\.(?:[A-Za-z]{2}(?:\s|$)|\\")'
+- extensions: ['.1in', '.1m', '.1x', '.3in', '.3m', '.3p', '.3pm', '.3qt', '.3x', '.man', '.mdoc']
+  rules:
+  - language: Roff Manpage
+    and:
+    - named_pattern: mdoc-date
+    - named_pattern: mdoc-title
+    - named_pattern: mdoc-heading
+  - language: Roff Manpage
+    and:
+    - named_pattern: man-title
+    - named_pattern: man-heading
   - language: Roff
 - extensions: ['.al']
   rules:
@@ -56,8 +64,7 @@ disambiguations:
 - extensions: ['.as']
   rules:
   - language: ActionScript
-    pattern: '^\s*(package\s+[a-z0-9_\.]+|import\s+[a-zA-Z0-9_\.]+;|class\s+[A-Za-z0-9_]+\s+extends\s+[A-Za-z0-9_]+)'
-  - language: AngelScript
+    pattern: '^\s*(?:package(?:\s+[\w.]+)?\s+(?:{|$)|import\s+[\w.*]+\s*;|(?=.*?(?:intrinsic|extends))(intrinsic\s+)?class\s+[\w<>.]+(?:\s+extends\s+[\w<>.]+)?|(?:(?:public|protected|private|static)\s+)*(?:(?:var|const|local)\s+\w+\s*:\s*[\w<>.]+(?:\s*=.*)?\s*;|function\s+\w+\s*\((?:\s*\w+\s*:\s*[\w<>.]+\s*(,\s*\w+\s*:\s*[\w<>.]+\s*)*)?\)))'
 - extensions: ['.asc']
   rules:
   - language: Public Key
@@ -75,12 +82,22 @@ disambiguations:
   - language: LTspice Symbol
     pattern: '^SymbolType[ \t]'
   - language: Asymptote
+- extensions: ['.bas']
+  rules:
+  - language: FreeBasic
+    pattern: '^[ \t]*#(?:define|endif|endmacro|ifn?def|if|include|lang|macro)\s'
+  - language: BASIC
+    pattern: '^\s*\d+'
 - extensions: ['.bb']
   rules:
   - language: BlitzBasic
     pattern: '(<^\s*; |End Function)'
   - language: BitBake
     pattern: '^\s*(# |include|require)\b'
+- extensions: ['.bi']
+  rules:
+  - language: FreeBasic
+    pattern: '^[ \t]*#(?:define|endif|endmacro|ifn?def|if|include|lang|macro)\s'
 - extensions: ['.builds']
   rules:
   - language: XML
@@ -100,15 +117,25 @@ disambiguations:
 - extensions: ['.cls']
   rules:
   - language: TeX
-    pattern: '\\\w+{'
+    pattern: '^\s*\\(?:NeedsTeXFormat|ProvidesClass){'
   - language: ObjectScript
     pattern: '^Class\s'
+- extensions: ['.cmp']
+  rules:
+  - language: Gerber Image
+    pattern: '^[DGMT][0-9]{2}\*\r?\n'
 - extensions: ['.cs']
   rules:
   - language: Smalltalk
     pattern: '![\w\s]+methodsFor: '
   - language: 'C#'
     pattern: '^(\s*namespace\s*[\w\.]+\s*{|\s*\/\/)'
+- extensions: ['.csl']
+  rules:
+  - language: XML
+    pattern: '(?i:^\s*(\<\?xml|xmlns))'
+  - language: Kusto
+    pattern: '(^\|\s*(where|extend|project|limit|summarize))|(^\.\w+)'
 - extensions: ['.d']
   rules:
   - language: D
@@ -186,14 +213,16 @@ disambiguations:
   - language: Graph Modeling Language
     pattern: '(?i:^\s*(graph|node)\s+\[$)'
   - language: Gerber Image
-    pattern: '\*\%$'
+    pattern: '^[DGMT][0-9]{2}\*$'
   - language: Game Maker Language
 - extensions: ['.gs']
   rules:
   - language: GLSL
     pattern: '^#version\s+[0-9]+\b'
   - language: Gosu
-    pattern: '^uses java\.'
+    pattern: '^uses (java|gw)\.'
+  - language: Genie
+    pattern: '^\[indent=[0-9]+\]'
 - extensions: ['.h']
   rules:
   - language: Objective-C
@@ -271,6 +300,13 @@ disambiguations:
     pattern: '^\s*%'
   - language: Limbo
     pattern: '^\w+\s*:\s*module\s*{'
+- extensions: ['.m4']
+  rules:
+  - language: M4Sugar
+    pattern:
+    - 'AC_DEFUN|AC_PREREQ|AC_INIT'
+    - '^_?m4_'
+  - language: 'M4'
 - extensions: ['.mask']
   rules:
   - language: Unity3d Asset
@@ -316,6 +352,8 @@ disambiguations:
   rules:
   - language: XML
     pattern: '^\s*<\?xml\s+version'
+  - language: Gerber Image
+    pattern: '^[DGMT][0-9]{2}\*\r?\n'
   - language: Text
     pattern: 'THE_TITLE'
 - extensions: ['.nl']
@@ -431,6 +469,12 @@ disambiguations:
     pattern:
     - '^\s*#(?:(?:if|ifdef|define|pragma)\s+\w|\s*include\s+<[^>]+>)'
     - '^\s*template\s*<'
+- extensions: ['.res']
+  rules:
+  - language: ReScript
+    pattern:
+    - '^\s*(let|module|type)\s+\w*\s+=\s+'
+    - '^\s*(?:include|open)\s+\w+\s*$'
 - extensions: ['.rno']
   rules:
   - language: RUNOFF
@@ -448,6 +492,8 @@ disambiguations:
     pattern: '^(use |fn |mod |pub |macro_rules|impl|#!?\[)'
   - language: RenderScript
     pattern: '#include|#pragma\s+(rs|version)|__attribute__'
+  - language: XML
+    pattern: '^\s*<\?xml'
 - extensions: ['.s']
   rules:
   - language: Motorola 68K Assembly
@@ -463,7 +509,7 @@ disambiguations:
   - language: Solidity
     pattern: '\bpragma\s+solidity\b|\b(?:abstract\s+)?contract\s+(?!\d)[a-zA-Z0-9$_]+(?:\s+is\s+(?:[a-zA-Z0-9$_][^\{]*?)?)?\s*\{'
   - language: Gerber Image
-    pattern: '^G75\*'
+    pattern: '^[DGMT][0-9]{2}\*\r?\n'
 - extensions: ['.sql']
   rules:
    # Postgres
@@ -483,6 +529,12 @@ disambiguations:
   rules:
   - language: SubRip Text
     pattern: '^(\d{2}:\d{2}:\d{2},\d{3})\s*(-->)\s*(\d{2}:\d{2}:\d{2},\d{3})$'
+- extensions: ['.st']
+  rules:
+  - language: StringTemplate
+    pattern: '\$\w+[($]|(.)!\s*.+?\s*!\1|<!\s*.+?\s*!>|\[!\s*.+?\s*!\]|\{!\s*.+?\s*!\}'
+  - language: Smalltalk
+    pattern: '\A\s*[\[{(^"''\w#]|[a-zA-Z_]\w*\s*:=\s*[a-zA-Z_]\w*|class\s*>>\s*[a-zA-Z_]\w*|^[a-zA-Z_]\w*\s+[a-zA-Z_]\w*:|^Class\s*{|if(?:True|False):\s*\['
 - extensions: ['.t']
   rules:
   - language: Perl
@@ -519,16 +571,16 @@ disambiguations:
     # The following RegExp is simply a collapsed and simplified form of the
     # VIM_MODELINE pattern in `./lib/linguist/strategy/modeline.rb`.
   - language: Vim Help File
-    pattern: '(?:(?:\s|^)vi(?:m[<=>]?\d+|m)?|\sex)(?=:(?=\s*set?\s[^\n:]+:)|:(?!\s* set?\s))(?:(?:\s|\s*:\s*)\w*(?:\s*=(?:[^\\\s]|\\.)*)?)*[\s:](?:filetype|ft|syntax)\s*=help(?=\s|:|$)'
+    pattern: '(?:(?:[ \t]|^)vi(?:m[<=>]?\d+|m)?|[ \t]ex)(?=:(?=[ \t]*set?[ \t][^\n:]+:)|:(?![ \t]* set?[ \t]))(?:(?:[ \t]|[ \t]*:[ \t]*)\w*(?:[ \t]*=(?:[^\\[ \t]]|\\.)*)?)*[[ \t]:](?:filetype|ft|syntax)[ \t]*=help(?=[ \t]|:|$)'
     # HACK: This is a contrived use of heuristics needed to address
     # an unusual edge-case. See https://git.io/JULye for discussion.
   - language: Text
 - extensions: ['.v']
   rules:
   - language: Coq
-    pattern: '\(\*.*?\*\)|(?:^|\s)(?:Proof|Qed)\.(?:$|\s)|(?:^|\s)Require[ \t]+Import\s'
+    pattern: '(?:^|\s)(?:Proof|Qed)\.(?:$|\s)|(?:^|\s)Require[ \t]+(Import|Export)\s'
   - language: Verilog
-    pattern: '^[ \t]*module\s+[^\s()]+\s+\#?\(|^[ \t]*`(?:ifdef|timescale)\s|^[ \t]*always[ \t]+@'
+    pattern: '^[ \t]*module\s+[^\s()]+\s+\#?\(|^[ \t]*`(?:define|ifdef|ifndef|include|timescale)|^[ \t]*always[ \t]+@|^[ \t]*initial[ \t]+(begin|@)'
   - language: V
     pattern: '\$(?:if|else)[ \t]|^[ \t]*fn\s+[^\s()]+\(.*?\).*?\{|^[ \t]*for\s*\{'
 - extensions: ['.vba']
@@ -576,6 +628,11 @@ named_patterns:
   - '(?im)^\s*move[mp](?:\.[wl])?\b'
   - '(?im)^\s*btst\b'
   - '(?im)^\s*dbra\b'
+  man-heading:  '^[.''][ \t]*SH +(?:[^"\s]+|"[^"\s]+)'
+  man-title:    '^[.''][ \t]*TH +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)'
+  mdoc-date:    '^[.''][ \t]*Dd +(?:[^"\s]+|"[^"]+")'
+  mdoc-heading: '^[.''][ \t]*Sh +(?:[^"\s]|"[^"]+")'
+  mdoc-title:   '^[.''][ \t]*Dt +(?:[^"\s]+|"[^"]+") +"?(?:[1-9]|@[^\s@]+@)'
   objectivec: '^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])'
   perl5: '\buse\s+(?:strict\b|v?5\.)'
   perl6: '^\s*(?:use\s+v6\b|\bmodule\b|\b(?:my\s+)?class\b)'