github-linguist 5.3.1 → 5.3.2

data/grammars/source.glsl.json

@@ -81,7 +81,7 @@
       "name": "storage.modifier.glsl"
     },
     {
-      "match": "\\b(gl_BackColor|gl_BackLightModelProduct|gl_BackLightProduct|gl_BackMaterial|gl_BackSecondaryColor|gl_ClipDistance|gl_ClipPlane|gl_ClipVertex|gl_Color|gl_DepthRange|gl_DepthRangeParameters|gl_EyePlaneQ|gl_EyePlaneR|gl_EyePlaneS|gl_EyePlaneT|gl_Fog|gl_FogCoord|gl_FogFragCoord|gl_FogParameters|gl_FragColor|gl_FragCoord|gl_FragDat|gl_FragDept|gl_FrontColor|gl_FrontFacing|gl_FrontLightModelProduct|gl_FrontLightProduct|gl_FrontMaterial|gl_FrontSecondaryColor|gl_InstanceID|gl_Layer|gl_LightModel|gl_LightModelParameters|gl_LightModelProducts|gl_LightProducts|gl_LightSource|gl_LightSourceParameters|gl_MaterialParameters|gl_ModelViewMatrix|gl_ModelViewMatrixInverse|gl_ModelViewMatrixInverseTranspose|gl_ModelViewMatrixTranspose|gl_ModelViewProjectionMatrix|gl_ModelViewProjectionMatrixInverse|gl_ModelViewProjectionMatrixInverseTranspose|gl_ModelViewProjectionMatrixTranspose|gl_MultiTexCoord[0-7]|gl_Normal|gl_NormalMatrix|gl_NormalScale|gl_ObjectPlaneQ|gl_ObjectPlaneR|gl_ObjectPlaneS|gl_ObjectPlaneT|gl_Point|gl_PointCoord|gl_PointParameters|gl_PointSize|gl_Position|gl_PrimitiveIDIn|gl_ProjectionMatrix|gl_ProjectionMatrixInverse|gl_ProjectionMatrixInverseTranspose|gl_ProjectionMatrixTranspose|gl_SecondaryColor|gl_TexCoord|gl_TextureEnvColor|gl_TextureMatrix|gl_TextureMatrixInverse|gl_TextureMatrixInverseTranspose|gl_TextureMatrixTranspose|gl_Vertex|gl_VertexID)\\b",
+      "match": "\\b(gl_BackColor|gl_BackLightModelProduct|gl_BackLightProduct|gl_BackMaterial|gl_BackSecondaryColor|gl_ClipDistance|gl_ClipPlane|gl_ClipVertex|gl_Color|gl_DepthRange|gl_DepthRangeParameters|gl_EyePlaneQ|gl_EyePlaneR|gl_EyePlaneS|gl_EyePlaneT|gl_Fog|gl_FogCoord|gl_FogFragCoord|gl_FogParameters|gl_FragColor|gl_FragCoord|gl_FragData|gl_FragDepth|gl_FrontColor|gl_FrontFacing|gl_FrontLightModelProduct|gl_FrontLightProduct|gl_FrontMaterial|gl_FrontSecondaryColor|gl_InstanceID|gl_Layer|gl_LightModel|gl_LightModelParameters|gl_LightModelProducts|gl_LightProducts|gl_LightSource|gl_LightSourceParameters|gl_MaterialParameters|gl_ModelViewMatrix|gl_ModelViewMatrixInverse|gl_ModelViewMatrixInverseTranspose|gl_ModelViewMatrixTranspose|gl_ModelViewProjectionMatrix|gl_ModelViewProjectionMatrixInverse|gl_ModelViewProjectionMatrixInverseTranspose|gl_ModelViewProjectionMatrixTranspose|gl_MultiTexCoord[0-7]|gl_Normal|gl_NormalMatrix|gl_NormalScale|gl_ObjectPlaneQ|gl_ObjectPlaneR|gl_ObjectPlaneS|gl_ObjectPlaneT|gl_Point|gl_PointCoord|gl_PointParameters|gl_PointSize|gl_Position|gl_PrimitiveIDIn|gl_ProjectionMatrix|gl_ProjectionMatrixInverse|gl_ProjectionMatrixInverseTranspose|gl_ProjectionMatrixTranspose|gl_SecondaryColor|gl_TexCoord|gl_TextureEnvColor|gl_TextureMatrix|gl_TextureMatrixInverse|gl_TextureMatrixInverseTranspose|gl_TextureMatrixTranspose|gl_Vertex|gl_VertexID)\\b",
       "name": "support.variable.glsl"
     },
     {

data/grammars/source.js.json

@@ -467,7 +467,7 @@
       ]
     },
     {
-      "begin": "(?x)\n(?=\n  (?!\n    (break|case|catch|continue|do|else|finally|for|function|if|\n    package|return|switch|throw|try|while|with)\n    [\\s\\(]\n  )\n  (\n    \\b(get|set)            # Property getter/setter: get foo(){}\n    (?:\\s+|(?=\\[))        # Followed by whitespace or square bracket\n  )?+\n  (                         # Method name\n    \\b[a-zA-Z_$][\\w$]*    # Fixed name\n    |\n    \\[                     # Computed property key\n      [^\\[\\]]++           # Contains at least one non-brace character\n    \\]\n  )\n  \\s*\\(\\s*               # Start of arguments list\n    (\n      \"[^\"]*\"  |            # Double-quoted string\n      '[^']*'  |            # Single-quoted string\n       [^\"()']              # Any non-bracket or non-quote\n    )*\n  \\)\\s*                   # End of arguments\n  {                         # Beginning of body\n)",
+      "begin": "(?x)\n(?=\n  (?!\n    (break|case|catch|continue|do|else|finally|for|function|if|\n    return|switch|throw|try|while|with)\n    [\\s\\(]\n  )\n  (\n    \\b(get|set)            # Property getter/setter: get foo(){}\n    (?:\\s+|(?=\\[))        # Followed by whitespace or square bracket\n  )?+\n  (                         # Method name\n    \\b[a-zA-Z_$][\\w$]*    # Fixed name\n    |\n    \\[                     # Computed property key\n      [^\\[\\]]++           # Contains at least one non-brace character\n    \\]\n  )\n  \\s*\\(\\s*               # Start of arguments list\n    (\n      \"[^\"]*\"  |            # Double-quoted string\n      '[^']*'  |            # Single-quoted string\n       [^\"()']              # Any non-bracket or non-quote\n    )*\n  \\)\\s*                   # End of arguments\n  {                         # Beginning of body\n)",
       "end": "(?<=})",
       "patterns": [
         {
@@ -899,7 +899,11 @@
       "name": "meta.control.yield.js"
     },
     {
-      "match": "(?<!\\.)\\b(await|break|catch|continue|do|else|finally|for|if|import|package|return|throw|try|while|with)(?!\\s*:)\\b",
+      "match": "(?:(?<=\\.{3})|(?<!\\.))\\b(await)(?!\\s*:)\\b",
+      "name": "keyword.control.js"
+    },
+    {
+      "match": "(?<!\\.)\\b(break|catch|continue|do|else|finally|for|if|import|package|return|throw|try|while|with)(?!\\s*:)\\b",
       "name": "keyword.control.js"
     },
     {

data/grammars/source.meson.json

@@ -60,7 +60,7 @@
       "name": "keyword.operator.arithmetic.meson"
     },
     {
-      "match": "(?x)\\b(add_global_arguments|add_global_link_arguments|add_languages|add_project_arguments|add_project_link_arguments|add_test_setup|benchmark|build_target|configuration_data|configure_file|custom_target|declare_dependency|dependency|error|environment|executable|find_program|find_library|files|generator|get_option|get_variable|import|include_directories|install_data|install_headers|install_man|install_subdir|is_variable|jar|join_paths|library|message|option|project|run_command|run_target|set_variable|shared_library|shared_module|static_library|subdir|subproject|test|vcs_tag\n)\\b\\s*(?=\\()",
+      "match": "(?x)\\b(add_global_arguments|add_project_arguments|add_global_link_arguments|add_project_link_arguments|add_test_setup|add_languages|assert|benchmark|build_target|configuration_data|configure_file|custom_target|declare_dependency|dependency|environment|error|executable|generator|gettext|get_option|get_variable|files|find_library|find_program|include_directories|import|install_data|install_headers|install_man|install_subdir|is_variable|jar|join_paths|library|message|option|project|run_target|run_command|set_variable|subdir|subproject|shared_library|shared_module|static_library|test|vcs_tag\n)\\b\\s*(?=\\()",
       "name": "support.function.builtin.meson"
     }
   ],

data/grammars/source.tsx.json

@@ -27,15 +27,12 @@
         },
         {
           "include": "#jsx-tag-in-expression"
-        },
-        {
-          "include": "#jsx-tag-invalid"
         }
       ]
     },
     "jsx-tag-without-attributes-in-expression": {
-      "begin": "(?x)\n  (?<=[({\\[,?=>:*]|&&|\\|\\||\\?|\\Wreturn|^return|\\Wdefault|^)\\s*\n  (?=(<)\\s*((?:[a-z][a-z0-9]*|([_$a-zA-Z][-$\\w.]*))(?<!\\.|-))\\s*(>))",
-      "end": "(?!\\s*(<)\\s*((?:[a-z][a-z0-9]*|([_$a-zA-Z][-$\\w.]*))(?<!\\.|-))\\s*(>))",
+      "begin": "(?x)\n  (?<=[({\\[,?=>:*]|&&|\\|\\||\\?|\\Wreturn|^return|\\Wdefault|^)\\s*\n  (?=(<)\\s*((?:[a-z][a-z0-9]*|([_$a-zA-Z][-$\\w.]*))(?<!\\.|-))?\\s*(>))",
+      "end": "(?!\\s*(<)\\s*((?:[a-z][a-z0-9]*|([_$a-zA-Z][-$\\w.]*))(?<!\\.|-))?\\s*(>))",
       "patterns": [
         {
           "include": "#jsx-tag-without-attributes"
@@ -44,8 +41,8 @@
     },
     "jsx-tag-without-attributes": {
       "name": "meta.tag.without-attributes.tsx",
-      "begin": "(<)\\s*((?:[a-z][a-z0-9]*|([_$a-zA-Z][-$\\w.]*))(?<!\\.|-))\\s*(>)",
-      "end": "(</)\\s*((?:[a-z][a-z0-9]*|([_$a-zA-Z][-$\\w.]*))(?<!\\.|-))\\s*(>)",
+      "begin": "(<)\\s*((?:[a-z][a-z0-9]*|([_$a-zA-Z][-$\\w.]*))(?<!\\.|-))?\\s*(>)",
+      "end": "(</)\\s*((?:[a-z][a-z0-9]*|([_$a-zA-Z][-$\\w.]*))(?<!\\.|-))?\\s*(>)",
       "beginCaptures": {
         "1": {
           "name": "punctuation.definition.tag.begin.tsx"
@@ -188,10 +185,6 @@
         }
       ]
     },
-    "jsx-tag-invalid": {
-      "name": "invalid.illegal.tag.incomplete.tsx",
-      "match": "<\\s*>"
-    },
     "jsx-children": {
       "patterns": [
         {
@@ -200,9 +193,6 @@
         {
           "include": "#jsx-child-tag"
         },
-        {
-          "include": "#jsx-tag-invalid"
-        },
         {
           "include": "#jsx-evaluated-code"
         },

data/grammars/source.wdl.json

@@ -48,7 +48,7 @@
   "repository": {
     "builtin_types": {
       "name": "support.type.wdl",
-      "match": "(?<!\\.)\\b(Array|Boolean|File|Float|Int|Map|Object|String|Uri)\\b"
+      "match": "(?<!\\.)\\b(Array|Boolean|File|Float|Int|Map|Object|String|Pair)\\b"
     },
     "comments": {
       "patterns": [
@@ -129,7 +129,7 @@
       "patterns": [
         {
           "name": "keyword.other.wdl",
-          "match": "\\s*(call|command|output|runtime|task|workflow|if|then|else|import|as|input|output|meta|parameter_meta)\\s+"
+          "match": "\\s*(call|command|output|runtime|task|workflow|if|then|else|import|as|input|output|meta|parameter_meta|scatter)\\s+"
         }
       ]
     },

data/grammars/text.roff.json

@@ -492,6 +492,9 @@
           },
           "contentName": "string.unquoted.roff",
           "patterns": [
+            {
+              "include": "#escapes-clipped"
+            },
             {
               "include": "#escapes"
             }
@@ -751,7 +754,7 @@
     "definition": {
       "patterns": [
         {
-          "name": "meta.function.definition.request.$2.roff",
+          "name": "meta.macro.definition.$2.roff",
           "begin": "^([.'])[ \t]*((dei?1?)|(ami?1?))\\s+(\\S+?)?\\s*(\\\\[\"#].*)?$",
           "end": "^(?:[ \t]*\\x5C{2})?\\.[ \t]*\\.",
           "beginCaptures": {
@@ -794,7 +797,7 @@
           ]
         },
         {
-          "name": "meta.function.definition.request.with-terminator.$2.roff",
+          "name": "meta.macro.definition.with-terminator.$2.roff",
           "begin": "^([.'])[ \t]*((dei?1?)|(ami?1?))\\s+(\\S+)\\s*(\"[^\"]+\"?|\\S+?(?=\\s|\\\\[\"#]))?(.*)$",
           "end": "^(\\.)[ \t]*((\\6)(?=$|\\s|\\\\(?:$|\")))",
           "beginCaptures": {
@@ -1139,14 +1142,7 @@
           }
         },
         {
-          "name": "constant.character.escape.newline.roff",
-          "begin": "(\\\\)$\\n?",
-          "end": "^(?:[.'])?",
-          "beginCaptures": {
-            "1": {
-              "name": "punctuation.definition.escape.roff"
-            }
-          }
+          "include": "#continuous-newline"
         },
         {
           "include": "#register-expansion"
@@ -1578,6 +1574,121 @@
         }
       ]
     },
+    "continuous-newline": {
+      "begin": "(\\\\)?(\\\\)$\\n?",
+      "end": "^(?:[.'])?",
+      "beginCaptures": {
+        "0": {
+          "name": "constant.character.escape.newline.roff"
+        },
+        "1": {
+          "name": "punctuation.definition.concealed.escape.backslash.roff"
+        },
+        "2": {
+          "name": "punctuation.definition.escape.roff"
+        }
+      }
+    },
+    "escapes-clipped": {
+      "patterns": [
+        {
+          "begin": "\\\\f(?:[I2]|\\(CI|\\[\\s*(?:[I2]|CI)\\s*\\])",
+          "end": "$|(?=\\\\f[\\[A-Za-z0-9])",
+          "beginCaptures": {
+            "0": {
+              "patterns": [
+                {
+                  "include": "#escapes"
+                }
+              ]
+            }
+          },
+          "patterns": [
+            {
+              "include": "#escaped-newline"
+            },
+            {
+              "include": "$self"
+            },
+            {
+              "include": "#italic-word"
+            }
+          ]
+        },
+        {
+          "begin": "\\\\f(?:[B3]|\\(CB|\\[\\s*(?:[B3]|CB)\\s*\\])",
+          "end": "$|(?=\\\\f[\\[A-Za-z0-9])",
+          "beginCaptures": {
+            "0": {
+              "patterns": [
+                {
+                  "include": "#escapes"
+                }
+              ]
+            }
+          },
+          "patterns": [
+            {
+              "include": "#escaped-newline"
+            },
+            {
+              "include": "$self"
+            },
+            {
+              "include": "#bold-word"
+            }
+          ]
+        },
+        {
+          "begin": "\\\\f(?:4|\\(BI|\\[\\s*BI\\s*\\])",
+          "end": "$|(?=\\\\f[\\[A-Za-z0-9])",
+          "beginCaptures": {
+            "0": {
+              "patterns": [
+                {
+                  "include": "#escapes"
+                }
+              ]
+            }
+          },
+          "patterns": [
+            {
+              "include": "#escaped-newline"
+            },
+            {
+              "include": "$self"
+            },
+            {
+              "include": "#bold-italic-word"
+            }
+          ]
+        },
+        {
+          "begin": "\\\\f(?:\\(C[WR]|\\[\\s*C[WR]\\s*\\])",
+          "end": "$|(?=\\\\f[\\[A-Za-z0-9])",
+          "beginCaptures": {
+            "0": {
+              "patterns": [
+                {
+                  "include": "#escapes"
+                }
+              ]
+            }
+          },
+          "patterns": [
+            {
+              "include": "#escaped-newline"
+            },
+            {
+              "include": "$self"
+            },
+            {
+              "include": "#monospace-word"
+            }
+          ]
+        }
+      ]
+    },
     "escapes-full": {
       "patterns": [
         {
@@ -2406,6 +2517,34 @@
                 }
               ]
             },
+            {
+              "name": "meta.shell-snippet.mdoc.roff",
+              "begin": "^(?:\\S*.*?\\s+)?(?i:Bash|(?:Bourne[\\s-]?)?Shell(?:[\\s-]?Script)?):\\s*$\\n?",
+              "end": "^(?!\\t|\\s*$)",
+              "beginCaptures": {
+                "0": {
+                  "patterns": [
+                    {
+                      "include": "#main"
+                    }
+                  ]
+                }
+              },
+              "patterns": [
+                {
+                  "match": ".+",
+                  "captures": {
+                    "0": {
+                      "patterns": [
+                        {
+                          "include": "source.shell"
+                        }
+                      ]
+                    }
+                  }
+                }
+              ]
+            },
             {
               "include": "#main"
             }
@@ -3349,7 +3488,7 @@
           ]
         },
         {
-          "begin": "^([.'])[ \t]*(TP)(?=\\s|\\\\[\"#])(.*)?$\\n?",
+          "begin": "^([.'])[ \t]*(TP|TQ)(?=\\s|\\\\[\"#])(.*)?$\\n?",
           "end": "^(.*)(?<!\\\\)$",
           "patterns": [
             {
@@ -3384,42 +3523,17 @@
             }
           },
           "endCaptures": {
-            "1": {
-              "patterns": [
-                {
-                  "include": "$self"
-                }
-              ]
-            }
-          }
-        },
-        {
-          "name": "markup.list.unnumbered.man.macro.gnu.roff",
-          "begin": "^([.'])[ \t]*(TQ)[ \t]*(\\\\[#\"].*)?$",
-          "end": "^(?=[.'][ \t]*TP(?:\\s|\\\\[#\"]))",
-          "beginCaptures": {
             "0": {
-              "name": "meta.function.header-continuation.man.macro.gnu.roff"
+              "name": "markup.heading.paragraph.roff"
             },
             "1": {
-              "name": "punctuation.definition.macro.roff"
-            },
-            "2": {
-              "name": "entity.function.name.gnu.roff"
-            },
-            "3": {
               "patterns": [
                 {
-                  "include": "#escapes"
+                  "include": "$self"
                 }
               ]
             }
-          },
-          "patterns": [
-            {
-              "include": "$self"
-            }
-          ]
+          }
         },
         {
           "name": "meta.deprecated.function.hanging-paragraph.man.macro.roff",
@@ -4827,8 +4941,8 @@
           }
         },
         {
-          "begin": "^([.'])[ \t]*(\\[)(.*?)\\s*(\\\\[\"#].*)?$",
-          "end": "^([.'])[ \t]*(\\])(.*?)(?=\\s|$|\\\\\")",
+          "begin": "^([.'])[ \t]*(\\[)\\s*([-$'\\w.\\\\]*?)\\s*(\\\\[\"#].*)?$",
+          "end": "^([.'])[ \t]*(\\])\\s*([-$'\\w.\\\\]*?)(?=\\s|$|\\\\\")",
           "contentName": "meta.citation.roff",
           "patterns": [
             {

data/grammars/text.shell-session.json

@@ -6,7 +6,7 @@
   ],
   "patterns": [
     {
-      "match": "(?x) ^ (?: ( (?:\\(\\S+\\))? (?: sh\\S*?                       | \\w+\\S+[@:]\\S+(?:\\s+\\S+)? | \\[\\S+[@:][^\\n]+\\].+ ) ) \\s* )? ( [>$#%] ) \\s+ (.*) $",
+      "match": "(?x) ^ (?: ( (?:\\(\\S+\\))? (?: sh\\S*?                       | \\w+\\S+[@:]\\S+(?:\\s+\\S+)? | \\[\\S+[@:][^\\n]+\\].+ ) ) \\s* )? ( [>$#%] | \\p{Greek} ) \\s+ (.*) $",
       "captures": {
         "1": {
           "name": "entity.other.prompt-prefix.shell-session"

data/lib/linguist/blob_helper.rb

@@ -275,10 +275,8 @@ module Linguist
           # also--importantly--without having to duplicate many (potentially
           # large) strings.
           begin
-            encoded_newlines = ["\r\n", "\r", "\n"].
-              map { |nl| nl.encode(ruby_encoding, "ASCII-8BIT").force_encoding(data.encoding) }
-
-            data.split(Regexp.union(encoded_newlines), -1)
+            
+            data.split(encoded_newlines_re, -1)
           rescue Encoding::ConverterNotFoundError
             # The data is not splittable in the detected encoding.  Assume it's
             # one big line.
@@ -289,6 +287,51 @@ module Linguist
         end
     end
 
+    def encoded_newlines_re
+      @encoded_newlines_re ||= Regexp.union(["\r\n", "\r", "\n"].
+                                              map { |nl| nl.encode(ruby_encoding, "ASCII-8BIT").force_encoding(data.encoding) })
+
+    end
+
+    def first_lines(n)
+      return lines[0...n] if defined? @lines
+      return [] unless viewable? && data
+
+      i, c = 0, 0
+      while c < n && j = data.index(encoded_newlines_re, i)
+        i = j + $&.length
+        c += 1
+      end
+      data[0...i].split(encoded_newlines_re, -1)
+    end
+
+    def last_lines(n)
+      if defined? @lines
+        if n >= @lines.length
+          @lines
+        else
+          lines[-n..-1]
+        end
+      end
+      return [] unless viewable? && data
+
+      no_eol = true
+      i, c = data.length, 0
+      k = i
+      while c < n && j = data.rindex(encoded_newlines_re, i - 1)
+        if c == 0 && j + $&.length == i
+          no_eol = false
+          n += 1
+        end
+        i = j
+        k = j + $&.length
+        c += 1
+      end
+      r = data[k..-1].split(encoded_newlines_re, -1)
+      r.pop if !no_eol
+      r
+    end
+
     # Public: Get number of lines of code
     #
     # Requires Blob#data

data/lib/linguist/classifier.rb

@@ -3,6 +3,8 @@ require 'linguist/tokenizer'
 module Linguist
   # Language bayesian classifier.
   class Classifier
+    CLASSIFIER_CONSIDER_BYTES = 50 * 1024
+
     # Public: Use the classifier to detect language of the blob.
     #
     # blob               - An object that quacks like a blob.
@@ -17,7 +19,7 @@ module Linguist
     # Returns an Array of Language objects, most probable first.
     def self.call(blob, possible_languages)
       language_names = possible_languages.map(&:name)
-      classify(Samples.cache, blob.data, language_names).map do |name, _|
+      classify(Samples.cache, blob.data[0...CLASSIFIER_CONSIDER_BYTES], language_names).map do |name, _|
         Language[name] # Return the actual Language objects
       end
     end