github-linguist 2.1.2 → 2.2.0

data/lib/linguist/samples.rb

@@ -45,6 +45,10 @@ module Linguist
               })
             end
           else
+            if File.extname(filename) == ""
+              raise "#{File.join(dirname, filename)} is missing an extension, maybe it belongs in filenames/ subdir"
+            end
+
             yield({
               :path     => File.join(dirname, filename),
               :language => category,
@@ -68,18 +72,16 @@ module Linguist
       each do |sample|
         language_name = sample[:language]
 
-        # TODO: For now skip empty extnames
-        if sample[:extname] && sample[:extname] != ""
+        if sample[:extname]
           db['extnames'][language_name] ||= []
           if !db['extnames'][language_name].include?(sample[:extname])
             db['extnames'][language_name] << sample[:extname]
           end
         end
 
-        # TODO: For now skip empty extnames
-        if fn = sample[:filename]
+        if sample[:filename]
           db['filenames'][language_name] ||= []
-          db['filenames'][language_name] << fn
+          db['filenames'][language_name] << sample[:filename]
         end
 
         data = File.read(sample[:path])

data/lib/linguist/tokenizer.rb

@@ -1,3 +1,5 @@
+require 'strscan'
+
 module Linguist
   # Generic programming language tokenizer.
   #
@@ -50,8 +52,13 @@ module Linguist
 
       tokens = []
       until s.eos?
+        if token = s.scan(/^#!.+$/)
+          if name = extract_shebang(token)
+            tokens << "SHEBANG#!#{name}"
+          end
+
         # Single line comment
-        if token = s.scan(START_SINGLE_LINE_COMMENT)
+        elsif token = s.scan(START_SINGLE_LINE_COMMENT)
           tokens << token.strip
           s.skip_until(/\n|\Z/)
 
@@ -64,19 +71,27 @@ module Linguist
 
         # Skip single or double quoted strings
         elsif s.scan(/"/)
-          s.skip_until(/[^\\]"/)
+          if s.peek(1) == "\""
+            s.getch
+          else
+            s.skip_until(/[^\\]"/)
+          end
         elsif s.scan(/'/)
-          s.skip_until(/[^\\]'/)
+          if s.peek(1) == "'"
+            s.getch
+          else
+            s.skip_until(/[^\\]'/)
+          end
 
         # Skip number literals
-        elsif s.scan(/(0x)?\d+/)
+        elsif s.scan(/(0x)?\d(\d|\.)*/)
 
         # SGML style brackets
         elsif token = s.scan(/<[^\s<>][^<>]*>/)
           extract_sgml_tokens(token).each { |t| tokens << t }
 
         # Common programming punctuation
-        elsif token = s.scan(/;|\{|\}|\(|\)/)
+        elsif token = s.scan(/;|\{|\}|\(|\)|\[|\]/)
           tokens << token
 
         # Regular token
@@ -95,6 +110,33 @@ module Linguist
       tokens
     end
 
+    # Internal: Extract normalized shebang command token.
+    #
+    # Examples
+    #
+    #   extract_shebang("#!/usr/bin/ruby")
+    #   # => "ruby"
+    #
+    #   extract_shebang("#!/usr/bin/env node")
+    #   # => "node"
+    #
+    # Returns String token or nil it couldn't be parsed.
+    def extract_shebang(data)
+      s = StringScanner.new(data)
+
+      if path = s.scan(/^#!\s*\S+/)
+        script = path.split('/').last
+        if script == 'env'
+          s.scan(/\s+/)
+          script = s.scan(/\S+/)
+        end
+        script = script[/[^\d]+/, 0]
+        return script
+      end
+
+      nil
+    end
+
     # Internal: Extract tokens from inside SGML tag.
     #
     # data - SGML tag String.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: github-linguist
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-23 00:00:00.000000000 Z
+date: 2012-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: charlock_holmes
@@ -133,6 +133,7 @@ files:
 - lib/linguist/blob_helper.rb
 - lib/linguist/classifier.rb
 - lib/linguist/file_blob.rb
+- lib/linguist/generated.rb
 - lib/linguist/language.rb
 - lib/linguist/languages.yml
 - lib/linguist/md5.rb